Skip to content

Commit

Permalink
fixed convergence by replacing the frozen BN for regular BN layers
Browse files Browse the repository at this point in the history
  • Loading branch information
aldo committed Nov 20, 2020
1 parent 39f6e84 commit 278ae05
Show file tree
Hide file tree
Showing 7 changed files with 79 additions and 66 deletions.
3 changes: 2 additions & 1 deletion aoe
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
'python', '-m', 'easy_attributes.do_train', '--input_data', '/disk1/mcs_physics_data_derender/', '--output_dir', 'bayesian_opt/output', '--bo_config_file', 'bayesian_opt/output/bo_cfg.yml', '--distributed'
'python', '-m', 'easy_attributes.do_train', '--input_data', '/disk1/mcs_physics_data_derender/', '--output_dir', 'bayesian_opt/output', '--bo_config_file', 'bayesian_opt/output/bo_cfg.yml', '--distributed'
detectron2/solver/build.py:147
11 changes: 6 additions & 5 deletions bayesian_opt/do_bayes_opt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import sys
from argparse import Namespace
from collections import defaultdict
from math import ceil
from pathlib import Path

from dragonfly.apis.api_utils import preprocess_multifidelity_arguments
Expand Down Expand Up @@ -43,7 +44,7 @@ def setup_opt(config):
"SOLVER.MOMENTUM": lambda x: float(1 - 10.0 ** x),
"SOLVER.ADAM_BETA": lambda x: float(1 - 10.0 ** x),
# "SOLVER.OPT_TYPE": lambda x: str(x),
"SOLVER.IMS_PER_BATCH": lambda x: int(2 ** x),
"SOLVER.IMS_PER_BATCH": lambda x: ceil(2 ** x / 2.) * 2,
"SOLVER.MAX_TIME_SECS": lambda x: int(x),
"DATALOADER.NUM_WORKERS": lambda x: int(2 ** x)})

Expand Down Expand Up @@ -89,9 +90,9 @@ def parse_args():
# parser.add_argument('--num_input_channels', type=int)
return parser.parse_args()

def evaluate_objective():
command = f'python -m easy_attributes.do_train --input_data /disk1/mcs_physics_data_derender/ --output_dir bayesian_opt/output --bo_config_file bayesian_opt/output/bo_cfg.yml --distributed'
return os.system(command)
# def evaluate_objective():
# command = f'python -m easy_attributes.do_train --input_data /disk1/mcs_physics_data_derender/ --output_dir bayesian_opt/output --bo_config_file bayesian_opt/output/bo_cfg.yml --distributed'
# return os.system(command)

if __name__ == "__main__":
config = load_config_file('bayesian_opt/params_domain.json')
Expand All @@ -113,7 +114,7 @@ def evaluate_objective():
# except Exception:
# ret_code = 1
try:
subprocess.run(['python', '-m', 'easy_attributes.do_train', '--input_data', '/disk1/mcs_physics_data_derender/', '--output_dir', 'bayesian_opt/output', '--bo_config_file', 'bayesian_opt/output/bo_cfg.yml', '--distributed'])
subprocess.run(['python', '-m', 'easy_attributes.do_train', '--input_data', args.input_data, '--output_dir', 'bayesian_opt/output', '--bo_config_file', 'bayesian_opt/output/bo_cfg.yml', '--distributed'])
ret_code = 0
except:
ret_code = 1
Expand Down
16 changes: 11 additions & 5 deletions data_processing/intphys_process.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import argparse
import os
import random
import shutil
import sys
from itertools import chain
Expand All @@ -11,14 +12,16 @@
from pycocotools import mask as mask_util
from PIL import Image



sys.path.insert(0, './')
from easy_attributes.utils.io import read_serialized, write_serialized
from easy_attributes.utils.istarmap_tqdm_patch import array_apply
from easy_attributes.utils.meta_data import get_discrete_metadata, get_pixels_mean_and_std
from easy_attributes.utils.visualize import visualize_data_dict

MIN_AREA = 50


def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--input_dir', type=str)
Expand Down Expand Up @@ -93,7 +96,6 @@ def process_frame(depth_file, mask_file, rgb_file, frame, out_dir, index):
**attributes,
'filename': str(input_file),
})

return objects


Expand All @@ -109,7 +111,8 @@ def process_video(video_path, vid_num, out_dir):
masks.append(video_path / 'masks' / ('masks_' + str(i).zfill(3) + '.png'))
frames.append(status['frames'][i - 1])

objects = [process_frame(d, m, r, f, out_dir, vid_num * 1000 + f_num) for f_num, (d, m, r, f) in
objects = [process_frame(d, m, r, f, out_dir, vid_num * 1000 + f_num)
for f_num, (d, m, r, f) in
enumerate(zip(depths, masks, rgbs, frames))]

return chain.from_iterable(objects)
Expand All @@ -125,13 +128,16 @@ def process_video(video_path, vid_num, out_dir):

data = {}
for f_set, folders in {'val': video_folders_val, 'train': video_folders_train}.items():
worker_args = [(v, i, args.output_dir) for i, v in enumerate(folders)]
worker_args = [(v,
i + len(video_folders_val) if f_set == 'train' else i,
args.output_dir) for i, v in enumerate(folders)]

objects = list(chain.from_iterable(array_apply(process_video,
# random.sample(worker_args,10),
worker_args,
args.parallel,
# cpu_frac=2,
chunksize=10,
chunksize=100,
description='processing intphys scenes')))

data[f_set] = objects
Expand Down
5 changes: 3 additions & 2 deletions data_processing/mcs/mcs_process.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

sys.path.insert(0, './')
from easy_attributes.utils.io import write_serialized
from easy_attributes.utils.meta_data import get_continuous_metadata, get_discrete_metadata
from easy_attributes.utils.meta_data import get_continuous_metadata, get_discrete_metadata, get_pixels_mean_and_std

MIN_AREA = 100

Expand Down Expand Up @@ -209,7 +209,8 @@ def ParallelSceneProcess(work_q: Queue, result_q: Queue, mcs_executable: Path, c
meta_data = {'inputs': {'file_name': {'type': 'input_tensor',
'num_channels': 4,
'height': 400,
'width': 600},
'width': 600,
**get_pixels_mean_and_std(val_dicts)},
'mask': {'type': 'bitmask'},
'bbox': {'type': 'bounding_box'}},
'outputs': {**{e: get_continuous_metadata(val_dicts, e)
Expand Down
80 changes: 41 additions & 39 deletions easy_attributes/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ def get_config(data_path: Path,
cfg.MODEL.META_ARCHITECTURE = 'CustomModel'

if model_weights_path is None:
# cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml")
cfg.MODEL.WEIGHTS = 'https://dl.fbaipublicfiles.com/detectron2/ImageNetPretrained/MSRA/R-50.pkl'
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml")
# cfg.MODEL.WEIGHTS = 'https://dl.fbaipublicfiles.com/detectron2/ImageNetPretrained/MSRA/R-50.pkl'
else:
cfg.MODEL.WEIGHTS = str(model_weights_path)

Expand All @@ -41,63 +41,65 @@ def get_config(data_path: Path,
# 'agent_position_y',
# 'agent_position_z',
# 'agent_rotation',
# 'dimension_0_x',
# 'dimension_0_y',
# 'dimension_0_z',
# 'dimension_1_x',
# 'dimension_1_y',
# 'dimension_1_z',
# 'dimension_2_x',
# 'dimension_2_y',
# 'dimension_2_z',
# 'dimension_3_x',
# 'dimension_3_y',
# 'dimension_3_z',
# 'dimension_4_x',
# 'dimension_4_y',
# 'dimension_4_z',
# 'dimension_5_x',
# 'dimension_5_y',
# 'dimension_5_z',
# 'dimension_6_x',
# 'dimension_6_y',
# 'dimension_6_z',
# 'dimension_7_x',
# 'dimension_7_y',
# 'dimension_7_z',
# 'position_x',
# 'position_y',
# 'position_z',
# 'rotation_x',
# 'rotation_y',
# 'rotation_z',
'dimension_0_x',
'dimension_0_y',
'dimension_0_z',
'dimension_1_x',
'dimension_1_y',
'dimension_1_z',
'dimension_2_x',
'dimension_2_y',
'dimension_2_z',
'dimension_3_x',
'dimension_3_y',
'dimension_3_z',
'dimension_4_x',
'dimension_4_y',
'dimension_4_z',
'dimension_5_x',
'dimension_5_y',
'dimension_5_z',
'dimension_6_x',
'dimension_6_y',
'dimension_6_z',
'dimension_7_x',
'dimension_7_y',
'dimension_7_z',
'position_x',
'position_y',
'position_z',
'rotation_x',
'rotation_y',
'rotation_z',
'shape',)

cfg.DEBUG = debug

metadata = read_serialized(data_path / 'metadata.yml')
num_input_channels = metadata['inputs']['file_name']['num_channels']
input_meta = metadata['inputs']['file_name']
num_input_channels = input_meta['num_channels']
num_repeats = sum([use_mask, use_bounding_box]) + 1
num_input_channels *= num_repeats

cfg.INPUT.FORMAT = "D" * num_input_channels
cfg.MODEL.PIXEL_MEAN = metadata['inputs']['file_name']['pixel_mean'] * num_repeats
cfg.MODEL.PIXEL_STD = metadata['inputs']['file_name']['pixel_std'] * num_repeats
cfg.MODEL.PIXEL_MEAN = input_meta['pixel_mean'] * num_repeats if 'pixel_mean' in input_meta else [0.5] * num_input_channels
cfg.MODEL.PIXEL_STD = input_meta['pixel_std'] * num_repeats if 'pixel_std' in input_meta else [1.0] * num_input_channels

cfg.MODEL.BACKBONE.FREEZE_AT = 0
cfg.MODEL.BACKBONE.NAME = 'build_resnet_backbone'
cfg.MODEL.RESNETS.OUT_FEATURES = ['res5']
cfg.MODEL.BACKBONE.NAME = 'build_resnet_fpn_backbone'
# cfg.MODEL.RESNETS.OUT_FEATURES = ['res5']
cfg.MODEL.RESNETS.NORM = "BN"
# cfg.MODEL.FPN_OUT_FEATS = ('p2', 'p3', 'p4', 'p5', 'p6')
cfg.MODEL.LAST_HIDDEN_LAYER_FEATS = 512
cfg.MODEL.POOLER_TYPE = 'max'

cfg.SOLVER.WARMUP_FACTOR = 1.0 / 100
cfg.SOLVER.WARMUP_ITERS = 100 # a warm up is necessary to avoid diverging training while keeping the goal learning rate as high as possible
cfg.SOLVER.IMS_PER_BATCH = 20 if not debug else 8
cfg.SOLVER.IMS_PER_BATCH = 50 if not debug else 8
# cfg.SOLVER.MAX_ITER = 80000
cfg.SOLVER.STEPS = (30000, 45000, 60000)
cfg.SOLVER.GAMMA = 0.5 # after each milestone in SOLVER.STEPS gets reached, the learning rate gets scaled by Gamma.
cfg.SOLVER.BASE_LR = 6.658777172739463e-5 / 4
cfg.SOLVER.BASE_LR = 6.658777172739463e-5

cfg.SOLVER.OPT_TYPE = "Adam" # options "Adam" "SGD"
cfg.SOLVER.MOMENTUM = 0.9960477666835778 # found via Bayesian Optimization
Expand Down
29 changes: 15 additions & 14 deletions easy_attributes/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def loss(self, inputs, targets):

loss = loss_method(l_input, l_target)

loss_dict[term] = loss
loss_dict[term] = loss/len(self.terms)

# loss_dict["total_loss"] = sum(loss_dict.values())
return loss_dict
Expand Down Expand Up @@ -144,33 +144,33 @@ def __init__(self,
input_height: int,
input_width: int,
# batch_size: int,
resnet_features: List[str],
# resnet_features: List[str],
last_hid_num_feats: int,
pooler_type: str
):
super().__init__()
self.backbone = backbone
self.backbone_features = resnet_features
self.backbone_features = backbone.output_shape().keys()

feat_strides = {p: backbone.output_shape()[p].stride for p in resnet_features}
feat_strides = {p: v.stride for p,v in backbone.output_shape().items()}
feat_heights = {p: ceil(input_height / fs) for p, fs in feat_strides.items()}
feat_widths = {p: ceil(input_width / fs) for p, fs in feat_strides.items()}

p_funcs = {'average': nn.AvgPool2d,
'max': nn.MaxPool2d}
self.poolers = {p: p_funcs[pooler_type]((feat_heights[p], feat_widths[p]))
for p in resnet_features}
for p in backbone.output_shape().keys()}

self.sum_feat_channels = sum([backbone.output_shape()[p].channels for p in resnet_features])
self.sum_feat_channels = sum([v.channels for v in backbone.output_shape().values()])

self.output_network = nn.Sequential(nn.Linear(self.sum_feat_channels, last_hid_num_feats),
nn.ReLU(),
nn.Linear(last_hid_num_feats, len(output_head)))

self.output_head = output_head

self.register_buffer("pixel_mean", torch.Tensor(pixel_mean).view(-1, 1, 1))
self.register_buffer("pixel_std", torch.Tensor(pixel_std).view(-1, 1, 1))
self.register_buffer("pixel_mean", torch.Tensor(pixel_mean).view(1, -1, 1, 1))
self.register_buffer("pixel_std", torch.Tensor(pixel_std).view(1, -1, 1, 1))
assert (
self.pixel_mean.shape == self.pixel_std.shape
), f"{self.pixel_mean} and {self.pixel_std} have different shapes!"
Expand All @@ -186,7 +186,7 @@ def from_config(cls, cfg):
"input_height": MetadataCatalog.get(cfg.DATASETS.TEST[0]).inputs['file_name']['height'],
"input_width": MetadataCatalog.get(cfg.DATASETS.TEST[0]).inputs['file_name']['width'],
# "batch_size": get_batch_size(cfg.SOLVER.IMS_PER_BATCH),
"resnet_features": cfg.MODEL.RESNETS.OUT_FEATURES,
# "resnet_features": cfg.MODEL.RESNETS.OUT_FEATURES,
"last_hid_num_feats": cfg.MODEL.LAST_HIDDEN_LAYER_FEATS,
'pooler_type': cfg.MODEL.POOLER_TYPE
}
Expand All @@ -198,7 +198,7 @@ def device(self):
def predict_only(self, batched_inputs):
inputs = self.preprocess_input([batched_inputs['inputs']['input_tensor']])

features = self.backbone(inputs.tensor[0])
features = self.backbone(inputs)

features = [self.poolers[p](features[p]) for p in self.backbone_features]

Expand Down Expand Up @@ -253,7 +253,8 @@ def preprocess_input(self, input_tensor):
"""
Normalize, pad and batch the input images.
"""
images = [input_tensor[0].to(self.device)]
images = [(x - self.pixel_mean) / self.pixel_std for x in images]
images = ImageList.from_tensors(images, self.backbone.size_divisibility)
return images
input_tensor = input_tensor[0].to(self.device)
input_tensor = (input_tensor - self.pixel_mean) / self.pixel_std
input_tensor = ImageList.from_tensors([input_tensor], self.backbone.size_divisibility)
input_tensor = input_tensor.tensor[0]
return input_tensor
1 change: 1 addition & 0 deletions easy_attributes/utils/visualize.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ def visualize_data_dict(d, save_path: Path = None, channels=(0, 1, 2), input_ten
assert img.shape[0] == 3
img = img.swapaxes(0, 1).swapaxes(1, 2)
img = img.astype(np.uint8)
MetadataCatalog.get('val').set(thing_classes=['object'])
visualizer = Visualizer(img, metadata=MetadataCatalog.get('val'))
out = visualizer.draw_dataset_dict({"annotations": [{"bbox": d['bbox'],
"bbox_mode": int(BoxMode.XYXY_ABS),
Expand Down

0 comments on commit 278ae05

Please sign in to comment.