I want to train SE-SSD on three class at the same time. I have modified your code. But I met a problem, and don't know how to solve it.
File "train.py", line 120, in <module>
main()
File "train.py", line 117, in main
train_detector(model, datasets, cfg, distributed=distributed, validate=args.validate, logger=logger,)
File "/data/SE-SSD/det3d/torchie/apis/train_sessd.py", line 324, in train_detector
trainer.run(data_loaders, cfg.workflow, cfg.total_epochs, local_rank=cfg.local_rank)
File "/data/SE-SSD/det3d/torchie/trainer/trainer_sessd.py", line 473, in run
epoch_runner(data_loaders[0], data_loaders[1], self.epoch, **kwargs)
File "/data/SE-SSD/det3d/torchie/trainer/trainer_sessd.py", line 347, in train
outputs = self.batch_processor_inline(self.model, self.model_ema, data_batch, consistency_weight, train_mode=True, **kwargs)
File "/data/SE-SSD/det3d/torchie/trainer/trainer_sessd.py", line 267, in batch_processor_inline
losses = model(example, is_ema=[False, output_ema], return_loss=True)
File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 550, in __call__
result = self.forward(*input, **kwargs)
File "/data/SE-SSD/det3d/models/detectors/voxelnet_sessd.py", line 41, in forward
return self.bbox_head.loss(example, preds, is_ema[1])
File "/data/SE-SSD/det3d/models/bbox_heads/mg_head_sessd.py", line 709, in loss
consistency_loss = self.consistency_loss(preds_dicts, preds_ema, example)
File "/data/SE-SSD/det3d/models/bbox_heads/mg_head_sessd.py", line 679, in consistency_loss
box_consistency_loss, idx1, idx2, mask1, mask2 = self.nn_distance(top_box_preds_stu, top_box_preds_tea)
File "/data/SE-SSD/det3d/models/bbox_heads/mg_head_sessd.py", line 578, in nn_distance
ans_iou = iou3d_utils.boxes_iou_bev_gpu(box1, box2)
RuntimeError: copy_if failed to synchronize: cudaErrorIllegalAddress: an illegal memory access was encountered
I have located the location of the program error. It's the following function. It throwed the error message at the beginning of training.
I don't know why this mistake happened. Looking forward yo your reply!
import itertools
import logging
from pathlib import Path
from det3d.builder import build_box_coder
from det3d.utils.config_tool import get_downsample_factor
data_root_prefix = "/data/SE-SSD/datasets"
# norm_cfg = dict(type='SyncBN', eps=1e-3, momentum=0.01)
norm_cfg = None
my_paras = dict(
batch_size=1,
data_mode="train", # "train" or "trainval": the set to train the model;
enable_ssl=True, # Ensure "False" in CIA-SSD training
eval_training_set=False, # True: eval on "data_mode" set; False: eval on validation set.[Ensure "False" in training; Switch in Testing]
# unused
enable_difficulty_level=False,
remove_difficulty_points=False, # act with neccessary condition: enable_difficulty_level=True.
gt_random_drop=-1,
data_aug_random_drop=-1,
far_points_first=False,
data_aug_with_context=-1, # enlarged size for w and l in data aug.
gt_aug_with_context=-1,
gt_aug_similar_type=False,
min_points_in_gt=-1,
loss_iou=None,
)
tasks = [
dict(num_class=1, class_names=["Car"]),
dict(num_class=1, class_names=["Pedestrian"]),
dict(num_class=1, class_names=["Cyclist"]),
]
class_names = list(itertools.chain(*[t["class_names"] for t in tasks]))
# training and testing settings
target_assigner = dict(
type="iou",
anchor_generators=[
dict(
type="anchor_generator_range",
sizes=[1.6, 3.9, 1.56],
anchor_ranges=[0, -40.0, -1.0, 70.4, 40.0, -1.0],
rotations=[0, 1.57],
matched_threshold=0.6,
unmatched_threshold=0.45,
class_name="Car",
),
dict(
type="anchor_generator_range",
sizes=[0.6, 0.8, 1.73],
anchor_ranges=[0, -40.0, -0.6, 70.4, 40.0, -0.6],
rotations=[0, 1.57],
matched_threshold=0.4,
unmatched_threshold=0.2,
class_name="Pedestrian",
),
dict(
type="anchor_generator_range",
sizes=[0.6, 1.76, 1.73],
anchor_ranges=[0, -40.0, -0.6, 70.4, 40.0, -0.6],
rotations=[0, 1.57],
matched_threshold=0.4,
unmatched_threshold=0.2,
class_name="Cyclist",
),
],
sample_positive_fraction=-1,
sample_size=512,
region_similarity_calculator=dict(type="nearest_iou_similarity",),
pos_area_threshold=-1,
tasks=tasks,
)
box_coder = dict(
type="ground_box3d_coder", n_dim=7, linear_dim=False, encode_angle_vector=False,
)
# model settings
model = dict(
type="VoxelNet",
pretrained=None,
reader=dict(
type="VoxelFeatureExtractorV3",
# type='SimpleVoxel',
num_input_features=4,
norm_cfg=norm_cfg,
),
backbone=dict(
type="SpMiddleFHD", num_input_features=4, ds_factor=8, norm_cfg=norm_cfg,
),
neck=dict(
type="SSFA",
layer_nums=[5,],
ds_layer_strides=[1,],
ds_num_filters=[128,],
us_layer_strides=[1,],
us_num_filters=[128,],
num_input_features=128,
norm_cfg=norm_cfg,
logger=logging.getLogger("RPN"),
),
bbox_head=dict(
# type='RPNHead',
type="MultiGroupHead",
mode="3d",
in_channels=sum([128,]),
norm_cfg=norm_cfg,
tasks=tasks,
weights=[1,],
box_coder=build_box_coder(box_coder),
encode_background_as_zeros=True,
loss_norm=dict(
type="NormByNumPositives", pos_cls_weight=1.0, neg_cls_weight=1.0,
),
loss_cls=dict(type="SigmoidFocalLoss", alpha=0.25, gamma=2.0, loss_weight=1.0,),
use_sigmoid_score=True,
loss_bbox=dict(
type="WeightedSmoothL1Loss",
sigma=3.0,
code_weights=[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
codewise=True,
loss_weight=2.0,
),
encode_rad_error_by_sin=True,
loss_aux=dict(
type="WeightedSoftmaxClassificationLoss",
name="direction_classifier",
loss_weight=0.2,
),
direction_offset=0.0,
),
)
assigner = dict(
box_coder=box_coder,
target_assigner=target_assigner,
out_size_factor=get_downsample_factor(model),
debug=False,
)
train_cfg = dict(assigner=assigner)
test_cfg = dict(
nms=dict(
use_rotate_nms=True,
use_multi_class_nms=False,
nms_pre_max_size=1000,
nms_post_max_size=100,
nms_iou_threshold=0.01,
),
score_threshold=0.3,
post_center_limit_range=[0, -40.0, -5.0, 70.4, 40.0, 5.0],
max_per_img=100,
)
# dataset settings
dataset_type = "KittiDataset"
data_root = data_root_prefix + "/KITTI/"
train_anno = data_root_prefix + "/KITTI/kitti_infos_train.pkl"
val_anno = data_root_prefix + "/KITTI/kitti_infos_val.pkl"
test_anno = data_root_prefix + "/KITTI/kitti_infos_test.pkl"
trainval_anno = data_root_prefix + "/KITTI/kitti_infos_trainval.pkl"
db_sampler = dict(
type="GT-AUG",
enable=True,
db_info_path=data_root_prefix + "/KITTI/dbinfos_train.pkl",
sample_groups=[dict(Car=15), dict(Pedestrian=8), dict(Cyclist=8),],
db_prep_steps=[
dict(filter_by_min_num_points=dict(Car=5, Pedestrian=5, Cyclist=5)),
dict(filter_by_difficulty=[-1],),
],
global_random_rotation_range_per_object=[0, 0],
rate=1.0,
gt_random_drop=my_paras['gt_random_drop'],
gt_aug_with_context=my_paras['gt_aug_with_context'],
gt_aug_similar_type=my_paras['gt_aug_similar_type'],
)
train_preprocessor = dict(
mode="train",
shuffle_points=True,
gt_loc_noise=[1.0, 1.0, 0.5],
gt_rot_noise=[-0.785, 0.785],
global_rot_noise=[-0.785, 0.785],
global_scale_noise=[0.95, 1.05],
global_rot_per_obj_range=[0, 0],
global_trans_noise=[0.0, 0.0, 0.0],
remove_points_after_sample=True,
gt_drop_percentage=0.0,
gt_drop_max_keep_points=15,
remove_unknown_examples=False,
remove_environment=False,
db_sampler=db_sampler,
class_names=class_names,
)
val_preprocessor = dict(
mode="val",
shuffle_points=False,
remove_environment=False,
remove_unknown_examples=False,
)
voxel_generator = dict(
range=[0, -40.0, -3.0, 70.4, 40.0, 1.0],
voxel_size=[0.05, 0.05, 0.1],
max_points_in_voxel=5,
max_voxel_num=40000,
far_points_first=my_paras['far_points_first'],
)
train_pipeline = [
dict(type="LoadPointCloudFromFile"),
dict(type="LoadPointCloudAnnotations", with_bbox=True),
dict(type="Preprocess", cfg=train_preprocessor),
dict(type="Voxelization", cfg=voxel_generator),
dict(type="AssignTarget", cfg=train_cfg["assigner"]),
dict(type="Reformat"),
# dict(type='PointCloudCollect', keys=['points', 'voxels', 'annotations', 'calib']),
]
test_pipeline = [
dict(type="LoadPointCloudFromFile"),
dict(type="LoadPointCloudAnnotations", with_bbox=True),
dict(type="Preprocess", cfg=val_preprocessor),
dict(type="Voxelization", cfg=voxel_generator),
dict(type="AssignTarget", cfg=train_cfg["assigner"]),
dict(type="Reformat"),
]
data = dict(
samples_per_gpu=my_paras['batch_size'], # batch_size
workers_per_gpu=1,
train=dict(
type=dataset_type,
root_path=data_root,
info_path=data_root + "/kitti_infos_train.pkl",
ann_file=train_anno,
class_names=class_names,
pipeline=train_pipeline,
),
val=dict(
type=dataset_type,
root_path=data_root,
info_path=data_root + "/kitti_infos_val.pkl",
ann_file=val_anno,
class_names=class_names,
pipeline=test_pipeline,
),
test=dict(
type=dataset_type,
root_path=data_root,
info_path=test_anno,
ann_file=test_anno,
class_names=class_names,
pipeline=test_pipeline,
),
train_unlabel_val=dict(
type=dataset_type,
root_path=data_root,
info_path=val_anno,
class_names=class_names,
pipeline=train_pipeline,
labeled=False,
),
train_unlabel_test=dict(
type=dataset_type,
root_path=data_root,
info_path=test_anno,
class_names=class_names,
pipeline=train_pipeline,
labeled=False,
),
)
# optimizer
optimizer = dict(
type="adam", amsgrad=0.0, wd=0.01, fixed_wd=True, moving_average=False,
)
"""training hooks """
optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
# learning policy in training hooks
lr_config = dict(
type="one_cycle", lr_max=0.003, moms=[0.95, 0.85], div_factor=10.0, pct_start=0.4,
)
checkpoint_config = dict(interval=1) # 几个epoch保存一次
# yapf:disable
log_config = dict(
interval=20,
hooks=[
dict(type="TextLoggerHook"),
# dict(type='TensorboardLoggerHook')
],
)
# yapf:enable
# runtime settings
TAG = 'se-ssd-3class'
total_epochs = 300
device_ids = range(8)
dist_params = dict(backend="nccl", init_method="env://")
log_level = "INFO"
work_dir = "/data/SE-SSD/datasets/" + TAG
load_from = None #预训练
resume_from = None
workflow = [("train", 5), ("val", 1)]
save_file = False if TAG == "debug" or TAG == "exp_debug" or Path(work_dir, "Det3D").is_dir() else True