🐛 Describe the bug <div class="snippet-clipboard-content notranslate position-r

It would be great to get some more details of the issue: <ul dir="aut

<a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/us

DataParallel Multi-gpu training problem about super-gradients HOT 4 OPEN

JaehwanOh-R commented on August 30, 2024

DataParallel Multi-gpu training problem

from super-gradients.

Comments (4)

BloodAxe commented on August 30, 2024

It would be great to get some more details of the issue:

What model are you training
How are you launching training (torchrun, train_from_recipe, maybe your custom train.py)
Launching from console or notebook?
SG version

from super-gradients.

JaehwanOh-R commented on August 30, 2024

It would be great to get some more details of the issue:

What model are you training

How are you launching training (torchrun, train_from_recipe, maybe your custom train.py)

Launching from console or notebook?

SG version

I trained the YOLO-NAS-L model
I trained below codes

train_loader = dataloaders.get(name="coco2017_train_yolo_nas",
                               dataset_params={"data_dir": "/workspace/data/coco_dataset"},
                               dataloader_params={"num_workers": 2})
valid_loader = dataloaders.get(name="coco2017_val_yolo_nas",
                               dataset_params={"data_dir": "/workspace/data/coco_dataset"},
                               dataloader_params={"num_workers": 2})

train_params = {
    "max_epochs": 300,
    "warmup_mode": "linear_epoch_step",
    "warmup_initial_lr": 1e-6,
    "lr_warmup_steps": 1000,
    "lr_warmup_epochs": 0,
    "initial_lr": 2e-4,
    "lr_mode": "cosine",
    "cosine_final_lr_ratio": 0.1,
    "zero_weight_decay_on_bias_and_bn": True,
    "batch_accumulate": 1,
    "save_ckpt_epoch_list": [100, 200, 250],
    "loss": PPYoloELoss(use_static_assigner=False, num_classes=80, reg_max=16),
    "optimizer": "AdamW",
    "optimizer_params": {"weight_decay": 0.00001},
    "ema": True,
    "ema_params": {"decay": 0.9997, "decay_type": "threshold"},
    "mixed_precision": False,
    "sync_bn": True,
    
    "valid_metrics_list": [
        DetectionMetrics(score_thres=0.1, top_k_predictions=300, num_cls=80, normalize_targets=True,
                         post_prediction_callback=PPYoloEPostPredictionCallback(score_threshold=0.01,
                                                                                nms_top_k=1000, max_predictions=300,
                                                                                nms_threshold=0.7))],
    "metric_to_watch": '[email protected]:0.95',
    "greater_metric_to_watch_is_better": True}

setup_device(multi_gpu='DP', num_gpus=2)

trainer = Trainer(experiment_name="yolo_nas_l_coco", ckpt_root_dir="./out/sg_checkpoints_dir/")
net = models.get(Models.YOLO_NAS_L, num_classes=80)
trainer.train(model=net, training_params=train_params, train_loader=train_loader, valid_loader=valid_loader)