henry000 commited on
Commit
5be4e0e
Β·
1 Parent(s): 70a1547

πŸ› [Fix] DDP bugs, sync mAP on diff rank

Browse files
Files changed (1) hide show
  1. yolo/tools/solver.py +3 -4
yolo/tools/solver.py CHANGED
@@ -147,7 +147,7 @@ class ModelTrainer:
147
  self.progress.finish_one_epoch(epoch_loss, epoch_idx=epoch_idx)
148
 
149
  mAPs = self.validator.solve(self.validation_dataloader, epoch_idx=epoch_idx)
150
- if mAPs is not None and self.good_epoch(mAPs):
151
  self.save_checkpoint(epoch_idx=epoch_idx)
152
  # TODO: save model if result are better than before
153
  self.progress.finish_train()
@@ -256,9 +256,8 @@ class ModelValidator:
256
 
257
  with open(self.json_path, "w") as f:
258
  predict_json = collect_prediction(predict_json, self.progress.local_rank)
259
- if self.progress.local_rank != 0:
260
- return
261
- json.dump(predict_json, f)
262
  if hasattr(self, "coco_gt"):
263
  self.progress.start_pycocotools()
264
  result = calculate_ap(self.coco_gt, predict_json)
 
147
  self.progress.finish_one_epoch(epoch_loss, epoch_idx=epoch_idx)
148
 
149
  mAPs = self.validator.solve(self.validation_dataloader, epoch_idx=epoch_idx)
150
+ if self.good_epoch(mAPs):
151
  self.save_checkpoint(epoch_idx=epoch_idx)
152
  # TODO: save model if result are better than before
153
  self.progress.finish_train()
 
256
 
257
  with open(self.json_path, "w") as f:
258
  predict_json = collect_prediction(predict_json, self.progress.local_rank)
259
+ if self.progress.local_rank == 0:
260
+ json.dump(predict_json, f)
 
261
  if hasattr(self, "coco_gt"):
262
  self.progress.start_pycocotools()
263
  result = calculate_ap(self.coco_gt, predict_json)