π [Fix] DDP bugs, sync mAP on diff rank
Browse files- yolo/tools/solver.py +3 -4
yolo/tools/solver.py
CHANGED
@@ -147,7 +147,7 @@ class ModelTrainer:
|
|
147 |
self.progress.finish_one_epoch(epoch_loss, epoch_idx=epoch_idx)
|
148 |
|
149 |
mAPs = self.validator.solve(self.validation_dataloader, epoch_idx=epoch_idx)
|
150 |
-
if
|
151 |
self.save_checkpoint(epoch_idx=epoch_idx)
|
152 |
# TODO: save model if result are better than before
|
153 |
self.progress.finish_train()
|
@@ -256,9 +256,8 @@ class ModelValidator:
|
|
256 |
|
257 |
with open(self.json_path, "w") as f:
|
258 |
predict_json = collect_prediction(predict_json, self.progress.local_rank)
|
259 |
-
if self.progress.local_rank
|
260 |
-
|
261 |
-
json.dump(predict_json, f)
|
262 |
if hasattr(self, "coco_gt"):
|
263 |
self.progress.start_pycocotools()
|
264 |
result = calculate_ap(self.coco_gt, predict_json)
|
|
|
147 |
self.progress.finish_one_epoch(epoch_loss, epoch_idx=epoch_idx)
|
148 |
|
149 |
mAPs = self.validator.solve(self.validation_dataloader, epoch_idx=epoch_idx)
|
150 |
+
if self.good_epoch(mAPs):
|
151 |
self.save_checkpoint(epoch_idx=epoch_idx)
|
152 |
# TODO: save model if result are better than before
|
153 |
self.progress.finish_train()
|
|
|
256 |
|
257 |
with open(self.json_path, "w") as f:
|
258 |
predict_json = collect_prediction(predict_json, self.progress.local_rank)
|
259 |
+
if self.progress.local_rank == 0:
|
260 |
+
json.dump(predict_json, f)
|
|
|
261 |
if hasattr(self, "coco_gt"):
|
262 |
self.progress.start_pycocotools()
|
263 |
result = calculate_ap(self.coco_gt, predict_json)
|