From 07c3ae4df44c6139dfe057e3175abed2821362f0 Mon Sep 17 00:00:00 2001 From: uermel Date: Mon, 8 Jun 2026 18:41:12 -0700 Subject: [PATCH 1/2] fix DDP race --- ddw/utils/unet.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ddw/utils/unet.py b/ddw/utils/unet.py index 6ad1eb2..7409d92 100644 --- a/ddw/utils/unet.py +++ b/ddw/utils/unet.py @@ -171,10 +171,17 @@ def update_hparam(self, hparam, value): """ Update a hyperparameter in the hparams.yaml file. """ + # DDP-safe: only the global-zero rank writes hparams.yaml. With more + # than one rank, every rank racing open(hparams_file, "w") truncated the + # file, so a concurrent reader saw an empty file (yaml.safe_load -> None + # -> "'NoneType' object does not support item assignment") or a partial + # write (ScannerError). `or {}` guards a transient empty read. + if getattr(self, "global_rank", 0) != 0: + return logger = self.trainer.logger logdir = f"{logger.save_dir}/{logger.name}/version_{logger.version}" hparams_file = f"{logdir}/hparams.yaml" - hparams = yaml.safe_load(open(hparams_file, "r")) + hparams = yaml.safe_load(open(hparams_file, "r")) or {} hparams[hparam] = value with open(hparams_file, "w") as f: yaml.dump(hparams, f) From 298deb13bde76f07936d3594ee1023a726370145 Mon Sep 17 00:00:00 2001 From: uermel Date: Mon, 8 Jun 2026 21:55:14 -0700 Subject: [PATCH 2/2] less verbose comment, remove potential silent error --- ddw/utils/unet.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ddw/utils/unet.py b/ddw/utils/unet.py index 7409d92..c1f5d59 100644 --- a/ddw/utils/unet.py +++ b/ddw/utils/unet.py @@ -171,17 +171,13 @@ def update_hparam(self, hparam, value): """ Update a hyperparameter in the hparams.yaml file. """ - # DDP-safe: only the global-zero rank writes hparams.yaml. With more - # than one rank, every rank racing open(hparams_file, "w") truncated the - # file, so a concurrent reader saw an empty file (yaml.safe_load -> None - # -> "'NoneType' object does not support item assignment") or a partial - # write (ScannerError). `or {}` guards a transient empty read. + # DDP-safe: only the global-zero rank writes hparams.yaml. if getattr(self, "global_rank", 0) != 0: return logger = self.trainer.logger logdir = f"{logger.save_dir}/{logger.name}/version_{logger.version}" hparams_file = f"{logdir}/hparams.yaml" - hparams = yaml.safe_load(open(hparams_file, "r")) or {} + hparams = yaml.safe_load(open(hparams_file, "r")) hparams[hparam] = value with open(hparams_file, "w") as f: yaml.dump(hparams, f)