Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion deepspeed/runtime/zero/stage3.py
Original file line number Diff line number Diff line change
Expand Up @@ -4011,7 +4011,9 @@ def load_hp_checkpoint_state(self, folder, key, param=None):
# AutoTP + ZeRO-3: the universal tensor is the FULL weight, but each tensor-parallel
# rank only owns its TP shard, and ZeRO then partitions that shard across the DP group.
# Resolve the TP shard first so the DP partition below operates on the per-rank shard.
if param is not None and isinstance(ckpt_dict, dict):
# A checkpoint converted from ZeRO-3 without TP stores a bare tensor, so the target's
# metadata, not the file format, decides whether to slice.
if param is not None:
from deepspeed.checkpoint.universal_checkpoint import _get_param_uc_restore_meta, _resolve_autotp_partition
if _get_param_uc_restore_meta(param):
tp_group = groups.get_tensor_model_parallel_group()
Expand Down
36 changes: 34 additions & 2 deletions tests/unit/checkpoint/test_autotp_uc_checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -1585,13 +1585,13 @@ def forward(self, x):
return self.fc2(torch.tanh(self.fc1(x)))


def _affine_resume_engine(tp_size, load_universal=False):
def _affine_resume_engine(tp_size, load_universal=False, zero_stage=1):
torch.manual_seed(1234)
model = AffineResumeModel()
config = {
"train_micro_batch_size_per_gpu": 2,
"zero_optimization": {
"stage": 1
"stage": zero_stage
},
"zero_allow_untested_optimizer": True,
"checkpoint": {
Expand Down Expand Up @@ -1735,6 +1735,38 @@ def test_resume_matches_uninterrupted_training(self, affine_resume_checkpoint, t
engine.destroy()


class zero3_no_tp_checkpoint(DistributedFixture):
world_size = 1

def run(self, tmpdir):
engine = _affine_resume_engine(1, zero_stage=3)
for step in range(4):
_affine_resume_step(engine, 1, step)
engine.save_checkpoint(tmpdir, tag="zero3_no_tp")
_convert_to_universal(os.path.join(tmpdir, "zero3_no_tp"), os.path.join(tmpdir, "zero3_universal"))
torch.save(_affine_resume_state(engine, 1), os.path.join(tmpdir, "zero3_saved_state.pt"))
engine.destroy()


@pytest.mark.parametrize("world_size", [1, 2], ids=["tp1", "tp2"])
class TestZero3UniversalCheckpointWithoutTp(DistributedTest):
"""ZeRO-3 conversion without TP stores each state as a bare tensor, not a dict.
An AutoTP target must still take its TP shard of it.
"""

def test_restores_saved_state(self, zero3_no_tp_checkpoint, tmpdir, world_size):
tp_size = dist.get_world_size()
expected = torch.load(os.path.join(tmpdir, "zero3_saved_state.pt"), weights_only=False)
engine = _affine_resume_engine(tp_size, load_universal=True, zero_stage=3)
engine.load_checkpoint(tmpdir, tag="zero3_universal", load_optimizer_states=True)
restored = _affine_resume_state(engine, tp_size)
assert restored.keys() == expected.keys()
# Conversion and restore only copy elements, so every state must come back bit for bit.
for name in expected:
assert torch.equal(torch.as_tensor(restored[name]), torch.as_tensor(expected[name])), name
engine.destroy()


class TestAffineMapProducer(DistributedTest):
"""The producer must emit the layout that `affine_resume_checkpoint` supplies by hand.

Expand Down
Loading