Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions deepspeed/checkpoint/ds_to_universal.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

from deepspeed.checkpoint import DeepSpeedCheckpoint
from deepspeed.checkpoint import (
DS_VERSION,
OPTIMIZER_STATE_DICT,
ZERO_STAGE,
BASE_OPTIMIZER_STATE,
Expand Down Expand Up @@ -1001,6 +1002,9 @@ def _save_optimizer_state_stage3(args, optim_files):
sd = torch.load(optim_files[0], map_location=torch.device('cpu'), weights_only=False)
output_sd = sd[OPTIMIZER_STATE_DICT]
output_sd[PARAM_GROUPS] = output_sd[OPTIMIZER_STATE_DICT][PARAM_GROUPS]
# ZeRO-3 keeps the version beside its optimizer state rather than in it; ZeRO-1/2 targets require it.
if sd.get(DS_VERSION):
output_sd[DS_VERSION] = sd[DS_VERSION]
zero_output_folder = os.path.join(args.output_folder, "zero")
output_file_path = os.path.join(zero_output_folder, "optimizer_state.pt")
_save_checkpoint(output_file_path, output_sd)
Expand Down
33 changes: 32 additions & 1 deletion deepspeed/checkpoint/universal_checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
EP_IS_EXPERT_PARAM, EP_NUM_EXPERTS, DS_AUTOEP_UC_META, DS_AUTOTP_UC_META,
AUTOEP_EXPERT_PLACEMENT, AUTOEP_PARAM_EP_RANK, AUTOEP_PARAM_LOCAL_EXPERTS,
AUTOEP_PARAM_LOGICAL_SHAPE, AUTOEP_PLACEMENT_EXPERTS, AUTOEP_PLACEMENT_RANK,
AUTOEP_PLACEMENT_RANKS, UNIVERSAL_CHECKPOINT_VERSION_KEY)
AUTOEP_PLACEMENT_RANKS, UNIVERSAL_CHECKPOINT_VERSION_KEY, OPTIMIZER_STATE_DICT)
from .autoep_affine import autoep_metadata_to_affine_map, extract_autoep_rank_tensor


Expand Down Expand Up @@ -233,6 +233,35 @@ def _resolve_autotp_partition(current_param, ckpt_dict, full_hp_param, tp_rank,
return slice_tensor.flatten()


def _shape_flat_universal_state(param, tensor, ckpt_file):
"""Give a state that ds_to_universal wrote as a bare flat tensor (a ZeRO-3 source) the parameter's shape.

Such a file records no tensor-parallel layout, so it can only fill a parameter that holds the whole tensor.
"""
if tensor.numel() != param.numel():
raise ValueError(f"{ckpt_file} holds a flat tensor of {tensor.numel()} elements with no tensor-parallel "
f"layout, but the parameter loading it has shape {tuple(param.shape)}. A universal "
"checkpoint converted from ZeRO-3 can be loaded into ZeRO-1/2 only without tensor "
"parallelism.")
return tensor.view(param.shape)


def load_universal_optimizer_step(optim_sd, checkpoint_dir, param_names):
"""The optimizer step a universal checkpoint was saved at, whichever ZeRO stage it came from.

A ZeRO-3 source keeps it in the global optimizer state; a ZeRO-1/2 source writes a step.pt next to
each parameter's states.
"""
state = optim_sd.get(OPTIMIZER_STATE_DICT, {}).get('state')
if state:
return next(iter(state.values())).get('step')

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Give each restored state an independent step tensor

When a cross-stage checkpoint was written by torch Adam (so step is a tensor) and the target optimizer has more than one group/subgroup, this returns one object that base_optimizer.py:405-408 caches across all groups and stage3.py:3752,3770 assigns across all subgroups. Adam increments each state's step in place, so a single resumed optimizer step advances this aliased scalar once per group/subgroup (for example, 2 to 4 for two groups) and uses incorrect bias correction; clone the value for each state before storing it.

AGENTS.md reference: AGENTS.md:L19-L19

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in ce758ab. Both universal loaders now give every group and sub-group its own copy of the step, so one optimizer step advances each state once. The cross-stage test splits ZeRO-3 into sub-groups and checks that every step goes up by exactly one after a step following the load.

for name in param_names:
path = os.path.join(checkpoint_dir, name, 'step.pt')
if os.path.isfile(path):
return torch.load(path, weights_only=False)
return None


def load_hp_checkpoint_state(self, folder, tp_rank, tp_world_size, ep_rank=0, ep_size=1, load_optimizer_states=True):
hp_mapping = self._hp_mapping
if load_optimizer_states:
Expand Down Expand Up @@ -265,6 +294,8 @@ def load_hp_checkpoint_state(self, folder, tp_rank, tp_world_size, ep_rank=0, ep
step = ckpt_dict
continue

if torch.is_tensor(ckpt_dict):
ckpt_dict = {PARAM: _shape_flat_universal_state(self, ckpt_dict, ckpt_file)}
full_hp_param = ckpt_dict[PARAM]

# EP-aware slicing for expert parameters saved in universal format.
Expand Down
10 changes: 9 additions & 1 deletion deepspeed/runtime/base_optimizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,7 @@ def load_hp_checkpoint_state_from_checkpoint_dir(self,
optim_state_path), f'{optim_state_path} containing optimizer global state is missing! Cannot proceed.'
optim_sd = torch.load(optim_state_path, weights_only=False)
self._load_global_state(optim_sd)
global_step = None

for i, (param_group,
loaded_param_group) in enumerate(zip(self.optimizer.param_groups, optim_sd['param_groups'])):
Expand All @@ -397,8 +398,15 @@ def load_hp_checkpoint_state_from_checkpoint_dir(self,

hp_param = param_group['params'][0]
assert all(step == steps[0] for step in steps), f"Steps {steps} are not equal"
if steps[0] is None:
# A ZeRO-3 source writes no per-parameter step.pt; its step is in the global state.
if global_step is None:
from deepspeed.checkpoint.universal_checkpoint import load_universal_optimizer_step
global_step = load_universal_optimizer_step(optim_sd, checkpoint_dir, [])
steps[0] = global_step
if steps[0] is not None:
self.optimizer.state[hp_param]['step'] = steps[0]
# The optimizer advances step in place, and a ZeRO-3 source's step is shared by every group.
self.optimizer.state[hp_param]['step'] = steps[0].clone() if torch.is_tensor(steps[0]) else steps[0]

map_to_flat_opt_states(hp_param, lp_groups[i], self.optimizer.state, opt_keys)

Expand Down
15 changes: 13 additions & 2 deletions deepspeed/runtime/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -4415,13 +4415,15 @@ def _get_zero_ckpt_name(self, checkpoints_path, tag):
bf16_mode = self.bfloat16_enabled()
return self._get_rank_zero_ckpt_name(checkpoints_path, tag, self.checkpoint_mp_rank, pp_rank, bf16_mode)

def _get_ckpt_name(self, checkpoints_path, tag, mp_placeholder=None, pp_placeholder=None):
def _get_ckpt_name(self, checkpoints_path, tag, mp_placeholder=None, pp_placeholder=None, partition_weights=None):
if mp_placeholder is not None:
mp_rank_str = mp_placeholder
else:
mp_rank_str = f"{self.checkpoint_mp_rank:02d}"

if self.zero_optimization_partition_weights():
if partition_weights is None:
partition_weights = self.zero_optimization_partition_weights()
if partition_weights:
if pp_placeholder is not None:
pp_rank = pp_placeholder
else:
Expand Down Expand Up @@ -4471,6 +4473,15 @@ def _get_all_ckpt_names(self, checkpoints_path, tag):
import glob

ckpt_files = glob.glob(ckpt_file_pattern)
if not ckpt_files and self.load_universal_checkpoint():
# The source's ZeRO stage named these files, and it may differ from ours. A universal load reads
# only metadata from them, so the other stage's naming serves as well.
other_pattern = self._get_ckpt_name(checkpoints_path,
tag,
mp_placeholder="*",
pp_placeholder="0",
partition_weights=not self.zero_optimization_partition_weights())
ckpt_files = glob.glob(other_pattern)
ckpt_files.sort()
return ckpt_files

Expand Down
20 changes: 15 additions & 5 deletions deepspeed/runtime/zero/stage3.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
EP_IS_EXPERT_PARAM,
EP_NUM_EXPERTS,
OPTIMIZER_STATE_DICT,
PARAM_GROUPS,
FP32_FLAT_GROUPS,
PARAM,
PARTITION_COUNT,
Expand Down Expand Up @@ -3747,7 +3748,8 @@ def load_hp_checkpoint_state_from_checkpoint_dir_stage3(self, checkpoint_dir):
self._load_global_state_stage3(optim_sd)

# Generally the step of each optimizer file should be the same, we can obtain from any parameter.
state_step = optim_sd[OPTIMIZER_STATE_DICT]['state'][0]['step']
from deepspeed.checkpoint.universal_checkpoint import load_universal_optimizer_step
state_step = load_universal_optimizer_step(optim_sd, checkpoint_dir, self.param_names.values())
for key in ["fp32", "exp_avg", "exp_avg_sq"]:
for sub_group_id, fp16_group in enumerate(self.fp16_groups):
fp32_param = self.fp32_partitioned_groups_flat[sub_group_id]
Expand All @@ -3764,13 +3766,21 @@ def load_hp_checkpoint_state_from_checkpoint_dir_stage3(self, checkpoint_dir):
offset += key_layer_state_partition.numel()
if key == "fp32":
self.fp32_partitioned_groups_flat[sub_group_id].data.copy_(key_tensor)
self.optimizer.state[fp32_param]['step'] = state_step
if state_step is not None:
# Each sub-group's optimizer advances its step in place, so give it its own copy.
self.optimizer.state[fp32_param]['step'] = (state_step.clone()
if torch.is_tensor(state_step) else state_step)
else:
self.optimizer.state[fp32_param][key] = key_tensor

for param_group in self.optimizer.param_groups:
# Generally, the hyperparameters of each parameter should be the same, we can obtain from any parameter.
for key, value in optim_sd[OPTIMIZER_STATE_DICT]["param_groups"][0].items():
# A ZeRO-1/2 source has no OPTIMIZER_STATE_DICT, only the top-level PARAM_GROUPS. Both list the
# optimizer's own parameter groups in order, as self.optimizer.param_groups does.
saved_param_groups = optim_sd.get(PARAM_GROUPS) or optim_sd[OPTIMIZER_STATE_DICT][PARAM_GROUPS]
if len(saved_param_groups) != len(self.optimizer.param_groups):
raise ValueError(f"The universal checkpoint in {checkpoint_dir} has {len(saved_param_groups)} optimizer "
f"parameter groups, but this optimizer has {len(self.optimizer.param_groups)}.")
for param_group, saved_param_group in zip(self.optimizer.param_groups, saved_param_groups):
for key, value in saved_param_group.items():
if key == 'params':
param_group['params'] = []
else:
Expand Down
Loading
Loading