Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion deepspeed/runtime/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from .zero.offload_config import OffloadDeviceEnum, OffloadStateTypeEnum
from deepspeed.runtime.base_optimizer import ZeROOptimizer
from deepspeed.runtime.zero.stage_1_and_2 import DeepSpeedZeroOptimizer
from deepspeed.runtime.zero import optimizer_state_swap as state_swap
from deepspeed.runtime.zenflow.zenflow_stage_1_and_2 import ZenFlowZeroOptimizer
from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
from deepspeed.runtime.zero.utils import is_zero_supported_optimizer, ZeRORuntimeException
Expand Down Expand Up @@ -1651,6 +1652,9 @@ def zero_compute_grad_norm(self):
def zero_retain_gpu_grad_partition_until_step(self):
return self._config.zero_config.retain_gpu_grad_partition_until_step

def zero_optimizer_state_swap(self):
return self._config.zero_config.optimizer_state_swap

def zero_optimization_partition_gradients(self):
return self.zero_optimization_stage() >= ZeroStageEnum.gradients

Expand Down Expand Up @@ -1965,6 +1969,10 @@ def _configure_lr_scheduler(self):
else:
log_dist('DeepSpeed using client LR scheduler', ranks=[0])
self.lr_scheduler = self.client_lr_scheduler
# Built on the optimizer's param groups before the streamed optimizer-state swap split them.
remap = getattr(self.optimizer, "remap_lr_scheduler", None)
if remap is not None:
remap(self.lr_scheduler)
else:
# load lr scheduler from json configuration if lr scheduler is not defined and passed in
lr_scheduler = self._scheduler_from_config(self.optimizer)
Expand Down Expand Up @@ -2385,6 +2393,9 @@ def _do_optimizer_sanity_check(self, basic_optimizer):
if not self.zero_compute_grad_norm():
raise ValueError("zero_optimization.compute_grad_norm=false does not support ZeRO Stage 1 with "
"BF16 parameters and FP32 gradient accumulation")
if self.zero_optimizer_state_swap() is not None:
raise ValueError("zero_optimization.optimizer_state_swap does not support ZeRO Stage 1 with BF16 "
"parameters and FP32 gradient accumulation")
return BFLOAT16
return ZERO_OPTIMIZATION
# data type checks
Expand Down Expand Up @@ -2782,7 +2793,8 @@ def _configure_zero_optimizer(self, optimizer):
elastic_checkpoint=self.zero_elastic_checkpoint(),
check_grad_overflow=check_grad_overflow,
compute_grad_norm=self.zero_compute_grad_norm(),
retain_gpu_grad_partition_until_step=self.zero_retain_gpu_grad_partition_until_step())
retain_gpu_grad_partition_until_step=self.zero_retain_gpu_grad_partition_until_step(),
optimizer_state_swap=self.zero_optimizer_state_swap())

elif zero_stage == ZeroStageEnum.weights:
self._validate_zero3_moe_compatibility()
Expand Down Expand Up @@ -4514,6 +4526,19 @@ def load_checkpoint(self,
before ``load_checkpoint()``.

"""
# The optimizer-state swap's hold, as a context rather than a decorator: callers verify this method's source.
with state_swap.held(self.optimizer, reselect=True, on_device=self.load_universal_checkpoint()):
return self._load_checkpoint_held(load_dir, tag, load_module_strict, load_optimizer_states,
load_lr_scheduler_states, load_module_only, custom_load_fn)

def _load_checkpoint_held(self,
load_dir,
tag=None,
load_module_strict=True,
load_optimizer_states=True,
load_lr_scheduler_states=True,
load_module_only=False,
custom_load_fn=None):

if tag is None:
latest_tag = "latest_universal" if self.load_universal_checkpoint() else "latest"
Expand Down Expand Up @@ -4993,6 +5018,12 @@ def save_checkpoint(self, save_dir, tag=None, client_state={}, save_latest=True,
process with rank 0.

"""
# The optimizer-state swap's hold, as a context rather than a decorator: callers verify this method's source.
with state_swap.held(self.optimizer):
return self._save_checkpoint_held(save_dir, tag, client_state, save_latest, exclude_frozen_parameters)

def _save_checkpoint_held(self, save_dir, tag=None, client_state={}, save_latest=True,
exclude_frozen_parameters=False):
if not save_dir:
raise ValueError(f"save_dir must be a non-empty string, got {save_dir!r}")

Expand Down
58 changes: 57 additions & 1 deletion deepspeed/runtime/zero/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import sys
from typing import Optional
from enum import Enum
from pydantic import Field, model_validator
from pydantic import Field, StrictBool, model_validator
from deepspeed.runtime.config_utils import get_scalar_param, pp_int, DeepSpeedConfigModel
from deepspeed.utils import logger
from .offload_config import DeepSpeedZeroOffloadParamConfig, DeepSpeedZeroOffloadOptimizerConfig, OffloadDeviceEnum
Expand Down Expand Up @@ -50,6 +50,7 @@
"override_module_apply": [true|false],
"log_trace_cache_warnings" : [true|false],
"enable_sanity_checks": [true|false],
"optimizer_state_swap": {"fraction": (0, 1], "stream": [true|false], "group_gib": 2.0},
}
}
"""
Expand Down Expand Up @@ -87,6 +88,40 @@ class ZeroStageEnum(int, Enum):
max_stage = 3


class DeepSpeedZeroOptimizerStateSwapConfig(DeepSpeedConfigModel):
"""
ZeRO stage 1 and 2: the FP32 master weights and optimizer state in pinned host memory outside the optimizer step
(see ``deepspeed/runtime/zero/optimizer_state_swap.py``). Replaces ``offload_optimizer``, which moves the step
itself to the CPU.
"""

fraction: float = Field(gt=0, le=1)
"""
Fraction of the FP32 masters and optimizer state, by bytes and in whole tensors in the optimizer's order, kept in
pinned host memory outside the step: fetched back from the end of the backward, evicted after the step. With
``stream``, the fraction of the streamed param groups; the rest stay on the GPU.
"""

stream: StrictBool = False
"""
Stream the step group by group: the param groups are split into slices of at most ``group_gib`` GiB of FP32
master, each slice's state on the GPU only around its own step, for state that cannot be resident all at once.
"""

group_gib: Optional[float] = Field(None, gt=0)
"""
With ``stream``: the most FP32 master per param-group slice, in GiB (2 when unset). Changing ``stream`` or
``group_gib`` changes the number of param groups, so a checkpoint does not load across that change.
"""

@model_validator(mode="after")
def modes_valid(self):
if self.group_gib is not None and not self.stream:
raise ValueError("optimizer_state_swap.group_gib sizes the streamed step's param groups; set "
"optimizer_state_swap.stream too")
return self


class DeepSpeedZeroConfig(DeepSpeedConfigModel):
"""
Sets parameters for ZeRO optimizations.
Expand Down Expand Up @@ -389,6 +424,12 @@ class DeepSpeedZeroConfig(DeepSpeedConfigModel):
Configuration for modules that should be treated as ZeRO3 leaf modules.
"""

optimizer_state_swap: Optional[DeepSpeedZeroOptimizerStateSwapConfig] = None
"""
Stage 1 and 2: keep the FP32 master weights and optimizer state in pinned host memory outside the optimizer step.
Expects a dictionary containing values for :any:`DeepSpeedZeroOptimizerStateSwapConfig`.
"""

# Validators
@model_validator(mode="after")
def overlap_comm_valid(self):
Expand All @@ -409,6 +450,21 @@ def offload_ratio_check(self):
assert self.stage == ZeroStageEnum.weights, "Partial offloading only supported for ZeRO Stage 3."
return self

@model_validator(mode="after")
def optimizer_state_swap_valid(self):
if self.optimizer_state_swap is not None:
if self.stage not in (ZeroStageEnum.optimizer_states, ZeroStageEnum.gradients):
raise ValueError(
f"zero_optimization.optimizer_state_swap needs ZeRO stage 1 or 2, got stage {int(self.stage)}")
offload = self.offload_optimizer
if offload is not None and offload.device != OffloadDeviceEnum.none:
raise ValueError(
"zero_optimization.optimizer_state_swap replaces zero_optimization.offload_optimizer; "
"set one of them")
if self.zenflow is not None:
raise ValueError("zero_optimization.optimizer_state_swap does not support ZenFlow")
return self

@model_validator(mode="after")
def elastic_checkpoint_deprecated(self):
if self.stage == ZeroStageEnum.weights and self.elastic_checkpoint:
Expand Down
Loading
Loading