Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions deepspeed/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -541,11 +541,19 @@ def initialize(
if not isinstance(config, dict):
config = load_ds_config(config)

zero_stage = config.get("zero_optimization", {}).get("stage", 0)
if _autosp_sequence_parallel_size(config) > 1 and zero_stage >= 2:
# The engine would fall back to the plain torch compiler, which does not shard the sequence, while
# still summing gradients over the sequence parallel ranks.
raise ValueError(f"AutoSP does not support ZeRO stage {zero_stage}; use ZeRO stage 0 or 1")
if _autosp_sequence_parallel_size(config) > 1:
# Without the AutoSP backend each rank trains on its full sequence, while the engine still sums
# gradients over the sequence parallel ranks. Only DeepCompile runs that backend, and not with ZeRO 2/3.
if not config.get("compile", {}).get("deepcompile", False):
raise ValueError("AutoSP requires DeepCompile; set compile.deepcompile to true")
zero_config = config.get("zero_optimization", {})
if isinstance(zero_config, bool):
# The deprecated boolean form selects stage 1 or 0.
zero_stage = int(zero_config)
else:
zero_stage = zero_config.get("stage", 0)
if zero_stage >= 2:
raise ValueError(f"AutoSP does not support ZeRO stage {zero_stage}; use ZeRO stage 0 or 1")

mesh_device = None
if mesh_param:
Expand Down
249 changes: 235 additions & 14 deletions deepspeed/compile/passes/sp_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from ..custom_ops import all_to_all, sp_dp_registry # noqa: F401
from ..fx import find_node_by_name, get_node_shape_meta, is_impure_for_dce
from ..util import (get_autosp_seq_dim, get_input_id_node, get_label_id_node, get_position_id_node, shard_tensor_node,
get_sdpa_nodes)
get_sdpa_nodes, insert_sequence_shard)


def prepare_autosp_inputs(input_id: torch.Tensor,
Expand Down Expand Up @@ -95,14 +95,12 @@ def pass_shard_seq_dim(gm: GraphModule, example_inputs):
with gm.graph.inserting_after(sym_seq_dim_node):
sharded_node = gm.graph.call_function(operator.floordiv, args=(sym_seq_dim_node, sp_size))

# Labels stay whole: pass_shard_label_ids gives each cross entropy the shard of its target.
sharded_input_nodes = set()
label_ids_node = get_label_id_node(gm)
position_ids_node = get_position_id_node(gm)

if input_ids_node is not None:
sharded_input_nodes.add(input_ids_node)
if label_ids_node is not None:
sharded_input_nodes.add(label_ids_node)
if position_ids_node is not None:
sharded_input_nodes.add(position_ids_node)

Expand Down Expand Up @@ -144,14 +142,234 @@ def _get_cross_entropy_arg(loss_node: Node, name: str, position: int, default):
return default


_RANK_PRESERVING_METHODS = {"contiguous", "float", "to", "long", "int", "half", "bfloat16"}
_RESHAPE_METHODS = {"view", "reshape", "flatten"}
_RESHAPE_FUNCTIONS = {torch.reshape, torch.flatten}
# Each rank holds only its shard of the logits, so these would reorder positions within the shard only.
_SEQUENCE_REORDERING_METHODS = {"roll", "flip", "fliplr", "flipud"}
_SEQUENCE_REORDERING_FUNCTIONS = {torch.roll, torch.flip, torch.fliplr, torch.flipud}


def _is_rank_preserving(node: Node) -> bool:
return node.op == "call_method" and node.target in _RANK_PRESERVING_METHODS


def _is_reshape(node: Node) -> bool:
if node.op == "call_method":
return node.target in _RESHAPE_METHODS
return node.op == "call_function" and node.target in _RESHAPE_FUNCTIONS


def _is_sequence_reordering(node: Node) -> bool:
if node.op == "call_method":
return node.target in _SEQUENCE_REORDERING_METHODS
return node.op == "call_function" and node.target in _SEQUENCE_REORDERING_FUNCTIONS


def _reshape_uses_symbol_of(node: Node, length: torch.SymInt) -> bool:
"""Whether a size argument of the reshape ``node`` is derived from the symbolic ``length``."""
symbols = length.node.expr.free_symbols
for size_node in node.all_input_nodes:
if size_node is node.args[0]:
continue
size = get_node_shape_meta(size_node)
if isinstance(size, torch.SymInt) and size.node.expr.free_symbols & symbols:
return True
return False


def _is_tensor_getitem(node: Node) -> bool:
if node.op != "call_function" or node.target is not operator.getitem:
return False
return isinstance(get_node_shape_meta(node.args[0]), torch.Tensor)


def _get_seq_slice(node: Node, seq_dim: int):
"""
Return ``(start, stop)`` of a getitem that only narrows ``seq_dim`` by static amounts, where
``start >= 0`` counts from the front and ``stop <= 0`` counts from the end.
"""
ndim = get_node_shape_meta(node.args[0]).ndim
index = node.args[1] if isinstance(node.args[1], tuple) else (node.args[1], )
if index.count(Ellipsis) == 1:
position = index.index(Ellipsis)
index = index[:position] + (slice(None), ) * (ndim - len(index) + 1) + index[position + 1:]
index = index + (slice(None), ) * (ndim - len(index))

unsupported = RuntimeError(f"AutoSP cannot shard cross entropy logits indexed by {node.args[1]!r}; only static "
f"slices of the sequence dimension such as logits[..., :-1, :] are supported")
if len(index) != ndim or not all(isinstance(i, slice) for i in index):
raise unsupported
for dim, dim_slice in enumerate(index):
if dim_slice.step not in (None, 1):
raise unsupported
if dim != seq_dim and (dim_slice.start not in (None, 0) or dim_slice.stop is not None):
raise unsupported

start = index[seq_dim].start or 0
stop = 0 if index[seq_dim].stop is None else index[seq_dim].stop
if not isinstance(start, int) or not isinstance(stop, int) or start < 0 or stop > 0:
raise unsupported
if index[seq_dim].stop == 0:
raise unsupported
return start, stop


def _find_seq_dim(node: Node, length: torch.SymInt) -> int:
"""Return the only dimension of ``node`` whose symbolic size is ``length``."""
val = get_node_shape_meta(node)
dims = [
dim for dim, size in enumerate(val.shape)
if isinstance(size, torch.SymInt) and size.node.expr == length.node.expr
]
if len(dims) != 1:
raise RuntimeError(f"AutoSP expected exactly one dimension of size {length} in {node.format_node()} "
f"with shape {tuple(val.shape)} to line up with the sequence")
return dims[0]


def _undo_logits_seq_slices(input_node: Node, seq_len: torch.SymInt, sharded_nodes):
"""
Remove the sequence slices between the logits and a cross entropy, as each rank must keep the
logits of its whole shard. Return the window ``[start, seq_len + stop)`` of the full sequence that
the cross entropy input covered before the slices were removed.
"""
chain = []
slices = []
node = input_node
while _is_rank_preserving(node) or _is_reshape(node) or _is_tensor_getitem(node):
if _is_tensor_getitem(node):
slices.append(node)
elif _is_reshape(node) and slices:
# A reshape that runs before a slice would let the slice cut across the sequence dimension.
raise RuntimeError("AutoSP only supports slicing the cross entropy logits before they are reshaped")
Comment on lines +243 to +245

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Reject explicit logits reshapes after removed slices

The fresh remaining reshape case is on the logits side rather than the target-side symbolic reshape fixed earlier: for logits[..., :-1, :].reshape(batch * (seq_len - 1), vocab), traversal sees the reshape before it records the underlying slice, so this check permits it. The pass then erases the slice but leaves the explicit S-1 reshape size, and fake propagation or the first forward fails while trying to reshape B*S*V elements to B*(S-1)*V; require inferred reshape dimensions or rewrite the symbolic size when removing the slice.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in b686c90: when a slice is removed, a reshape after it whose sizes are derived from the symbolic sequence length is rejected with a hint to use view(-1, vocab_size). Static sizes such as view(-1, logits.size(-1)) stay supported. Covered by the explicit_logits_view row (logits[..., :-1, :].reshape(B * (S - 1), V)).

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the fix! The same rotation can be written in other ways, such as torch.cat((logits[:, 1:], logits[:, :1]), dim=1), which still runs and silently produces an incorrect loss.
Since it’s hard to detect every equivalent pattern, could we revise “AutoSP raises an error for any other pattern” to warn that some unsupported patterns may go undetected?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay thanks, I reword in f275054 as follow:

AutoSP raises for the unsupported patterns it detects, but some equivalent rewrites, such as  torch.cat((logits[:, 1:], logits[:, :1]), dim=1) , may go undetected and silently produce an incorrect loss.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tohtana updated.

chain.append(node)
node = node.args[0]

if node not in sharded_nodes:
raise RuntimeError(f"AutoSP could not trace the cross entropy input back to sequence-sharded logits; "
f"stopped at {node.format_node()}")
Comment on lines +249 to +251

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Reject sequence reorders before accepting the logits root

When a model applies a sequence-wide transform such as torch.roll to logits before the allowed reshape/cross-entropy chain, this condition accepts the roll node because sharded_nodes contains every transitive consumer of the sharded inputs, not just the raw logits. The logits roll then runs independently inside each local S/P shard, whereas the replicated labels can be rolled globally and only then sharded, so boundary positions produce silently incorrect loss and gradients instead of the documented unsupported-pattern error; continue tracing or explicitly reject sequence-reordering operations.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in b686c90: the pass now rejects logits reordered by roll/flip/fliplr/flipud (function or method form) where it accepts the logits root, since each rank would only reorder its own shard. Covered by the roll row of test_rejects_labels_it_cannot_align.

if _is_sequence_reordering(node):
raise RuntimeError(f"AutoSP cannot shard cross entropy logits reordered by {node.format_node()}, as each "
f"rank would only reorder its own shard of the sequence")

seq_dim = _find_seq_dim(node, seq_len)
start, stop = 0, 0
for slice_node in slices:
slice_start, slice_stop = _get_seq_slice(slice_node, seq_dim)
start += slice_start
stop += slice_stop
if slices and any(len(chain_node.users) != 1 for chain_node in chain):
raise RuntimeError("AutoSP cannot remove the sequence slice of cross entropy logits used elsewhere")
for chain_node in chain:
if slices and _is_reshape(chain_node) and _reshape_uses_symbol_of(chain_node, seq_len):
# Once the slice is removed, a size computed from the sliced sequence no longer fits the logits.
raise RuntimeError(f"AutoSP cannot reshape sliced cross entropy logits with sizes derived from the "
f"sequence length in {chain_node.format_node()}; flatten them with "
f"view(-1, vocab_size) instead")
for slice_node in slices:
slice_node.replace_all_uses_with(slice_node.args[0])
slice_node.graph.erase_node(slice_node)
return start, stop


def _localize_cross_entropy_target(gm: GraphModule, loss_node: Node, window, seq_len: torch.SymInt, ignore_index):
"""
Replace the full-sequence target of a cross entropy with the part that lines up with this rank's
logits. Positions outside the logits window get ``ignore_index``.
"""
consumer = loss_node
target = _get_cross_entropy_arg(loss_node, "target", 1, None)
while _is_rank_preserving(target) or _is_reshape(target):
Comment thread
tohtana marked this conversation as resolved.
Resolved
if _is_reshape(target) and len(target.all_input_nodes) > 1:
# A size taken from the full sequence would not fit this rank's shorter target.
raise RuntimeError(f"AutoSP cannot reshape a cross entropy target with explicit symbolic sizes in "
f"{target.format_node()}; flatten it with view(-1) instead")
if len(target.users) > 1:
# Another cross entropy may share this transform. Rewire a copy so that its target stays whole.
with gm.graph.inserting_after(target):
target_copy = gm.graph.node_copy(target)
consumer.replace_input_with(target, target_copy)
target = target_copy
consumer = target
target = target.args[0]

start, stop = window
# The target pairs with the logits position by position, so it covers as many positions as the window.
seq_dim = _find_seq_dim(target, seq_len + stop - start)
target_val = get_node_shape_meta(target)

padded = target
if start or stop:
anchor = target
if target.op == "placeholder":
anchor = [node for node in gm.graph.nodes if node.op == "placeholder"][-1]
# Pad with cat rather than F.pad, which would specialize the symbolic sequence length.
pieces = [target]
for width, at_front in ((start, True), (-stop, False)):
if width == 0:
continue
# Expand a single position, as a large shift can be wider than the target itself.
with gm.graph.inserting_after(anchor):
anchor = gm.graph.call_function(torch.narrow, args=(target, seq_dim, 0, 1))
with gm.graph.inserting_after(anchor):
anchor = gm.graph.call_function(torch.full_like, args=(anchor, ignore_index))
sizes = [-1] * target_val.ndim
sizes[seq_dim] = width
with gm.graph.inserting_after(anchor):
anchor = gm.graph.call_method("expand", args=(anchor, *sizes))
pieces.insert(0 if at_front else len(pieces), anchor)
with gm.graph.inserting_after(anchor):
padded = gm.graph.call_function(torch.cat, args=(pieces, seq_dim))
local_target = insert_sequence_shard(gm, padded, target_val.ndim, seq_dim, find_node_by_name(gm, str(seq_len)))
consumer.replace_input_with(target, local_target)
Comment thread
tohtana marked this conversation as resolved.
Resolved


def _check_label_consumers(label_ids_node: Node, sharded_nodes):
"""Labels stay unsharded, so they may only flow into cross entropy targets."""
worklist = [label_ids_node]
label_nodes = set()
while worklist:
node = worklist.pop()
if node in label_nodes:
continue
label_nodes.add(node)
if any(input_node in sharded_nodes for input_node in node.all_input_nodes):
raise RuntimeError(f"AutoSP labels cannot be combined with sequence-sharded tensors in "
f"{node.format_node()}")
for user in node.users:
if user.target is torch.nn.functional.cross_entropy:
if _get_cross_entropy_arg(user, "target", 1, None) is not node:
raise RuntimeError(f"AutoSP labels may only be the target of a cross entropy, got "
f"{user.format_node()}")
elif user.op == "output":
raise RuntimeError("AutoSP labels cannot be returned from the compiled module")
else:
worklist.append(user)


def pass_shard_label_ids(gm: GraphModule, example_inputs):
"""
Shard the labels and make every cross entropy on them return the loss of the full sequence.
Make every cross entropy on the labels return the loss of the full sequence.

Each cross entropy is summed per shard and combined over the SP group, which weights shards
with different numbers of valid tokens correctly.
The labels stay whole, so any shift the model applies to them sees the full sequence. Each cross
entropy then takes the part of its target that lines up with this rank's logits, sums its loss and
combines the sums over the SP group, which weights shards with different numbers of valid tokens
correctly.
"""
label_ids_node = get_label_id_node(gm)
input_ids_node = get_input_id_node(gm)
seq_len = get_node_shape_meta(input_ids_node).shape[get_autosp_seq_dim(input_ids_node)]

sharded_nodes = set()
worklist = [node for node in (input_ids_node, get_position_id_node(gm)) if node is not None]
while worklist:
node = worklist.pop()
if node not in sharded_nodes:
sharded_nodes.add(node)
worklist.extend(node.users)

_check_label_consumers(label_ids_node, sharded_nodes)

label_users = set()
worklist = [label_ids_node]
Expand All @@ -165,7 +383,6 @@ def pass_shard_label_ids(gm: GraphModule, example_inputs):
node for node in gm.graph.nodes if node.target is torch.nn.functional.cross_entropy
and _get_cross_entropy_arg(node, "target", 1, None) in label_users
]
shard_tensor_node(gm, label_ids_node)

for loss_node in loss_nodes:
weight = _get_cross_entropy_arg(loss_node, "weight", 2, None)
Expand All @@ -180,20 +397,24 @@ def pass_shard_label_ids(gm: GraphModule, example_inputs):
if reduction not in ("mean", "sum"):
raise RuntimeError(f"AutoSP supports cross entropy with reduction 'mean' or 'sum', got {reduction!r}")
ignore_index = _get_cross_entropy_arg(loss_node, "ignore_index", 4, -100)

target_node = _get_cross_entropy_arg(loss_node, "target", 1, None)
if get_node_shape_meta(target_node).dtype.is_floating_point:
if get_node_shape_meta(_get_cross_entropy_arg(loss_node, "target", 1, None)).dtype.is_floating_point:
# Class probability targets have no ignore_index, so counting valid tokens does not apply.
raise RuntimeError("AutoSP supports cross entropy with class index targets only, not class probabilities")

window = _undo_logits_seq_slices(_get_cross_entropy_arg(loss_node, "input", 0, None), seq_len, sharded_nodes)
_localize_cross_entropy_target(gm, loss_node, window, seq_len, ignore_index)

target_node = _get_cross_entropy_arg(loss_node, "target", 1, None)
# Under autocast, cross entropy returns FP32 even for half-precision logits.
loss_dtype = get_node_shape_meta(loss_node).dtype
with gm.graph.inserting_after(target_node):
valid_mask = gm.graph.call_function(operator.ne, args=(target_node, ignore_index))
with gm.graph.inserting_after(valid_mask):
valid_tokens = gm.graph.call_function(torch.sum, args=(valid_mask, ))
Comment thread
tohtana marked this conversation as resolved.
Resolved

# Sum the per-token losses in FP32: a half-precision sum over a long shard overflows even
# where the mean the model asked for would not.
# Sum the per-token losses in at least FP32: a half-precision sum over a long shard overflows
# even where the mean the model asked for would not.
sum_dtype = torch.promote_types(loss_dtype, torch.float32)
if "reduction" in loss_node.kwargs or len(loss_node.args) <= 6:
loss_node.update_kwarg("reduction", "none")
else:
Expand All @@ -205,7 +426,7 @@ def pass_shard_label_ids(gm: GraphModule, example_inputs):
loss_node.update_arg(position, None)

with gm.graph.inserting_after(loss_node):
token_losses = gm.graph.call_method("float", args=(loss_node, ))
token_losses = gm.graph.call_method("to", args=(loss_node, sum_dtype))
with gm.graph.inserting_after(token_losses):
local_loss_sum = gm.graph.call_function(torch.sum, args=(token_losses, ))
with gm.graph.inserting_after(local_loss_sum):
Expand Down
12 changes: 9 additions & 3 deletions deepspeed/compile/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -608,8 +608,15 @@ def shard_tensor_node(gm: GraphModule, tensor_node: Node):
symb_seq_int_node = find_node_by_name(gm, str(seq_len))
assert symb_seq_int_node, f"Unable to find symbolic placeholder for {seq_len}"

sliced_node = insert_sequence_shard(gm, tensor_node, val.ndim, seq_dim, symb_seq_int_node)
replace_node_users(tensor_node, sliced_node, exclude=[sliced_node])


def insert_sequence_shard(gm: GraphModule, tensor_node: Node, ndim: int, seq_dim: int,
symb_seq_int_node: Node) -> Node:
"""Insert a node that takes this rank's shard of ``tensor_node`` along ``seq_dim`` and return it."""
slice_all, slice_range = create_symbolic_slice_indices(gm, symb_seq_int_node)
indices = tuple(slice_range if dim == seq_dim else slice_all for dim in range(val.ndim))
indices = tuple(slice_range if dim == seq_dim else slice_all for dim in range(ndim))

positions = {node: i for i, node in enumerate(gm.graph.nodes)}
# Insert after the later dependency so the new getitem does not appear
Expand All @@ -621,5 +628,4 @@ def shard_tensor_node(gm: GraphModule, tensor_node: Node):
operator.getitem,
args=(tensor_node, indices),
)

replace_node_users(tensor_node, sliced_node, exclude=[sliced_node])
return sliced_node
3 changes: 3 additions & 0 deletions deepspeed/runtime/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -2927,6 +2927,9 @@ def deepspeed_io(self,
if self.mpu is not None:
data_parallel_world_size = self.mpu.get_data_parallel_world_size()
data_parallel_rank = self.mpu.get_data_parallel_rank()
elif self.mesh_device is not None:
# The ranks of a sequence parallel group must load the same samples.
data_parallel_rank = dist.get_rank(group=self.mesh_device.get_group(mesh_dim="data_parallel"))

if data_sampler is None and (route == ROUTE_PREDICT or route == ROUTE_EVAL):
data_sampler = torch.utils.data.DistributedSampler(
Expand Down
Loading
Loading