Repository navigation
[AutoSP] Align cross entropy targets after an in-model causal label shift #8811
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
698c9b0
b686c90
f275054
eac45e5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,7 +19,7 @@ | |
| from ..custom_ops import all_to_all, sp_dp_registry # noqa: F401 | ||
| from ..fx import find_node_by_name, get_node_shape_meta, is_impure_for_dce | ||
| from ..util import (get_autosp_seq_dim, get_input_id_node, get_label_id_node, get_position_id_node, shard_tensor_node, | ||
| get_sdpa_nodes) | ||
| get_sdpa_nodes, insert_sequence_shard) | ||
|
|
||
|
|
||
| def prepare_autosp_inputs(input_id: torch.Tensor, | ||
|
|
@@ -95,14 +95,12 @@ def pass_shard_seq_dim(gm: GraphModule, example_inputs): | |
| with gm.graph.inserting_after(sym_seq_dim_node): | ||
| sharded_node = gm.graph.call_function(operator.floordiv, args=(sym_seq_dim_node, sp_size)) | ||
|
|
||
| # Labels stay whole: pass_shard_label_ids gives each cross entropy the shard of its target. | ||
| sharded_input_nodes = set() | ||
| label_ids_node = get_label_id_node(gm) | ||
| position_ids_node = get_position_id_node(gm) | ||
|
|
||
| if input_ids_node is not None: | ||
| sharded_input_nodes.add(input_ids_node) | ||
| if label_ids_node is not None: | ||
| sharded_input_nodes.add(label_ids_node) | ||
| if position_ids_node is not None: | ||
| sharded_input_nodes.add(position_ids_node) | ||
|
|
||
|
|
@@ -144,14 +142,234 @@ def _get_cross_entropy_arg(loss_node: Node, name: str, position: int, default): | |
| return default | ||
|
|
||
|
|
||
| _RANK_PRESERVING_METHODS = {"contiguous", "float", "to", "long", "int", "half", "bfloat16"} | ||
| _RESHAPE_METHODS = {"view", "reshape", "flatten"} | ||
| _RESHAPE_FUNCTIONS = {torch.reshape, torch.flatten} | ||
| # Each rank holds only its shard of the logits, so these would reorder positions within the shard only. | ||
| _SEQUENCE_REORDERING_METHODS = {"roll", "flip", "fliplr", "flipud"} | ||
| _SEQUENCE_REORDERING_FUNCTIONS = {torch.roll, torch.flip, torch.fliplr, torch.flipud} | ||
|
|
||
|
|
||
| def _is_rank_preserving(node: Node) -> bool: | ||
| return node.op == "call_method" and node.target in _RANK_PRESERVING_METHODS | ||
|
|
||
|
|
||
| def _is_reshape(node: Node) -> bool: | ||
| if node.op == "call_method": | ||
| return node.target in _RESHAPE_METHODS | ||
| return node.op == "call_function" and node.target in _RESHAPE_FUNCTIONS | ||
|
|
||
|
|
||
| def _is_sequence_reordering(node: Node) -> bool: | ||
| if node.op == "call_method": | ||
| return node.target in _SEQUENCE_REORDERING_METHODS | ||
| return node.op == "call_function" and node.target in _SEQUENCE_REORDERING_FUNCTIONS | ||
|
|
||
|
|
||
| def _reshape_uses_symbol_of(node: Node, length: torch.SymInt) -> bool: | ||
| """Whether a size argument of the reshape ``node`` is derived from the symbolic ``length``.""" | ||
| symbols = length.node.expr.free_symbols | ||
| for size_node in node.all_input_nodes: | ||
| if size_node is node.args[0]: | ||
| continue | ||
| size = get_node_shape_meta(size_node) | ||
| if isinstance(size, torch.SymInt) and size.node.expr.free_symbols & symbols: | ||
| return True | ||
| return False | ||
|
|
||
|
|
||
| def _is_tensor_getitem(node: Node) -> bool: | ||
| if node.op != "call_function" or node.target is not operator.getitem: | ||
| return False | ||
| return isinstance(get_node_shape_meta(node.args[0]), torch.Tensor) | ||
|
|
||
|
|
||
| def _get_seq_slice(node: Node, seq_dim: int): | ||
| """ | ||
| Return ``(start, stop)`` of a getitem that only narrows ``seq_dim`` by static amounts, where | ||
| ``start >= 0`` counts from the front and ``stop <= 0`` counts from the end. | ||
| """ | ||
| ndim = get_node_shape_meta(node.args[0]).ndim | ||
| index = node.args[1] if isinstance(node.args[1], tuple) else (node.args[1], ) | ||
| if index.count(Ellipsis) == 1: | ||
| position = index.index(Ellipsis) | ||
| index = index[:position] + (slice(None), ) * (ndim - len(index) + 1) + index[position + 1:] | ||
| index = index + (slice(None), ) * (ndim - len(index)) | ||
|
|
||
| unsupported = RuntimeError(f"AutoSP cannot shard cross entropy logits indexed by {node.args[1]!r}; only static " | ||
| f"slices of the sequence dimension such as logits[..., :-1, :] are supported") | ||
| if len(index) != ndim or not all(isinstance(i, slice) for i in index): | ||
| raise unsupported | ||
| for dim, dim_slice in enumerate(index): | ||
| if dim_slice.step not in (None, 1): | ||
| raise unsupported | ||
| if dim != seq_dim and (dim_slice.start not in (None, 0) or dim_slice.stop is not None): | ||
| raise unsupported | ||
|
|
||
| start = index[seq_dim].start or 0 | ||
| stop = 0 if index[seq_dim].stop is None else index[seq_dim].stop | ||
| if not isinstance(start, int) or not isinstance(stop, int) or start < 0 or stop > 0: | ||
| raise unsupported | ||
| if index[seq_dim].stop == 0: | ||
| raise unsupported | ||
| return start, stop | ||
|
|
||
|
|
||
| def _find_seq_dim(node: Node, length: torch.SymInt) -> int: | ||
| """Return the only dimension of ``node`` whose symbolic size is ``length``.""" | ||
| val = get_node_shape_meta(node) | ||
| dims = [ | ||
| dim for dim, size in enumerate(val.shape) | ||
| if isinstance(size, torch.SymInt) and size.node.expr == length.node.expr | ||
| ] | ||
| if len(dims) != 1: | ||
| raise RuntimeError(f"AutoSP expected exactly one dimension of size {length} in {node.format_node()} " | ||
| f"with shape {tuple(val.shape)} to line up with the sequence") | ||
| return dims[0] | ||
|
|
||
|
|
||
| def _undo_logits_seq_slices(input_node: Node, seq_len: torch.SymInt, sharded_nodes): | ||
| """ | ||
| Remove the sequence slices between the logits and a cross entropy, as each rank must keep the | ||
| logits of its whole shard. Return the window ``[start, seq_len + stop)`` of the full sequence that | ||
| the cross entropy input covered before the slices were removed. | ||
| """ | ||
| chain = [] | ||
| slices = [] | ||
| node = input_node | ||
| while _is_rank_preserving(node) or _is_reshape(node) or _is_tensor_getitem(node): | ||
| if _is_tensor_getitem(node): | ||
| slices.append(node) | ||
| elif _is_reshape(node) and slices: | ||
| # A reshape that runs before a slice would let the slice cut across the sequence dimension. | ||
| raise RuntimeError("AutoSP only supports slicing the cross entropy logits before they are reshaped") | ||
| chain.append(node) | ||
| node = node.args[0] | ||
|
|
||
| if node not in sharded_nodes: | ||
| raise RuntimeError(f"AutoSP could not trace the cross entropy input back to sequence-sharded logits; " | ||
| f"stopped at {node.format_node()}") | ||
|
Comment on lines
+249
to
+251
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a model applies a sequence-wide transform such as Useful? React with 👍 / 👎.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in b686c90: the pass now rejects logits reordered by |
||
| if _is_sequence_reordering(node): | ||
| raise RuntimeError(f"AutoSP cannot shard cross entropy logits reordered by {node.format_node()}, as each " | ||
| f"rank would only reorder its own shard of the sequence") | ||
|
|
||
| seq_dim = _find_seq_dim(node, seq_len) | ||
| start, stop = 0, 0 | ||
| for slice_node in slices: | ||
| slice_start, slice_stop = _get_seq_slice(slice_node, seq_dim) | ||
| start += slice_start | ||
| stop += slice_stop | ||
| if slices and any(len(chain_node.users) != 1 for chain_node in chain): | ||
| raise RuntimeError("AutoSP cannot remove the sequence slice of cross entropy logits used elsewhere") | ||
| for chain_node in chain: | ||
| if slices and _is_reshape(chain_node) and _reshape_uses_symbol_of(chain_node, seq_len): | ||
| # Once the slice is removed, a size computed from the sliced sequence no longer fits the logits. | ||
| raise RuntimeError(f"AutoSP cannot reshape sliced cross entropy logits with sizes derived from the " | ||
| f"sequence length in {chain_node.format_node()}; flatten them with " | ||
| f"view(-1, vocab_size) instead") | ||
| for slice_node in slices: | ||
| slice_node.replace_all_uses_with(slice_node.args[0]) | ||
| slice_node.graph.erase_node(slice_node) | ||
| return start, stop | ||
|
|
||
|
|
||
| def _localize_cross_entropy_target(gm: GraphModule, loss_node: Node, window, seq_len: torch.SymInt, ignore_index): | ||
| """ | ||
| Replace the full-sequence target of a cross entropy with the part that lines up with this rank's | ||
| logits. Positions outside the logits window get ``ignore_index``. | ||
| """ | ||
| consumer = loss_node | ||
| target = _get_cross_entropy_arg(loss_node, "target", 1, None) | ||
| while _is_rank_preserving(target) or _is_reshape(target): | ||
|
tohtana marked this conversation as resolved.
Resolved
|
||
| if _is_reshape(target) and len(target.all_input_nodes) > 1: | ||
| # A size taken from the full sequence would not fit this rank's shorter target. | ||
| raise RuntimeError(f"AutoSP cannot reshape a cross entropy target with explicit symbolic sizes in " | ||
| f"{target.format_node()}; flatten it with view(-1) instead") | ||
| if len(target.users) > 1: | ||
| # Another cross entropy may share this transform. Rewire a copy so that its target stays whole. | ||
| with gm.graph.inserting_after(target): | ||
| target_copy = gm.graph.node_copy(target) | ||
| consumer.replace_input_with(target, target_copy) | ||
| target = target_copy | ||
| consumer = target | ||
| target = target.args[0] | ||
|
|
||
| start, stop = window | ||
| # The target pairs with the logits position by position, so it covers as many positions as the window. | ||
| seq_dim = _find_seq_dim(target, seq_len + stop - start) | ||
| target_val = get_node_shape_meta(target) | ||
|
|
||
| padded = target | ||
| if start or stop: | ||
| anchor = target | ||
| if target.op == "placeholder": | ||
| anchor = [node for node in gm.graph.nodes if node.op == "placeholder"][-1] | ||
| # Pad with cat rather than F.pad, which would specialize the symbolic sequence length. | ||
| pieces = [target] | ||
| for width, at_front in ((start, True), (-stop, False)): | ||
| if width == 0: | ||
| continue | ||
| # Expand a single position, as a large shift can be wider than the target itself. | ||
| with gm.graph.inserting_after(anchor): | ||
| anchor = gm.graph.call_function(torch.narrow, args=(target, seq_dim, 0, 1)) | ||
| with gm.graph.inserting_after(anchor): | ||
| anchor = gm.graph.call_function(torch.full_like, args=(anchor, ignore_index)) | ||
| sizes = [-1] * target_val.ndim | ||
| sizes[seq_dim] = width | ||
| with gm.graph.inserting_after(anchor): | ||
| anchor = gm.graph.call_method("expand", args=(anchor, *sizes)) | ||
| pieces.insert(0 if at_front else len(pieces), anchor) | ||
| with gm.graph.inserting_after(anchor): | ||
| padded = gm.graph.call_function(torch.cat, args=(pieces, seq_dim)) | ||
| local_target = insert_sequence_shard(gm, padded, target_val.ndim, seq_dim, find_node_by_name(gm, str(seq_len))) | ||
| consumer.replace_input_with(target, local_target) | ||
|
tohtana marked this conversation as resolved.
Resolved
|
||
|
|
||
|
|
||
| def _check_label_consumers(label_ids_node: Node, sharded_nodes): | ||
| """Labels stay unsharded, so they may only flow into cross entropy targets.""" | ||
| worklist = [label_ids_node] | ||
| label_nodes = set() | ||
| while worklist: | ||
| node = worklist.pop() | ||
| if node in label_nodes: | ||
| continue | ||
| label_nodes.add(node) | ||
| if any(input_node in sharded_nodes for input_node in node.all_input_nodes): | ||
| raise RuntimeError(f"AutoSP labels cannot be combined with sequence-sharded tensors in " | ||
| f"{node.format_node()}") | ||
| for user in node.users: | ||
| if user.target is torch.nn.functional.cross_entropy: | ||
| if _get_cross_entropy_arg(user, "target", 1, None) is not node: | ||
| raise RuntimeError(f"AutoSP labels may only be the target of a cross entropy, got " | ||
| f"{user.format_node()}") | ||
| elif user.op == "output": | ||
| raise RuntimeError("AutoSP labels cannot be returned from the compiled module") | ||
| else: | ||
| worklist.append(user) | ||
|
|
||
|
|
||
| def pass_shard_label_ids(gm: GraphModule, example_inputs): | ||
| """ | ||
| Shard the labels and make every cross entropy on them return the loss of the full sequence. | ||
| Make every cross entropy on the labels return the loss of the full sequence. | ||
|
|
||
| Each cross entropy is summed per shard and combined over the SP group, which weights shards | ||
| with different numbers of valid tokens correctly. | ||
| The labels stay whole, so any shift the model applies to them sees the full sequence. Each cross | ||
| entropy then takes the part of its target that lines up with this rank's logits, sums its loss and | ||
| combines the sums over the SP group, which weights shards with different numbers of valid tokens | ||
| correctly. | ||
| """ | ||
| label_ids_node = get_label_id_node(gm) | ||
| input_ids_node = get_input_id_node(gm) | ||
| seq_len = get_node_shape_meta(input_ids_node).shape[get_autosp_seq_dim(input_ids_node)] | ||
|
|
||
| sharded_nodes = set() | ||
| worklist = [node for node in (input_ids_node, get_position_id_node(gm)) if node is not None] | ||
| while worklist: | ||
| node = worklist.pop() | ||
| if node not in sharded_nodes: | ||
| sharded_nodes.add(node) | ||
| worklist.extend(node.users) | ||
|
|
||
| _check_label_consumers(label_ids_node, sharded_nodes) | ||
|
|
||
| label_users = set() | ||
| worklist = [label_ids_node] | ||
|
|
@@ -165,7 +383,6 @@ def pass_shard_label_ids(gm: GraphModule, example_inputs): | |
| node for node in gm.graph.nodes if node.target is torch.nn.functional.cross_entropy | ||
| and _get_cross_entropy_arg(node, "target", 1, None) in label_users | ||
| ] | ||
| shard_tensor_node(gm, label_ids_node) | ||
|
|
||
| for loss_node in loss_nodes: | ||
| weight = _get_cross_entropy_arg(loss_node, "weight", 2, None) | ||
|
|
@@ -180,20 +397,24 @@ def pass_shard_label_ids(gm: GraphModule, example_inputs): | |
| if reduction not in ("mean", "sum"): | ||
| raise RuntimeError(f"AutoSP supports cross entropy with reduction 'mean' or 'sum', got {reduction!r}") | ||
| ignore_index = _get_cross_entropy_arg(loss_node, "ignore_index", 4, -100) | ||
|
|
||
| target_node = _get_cross_entropy_arg(loss_node, "target", 1, None) | ||
| if get_node_shape_meta(target_node).dtype.is_floating_point: | ||
| if get_node_shape_meta(_get_cross_entropy_arg(loss_node, "target", 1, None)).dtype.is_floating_point: | ||
| # Class probability targets have no ignore_index, so counting valid tokens does not apply. | ||
| raise RuntimeError("AutoSP supports cross entropy with class index targets only, not class probabilities") | ||
|
|
||
| window = _undo_logits_seq_slices(_get_cross_entropy_arg(loss_node, "input", 0, None), seq_len, sharded_nodes) | ||
| _localize_cross_entropy_target(gm, loss_node, window, seq_len, ignore_index) | ||
|
|
||
| target_node = _get_cross_entropy_arg(loss_node, "target", 1, None) | ||
| # Under autocast, cross entropy returns FP32 even for half-precision logits. | ||
| loss_dtype = get_node_shape_meta(loss_node).dtype | ||
| with gm.graph.inserting_after(target_node): | ||
| valid_mask = gm.graph.call_function(operator.ne, args=(target_node, ignore_index)) | ||
| with gm.graph.inserting_after(valid_mask): | ||
| valid_tokens = gm.graph.call_function(torch.sum, args=(valid_mask, )) | ||
|
tohtana marked this conversation as resolved.
Resolved
|
||
|
|
||
| # Sum the per-token losses in FP32: a half-precision sum over a long shard overflows even | ||
| # where the mean the model asked for would not. | ||
| # Sum the per-token losses in at least FP32: a half-precision sum over a long shard overflows | ||
| # even where the mean the model asked for would not. | ||
| sum_dtype = torch.promote_types(loss_dtype, torch.float32) | ||
| if "reduction" in loss_node.kwargs or len(loss_node.args) <= 6: | ||
| loss_node.update_kwarg("reduction", "none") | ||
| else: | ||
|
|
@@ -205,7 +426,7 @@ def pass_shard_label_ids(gm: GraphModule, example_inputs): | |
| loss_node.update_arg(position, None) | ||
|
|
||
| with gm.graph.inserting_after(loss_node): | ||
| token_losses = gm.graph.call_method("float", args=(loss_node, )) | ||
| token_losses = gm.graph.call_method("to", args=(loss_node, sum_dtype)) | ||
| with gm.graph.inserting_after(token_losses): | ||
| local_loss_sum = gm.graph.call_function(torch.sum, args=(token_losses, )) | ||
| with gm.graph.inserting_after(local_loss_sum): | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The fresh remaining reshape case is on the logits side rather than the target-side symbolic reshape fixed earlier: for
logits[..., :-1, :].reshape(batch * (seq_len - 1), vocab), traversal sees the reshape before it records the underlying slice, so this check permits it. The pass then erases the slice but leaves the explicitS-1reshape size, and fake propagation or the first forward fails while trying to reshapeB*S*Velements toB*(S-1)*V; require inferred reshape dimensions or rewrite the symbolic size when removing the slice.Useful? React with 👍 / 👎.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed in b686c90: when a slice is removed, a reshape after it whose sizes are derived from the symbolic sequence length is rejected with a hint to use
view(-1, vocab_size). Static sizes such asview(-1, logits.size(-1))stay supported. Covered by theexplicit_logits_viewrow (logits[..., :-1, :].reshape(B * (S - 1), V)).There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for the fix! The same rotation can be written in other ways, such as
torch.cat((logits[:, 1:], logits[:, :1]), dim=1), which still runs and silently produces an incorrect loss.Since it’s hard to detect every equivalent pattern, could we revise “AutoSP raises an error for any other pattern” to warn that some unsupported patterns may go undetected?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Okay thanks, I reword in f275054 as follow:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@tohtana updated.