From 156c1a89ef26c084f89fde57225db584f8bca046 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Mon, 16 Nov 2026 10:49:48 +0100 Subject: [PATCH 1/2] [checkpoint] Prepare an affine transfer once and run it per step A weight sync moves the same batch every step, and every execute_transfers call repeats validation, the alias check, scheduling, the digest agreement, arena allocation and region views before the first byte moves. prepare_transfers does that work once and returns a PreparedTransfer whose run only moves data. The handle keeps the arenas that #9027's schedule reports, and for each round one list of (destination, source) views: the packs before a send and the unpacks after a receive, plus one list of local copies. execute_transfers is now prepare_transfers followed by one run, with the same collectives as before. Python's cyclic collector is paused while the views are built: tens of thousands stay alive together, and the collections their allocation triggered made a one-shot call at 8 processes on CPU about a third slower than master. A buffer that moves after prepare, as reload_states moves offloaded ZeRO-3 partitions, would leave those views writing into the old storage. So run checks by default: each process compares its buffers (the held tensors, or requests rebuilt from the live state) with what it prepared, and one small all-gather refuses on every process, before data moves, if a buffer moved on any of them or the processes are not running the same handle for the same time. The shape of prepare_transfers and the buffer layout check follow jinyouzhi/DeepSpeed#8 (a8c34e3); this is its staged half, without the window transport. Signed-off-by: 0z5a Co-authored-by: Jin, Youzhi --- .../checkpoint/affine_transfer_executor.py | 329 +++++++++++++----- docs/code-docs/source/affine-transfer.rst | 34 +- .../test_affine_transfer_executor.py | 181 +++++++++- 3 files changed, 447 insertions(+), 97 deletions(-) diff --git a/deepspeed/checkpoint/affine_transfer_executor.py b/deepspeed/checkpoint/affine_transfer_executor.py index 92820cf76a62..b73b178ada52 100644 --- a/deepspeed/checkpoint/affine_transfer_executor.py +++ b/deepspeed/checkpoint/affine_transfer_executor.py @@ -42,6 +42,8 @@ mover should be perturbing. """ +import contextlib +import gc import sys import zlib from dataclasses import dataclass @@ -55,7 +57,8 @@ __all__ = [ 'AffineTransferExecutionError', 'TransferEndpoints', 'StagingBudget', 'TransferStats', 'TransferRequest', - 'plan_digest', 'execute_transfer', 'execute_transfers', 'ScheduledRound', 'ProcessSchedule', 'schedule_transfers' + 'plan_digest', 'execute_transfer', 'execute_transfers', 'prepare_transfers', 'PreparedTransfer', 'ScheduledRound', + 'ProcessSchedule', 'schedule_transfers' ] @@ -403,7 +406,7 @@ class ScheduledRound: @dataclass(frozen=True) class ProcessSchedule: - """What one process does for a batch, exactly as ``execute_transfers`` runs it. + """What one process does for a batch, exactly as ``execute_transfers`` or a prepared transfer runs it. ``sends`` and ``receives`` map each peer to its rounds in order. ``send_arena_elements`` and ``receive_arena_elements`` give the staging arena, in elements, that each peer direction @@ -482,6 +485,35 @@ def execute_transfers(requests: Sequence[TransferRequest], Each direction stages at most ``bytes_per_peer`` per active peer. All requests and staging allocations are checked before payload, including read/write aliases between parameters. The caller stream's final unpack and local copies complete before this function returns. + A batch moved repeatedly can be prepared once with ``prepare_transfers`` instead. + """ + # The agreement inside prepare_transfers has just run, so a handle check would repeat it. + return prepare_transfers(requests, endpoints, dtype, budget, group).run(check=False) + + +@dataclass(frozen=True) +class _StagedRound: + """One peer's message in one round, and the copies that fill or empty it.""" + peer: int + # What goes on the wire: a slice of the peer's arena, or the region itself when it is sent in place. + payload: torch.Tensor + # ``(destination, source)`` view pairs: packs into the arena before a send, unpacks after a receive. + # Empty when the round is sent or received in place. + copies: Tuple[Tuple[torch.Tensor, torch.Tensor], ...] + + +@torch.no_grad() +def prepare_transfers(requests: Sequence[TransferRequest], + endpoints: TransferEndpoints, + dtype: torch.dtype, + budget: StagingBudget = StagingBudget(), + group: Optional[object] = None) -> 'PreparedTransfer': + """Everything ``execute_transfers`` does before data moves, done once for repeated runs. + + Collective, with the same preflight and refusal contract as ``execute_transfers``. The handle keeps + the requests' buffers, the staging arenas and a view of every region each round packs, unpacks or + copies locally, so ``PreparedTransfer.run`` only moves data. The buffers must stay where they are, + and every process must run the handle the same number of times; ``run`` checks both by default. """ import deepspeed.comm as dist @@ -496,10 +528,8 @@ def execute_transfers(requests: Sequence[TransferRequest], device = torch.device(get_accelerator().current_device_name()) if initialized else torch.device('cpu') # Without a process group every buffer is on the host, whatever the accelerator is. stream_ordered = initialized and not get_accelerator().is_synchronized_device() - indexed, local_copies = [], [] - recv_rounds, send_rounds, recv_arenas, send_arenas = {}, {}, {}, {} - recv_direct, send_direct = {}, {} - digest, rejected = 0, 0 + indexed, rounds, local_copies, layout = [], [], (), () + element_size, digest, rejected = 0, 0, 0 error = None try: element_size = torch.empty((), dtype=dtype).element_size() @@ -535,28 +565,31 @@ def execute_transfers(requests: Sequence[TransferRequest], tuple((request.key, digests[id(request.plan)]) for request in requests)) digest = zlib.crc32(repr(fingerprint).encode()) # The same computation as schedule_transfers, so what it reports is what runs here. - lane = _lanes(indexed, [rank], budget.bytes_per_peer // element_size)[rank] - local_copies, recv_rounds, send_rounds = lane.local_copies, lane.recv_rounds, lane.send_rounds - recv_direct = { - peer: [ - _direct_payload(entries, requests, dtype, False) if direct else None - for (entries, _), direct in zip(rounds, lane.recv_direct[peer]) - ] - for peer, rounds in recv_rounds.items() - } - send_direct = { - peer: [ - _direct_payload(entries, requests, dtype, True) if direct else None - for (entries, _), direct in zip(rounds, lane.send_direct[peer]) - ] - for peer, rounds in send_rounds.items() - } - recv_arenas = {peer: torch.empty(size, dtype=dtype, device=device) for peer, size in lane.recv_arena.items()} - send_arenas = {peer: torch.empty(size, dtype=dtype, device=device) for peer, size in lane.send_arena.items()} + with _collector_paused(): + lane = _lanes(indexed, [rank], budget.bytes_per_peer // element_size)[rank] + recv_arenas = { + peer: torch.empty(size, dtype=dtype, device=device) + for peer, size in lane.recv_arena.items() + } + send_arenas = { + peer: torch.empty(size, dtype=dtype, device=device) + for peer, size in lane.send_arena.items() + } + recvs = _staged_rounds(lane.recv_rounds, lane.recv_direct, recv_arenas, requests, dtype, source=False) + sends = _staged_rounds(lane.send_rounds, lane.send_direct, send_arenas, requests, dtype, source=True) + count = max((len(value) for value in list(recvs.values()) + list(sends.values())), default=0) + rounds = [([peer_rounds[index] for _, peer_rounds in sorted(recvs.items()) if index < len(peer_rounds)], + [peer_rounds[index] for _, peer_rounds in sorted(sends.items()) if index < len(peer_rounds)]) + for index in range(count)] + local_copies = tuple( + (_operation_region(requests, item, operation, dtype, source=False), + _operation_region(requests, item, operation, dtype, source=True), operation.payload_bytes) + for item, operation in lane.local_copies) + layout = _buffer_layout(requests) except (ValueError, TypeError, KeyError, AttributeError, RuntimeError, OverflowError, MemoryError) as caught: error = caught rejected = 1 - recv_arenas, send_arenas = {}, {} + rounds = [] print(f'[transfer] rank {rank} rejected the batch: {caught}', file=sys.stderr, flush=True) # Invalid metadata must not be inspected again while constructing the agreement payload. @@ -574,77 +607,185 @@ def execute_transfers(requests: Sequence[TransferRequest], elif rejected: raise AffineTransferExecutionError(f'the transfer batch was rejected before data moved: {error}') from error - rounds = max((len(value) for value in list(recv_rounds.values()) + list(send_rounds.values())), default=0) - sent = received = copied = 0 - pending = [] - - for round_index in range(rounds): - p2p = [] - for peer, peer_rounds in sorted(recv_rounds.items()): - if round_index < len(peer_rounds): - payload = recv_direct[peer][round_index] - if payload is None: - payload = recv_arenas[peer][:peer_rounds[round_index][1]] - p2p.append(('recv', payload, peer)) - for peer, peer_rounds in sorted(send_rounds.items()): - if round_index >= len(peer_rounds): - continue - entries, count = peer_rounds[round_index] - payload = send_direct[peer][round_index] - if payload is None: - arena = send_arenas[peer] - for item, operation, offset in entries: - source = _flat(operation.segment.source_rank, requests[item].source_buffers, dtype) - arena[offset:offset + operation.elements].view(operation.chunk_shape).copy_( - _region(source, operation.source_offset, operation.chunk_shape, - operation.segment.source_strides)) - payload = arena[:count] - sent += count * element_size - p2p.append(('send', payload, peer)) - operations = [ - dist.P2POp(dist.irecv if direction == 'recv' else dist.isend, - buffer, - dist.get_global_rank(group, peer) if group is not None else peer, - group=group) for direction, buffer, peer in p2p - ] - if operations: - for handle in dist.batch_isend_irecv(operations): - block_stream = getattr(handle, 'block_current_stream', None) - if stream_ordered and callable(block_stream): - # Order arena reuse on the caller stream; transport errors surface at the bounded drain below. - block_stream() - pending.append(handle) - else: + return PreparedTransfer(_requests=tuple(requests), + _layout=layout, + _rounds=rounds, + _local_copies=local_copies, + _segments=sum(len(request.plan.segments) for request in requests), + _chunks=len(indexed), + _element_size=element_size, + _digest=digest, + _group=group, + _world_size=world_size, + _device=device, + _stream_ordered=stream_ordered) + + +@dataclass(eq=False) +class PreparedTransfer: + """A batch validated, agreed and laid out by ``prepare_transfers``; ``run`` only moves data. + + The handle holds the requests' buffers and views into them. A buffer that moves to new storage + afterwards, as ZeRO-3 ``offload_states`` / ``reload_states`` move offloaded partitions, would + leave those views writing into the old storage, so ``run`` checks first by default. + """ + _requests: Tuple[TransferRequest, ...] + _layout: Tuple + # Per round: the receives, then the sends, each ordered by peer. + _rounds: List[Tuple[List[_StagedRound], List[_StagedRound]]] + # ``(target region, source region, bytes)`` for each copy that stays on this process. + _local_copies: Tuple[Tuple[torch.Tensor, torch.Tensor, int], ...] + _segments: int + _chunks: int + _element_size: int + _digest: int + _group: Optional[object] + _world_size: int + _device: torch.device + _stream_ordered: bool + _runs: int = 0 + + @torch.no_grad() + def run(self, requests: Optional[Sequence[TransferRequest]] = None, *, check: bool = True) -> TransferStats: + """Move the batch again from the buffers' current contents. Collective; synchronous on return. + + With ``check`` (the default), every process first compares its buffers with what it prepared: + ``requests``, rebuilt from the live state such as fresh ``zero_param_storage_shard`` views, or + else the tensors the handle holds, which catches a buffer re-pointed in place (``.data = ...``). + One small all-gather then refuses, on every process and before any data moves, if a buffer + moved or changed shape on any process, or if the processes are not running the same handle for + the same time. Prepare the batch again after a refusal. ``check=False`` skips that collective, + for a caller that already guarantees all three; every process must pass the same ``check``. + """ + if check: + self._check(requests) + return self._move() + + def _check(self, requests: Optional[Sequence[TransferRequest]]) -> None: + import deepspeed.comm as dist + try: + moved = int(_buffer_layout(self._requests if requests is None else requests) != self._layout) + except (AttributeError, TypeError, RuntimeError): + moved = 1 + if self._world_size > 1: + # One process raising alone would leave its peers waiting in the transfer. + local = torch.tensor([self._digest, self._runs, moved], dtype=torch.int64, device=self._device) + seen = torch.empty((self._world_size, 3), dtype=torch.int64, device=self._device) + dist.all_gather(list(seen.unbind(0)), local, group=self._group) + decisions = seen.cpu().tolist() + if any(row[2] for row in decisions): + moved = 1 + elif len({(row[0], row[1]) for row in decisions}) != 1: + raise AffineTransferExecutionError('processes are not running the same prepared transfer for the ' + 'same time; no data moved') + if moved: + raise AffineTransferExecutionError('a buffer of the prepared transfer moved or changed since ' + 'prepare_transfers; prepare the batch again. No data moved.') + + def _move(self) -> TransferStats: + import deepspeed.comm as dist + group, element_size, stream_ordered = self._group, self._element_size, self._stream_ordered + sent = received = copied = 0 + pending = [] + for round_index, (recvs, sends) in enumerate(self._rounds): + for staged in sends: + for destination, source in staged.copies: + destination.copy_(source) + p2p = [(dist.irecv, staged) for staged in recvs] + [(dist.isend, staged) for staged in sends] + operations = [ + dist.P2POp(direction, + staged.payload, + dist.get_global_rank(group, staged.peer) if group is not None else staged.peer, + group=group) for direction, staged in p2p + ] + if operations: + for handle in dist.batch_isend_irecv(operations): + block_stream = getattr(handle, 'block_current_stream', None) + if stream_ordered and callable(block_stream): + # Order arena reuse on the caller stream; transport errors surface at the bounded drain below. + block_stream() + pending.append(handle) + else: + handle.wait() + # Stream ordering already protects arena reuse. Bound retained handles while avoiding + # a host polling delay after every small group of rounds; all errors are observed on return. + if (round_index + 1) % 32 == 0 or round_index + 1 == len(self._rounds): + for handle in pending: handle.wait() - # Stream ordering already protects arena reuse. Bound retained handles while avoiding - # a host polling delay after every small group of rounds; all errors are observed on return. - if (round_index + 1) % 32 == 0 or round_index + 1 == rounds: - for handle in pending: - handle.wait() - pending.clear() - for peer, peer_rounds in recv_rounds.items(): - if round_index >= len(peer_rounds): - continue - received += peer_rounds[round_index][1] * element_size - if recv_direct[peer][round_index] is not None: + pending.clear() + sent += sum(staged.payload.numel() for staged in sends) * element_size + received += sum(staged.payload.numel() for staged in recvs) * element_size + for staged in recvs: + for destination, source in staged.copies: + destination.copy_(source) + + for target, source, size in self._local_copies: + target.copy_(source) + copied += size + + if stream_ordered: + get_accelerator().current_stream().synchronize() + self._runs += 1 + return TransferStats(self._segments, self._chunks, len(self._rounds), copied, sent, received) + + +@contextlib.contextmanager +def _collector_paused(): + """Pause Python's cyclic garbage collector while a batch's views are built. + + Tens of thousands of views and tuples stay alive together there, and every collection their + allocation triggers walks them all again: at 8 processes on CPU that was a third of a one-shot + ``execute_transfers``. They hold no reference cycles, so there is nothing for it to collect. + """ + enabled = gc.isenabled() + gc.disable() + try: + yield + finally: + if enabled: + gc.enable() + + +def _operation_region(requests, item, operation, dtype, source: bool) -> torch.Tensor: + """One operation's region in its source or target buffer, as a view.""" + segment = operation.segment + if source: + flat = _flat(segment.source_rank, requests[item].source_buffers, dtype) + return _region(flat, operation.source_offset, operation.chunk_shape, segment.source_strides) + flat = _flat(segment.target_rank, requests[item].target_buffers, dtype) + return _region(flat, operation.target_offset, operation.chunk_shape, segment.target_strides) + + +def _staged_rounds(rounds_by_peer, direct_by_peer, arenas, requests, dtype, source: bool): + """Each peer's rounds in one direction, resolved to wire payloads and the copies around them.""" + staged = {} + for peer, peer_rounds in rounds_by_peer.items(): + built = [] + for (entries, count), direct in zip(peer_rounds, direct_by_peer[peer]): + if direct: + built.append(_StagedRound(peer, _direct_payload(entries, requests, dtype, source), ())) continue - for item, operation, offset in peer_rounds[round_index][0]: - target = _flat(operation.segment.target_rank, requests[item].target_buffers, dtype) - payload = recv_arenas[peer][offset:offset + operation.elements].view(operation.chunk_shape) - _region(target, operation.target_offset, operation.chunk_shape, - operation.segment.target_strides).copy_(payload) - - for item, operation in local_copies: - source = _flat(operation.segment.source_rank, requests[item].source_buffers, dtype) - target = _flat(operation.segment.target_rank, requests[item].target_buffers, dtype) - _region(target, operation.target_offset, operation.chunk_shape, operation.segment.target_strides).copy_( - _region(source, operation.source_offset, operation.chunk_shape, operation.segment.source_strides)) - copied += operation.payload_bytes - - if stream_ordered: - get_accelerator().current_stream().synchronize() - return TransferStats(sum(len(request.plan.segments) for request in requests), len(indexed), rounds, copied, sent, - received) + arena = arenas[peer] + copies = [] + for item, operation, offset in entries: + slot = arena[offset:offset + operation.elements].view(operation.chunk_shape) + region = _operation_region(requests, item, operation, dtype, source) + copies.append((slot, region) if source else (region, slot)) + built.append(_StagedRound(peer, arena[:count], tuple(copies))) + staged[peer] = built + return staged + + +def _buffer_layout(requests: Sequence[TransferRequest]) -> Tuple: + """Where each request's local buffers are, and under which plan; a moved buffer changes it.""" + layout = [] + for request in requests: + for side, buffers in (('source', request.source_buffers), ('target', request.target_buffers)): + for location in sorted(buffers): + buffer = buffers[location] + layout.append((request.key, request.plan, side, location, buffer.data_ptr(), tuple(buffer.shape), + buffer.stride(), buffer.dtype, buffer.device)) + return tuple(layout) def _data_range(buffer: torch.Tensor) -> Tuple[int, int]: diff --git a/docs/code-docs/source/affine-transfer.rst b/docs/code-docs/source/affine-transfer.rst index 2987676035f3..c8252befe59b 100644 --- a/docs/code-docs/source/affine-transfer.rst +++ b/docs/code-docs/source/affine-transfer.rst @@ -33,6 +33,35 @@ The budget applies to each peer direction, not total device memory. Weights must remain frozen during transfer; optimizer migration and live AutoEP swaps are outside this API. Maintenance contact: ``@0z5a``. +Repeated transfers +------------------ + +A weight sync moves the same batch every step. ``prepare_transfers`` does +everything ``execute_transfers`` does before data moves -- validation, the +schedule agreement, the staging arenas and a view of every region each round +packs, unpacks or copies locally -- once, and returns a handle whose ``run`` only +moves data:: + + from deepspeed.checkpoint.affine_transfer_executor import prepare_transfers + + prepared = prepare_transfers(requests, endpoints, dtype, budget) + for step in steps: + ... # update the source buffers in place + prepared.run() + +``prepare_transfers`` is collective and refuses a batch exactly as +``execute_transfers`` does; ``execute_transfers`` is ``prepare_transfers`` +followed by one run. The handle keeps the buffers it was prepared with and its +arenas stay allocated, so buffers must stay where they are and every process +must run the handle the same number of times. By default ``run`` checks both +before any data moves, with one small all-gather: if a buffer moved or changed +shape on any process, or the processes are not running the same handle for the +same time, every process raises and the batch has to be prepared again. Pass the +requests rebuilt from the live state, such as fresh ``zero_param_storage_shard`` +views after ``reload_states``, as ``run(requests)`` to check those instead of +the tensors the handle holds. ``run(check=False)`` skips the all-gather, so +every process must pass the same ``check``. + Planning a sync with replica policies ------------------------------------- @@ -104,6 +133,7 @@ Pass the plans in request order, since that order fixes how chunks are packed into rounds. Each ``ProcessSchedule`` lists, per peer and direction, the rounds with their element and chunk counts and whether the round is sent or received in place, the staging arena each peer direction allocates, the local copies, and the -round count that ``TransferStats.rounds`` reports. ``execute_transfers`` builds -its rounds and arenas from the same computation, so the two cannot drift apart. +round count that ``TransferStats.rounds`` reports. ``execute_transfers`` and +``prepare_transfers`` build their rounds and arenas from the same computation, +so the two cannot drift apart. The schedule follows the current transport described above and changes with it. diff --git a/tests/unit/checkpoint/test_affine_transfer_executor.py b/tests/unit/checkpoint/test_affine_transfer_executor.py index 2a87958ce009..edf1d2440c44 100644 --- a/tests/unit/checkpoint/test_affine_transfer_executor.py +++ b/tests/unit/checkpoint/test_affine_transfer_executor.py @@ -4,6 +4,7 @@ # DeepSpeed Team """Check transfer payloads against extraction from the logical tensor.""" +import gc import math import os from dataclasses import replace @@ -16,7 +17,8 @@ from deepspeed.checkpoint.affine_transfer import PlanBudget, plan_transfer from deepspeed.checkpoint.affine_transfer_executor import (AffineTransferExecutionError, StagingBudget, TransferEndpoints, TransferRequest, _is_packed, - execute_transfer, execute_transfers, schedule_transfers) + execute_transfer, execute_transfers, prepare_transfers, + schedule_transfers) from unit.common import DistributedTest UNWRITTEN = float('nan') @@ -269,6 +271,131 @@ def test_schedule_is_what_the_executor_runs(self, world_size): assert any(r.direct for r in every) and any(not r.direct for r in every) +def _arena_elements(rounds, element_size): + """Elements of each peer's staging arena, read back from the rounds that stage through it.""" + return { + staged.peer: staged.payload.untyped_storage().nbytes() // element_size + for staged in rounds if staged.copies + } + + +def _refill(requests, offset): + """Shift every source by ``offset`` in place, keeping its storage.""" + for request in requests: + for buffer in request.source_buffers.values(): + buffer.add_(offset) + + +class TestB2Prepared(DistributedTest): + world_size = 2 + + @pytest.mark.parametrize('world_size', [2, 4]) + def test_repeated_runs_move_the_current_contents(self, world_size): + _join_transfer_group() + rank = dist.get_rank() + endpoints, requests, checks, guards = _mixed_batch(world_size, rank) + # 16 elements per message: packed and in-place rounds, as in the schedule test. + budget = StagingBudget(64) + prepared = prepare_transfers(requests, endpoints, torch.float32, budget) + schedule = schedule_transfers([request.plan for request in requests], endpoints, budget, 4, process=rank)[rank] + recvs = [staged for receives, _ in prepared._rounds for staged in receives] + sends = [staged for _, sending in prepared._rounds for staged in sending] + # The handle keeps exactly the arenas the schedule reports. + assert _arena_elements(recvs, 4) == dict(schedule.receive_arena_elements) + assert _arena_elements(sends, 4) == dict(schedule.send_arena_elements) + + once = execute_transfers(requests, endpoints, torch.float32, budget) + for run in range(3): + if run: + _refill(requests, 1000.0) + stats = prepared.run() + assert stats == once + for targets, target_map, full in checks: + _assert_matches(targets, target_map, full + 1000.0 * run) + for storage, size in guards: + assert torch.equal(storage[:5], torch.full((5, ), -17.0, device=storage.device)) + assert torch.equal(storage[5 + size:], torch.full((12 - 5, ), -17.0, device=storage.device)) + + @pytest.mark.parametrize('world_size', [2, 4]) + def test_a_buffer_moved_on_one_process_stops_every_process(self, world_size): + _join_transfer_group() + rank = dist.get_rank() + source, target = _row_to_column(world_size) + full = _full(16, 16, torch.float32) + endpoints = TransferEndpoints({location: location + for location in source.shard_shapes}, + {location: location + for location in target.shard_shapes}) + sources = { + location: source.extract(full, location).contiguous() + for location, process in endpoints.source.items() if process == rank + } + targets = _targets(target, endpoints, torch.float32) + request = TransferRequest('weight', plan_transfer(target, source), sources, targets) + prepared = prepare_transfers([request], endpoints, torch.float32, StagingBudget(64)) + prepared.run() + _assert_matches(targets, target, full) + + # The last process re-points one target in place, as a reload into new storage would. + moved = None + if rank == world_size - 1: + location = next(iter(targets)) + old = targets[location].data + targets[location].data = torch.full_like(old, -3.0) + moved = (location, old) + before = {location: buffer.clone() for location, buffer in targets.items()} + for buffer in sources.values(): + buffer.add_(1.0) + with pytest.raises(AffineTransferExecutionError, match='moved or changed'): + prepared.run() + for location, buffer in targets.items(): + assert torch.equal(buffer, before[location]) + if moved is not None: + assert torch.equal(moved[1], target.extract(full, moved[0])) + + @pytest.mark.parametrize('world_size', [2, 4]) + def test_requests_rebuilt_from_the_live_state_are_checked(self, world_size): + _join_transfer_group() + rank = dist.get_rank() + endpoints, requests, checks, _ = _mixed_batch(world_size, rank) + prepared = prepare_transfers(requests, endpoints, torch.float32, StagingBudget(64)) + + def rebuilt(view): + return [ + replace(request, + source_buffers={ + location: view(buffer) + for location, buffer in request.source_buffers.items() + }, + target_buffers=dict(request.target_buffers)) for request in requests + ] + + # Fresh views of the same storage, as accessors return them, pass. + prepared.run(rebuilt(lambda buffer: buffer.view(buffer.shape))) + for targets, target_map, full in checks: + _assert_matches(targets, target_map, full) + # A source copied to new storage on one process stops every process. + moved = rebuilt(lambda buffer: buffer.clone() if rank == 0 else buffer) + with pytest.raises(AffineTransferExecutionError, match='moved or changed'): + prepared.run(moved) + # So does a batch that is not the prepared one. + with pytest.raises(AffineTransferExecutionError, match='moved or changed'): + prepared.run(requests[:-1] if rank == 0 else requests) + prepared.run(requests) + + def test_processes_running_different_handles_are_refused(self): + _join_transfer_group() + rank = dist.get_rank() + endpoints, requests, _, _ = _mixed_batch(self.world_size, rank) + whole = prepare_transfers(requests, endpoints, torch.float32, StagingBudget(64)) + first = prepare_transfers(requests[:1], endpoints, torch.float32, StagingBudget(64)) + with pytest.raises(AffineTransferExecutionError, match='not running the same prepared transfer'): + (whole if rank == 0 else first).run() + # The refusal came before any data moved, so both handles still run. + assert whole.run() == execute_transfers(requests, endpoints, torch.float32, StagingBudget(64)) + first.run() + + class TestB2DisjointEndpoints(DistributedTest): """Every process is either a reader or a writer, never both. @@ -501,6 +628,58 @@ def test_disjoint_windows_of_one_storage_are_valid(self): assert torch.equal(target, expected) assert torch.equal(source, expected) + def test_a_prepared_transfer_writes_where_it_was_prepared(self): + # Why run checks by default: a re-pointed target keeps receiving into its old storage. + map_ = replicated_map((2, 3), 1) + source = torch.arange(6, dtype=torch.float32).reshape(2, 3) + target = torch.full((2, 3), -1.0) + prepared = prepare_transfers([TransferRequest('weight', plan_transfer(map_, map_), {0: source}, {0: target})], + TransferEndpoints({0: 0}, {0: 0}), torch.float32, StagingBudget(8)) + old = target.data + target.data = torch.full((2, 3), -2.0) + with pytest.raises(AffineTransferExecutionError, match='moved or changed'): + prepared.run() + assert torch.equal(old, torch.full((2, 3), -1.0)) and torch.equal(target, torch.full((2, 3), -2.0)) + prepared.run(check=False) + assert torch.equal(old, source) and torch.equal(target, torch.full((2, 3), -2.0)) + + def test_prepared_runs_keep_frozen_parameters_in_place(self): + map_ = replicated_map((2, 3), 1) + source = torch.nn.Parameter(torch.arange(6, dtype=torch.float32).reshape(2, 3)) + target = torch.nn.Parameter(torch.full((2, 3), -1.0)) + identity, pointer = id(target), target.data_ptr() + prepared = prepare_transfers([TransferRequest('weight', plan_transfer(map_, map_), {0: source}, {0: target})], + TransferEndpoints({0: 0}, {0: 0}), torch.float32, StagingBudget(8)) + for step in range(2): + with torch.no_grad(): + source.add_(step) + prepared.run() + assert id(target) == identity and target.data_ptr() == pointer + assert target.requires_grad and target.grad is None + assert torch.equal(target, source) + + @pytest.mark.parametrize('enabled', [True, False]) + def test_preparing_leaves_the_garbage_collector_as_it_was(self, enabled, monkeypatch): + import deepspeed.checkpoint.affine_transfer_executor as executor + map_ = replicated_map((2, 3), 1) + request = TransferRequest('weight', plan_transfer(map_, map_), {0: torch.ones(2, 3)}, {0: torch.zeros(2, 3)}) + endpoints = TransferEndpoints({0: 0}, {0: 0}) + + def refuse(requests): + raise RuntimeError('layout failed') + + was = gc.isenabled() + try: + gc.enable() if enabled else gc.disable() + prepare_transfers([request], endpoints, torch.float32, StagingBudget(8)) + assert gc.isenabled() == enabled + monkeypatch.setattr(executor, '_buffer_layout', refuse) + with pytest.raises(AffineTransferExecutionError, match='layout failed'): + prepare_transfers([request], endpoints, torch.float32, StagingBudget(8)) + assert gc.isenabled() == enabled + finally: + gc.enable() if was else gc.disable() + def test_frozen_parameters_keep_their_identity_and_storage(self): map_ = replicated_map((2, 3), 1) source = torch.nn.Parameter(torch.arange(6, dtype=torch.float32).reshape(2, 3)) From 57476e8f99f19524235ae0f0f8f4a76f04daf5af Mon Sep 17 00:00:00 2001 From: 0z5a Date: Tue, 17 Nov 2026 17:14:40 +0100 Subject: [PATCH 2/2] [checkpoint] Tell prepared transfers apart; build one-shot rounds as they run Two handles prepared for the same batch over different buffers had the same schedule digest, so run's check passed when each process ran a different one, and the peers exchanged payloads of different batches. Each process now counts its prepare_transfers calls and adds the count to the prepare all-gather. A handle's identity is a hash of the digest and the gathered counts, the same on every process, and the check compares that instead of the digest. The scheme is jinyouzhi's, from the review. execute_transfers no longer goes through a handle. It runs the same preflight and agreement, then builds each round's views when the move reaches that round, as it did before prepare_transfers, so building them overlaps the previous round's transfer. On L20 GPUs, building them all first made a one-shot call 10-16 ms slower than master. The docs and the handle's docstring now say that run() checks only the tensors the handle holds: a view such as zero_param_storage_shard returns keeps the old storage when reload_states re-points the partition, so such handles need run(requests) rebuilt from the live state. They also say that a handle keeps its buffers allocated until it is dropped. The prepared-run test no longer reads the handle's private rounds, and the garbage-collector test refuses through a too-small budget instead of patching a private helper. Signed-off-by: 0z5a --- .../checkpoint/affine_transfer_executor.py | 304 ++++++++++++------ docs/code-docs/source/affine-transfer.rst | 35 +- .../test_affine_transfer_executor.py | 69 ++-- 3 files changed, 265 insertions(+), 143 deletions(-) diff --git a/deepspeed/checkpoint/affine_transfer_executor.py b/deepspeed/checkpoint/affine_transfer_executor.py index b73b178ada52..bb4824bf4a46 100644 --- a/deepspeed/checkpoint/affine_transfer_executor.py +++ b/deepspeed/checkpoint/affine_transfer_executor.py @@ -44,10 +44,11 @@ import contextlib import gc +import hashlib import sys import zlib from dataclasses import dataclass -from itertools import product +from itertools import count, product from typing import Dict, List, Mapping, Optional, Sequence, Tuple import torch @@ -487,8 +488,15 @@ def execute_transfers(requests: Sequence[TransferRequest], The caller stream's final unpack and local copies complete before this function returns. A batch moved repeatedly can be prepared once with ``prepare_transfers`` instead. """ - # The agreement inside prepare_transfers has just run, so a handle check would repeat it. - return prepare_transfers(requests, endpoints, dtype, budget, group).run(check=False) + agreed = _agree(requests, endpoints, dtype, budget, group, keep=False) + lane = agreed.lane + # Nothing outlives this call, so each round's views are built when the move reaches it, while the + # round before may still be in flight, instead of all of them before the first byte moves. + rounds = _round_views(lane, agreed.recv_arenas, agreed.send_arenas, requests, dtype) + local_copies = _local_copy_views(lane, requests, dtype) + sent, received, copied = _run_rounds(rounds, _round_count(lane), local_copies, group, agreed.element_size, + agreed.stream_ordered) + return TransferStats(agreed.segments, agreed.chunks, _round_count(lane), copied, sent, received) @dataclass(frozen=True) @@ -502,21 +510,40 @@ class _StagedRound: copies: Tuple[Tuple[torch.Tensor, torch.Tensor], ...] -@torch.no_grad() -def prepare_transfers(requests: Sequence[TransferRequest], - endpoints: TransferEndpoints, - dtype: torch.dtype, - budget: StagingBudget = StagingBudget(), - group: Optional[object] = None) -> 'PreparedTransfer': - """Everything ``execute_transfers`` does before data moves, done once for repeated runs. +@dataclass +class _Agreement: + """A batch every process agreed to move, and this process's share of it.""" + lane: Optional[_Lane] + recv_arenas: Dict[int, torch.Tensor] + send_arenas: Dict[int, torch.Tensor] + # Built before the agreement only when a prepared handle keeps them. + rounds: List[Tuple[List[_StagedRound], List[_StagedRound]]] + local_copies: Tuple[Tuple[torch.Tensor, torch.Tensor, int], ...] + layout: Tuple + segments: int + chunks: int + element_size: int + identity: int + world_size: int + device: torch.device + stream_ordered: bool - Collective, with the same preflight and refusal contract as ``execute_transfers``. The handle keeps - the requests' buffers, the staging arenas and a view of every region each round packs, unpacks or - copies locally, so ``PreparedTransfer.run`` only moves data. The buffers must stay where they are, - and every process must run the handle the same number of times; ``run`` checks both by default. + +# This process's preparations so far. Two batches prepared alike share a digest, so what tells their +# handles apart is which call prepared them on each process. +_preparations = count() + + +def _agree(requests: Sequence[TransferRequest], endpoints: TransferEndpoints, dtype: torch.dtype, + budget: StagingBudget, group: Optional[object], keep: bool) -> _Agreement: + """Validate a batch, lay out this process's share and agree on it with every process. + + Collective. With ``keep``, the views a prepared handle keeps are built here too, before the + agreement, so a process that cannot build them refuses on every process. """ import deepspeed.comm as dist + preparation = next(_preparations) initialized = dist.is_initialized() if initialized and group is not None and callable(getattr(group, 'rank', None)): # Ask the group itself: a weight-update group built outside the default world (as trainer <-> engine @@ -528,6 +555,7 @@ def prepare_transfers(requests: Sequence[TransferRequest], device = torch.device(get_accelerator().current_device_name()) if initialized else torch.device('cpu') # Without a process group every buffer is on the host, whatever the accelerator is. stream_ordered = initialized and not get_accelerator().is_synchronized_device() + lane, recv_arenas, send_arenas = None, {}, {} indexed, rounds, local_copies, layout = [], [], (), () element_size, digest, rejected = 0, 0, 0 error = None @@ -565,7 +593,8 @@ def prepare_transfers(requests: Sequence[TransferRequest], tuple((request.key, digests[id(request.plan)]) for request in requests)) digest = zlib.crc32(repr(fingerprint).encode()) # The same computation as schedule_transfers, so what it reports is what runs here. - with _collector_paused(): + paused = _collector_paused() if keep else contextlib.nullcontext() + with paused: lane = _lanes(indexed, [rank], budget.bytes_per_peer // element_size)[rank] recv_arenas = { peer: torch.empty(size, dtype=dtype, device=device) @@ -575,59 +604,93 @@ def prepare_transfers(requests: Sequence[TransferRequest], peer: torch.empty(size, dtype=dtype, device=device) for peer, size in lane.send_arena.items() } - recvs = _staged_rounds(lane.recv_rounds, lane.recv_direct, recv_arenas, requests, dtype, source=False) - sends = _staged_rounds(lane.send_rounds, lane.send_direct, send_arenas, requests, dtype, source=True) - count = max((len(value) for value in list(recvs.values()) + list(sends.values())), default=0) - rounds = [([peer_rounds[index] for _, peer_rounds in sorted(recvs.items()) if index < len(peer_rounds)], - [peer_rounds[index] for _, peer_rounds in sorted(sends.items()) if index < len(peer_rounds)]) - for index in range(count)] - local_copies = tuple( - (_operation_region(requests, item, operation, dtype, source=False), - _operation_region(requests, item, operation, dtype, source=True), operation.payload_bytes) - for item, operation in lane.local_copies) - layout = _buffer_layout(requests) + if keep: + rounds = list(_round_views(lane, recv_arenas, send_arenas, requests, dtype)) + local_copies = tuple(_local_copy_views(lane, requests, dtype)) + layout = _buffer_layout(requests) except (ValueError, TypeError, KeyError, AttributeError, RuntimeError, OverflowError, MemoryError) as caught: error = caught rejected = 1 - rounds = [] + recv_arenas, send_arenas, rounds = {}, {}, [] print(f'[transfer] rank {rank} rejected the batch: {caught}', file=sys.stderr, flush=True) + preparations = [preparation] # Invalid metadata must not be inspected again while constructing the agreement payload. if world_size > 1: - local = torch.tensor([digest, rejected], dtype=torch.int64, device=device) - seen = torch.empty((world_size, 2), dtype=torch.int64, device=device) + local = torch.tensor([digest, rejected, preparation], dtype=torch.int64, device=device) + seen = torch.empty((world_size, 3), dtype=torch.int64, device=device) dist.all_gather(list(seen.unbind(0)), local, group=group) # Read the complete decision once instead of synchronizing each rank's scalar separately. decisions = seen.cpu().tolist() - if any(pair[1] for pair in decisions): + if any(row[1] for row in decisions): raise AffineTransferExecutionError( 'a rank rejected the transfer batch before it began; no data moved') from error - if len({pair[0] for pair in decisions}) != 1: + if len({row[0] for row in decisions}) != 1: raise AffineTransferExecutionError('ranks disagree about the transfer schedule (batch schedule)') + preparations = [row[2] for row in decisions] elif rejected: raise AffineTransferExecutionError(f'the transfer batch was rejected before data moved: {error}') from error + return _Agreement(lane=lane, + recv_arenas=recv_arenas, + send_arenas=send_arenas, + rounds=rounds, + local_copies=local_copies, + layout=layout, + segments=sum(len(request.plan.segments) for request in requests), + chunks=len(indexed), + element_size=element_size, + identity=_identity(digest, preparations), + world_size=world_size, + device=device, + stream_ordered=stream_ordered) + + +def _identity(digest: int, preparations: Sequence[int]) -> int: + """One preparation, named the same way on every process: its digest and each process's call count.""" + hashed = hashlib.blake2b(repr((digest, tuple(preparations))).encode(), digest_size=8).digest() + # Shifted to fit the signed 64-bit element that carries it in run's check. + return int.from_bytes(hashed, 'little') >> 1 + + +@torch.no_grad() +def prepare_transfers(requests: Sequence[TransferRequest], + endpoints: TransferEndpoints, + dtype: torch.dtype, + budget: StagingBudget = StagingBudget(), + group: Optional[object] = None) -> 'PreparedTransfer': + """Everything ``execute_transfers`` does before data moves, done once for repeated runs. + + Collective, with the same preflight and refusal contract as ``execute_transfers``. The handle keeps + the requests' buffers, the staging arenas and a view of every region each round packs, unpacks or + copies locally, so ``PreparedTransfer.run`` only moves data. The buffers must stay where they are, + and every process must run the handle the same number of times; ``run`` checks both by default. + """ + agreed = _agree(requests, endpoints, dtype, budget, group, keep=True) return PreparedTransfer(_requests=tuple(requests), - _layout=layout, - _rounds=rounds, - _local_copies=local_copies, - _segments=sum(len(request.plan.segments) for request in requests), - _chunks=len(indexed), - _element_size=element_size, - _digest=digest, + _layout=agreed.layout, + _rounds=agreed.rounds, + _local_copies=agreed.local_copies, + _segments=agreed.segments, + _chunks=agreed.chunks, + _element_size=agreed.element_size, + _identity=agreed.identity, _group=group, - _world_size=world_size, - _device=device, - _stream_ordered=stream_ordered) + _world_size=agreed.world_size, + _device=agreed.device, + _stream_ordered=agreed.stream_ordered) @dataclass(eq=False) class PreparedTransfer: """A batch validated, agreed and laid out by ``prepare_transfers``; ``run`` only moves data. - The handle holds the requests' buffers and views into them. A buffer that moves to new storage - afterwards, as ZeRO-3 ``offload_states`` / ``reload_states`` move offloaded partitions, would - leave those views writing into the old storage, so ``run`` checks first by default. + The handle holds the requests' buffers and views into them, so their storage stays allocated while + the handle lives: drop it before ZeRO-3 ``offload_states`` if that memory should be freed, and + prepare again after ``reload_states``. A buffer that moves to new storage would leave the views + writing into the old one, so ``run`` checks first by default, but only the tensors it is shown. A + view such as ``zero_param_storage_shard`` returns keeps its old storage when the partition behind it + is re-pointed, so pass views like it rebuilt from the live state, as ``run(requests)``. """ _requests: Tuple[TransferRequest, ...] _layout: Tuple @@ -638,7 +701,8 @@ class PreparedTransfer: _segments: int _chunks: int _element_size: int - _digest: int + # Differs between any two prepare_transfers calls, even for the same batch; see _preparations. + _identity: int _group: Optional[object] _world_size: int _device: torch.device @@ -669,7 +733,7 @@ def _check(self, requests: Optional[Sequence[TransferRequest]]) -> None: moved = 1 if self._world_size > 1: # One process raising alone would leave its peers waiting in the transfer. - local = torch.tensor([self._digest, self._runs, moved], dtype=torch.int64, device=self._device) + local = torch.tensor([self._identity, self._runs, moved], dtype=torch.int64, device=self._device) seen = torch.empty((self._world_size, 3), dtype=torch.int64, device=self._device) dist.all_gather(list(seen.unbind(0)), local, group=self._group) decisions = seen.cpu().tolist() @@ -683,52 +747,59 @@ def _check(self, requests: Optional[Sequence[TransferRequest]]) -> None: 'prepare_transfers; prepare the batch again. No data moved.') def _move(self) -> TransferStats: - import deepspeed.comm as dist - group, element_size, stream_ordered = self._group, self._element_size, self._stream_ordered - sent = received = copied = 0 - pending = [] - for round_index, (recvs, sends) in enumerate(self._rounds): - for staged in sends: - for destination, source in staged.copies: - destination.copy_(source) - p2p = [(dist.irecv, staged) for staged in recvs] + [(dist.isend, staged) for staged in sends] - operations = [ - dist.P2POp(direction, - staged.payload, - dist.get_global_rank(group, staged.peer) if group is not None else staged.peer, - group=group) for direction, staged in p2p - ] - if operations: - for handle in dist.batch_isend_irecv(operations): - block_stream = getattr(handle, 'block_current_stream', None) - if stream_ordered and callable(block_stream): - # Order arena reuse on the caller stream; transport errors surface at the bounded drain below. - block_stream() - pending.append(handle) - else: - handle.wait() - # Stream ordering already protects arena reuse. Bound retained handles while avoiding - # a host polling delay after every small group of rounds; all errors are observed on return. - if (round_index + 1) % 32 == 0 or round_index + 1 == len(self._rounds): - for handle in pending: - handle.wait() - pending.clear() - sent += sum(staged.payload.numel() for staged in sends) * element_size - received += sum(staged.payload.numel() for staged in recvs) * element_size - for staged in recvs: - for destination, source in staged.copies: - destination.copy_(source) - - for target, source, size in self._local_copies: - target.copy_(source) - copied += size - - if stream_ordered: - get_accelerator().current_stream().synchronize() + sent, received, copied = _run_rounds(self._rounds, len(self._rounds), self._local_copies, self._group, + self._element_size, self._stream_ordered) self._runs += 1 return TransferStats(self._segments, self._chunks, len(self._rounds), copied, sent, received) +def _run_rounds(rounds, round_count: int, local_copies, group: Optional[object], element_size: int, + stream_ordered: bool) -> Tuple[int, int, int]: + """Move the rounds in order, then the local copies; returns the bytes sent, received and copied.""" + import deepspeed.comm as dist + sent = received = copied = 0 + pending = [] + for round_index, (recvs, sends) in enumerate(rounds): + for staged in sends: + for destination, source in staged.copies: + destination.copy_(source) + p2p = [(dist.irecv, staged) for staged in recvs] + [(dist.isend, staged) for staged in sends] + operations = [ + dist.P2POp(direction, + staged.payload, + dist.get_global_rank(group, staged.peer) if group is not None else staged.peer, + group=group) for direction, staged in p2p + ] + if operations: + for handle in dist.batch_isend_irecv(operations): + block_stream = getattr(handle, 'block_current_stream', None) + if stream_ordered and callable(block_stream): + # Order arena reuse on the caller stream; transport errors surface at the bounded drain below. + block_stream() + pending.append(handle) + else: + handle.wait() + # Stream ordering already protects arena reuse. Bound retained handles while avoiding + # a host polling delay after every small group of rounds; all errors are observed on return. + if (round_index + 1) % 32 == 0 or round_index + 1 == round_count: + for handle in pending: + handle.wait() + pending.clear() + sent += sum(staged.payload.numel() for staged in sends) * element_size + received += sum(staged.payload.numel() for staged in recvs) * element_size + for staged in recvs: + for destination, source in staged.copies: + destination.copy_(source) + + for target, source, size in local_copies: + target.copy_(source) + copied += size + + if stream_ordered: + get_accelerator().current_stream().synchronize() + return sent, received, copied + + @contextlib.contextmanager def _collector_paused(): """Pause Python's cyclic garbage collector while a batch's views are built. @@ -756,24 +827,45 @@ def _operation_region(requests, item, operation, dtype, source: bool) -> torch.T return _region(flat, operation.target_offset, operation.chunk_shape, segment.target_strides) -def _staged_rounds(rounds_by_peer, direct_by_peer, arenas, requests, dtype, source: bool): - """Each peer's rounds in one direction, resolved to wire payloads and the copies around them.""" - staged = {} - for peer, peer_rounds in rounds_by_peer.items(): - built = [] - for (entries, count), direct in zip(peer_rounds, direct_by_peer[peer]): - if direct: - built.append(_StagedRound(peer, _direct_payload(entries, requests, dtype, source), ())) - continue - arena = arenas[peer] - copies = [] - for item, operation, offset in entries: - slot = arena[offset:offset + operation.elements].view(operation.chunk_shape) - region = _operation_region(requests, item, operation, dtype, source) - copies.append((slot, region) if source else (region, slot)) - built.append(_StagedRound(peer, arena[:count], tuple(copies))) - staged[peer] = built - return staged +def _round_count(lane: _Lane) -> int: + return max((len(rounds) for rounds in list(lane.recv_rounds.values()) + list(lane.send_rounds.values())), + default=0) + + +def _round_views(lane: _Lane, recv_arenas, send_arenas, requests, dtype): + """Each round's receives, then its sends, ordered by peer, as wire payloads and the copies around them.""" + for index in range(_round_count(lane)): + recvs, sends = [], [] + for peer, peer_rounds in sorted(lane.recv_rounds.items()): + if index < len(peer_rounds): + direct, arena = lane.recv_direct[peer][index], recv_arenas.get(peer) + recvs.append(_staged_round(peer, peer_rounds[index], direct, arena, requests, dtype, source=False)) + for peer, peer_rounds in sorted(lane.send_rounds.items()): + if index < len(peer_rounds): + direct, arena = lane.send_direct[peer][index], send_arenas.get(peer) + sends.append(_staged_round(peer, peer_rounds[index], direct, arena, requests, dtype, source=True)) + yield recvs, sends + + +def _staged_round(peer, peer_round, direct, arena, requests, dtype, source: bool) -> _StagedRound: + """One peer's message in one direction, resolved to its wire payload and the copies around it.""" + entries, elements = peer_round + if direct: + return _StagedRound(peer, _direct_payload(entries, requests, dtype, source), ()) + copies = [] + for item, operation, offset in entries: + slot = arena[offset:offset + operation.elements].view(operation.chunk_shape) + region = _operation_region(requests, item, operation, dtype, source) + copies.append((slot, region) if source else (region, slot)) + return _StagedRound(peer, arena[:elements], tuple(copies)) + + +def _local_copy_views(lane: _Lane, requests, dtype): + """``(target region, source region, bytes)`` for each copy that stays on this process.""" + for item, operation in lane.local_copies: + target = _operation_region(requests, item, operation, dtype, source=False) + source = _operation_region(requests, item, operation, dtype, source=True) + yield target, source, operation.payload_bytes def _buffer_layout(requests: Sequence[TransferRequest]) -> Tuple: diff --git a/docs/code-docs/source/affine-transfer.rst b/docs/code-docs/source/affine-transfer.rst index c8252befe59b..e0ecd5a2ea18 100644 --- a/docs/code-docs/source/affine-transfer.rst +++ b/docs/code-docs/source/affine-transfer.rst @@ -44,23 +44,32 @@ moves data:: from deepspeed.checkpoint.affine_transfer_executor import prepare_transfers - prepared = prepare_transfers(requests, endpoints, dtype, budget) + prepared = prepare_transfers(build_requests(), endpoints, dtype, budget) for step in steps: ... # update the source buffers in place - prepared.run() + prepared.run(build_requests()) +Here ``build_requests`` stands for building the requests from the live state, +for example with fresh ``zero_param_storage_shard`` views. ``prepare_transfers`` is collective and refuses a batch exactly as -``execute_transfers`` does; ``execute_transfers`` is ``prepare_transfers`` -followed by one run. The handle keeps the buffers it was prepared with and its -arenas stay allocated, so buffers must stay where they are and every process -must run the handle the same number of times. By default ``run`` checks both -before any data moves, with one small all-gather: if a buffer moved or changed -shape on any process, or the processes are not running the same handle for the -same time, every process raises and the batch has to be prepared again. Pass the -requests rebuilt from the live state, such as fresh ``zero_param_storage_shard`` -views after ``reload_states``, as ``run(requests)`` to check those instead of -the tensors the handle holds. ``run(check=False)`` skips the all-gather, so -every process must pass the same ``check``. +``execute_transfers`` does; ``execute_transfers`` runs the same preflight but +builds each round's views only when it reaches that round, and keeps none of +them. The handle keeps the buffers it was prepared with and its arenas stay +allocated, so buffers must stay where they are and every process must run the +handle the same number of times. By default ``run`` checks both before any data +moves, with one small all-gather: if a buffer moved or changed shape on any +process, or the processes are not running the same handle for the same time, +every process raises and the batch has to be prepared again. + +``run(requests)`` checks the requests it is given; ``run()`` checks the tensors +the handle holds, which catches a buffer re-pointed in place (``.data = ...``) +but not a view taken before its storage moved. ``zero_param_storage_shard`` +returns such a view, and ZeRO-3 ``reload_states`` re-points the partition behind +it, so a handle over these views needs the rebuilt requests to notice the move. +The handle also keeps every buffer it holds allocated: drop it before +``offload_states`` if the partitions' memory should be freed, and prepare the +batch again after ``reload_states``. ``run(check=False)`` skips the all-gather, +so every process must pass the same ``check``. Planning a sync with replica policies ------------------------------------- diff --git a/tests/unit/checkpoint/test_affine_transfer_executor.py b/tests/unit/checkpoint/test_affine_transfer_executor.py index edf1d2440c44..3d076b4f9451 100644 --- a/tests/unit/checkpoint/test_affine_transfer_executor.py +++ b/tests/unit/checkpoint/test_affine_transfer_executor.py @@ -271,14 +271,6 @@ def test_schedule_is_what_the_executor_runs(self, world_size): assert any(r.direct for r in every) and any(not r.direct for r in every) -def _arena_elements(rounds, element_size): - """Elements of each peer's staging arena, read back from the rounds that stage through it.""" - return { - staged.peer: staged.payload.untyped_storage().nbytes() // element_size - for staged in rounds if staged.copies - } - - def _refill(requests, offset): """Shift every source by ``offset`` in place, keeping its storage.""" for request in requests: @@ -297,13 +289,6 @@ def test_repeated_runs_move_the_current_contents(self, world_size): # 16 elements per message: packed and in-place rounds, as in the schedule test. budget = StagingBudget(64) prepared = prepare_transfers(requests, endpoints, torch.float32, budget) - schedule = schedule_transfers([request.plan for request in requests], endpoints, budget, 4, process=rank)[rank] - recvs = [staged for receives, _ in prepared._rounds for staged in receives] - sends = [staged for _, sending in prepared._rounds for staged in sending] - # The handle keeps exactly the arenas the schedule reports. - assert _arena_elements(recvs, 4) == dict(schedule.receive_arena_elements) - assert _arena_elements(sends, 4) == dict(schedule.send_arena_elements) - once = execute_transfers(requests, endpoints, torch.float32, budget) for run in range(3): if run: @@ -395,6 +380,27 @@ def test_processes_running_different_handles_are_refused(self): assert whole.run() == execute_transfers(requests, endpoints, torch.float32, StagingBudget(64)) first.run() + def test_handles_prepared_alike_are_told_apart(self): + # Same keys, plans and endpoints, so the same schedule and digest; only the buffers differ. + _join_transfer_group() + rank = dist.get_rank() + endpoints, one, checks, _ = _mixed_batch(self.world_size, rank) + _, other, other_checks, _ = _mixed_batch(self.world_size, rank) + _refill(other, 1000.0) + first = prepare_transfers(one, endpoints, torch.float32, StagingBudget(64)) + second = prepare_transfers(other, endpoints, torch.float32, StagingBudget(64)) + with pytest.raises(AffineTransferExecutionError, match='not running the same prepared transfer'): + (first if rank == 0 else second).run() + for targets, _, _ in checks + other_checks: + for buffer in targets.values(): + assert torch.equal(buffer, torch.full_like(buffer, -17.0)) + first.run() + second.run() + for targets, target_map, full in checks: + _assert_matches(targets, target_map, full) + for targets, target_map, full in other_checks: + _assert_matches(targets, target_map, full + 1000.0) + class TestB2DisjointEndpoints(DistributedTest): """Every process is either a reader or a writer, never both. @@ -643,6 +649,27 @@ def test_a_prepared_transfer_writes_where_it_was_prepared(self): prepared.run(check=False) assert torch.equal(old, source) and torch.equal(target, torch.full((2, 3), -2.0)) + def test_requests_rebuilt_after_a_reload_show_the_move(self): + # As zero_param_storage_shard: a fresh view of ds_tensor on each call, which ZeRO-3 reload_states + # re-points with .data = ...; a view taken before that keeps the old storage. + map_ = replicated_map((2, 3), 1) + plan = plan_transfer(map_, map_) + endpoints = TransferEndpoints({0: 0}, {0: 0}) + source = torch.arange(6, dtype=torch.float32).reshape(2, 3) + ds_tensor = torch.full((6, ), -1.0) + + def requests(): + return [TransferRequest('weight', plan, {0: source}, {0: ds_tensor.data.view(-1).narrow(0, 0, 6)})] + + prepared = prepare_transfers(requests(), endpoints, torch.float32, StagingBudget(8)) + ds_tensor.data = torch.full((6, ), -2.0) + with pytest.raises(AffineTransferExecutionError, match='moved or changed'): + prepared.run(requests()) + assert torch.equal(ds_tensor, torch.full((6, ), -2.0)) + # Prepared again after the reload, the batch lands in the new storage. + prepare_transfers(requests(), endpoints, torch.float32, StagingBudget(8)).run(requests()) + assert torch.equal(ds_tensor, source.view(-1)) + def test_prepared_runs_keep_frozen_parameters_in_place(self): map_ = replicated_map((2, 3), 1) source = torch.nn.Parameter(torch.arange(6, dtype=torch.float32).reshape(2, 3)) @@ -659,23 +686,17 @@ def test_prepared_runs_keep_frozen_parameters_in_place(self): assert torch.equal(target, source) @pytest.mark.parametrize('enabled', [True, False]) - def test_preparing_leaves_the_garbage_collector_as_it_was(self, enabled, monkeypatch): - import deepspeed.checkpoint.affine_transfer_executor as executor + def test_preparing_leaves_the_garbage_collector_as_it_was(self, enabled): map_ = replicated_map((2, 3), 1) request = TransferRequest('weight', plan_transfer(map_, map_), {0: torch.ones(2, 3)}, {0: torch.zeros(2, 3)}) endpoints = TransferEndpoints({0: 0}, {0: 0}) - - def refuse(requests): - raise RuntimeError('layout failed') - was = gc.isenabled() try: gc.enable() if enabled else gc.disable() prepare_transfers([request], endpoints, torch.float32, StagingBudget(8)) assert gc.isenabled() == enabled - monkeypatch.setattr(executor, '_buffer_layout', refuse) - with pytest.raises(AffineTransferExecutionError, match='layout failed'): - prepare_transfers([request], endpoints, torch.float32, StagingBudget(8)) + with pytest.raises(AffineTransferExecutionError, match='at least one element'): + prepare_transfers([request], endpoints, torch.float32, StagingBudget(2)) assert gc.isenabled() == enabled finally: gc.enable() if was else gc.disable()