Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -979,6 +979,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Fixed

- The inherited-CUDA guard's test (`tests/pipeline/test_host_worker_setup.py::`
`TestWorkerSetUpPositiveControls`) checks what it states. Its script set `jax_platforms` to the
CPU at module level, which every spawned worker re-runs, so no worker saw the inherited
`JAX_PLATFORMS=cuda,cpu`; and its control took the read after serving the run, so the bare
Grain worker failed on a run with no units (`run_units.py:216`). With `WorkerSetUp`'s
`jax_platforms` line removed, the test still passed. The setting is now under the script's
`__main__` guard, and the test compares the platform list each worker holds, which needs no
GPU: datarax's workers hold `cpu`, a bare worker holds the inherited `cuda,cpu` (and `cpu` when
that is inherited). A failing CUDA start would not do, as jax 0.11.1 skips a listed CUDA
platform on a machine without NVIDIA device files. Removing that line from `WorkerSetUp` now
fails the test, on a machine with a GPU and on one without.
- A TFDS stream's workers no longer walk the pass's order record by record. Each of `K` worker
processes computed the shuffled order of every record of the run, one Python generator step
and one scalar NumPy draw per record, to find the units of its own slice, so the stream's CPU
Expand Down
41 changes: 27 additions & 14 deletions tests/pipeline/test_host_worker_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,28 +157,33 @@ def test_every_worker_watches_its_training_process(self, png_paths: list[str]) -
import json, sys
import jax
from substrax.testing import report
jax.config.update("jax_platforms", "cpu") # set in code: a spawned worker does not inherit it
import grain
from flax import nnx
from datarax.pipeline import Pipeline
from tests.test_common.worker_reads import png_source, resources, WorkerFactsDecoder


def platforms_seen(_):
# A bare worker's JAX platform list, read without starting a backend: whether a listed
# platform would start depends on the machine (JAX skips CUDA where no GPU device exists).
import jax
return jax.config.jax_platforms


if __name__ == "__main__":
# Set in code under the guard: a spawned worker re-imports this module without running the
# guard, so it keeps the inherited JAX_PLATFORMS unless something in the worker sets it.
jax.config.update("jax_platforms", "cpu")
paths = sys.argv[1:]
pipe = Pipeline(source=png_source(paths, WorkerFactsDecoder()), stages=[], batch_size=3,
rngs=nnx.Rngs(0), host_resources=resources(2))
facts = sorted({json.loads(r["worker"])["backend"]
for _, p in pipe.raw_batches(with_provenance=True) for r in p})
control = "served"
read = pipe.host_stage.read_for_workers(pipe)
facts = [json.loads(r["worker"]) for _, p in pipe.raw_batches(with_provenance=True) for r in p]
pipe.close()
bare = grain.MapDataset.range(2).map(read).to_iter_dataset().mp_prefetch(
bare = grain.MapDataset.range(1).map(platforms_seen).to_iter_dataset().mp_prefetch(
grain.multiprocessing.MultiprocessingOptions(num_workers=1))
try:
list(bare)
except Exception as error: # the control: what a worker without datarax's set-up raises
control = f"raised {type(error).__name__}: {str(error)[:300]}"
report({"backends": facts, "control": control})
report({"backends": sorted({f["backend"] for f in facts}),
"platforms": sorted({f["jax_platforms"] for f in facts}),
"control": list(bare)})
"""

_UNGUARDED_MAIN = """
Expand Down Expand Up @@ -219,8 +224,7 @@ def test_an_inherited_cuda_platform_list_never_reaches_a_worker(
) -> None:
"""The GPU recipe's ``JAX_PLATFORMS=cuda,cpu`` is inherited; datarax's workers read on CPU.

The same read in a bare Grain worker, without datarax's set-up, starts the listed
platforms and fails: the control.
A bare Grain worker, without datarax's set-up, holds the inherited list: the control.
"""
report = _run_script(
tmp_path,
Expand All @@ -229,7 +233,16 @@ def test_an_inherited_cuda_platform_list_never_reaches_a_worker(
env={"JAX_PLATFORMS": "cuda,cpu", "CUDA_VISIBLE_DEVICES": ""},
)
assert report["backends"] == ["cpu"]
assert report["control"].startswith("raised"), report["control"]
assert report["platforms"] == ["cpu"]
assert report["control"] == ["cuda,cpu"]

def test_the_control_holds_whatever_list_the_worker_inherits(
self, png_paths: list[str], tmp_path: Path
) -> None:
"""With ``cpu`` inherited the bare worker holds ``cpu``: it reads what it inherits."""
report = _run_script(tmp_path, _INHERITED_CUDA, *png_paths, env={"JAX_PLATFORMS": "cpu"})
assert report["platforms"] == ["cpu"]
assert report["control"] == ["cpu"]

def test_a_main_module_starting_jax_at_import_is_refused_naming_the_guard(
self, png_paths: list[str], tmp_path: Path
Expand Down
Loading