diff --git a/src/hyperloom/inference_optimizer/cli/preflight.py b/src/hyperloom/inference_optimizer/cli/preflight.py index 59ef9fa222..50d05b237e 100644 --- a/src/hyperloom/inference_optimizer/cli/preflight.py +++ b/src/hyperloom/inference_optimizer/cli/preflight.py @@ -33,6 +33,7 @@ has_anthropic_credential, provider_model_defaults, ) +from hyperloom.common.fs_utils import is_network_fs from hyperloom.common.gpu_identity import AMD_GPU_DISPATCH_IDENTITIES from hyperloom.common.platform_probe import probe_cpu_platform from hyperloom.common.pr_monitor_urls import kb_store_url @@ -2300,6 +2301,18 @@ def _prepare_kb_install_step() -> dict[str, Any]: raise exc # Always overwrite (not setdefault): a stale/broken INFERENCEX_PATH must not survive into the child env. os.environ["INFERENCEX_PATH"] = inferencex_path + # A round cd's into this checkout and bash reads the benchmark script off it for the whole run, so a revocable + # mount that flaps discards a measurement that already completed. Recording it here is what tells the next + # magpie_nonzero_after_valid_measurement apart from a variant that genuinely cannot serve. + inferencex_network_fs = is_network_fs(inferencex_path) + if inferencex_network_fs: + print( + f"Preflight: WARNING — INFERENCEX_PATH={inferencex_path} is on a network filesystem. A mount flap " + f"mid-round discards a measurement that already completed, and the round is recorded as " + f"magpie_nonzero_after_valid_measurement. Point INFERENCEX_PATH at local disk, or unset it and put " + f"HYPERLOOM_CACHE_DIR on local disk.", + file=sys.stderr, + ) _record_install_step( install_event, step_id="clone_inferencex", @@ -2312,6 +2325,7 @@ def _prepare_kb_install_step() -> dict[str, Any]: "ref": os.environ.get("INFERENCEX_REF") or _INFERENCEX_REF_DEFAULT, "dest": inferencex_path, "writable": os.access(inferencex_path, os.W_OK), + "network_fs": inferencex_network_fs, "exit_code": 0, }, ) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py b/src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py index bcbdd9cee6..4cc6cd8220 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py @@ -695,7 +695,6 @@ def fake_run(cmd, *args, **kwargs): def test_baseline_executor_pins_magpie_inferencex_path(tmp_path, monkeypatch): """The baseline executor's Magpie subprocess must inherit ``MAGPIE_INFERENCEX_PATH=$INFERENCEX_PATH`` so Magpie loads the patched checkout.""" - monkeypatch.setenv("INFERENCE_OPTIMIZER_DISABLE_LOCAL_INFERENCEX", "1") monkeypatch.setenv("INFERENCEX_PATH", "/path/hyperloom/InferenceX") base = tmp_path / "base.yaml" _write_yaml(base) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py index 109726b46f..b5f68d0cff 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py @@ -2048,193 +2048,6 @@ def fake_run(cmd, *args, **kwargs): assert result.get("output_throughput") == pytest.approx(4000.0) -def test_ensure_local_inferencex_noop_for_local_path(tmp_path, monkeypatch): - """A checkout already on a local filesystem is returned unchanged.""" - from hyperloom.orchestrator.actions.executors import baseline as bl - - src = tmp_path / "InferenceX" - (src / "benchmarks").mkdir(parents=True) - (src / "benchmarks" / "benchmark_lib.sh").write_text("# stub") - monkeypatch.setattr(bl, "is_network_fs", lambda p: False) - - assert bl._ensure_local_inferencex(str(src)) == str(src) - - -def test_ensure_local_inferencex_mirrors_network_path(tmp_path, monkeypatch): - """A checkout on a simulated network mount is mirrored to local disk and the returned path points at the local copy, not the original.""" - from hyperloom.orchestrator.actions.executors import baseline as bl - - src = tmp_path / "wekafs_InferenceX" - (src / "benchmarks").mkdir(parents=True) - (src / "benchmarks" / "benchmark_lib.sh").write_text("# patched lib") - (src / "utils").mkdir() - (src / "utils" / "marker.txt").write_text("payload") - - local_root = tmp_path / "local_cache" - monkeypatch.setattr(bl, "is_network_fs", lambda p: True) - monkeypatch.setenv( - "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT", - str(local_root), - ) - - dest = bl._ensure_local_inferencex(str(src)) - - assert dest != str(src) - assert str(local_root) in dest - assert (Path(dest) / "benchmarks" / "benchmark_lib.sh").read_text() == ("# patched lib") - assert (Path(dest) / "utils" / "marker.txt").read_text() == "payload" - - -def test_ensure_local_inferencex_isolates_per_task_mirrors( - tmp_path, - monkeypatch, -): - """Callers can include a task/output-dir key in the mirror hash so two overlapping baselines sharing one wekafs checkout never rmtree/replace a directory that another server is currently ``cd``-ed into.""" - from hyperloom.orchestrator.actions.executors import baseline as bl - - src = tmp_path / "wekafs_InferenceX" - (src / "benchmarks").mkdir(parents=True) - (src / "benchmarks" / "benchmark_lib.sh").write_text("# patched lib") - local_root = tmp_path / "local_cache" - monkeypatch.setattr(bl, "is_network_fs", lambda p: True) - monkeypatch.setenv( - "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT", - str(local_root), - ) - - dest_a = bl._ensure_local_inferencex(str(src), mirror_key="task-a") - dest_b = bl._ensure_local_inferencex(str(src), mirror_key="task-b") - - assert dest_a != dest_b - assert (Path(dest_a) / "benchmarks" / "benchmark_lib.sh").is_file() - assert (Path(dest_b) / "benchmarks" / "benchmark_lib.sh").is_file() - - -def test_ensure_local_inferencex_disabled_by_env(tmp_path, monkeypatch): - """The relocation can be opted out of via env even on a network mount.""" - from hyperloom.orchestrator.actions.executors import baseline as bl - - src = tmp_path / "wekafs_InferenceX" - (src / "benchmarks").mkdir(parents=True) - (src / "benchmarks" / "benchmark_lib.sh").write_text("# stub") - monkeypatch.setattr(bl, "is_network_fs", lambda p: True) - monkeypatch.setenv("INFERENCE_OPTIMIZER_DISABLE_LOCAL_INFERENCEX", "1") - - assert bl._ensure_local_inferencex(str(src)) == str(src) - - -def test_ensure_local_inferencex_falls_back_on_copy_failure( - tmp_path, - monkeypatch, -): - """When the mirror copy itself fails (e.g. local disk full), the helper degrades to the original network-mount path instead of raising, so the run still proceeds rather than aborting.""" - from hyperloom.orchestrator.actions.executors import baseline as bl - - src = tmp_path / "wekafs_InferenceX" - (src / "benchmarks").mkdir(parents=True) - (src / "benchmarks" / "benchmark_lib.sh").write_text("# patched") - local_root = tmp_path / "local_cache" - monkeypatch.setattr(bl, "is_network_fs", lambda p: True) - monkeypatch.setenv( - "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT", - str(local_root), - ) - - def _boom(*_a, **_k): - raise OSError("no space left on device") - - monkeypatch.setattr(bl.shutil, "copytree", _boom) - - assert bl._ensure_local_inferencex(str(src)) == str(src) - - -def test_ensure_local_inferencex_falls_back_when_mirror_incomplete( - tmp_path, - monkeypatch, -): - """If the copy lands but the mirror is missing the load-bearing ``benchmarks/benchmark_lib.sh``, the helper rejects it and returns the original path rather than handing Magpie a broken ``cd`` target.""" - from hyperloom.orchestrator.actions.executors import baseline as bl - - src = tmp_path / "wekafs_InferenceX" - (src / "utils").mkdir(parents=True) - (src / "utils" / "marker.txt").write_text("payload") - local_root = tmp_path / "local_cache" - monkeypatch.setattr(bl, "is_network_fs", lambda p: True) - monkeypatch.setenv( - "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT", - str(local_root), - ) - - assert bl._ensure_local_inferencex(str(src)) == str(src) - assert not [p for p in local_root.iterdir() if p.is_dir()] - - -def test_baseline_points_magpie_at_local_inferencex(tmp_path, monkeypatch): - """When INFERENCEX_PATH is on a network mount, the local mirror is what Magpie actually ``cd``-s into.""" - from hyperloom.orchestrator.actions.executors import baseline as bl - - base = tmp_path / "base.yaml" - _write_yaml(base, framework="sglang") - output_dir = tmp_path / "ws" - - ix_src = tmp_path / "wekafs_InferenceX" - (ix_src / "benchmarks").mkdir(parents=True) - # This test is about which InferenceX dir Magpie cd-s into, but the launch path runs the real patcher, which - # refuses to start an eval whose patches cannot be applied. - (ix_src / "benchmarks" / "benchmark_lib.sh").write_text( - "# patched\n" - "run_eval() {\n" - ' export EVAL_RESULT_DIR="$results_dir"\n' - "}\n" - "append_lm_eval_summary() {\n" - ' mv -f "$jf" ./ || echo "WARN: failed to move ${jf}" >&2\n' - "}\n" - ) - local_root = tmp_path / "local_cache" - monkeypatch.setattr(bl, "is_network_fs", lambda p: True) - monkeypatch.setenv("INFERENCEX_PATH", str(ix_src)) - monkeypatch.setenv( - "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT", - str(local_root), - ) - - seen: dict = {} - - def fake_run(cmd, *args, **kwargs): - seen["env"] = kwargs.get("env") - cfg_idx = cmd.index("--benchmark-config") - seen["materialized_cfg"] = yaml.safe_load(Path(cmd[cfg_idx + 1]).read_text()) - out_idx = cmd.index("--output-dir") - slot = Path(cmd[out_idx + 1]) - _fake_workspace(slot, tput=_HOT_TPUT) - return subprocess.CompletedProcess(cmd, 0, "ok", "") - - executor = _executor(base, tmp_path, baseline_double_run=False) - ctx = _make_ctx( - { - "output_dir": str(output_dir), - "timeout_sec": 10, - "gpu_type": "mi300x", - } - ) - - with patch( - "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", - side_effect=fake_run, - ): - result = _run(executor(ctx)) - - assert result["status"] == "succeeded" - yaml_ix = seen["materialized_cfg"]["benchmark"]["inferencex_path"] - assert yaml_ix != str(ix_src), seen["materialized_cfg"] - assert str(local_root) in yaml_ix - magpie_ix = seen["env"]["MAGPIE_INFERENCEX_PATH"] - assert magpie_ix != str(ix_src), seen["env"] - assert str(local_root) in magpie_ix - # Relocation is task-local; process-wide env stays the original source path. - assert os.environ["INFERENCEX_PATH"] == str(ix_src) - - def test_baseline_anchors_server_cwd_to_output_dir(tmp_path, monkeypatch): """The Magpie parent subprocess cwd is anchored to the stable task output_dir (never the default ``/tmp``) as defence-in-depth.""" base = tmp_path / "base.yaml" diff --git a/src/hyperloom/inference_optimizer/tests/test_inferencex_preflight_clone.py b/src/hyperloom/inference_optimizer/tests/test_inferencex_preflight_clone.py index 1e47699881..3008723016 100644 --- a/src/hyperloom/inference_optimizer/tests/test_inferencex_preflight_clone.py +++ b/src/hyperloom/inference_optimizer/tests/test_inferencex_preflight_clone.py @@ -140,6 +140,13 @@ def test_validated_inferencex_path_overwrites_env_not_setdefault(): assert 'os.environ.setdefault("INFERENCEX_PATH"' not in src +def test_a_network_mount_checkout_is_recorded_rather_than_copied(): + """Nothing relocates the checkout, so the ledger is what tells a flapped round from one that cannot serve.""" + src = Path(cli_preflight.__file__).read_text(encoding="utf-8") + assert "inferencex_network_fs = is_network_fs(inferencex_path)" in src + assert '"network_fs": inferencex_network_fs,' in src + + def test_auto_detected_inferencex_candidates_must_be_writable(): """Auto-detected read-only checkouts are skipped so preflight can clone.""" src = Path(cli_preflight.__file__).read_text(encoding="utf-8") diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index da5de4ea4a..9158ae52ed 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -7,7 +7,6 @@ import asyncio import gzip -import hashlib import json import logging import os @@ -25,7 +24,6 @@ import yaml from hyperloom.common.env import is_truthy -from hyperloom.common.fs_utils import is_network_fs from hyperloom.common.env_safety import redact_secret_values, scrub_benchmark_process_env from hyperloom.common.git_safety import safe_directory_args from hyperloom.common.model_paths import resolve_session_model_path @@ -51,7 +49,6 @@ StoppedByTheRun, ) from . import _server_lifecycle as _lifecycle -from ._file_lock import best_effort_file_lock from ._aiter_jit import ( AITER_JIT_PROBE_PATHS, BASELINE_COLD_START_TIMEOUT_SEC, @@ -852,91 +849,6 @@ def _is_double_run_accuracy_handoff( return _WARMUP_ROUND_DIR in Path(source).parts -def _ensure_local_inferencex(src: str, *, mirror_key: str = "") -> str: - """Mirror an InferenceX checkout onto stable local disk.""" - src = str(src) - if ( - os.environ.get( - "INFERENCE_OPTIMIZER_DISABLE_LOCAL_INFERENCEX", - "", - ).strip() - == "1" - ): - return src - try: - if not is_network_fs(src): - return src - except Exception: # noqa: BLE001 — detection is best-effort - return src - - real_src = os.path.realpath(src) - local_root = Path( - os.environ.get("INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT", "") - or os.path.join( - os.path.expanduser("~"), - ".cache", - "hyperloom", - "inferencex_local", - ) - ) - src_hash = hashlib.sha1(real_src.encode("utf-8"), usedforsecurity=False).hexdigest()[:16] - key_hash = hashlib.sha1(str(mirror_key or "").encode("utf-8"), usedforsecurity=False).hexdigest()[:16] - dest_name = src_hash if not mirror_key else f"{src_hash}-{key_hash}" - dest = local_root / dest_name - try: - local_root.mkdir(parents=True, exist_ok=True) - except OSError as exc: - log.warning( - "baseline_executor: could not create local InferenceX root %s (%s); using the network-mount checkout.", - local_root, - exc, - ) - return src - # Lock keyed on dest so concurrent tasks mirroring the same source serialize their rmtree/replace instead of - # racing. - lock_path = str(local_root / f".{dest.name}.lock") - staging: Path | None = None - try: - with best_effort_file_lock(lock_path, label="baseline_executor: InferenceX mirror lock"): - staging = Path(tempfile.mkdtemp(dir=str(local_root))) - staged_ix = staging / "InferenceX" - # Copy the tree fresh every run because the per-task patch step rewrites the mirror in place. - shutil.copytree(real_src, staged_ix, symlinks=True) - if dest.exists(): - shutil.rmtree(dest, ignore_errors=True) - os.replace(staged_ix, dest) - except OSError as exc: - log.warning( - "baseline_executor: could not mirror InferenceX %s to local disk " - "(%s); using the network-mount checkout. The #523 cuda-graph " - "pickle dump may ENOENT if the mount flaps mid-run.", - real_src, - exc, - ) - return src - finally: - # Always clear the staging dir so it doesn't accumulate across runs. - if staging is not None: - shutil.rmtree(staging, ignore_errors=True) - - if not (dest / "benchmarks" / "benchmark_lib.sh").is_file(): - log.warning( - "baseline_executor: local InferenceX mirror at %s is incomplete; using original %s", - dest, - real_src, - ) - shutil.rmtree(dest, ignore_errors=True) - return src - log.info( - "baseline_executor: #523 — mirrored InferenceX from network mount %s " - "to local disk %s so the server cwd (cuda-graph pickle dump target) " - "survives a wekafs/NFS flap.", - real_src, - dest, - ) - return str(dest) - - def _git_head_sha(repo_path: str) -> str: """Return the current HEAD sha of a git repo, or empty string on failure.""" if not repo_path: @@ -2695,11 +2607,6 @@ async def _run_once( output_dir = self._resolve_workspace(ctx, "baseline") output_dir.mkdir(parents=True, exist_ok=True) - # Keep the InferenceX checkout Magpie ``cd``s into on stable local disk so SGLang's relative-path cuda-graph - # dump survives a wekafs/NFS flap. - ix_env = os.environ.get("INFERENCEX_PATH", "").strip() - effective_inferencex_path = _ensure_local_inferencex(ix_env, mirror_key=str(output_dir)) if ix_env else "" - # Warm patches are prepared after config/runtime preflight, immediately before the single final benchmark. patch_application: list[dict[str, str]] | dict[str, Any] = [] applied_patches: list[dict[str, str]] = [] @@ -2749,7 +2656,6 @@ async def _run_once( args_mode=str(params.get("args_mode") or "append"), model_path=resolved_model, gpu_type=resolved_gpu, - inferencex_path=effective_inferencex_path, benchmark_script=override_script, establish_quality_ref=is_genuine_baseline, drop_moe_runner_backend=force_drop_moe_runner_backend, @@ -2766,6 +2672,7 @@ async def _run_once( } # Stash for the result so Coordinator can reuse it downstream. materialized_config_path = config_path + effective_inferencex_path = os.environ.get("INFERENCEX_PATH", "").strip() # Apply runtime_override from params into the materialized YAML so the revalidation baseline boots under the # same framework runtime as the KEEP'd candidate (PATH/PYTHONPATH/framework_bin etc.). _rt_from_params = params.get("runtime_override")