fix(bench): stop baseline benchmarking from a private InferenceX copy

src/hyperloom/inference_optimizer/cli/preflight.py

@@ -33,6 +33,7 @@
     has_anthropic_credential,
     provider_model_defaults,
 )
+from hyperloom.common.fs_utils import is_network_fs
 from hyperloom.common.gpu_identity import AMD_GPU_DISPATCH_IDENTITIES
 from hyperloom.common.platform_probe import probe_cpu_platform
 from hyperloom.common.pr_monitor_urls import kb_store_url
@@ -2300,6 +2301,18 @@ def _prepare_kb_install_step() -> dict[str, Any]:
         raise exc
     # Always overwrite (not setdefault): a stale/broken INFERENCEX_PATH must not survive into the child env.
     os.environ["INFERENCEX_PATH"] = inferencex_path
+    # A round cd's into this checkout and bash reads the benchmark script off it for the whole run, so a revocable
+    # mount that flaps discards a measurement that already completed. Recording it here is what tells the next
+    # magpie_nonzero_after_valid_measurement apart from a variant that genuinely cannot serve.
+    inferencex_network_fs = is_network_fs(inferencex_path)
+    if inferencex_network_fs:
+        print(
+            f"Preflight: WARNING — INFERENCEX_PATH={inferencex_path} is on a network filesystem. A mount flap "
+            f"mid-round discards a measurement that already completed, and the round is recorded as "
+            f"magpie_nonzero_after_valid_measurement. Point INFERENCEX_PATH at local disk, or unset it and put "
+            f"HYPERLOOM_CACHE_DIR on local disk.",
+            file=sys.stderr,
+        )
     _record_install_step(
         install_event,
         step_id="clone_inferencex",
@@ -2312,6 +2325,7 @@ def _prepare_kb_install_step() -> dict[str, Any]:
             "ref": os.environ.get("INFERENCEX_REF") or _INFERENCEX_REF_DEFAULT,
             "dest": inferencex_path,
             "writable": os.access(inferencex_path, os.W_OK),
+            "network_fs": inferencex_network_fs,
             "exit_code": 0,
         },
     )

src/hyperloom/inference_optimizer/tests/test_baseline_param_overrides.py

@@ -695,7 +695,6 @@ def fake_run(cmd, *args, **kwargs):
 
 def test_baseline_executor_pins_magpie_inferencex_path(tmp_path, monkeypatch):
     """The baseline executor's Magpie subprocess must inherit ``MAGPIE_INFERENCEX_PATH=$INFERENCEX_PATH`` so Magpie loads the patched checkout."""
-    monkeypatch.setenv("INFERENCE_OPTIMIZER_DISABLE_LOCAL_INFERENCEX", "1")
     monkeypatch.setenv("INFERENCEX_PATH", "/path/hyperloom/InferenceX")
     base = tmp_path / "base.yaml"
     _write_yaml(base)

src/hyperloom/inference_optimizer/tests/test_baseline_warmup_double_run.py

@@ -2048,193 +2048,6 @@ def fake_run(cmd, *args, **kwargs):
     assert result.get("output_throughput") == pytest.approx(4000.0)
 
 
-def test_ensure_local_inferencex_noop_for_local_path(tmp_path, monkeypatch):
-    """A checkout already on a local filesystem is returned unchanged."""
-    from hyperloom.orchestrator.actions.executors import baseline as bl
-
-    src = tmp_path / "InferenceX"
-    (src / "benchmarks").mkdir(parents=True)
-    (src / "benchmarks" / "benchmark_lib.sh").write_text("# stub")
-    monkeypatch.setattr(bl, "is_network_fs", lambda p: False)
-
-    assert bl._ensure_local_inferencex(str(src)) == str(src)
-
-
-def test_ensure_local_inferencex_mirrors_network_path(tmp_path, monkeypatch):
-    """A checkout on a simulated network mount is mirrored to local disk and the returned path points at the local copy, not the original."""
-    from hyperloom.orchestrator.actions.executors import baseline as bl
-
-    src = tmp_path / "wekafs_InferenceX"
-    (src / "benchmarks").mkdir(parents=True)
-    (src / "benchmarks" / "benchmark_lib.sh").write_text("# patched lib")
-    (src / "utils").mkdir()
-    (src / "utils" / "marker.txt").write_text("payload")
-
-    local_root = tmp_path / "local_cache"
-    monkeypatch.setattr(bl, "is_network_fs", lambda p: True)
-    monkeypatch.setenv(
-        "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT",
-        str(local_root),
-    )
-
-    dest = bl._ensure_local_inferencex(str(src))
-
-    assert dest != str(src)
-    assert str(local_root) in dest
-    assert (Path(dest) / "benchmarks" / "benchmark_lib.sh").read_text() == ("# patched lib")
-    assert (Path(dest) / "utils" / "marker.txt").read_text() == "payload"
-
-
-def test_ensure_local_inferencex_isolates_per_task_mirrors(
-    tmp_path,
-    monkeypatch,
-):
-    """Callers can include a task/output-dir key in the mirror hash so two overlapping baselines sharing one wekafs checkout never rmtree/replace a directory that another server is currently ``cd``-ed into."""
-    from hyperloom.orchestrator.actions.executors import baseline as bl
-
-    src = tmp_path / "wekafs_InferenceX"
-    (src / "benchmarks").mkdir(parents=True)
-    (src / "benchmarks" / "benchmark_lib.sh").write_text("# patched lib")
-    local_root = tmp_path / "local_cache"
-    monkeypatch.setattr(bl, "is_network_fs", lambda p: True)
-    monkeypatch.setenv(
-        "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT",
-        str(local_root),
-    )
-
-    dest_a = bl._ensure_local_inferencex(str(src), mirror_key="task-a")
-    dest_b = bl._ensure_local_inferencex(str(src), mirror_key="task-b")
-
-    assert dest_a != dest_b
-    assert (Path(dest_a) / "benchmarks" / "benchmark_lib.sh").is_file()
-    assert (Path(dest_b) / "benchmarks" / "benchmark_lib.sh").is_file()
-
-
-def test_ensure_local_inferencex_disabled_by_env(tmp_path, monkeypatch):
-    """The relocation can be opted out of via env even on a network mount."""
-    from hyperloom.orchestrator.actions.executors import baseline as bl
-
-    src = tmp_path / "wekafs_InferenceX"
-    (src / "benchmarks").mkdir(parents=True)
-    (src / "benchmarks" / "benchmark_lib.sh").write_text("# stub")
-    monkeypatch.setattr(bl, "is_network_fs", lambda p: True)
-    monkeypatch.setenv("INFERENCE_OPTIMIZER_DISABLE_LOCAL_INFERENCEX", "1")
-
-    assert bl._ensure_local_inferencex(str(src)) == str(src)
-
-
-def test_ensure_local_inferencex_falls_back_on_copy_failure(
-    tmp_path,
-    monkeypatch,
-):
-    """When the mirror copy itself fails (e.g. local disk full), the helper degrades to the original network-mount path instead of raising, so the run still proceeds rather than aborting."""
-    from hyperloom.orchestrator.actions.executors import baseline as bl
-
-    src = tmp_path / "wekafs_InferenceX"
-    (src / "benchmarks").mkdir(parents=True)
-    (src / "benchmarks" / "benchmark_lib.sh").write_text("# patched")
-    local_root = tmp_path / "local_cache"
-    monkeypatch.setattr(bl, "is_network_fs", lambda p: True)
-    monkeypatch.setenv(
-        "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT",
-        str(local_root),
-    )
-
-    def _boom(*_a, **_k):
-        raise OSError("no space left on device")
-
-    monkeypatch.setattr(bl.shutil, "copytree", _boom)
-
-    assert bl._ensure_local_inferencex(str(src)) == str(src)
-
-
-def test_ensure_local_inferencex_falls_back_when_mirror_incomplete(
-    tmp_path,
-    monkeypatch,
-):
-    """If the copy lands but the mirror is missing the load-bearing ``benchmarks/benchmark_lib.sh``, the helper rejects it and returns the original path rather than handing Magpie a broken ``cd`` target."""
-    from hyperloom.orchestrator.actions.executors import baseline as bl
-
-    src = tmp_path / "wekafs_InferenceX"
-    (src / "utils").mkdir(parents=True)
-    (src / "utils" / "marker.txt").write_text("payload")
-    local_root = tmp_path / "local_cache"
-    monkeypatch.setattr(bl, "is_network_fs", lambda p: True)
-    monkeypatch.setenv(
-        "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT",
-        str(local_root),
-    )
-
-    assert bl._ensure_local_inferencex(str(src)) == str(src)
-    assert not [p for p in local_root.iterdir() if p.is_dir()]
-
-
-def test_baseline_points_magpie_at_local_inferencex(tmp_path, monkeypatch):
-    """When INFERENCEX_PATH is on a network mount, the local mirror is what Magpie actually ``cd``-s into."""
-    from hyperloom.orchestrator.actions.executors import baseline as bl
-
-    base = tmp_path / "base.yaml"
-    _write_yaml(base, framework="sglang")
-    output_dir = tmp_path / "ws"
-
-    ix_src = tmp_path / "wekafs_InferenceX"
-    (ix_src / "benchmarks").mkdir(parents=True)
-    # This test is about which InferenceX dir Magpie cd-s into, but the launch path runs the real patcher, which
-    # refuses to start an eval whose patches cannot be applied.
-    (ix_src / "benchmarks" / "benchmark_lib.sh").write_text(
-        "# patched\n"
-        "run_eval() {\n"
-        '    export EVAL_RESULT_DIR="$results_dir"\n'
-        "}\n"
-        "append_lm_eval_summary() {\n"
-        '    mv -f "$jf" ./ || echo "WARN: failed to move ${jf}" >&2\n'
-        "}\n"
-    )
-    local_root = tmp_path / "local_cache"
-    monkeypatch.setattr(bl, "is_network_fs", lambda p: True)
-    monkeypatch.setenv("INFERENCEX_PATH", str(ix_src))
-    monkeypatch.setenv(
-        "INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT",
-        str(local_root),
-    )
-
-    seen: dict = {}
-
-    def fake_run(cmd, *args, **kwargs):
-        seen["env"] = kwargs.get("env")
-        cfg_idx = cmd.index("--benchmark-config")
-        seen["materialized_cfg"] = yaml.safe_load(Path(cmd[cfg_idx + 1]).read_text())
-        out_idx = cmd.index("--output-dir")
-        slot = Path(cmd[out_idx + 1])
-        _fake_workspace(slot, tput=_HOT_TPUT)
-        return subprocess.CompletedProcess(cmd, 0, "ok", "")
-
-    executor = _executor(base, tmp_path, baseline_double_run=False)
-    ctx = _make_ctx(
-        {
-            "output_dir": str(output_dir),
-            "timeout_sec": 10,
-            "gpu_type": "mi300x",
-        }
-    )
-
-    with patch(
-        "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill",
-        side_effect=fake_run,
-    ):
-        result = _run(executor(ctx))
-
-    assert result["status"] == "succeeded"
-    yaml_ix = seen["materialized_cfg"]["benchmark"]["inferencex_path"]
-    assert yaml_ix != str(ix_src), seen["materialized_cfg"]
-    assert str(local_root) in yaml_ix
-    magpie_ix = seen["env"]["MAGPIE_INFERENCEX_PATH"]
-    assert magpie_ix != str(ix_src), seen["env"]
-    assert str(local_root) in magpie_ix
-    # Relocation is task-local; process-wide env stays the original source path.
-    assert os.environ["INFERENCEX_PATH"] == str(ix_src)
-
-
 def test_baseline_anchors_server_cwd_to_output_dir(tmp_path, monkeypatch):
     """The Magpie parent subprocess cwd is anchored to the stable task output_dir (never the default ``/tmp``) as defence-in-depth."""
     base = tmp_path / "base.yaml"

src/hyperloom/inference_optimizer/tests/test_inferencex_preflight_clone.py

@@ -140,6 +140,13 @@ def test_validated_inferencex_path_overwrites_env_not_setdefault():
     assert 'os.environ.setdefault("INFERENCEX_PATH"' not in src
 
 
+def test_a_network_mount_checkout_is_recorded_rather_than_copied():
+    """Nothing relocates the checkout, so the ledger is what tells a flapped round from one that cannot serve."""
+    src = Path(cli_preflight.__file__).read_text(encoding="utf-8")
+    assert "inferencex_network_fs = is_network_fs(inferencex_path)" in src
+    assert '"network_fs": inferencex_network_fs,' in src
+
+
 def test_auto_detected_inferencex_candidates_must_be_writable():
     """Auto-detected read-only checkouts are skipped so preflight can clone."""
     src = Path(cli_preflight.__file__).read_text(encoding="utf-8")

src/hyperloom/orchestrator/actions/executors/baseline.py

@@ -7,7 +7,6 @@
 
 import asyncio
 import gzip
-import hashlib
 import json
 import logging
 import os
@@ -25,7 +24,6 @@
 import yaml
 
 from hyperloom.common.env import is_truthy
-from hyperloom.common.fs_utils import is_network_fs
 from hyperloom.common.env_safety import redact_secret_values, scrub_benchmark_process_env
 from hyperloom.common.git_safety import safe_directory_args
 from hyperloom.common.model_paths import resolve_session_model_path
@@ -51,7 +49,6 @@
     StoppedByTheRun,
 )
 from . import _server_lifecycle as _lifecycle
-from ._file_lock import best_effort_file_lock
 from ._aiter_jit import (
     AITER_JIT_PROBE_PATHS,
     BASELINE_COLD_START_TIMEOUT_SEC,
@@ -852,91 +849,6 @@ def _is_double_run_accuracy_handoff(
     return _WARMUP_ROUND_DIR in Path(source).parts
 
 
-def _ensure_local_inferencex(src: str, *, mirror_key: str = "") -> str:
-    """Mirror an InferenceX checkout onto stable local disk."""
-    src = str(src)
-    if (
-        os.environ.get(
-            "INFERENCE_OPTIMIZER_DISABLE_LOCAL_INFERENCEX",
-            "",
-        ).strip()
-        == "1"
-    ):
-        return src
-    try:
-        if not is_network_fs(src):
-            return src
-    except Exception:  # noqa: BLE001 — detection is best-effort
-        return src
-
-    real_src = os.path.realpath(src)
-    local_root = Path(
-        os.environ.get("INFERENCE_OPTIMIZER_LOCAL_INFERENCEX_ROOT", "")
-        or os.path.join(
-            os.path.expanduser("~"),
-            ".cache",
-            "hyperloom",
-            "inferencex_local",
-        )
-    )
-    src_hash = hashlib.sha1(real_src.encode("utf-8"), usedforsecurity=False).hexdigest()[:16]
-    key_hash = hashlib.sha1(str(mirror_key or "").encode("utf-8"), usedforsecurity=False).hexdigest()[:16]
-    dest_name = src_hash if not mirror_key else f"{src_hash}-{key_hash}"
-    dest = local_root / dest_name
-    try:
-        local_root.mkdir(parents=True, exist_ok=True)
-    except OSError as exc:
-        log.warning(
-            "baseline_executor: could not create local InferenceX root %s (%s); using the network-mount checkout.",
-            local_root,
-            exc,
-        )
-        return src
-    # Lock keyed on dest so concurrent tasks mirroring the same source serialize their rmtree/replace instead of
-    # racing.
-    lock_path = str(local_root / f".{dest.name}.lock")
-    staging: Path | None = None
-    try:
-        with best_effort_file_lock(lock_path, label="baseline_executor: InferenceX mirror lock"):
-            staging = Path(tempfile.mkdtemp(dir=str(local_root)))
-            staged_ix = staging / "InferenceX"
-            # Copy the tree fresh every run because the per-task patch step rewrites the mirror in place.
-            shutil.copytree(real_src, staged_ix, symlinks=True)
-            if dest.exists():
-                shutil.rmtree(dest, ignore_errors=True)
-            os.replace(staged_ix, dest)
-    except OSError as exc:
-        log.warning(
-            "baseline_executor: could not mirror InferenceX %s to local disk "
-            "(%s); using the network-mount checkout. The #523 cuda-graph "
-            "pickle dump may ENOENT if the mount flaps mid-run.",
-            real_src,
-            exc,
-        )
-        return src
-    finally:
-        # Always clear the staging dir so it doesn't accumulate across runs.
-        if staging is not None:
-            shutil.rmtree(staging, ignore_errors=True)
-
-    if not (dest / "benchmarks" / "benchmark_lib.sh").is_file():
-        log.warning(
-            "baseline_executor: local InferenceX mirror at %s is incomplete; using original %s",
-            dest,
-            real_src,
-        )
-        shutil.rmtree(dest, ignore_errors=True)
-        return src
-    log.info(
-        "baseline_executor: #523 — mirrored InferenceX from network mount %s "
-        "to local disk %s so the server cwd (cuda-graph pickle dump target) "
-        "survives a wekafs/NFS flap.",
-        real_src,
-        dest,
-    )
-    return str(dest)
-
-
 def _git_head_sha(repo_path: str) -> str:
     """Return the current HEAD sha of a git repo, or empty string on failure."""
     if not repo_path:
@@ -2695,11 +2607,6 @@ async def _run_once(
         output_dir = self._resolve_workspace(ctx, "baseline")
         output_dir.mkdir(parents=True, exist_ok=True)
 
-        # Keep the InferenceX checkout Magpie ``cd``s into on stable local disk so SGLang's relative-path cuda-graph
-        # dump survives a wekafs/NFS flap.
-        ix_env = os.environ.get("INFERENCEX_PATH", "").strip()
-        effective_inferencex_path = _ensure_local_inferencex(ix_env, mirror_key=str(output_dir)) if ix_env else ""
-
         # Warm patches are prepared after config/runtime preflight, immediately before the single final benchmark.
         patch_application: list[dict[str, str]] | dict[str, Any] = []
         applied_patches: list[dict[str, str]] = []
@@ -2749,7 +2656,6 @@ async def _run_once(
                 args_mode=str(params.get("args_mode") or "append"),
                 model_path=resolved_model,
                 gpu_type=resolved_gpu,
-                inferencex_path=effective_inferencex_path,
                 benchmark_script=override_script,
                 establish_quality_ref=is_genuine_baseline,
                 drop_moe_runner_backend=force_drop_moe_runner_backend,
@@ -2766,6 +2672,7 @@ async def _run_once(
             }
         # Stash for the result so Coordinator can reuse it downstream.
         materialized_config_path = config_path
+        effective_inferencex_path = os.environ.get("INFERENCEX_PATH", "").strip()
         # Apply runtime_override from params into the materialized YAML so the revalidation baseline boots under the
         # same framework runtime as the KEEP'd candidate (PATH/PYTHONPATH/framework_bin etc.).
         _rt_from_params = params.get("runtime_override")