Merge remote-tracking branch 'origin/main' into ajost/host-launch

Author: Andy-Jost

.github/workflows/test-wheel-linux.yml

@@ -146,12 +146,22 @@ jobs:
           dependencies: "jq wget libgl1 libegl1 g++ util-linux yq"
           dependent_exes: "jq wget yq"
 
+      - name: Checkout driver install scripts (workflow revision)
+        if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          path: .driver-tools
+          sparse-checkout: ci/tools
+          persist-credentials: false
+
       - name: Install GPU driver
         if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
         env:
           DRIVER: ${{ matrix.DRIVER }}
+          DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }}
           GPU_TYPE: ${{ matrix.GPU }}
-        run: ./ci/tools/install_gpu_driver.sh
+        run: ./.driver-tools/ci/tools/install_gpu_driver.sh
 
       - name: Ensure GPU is working
         run: nvidia-smi

.github/workflows/test-wheel-windows.yml

@@ -119,13 +119,23 @@ jobs:
         with:
           enable-apt: true
 
+      - name: Checkout driver install scripts (workflow revision)
+        if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          path: .driver-tools
+          sparse-checkout: ci/tools
+          persist-credentials: false
+
       - name: Install GPU driver
         if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
         env:
           DRIVER: ${{ matrix.DRIVER }}
+          DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }}
           GPU_TYPE: ${{ matrix.GPU }}
         run: |
-          ci/tools/install_gpu_driver.ps1
+          .driver-tools/ci/tools/install_gpu_driver.ps1
 
       - name: Configure driver mode
         env:

ci/test-matrix.yml

@@ -20,7 +20,16 @@
 #                  - install that version via ci/tools/install_gpu_driver.sh (Linux)
 #                    or ci/tools/install_gpu_driver.ps1 (Windows) at the start of the
 #                    job. The matrix row is routed to the 'latest' runner image (the
-#                    install scripts swap the driver themselves).
+#                    install scripts swap the driver themselves). A version-string
+#                    row must also set DRIVER_SHA256: the SHA-256 of the exact
+#                    installer the script downloads (see the URL construction in
+#                    install_gpu_driver.sh / .ps1; Windows data-center and desktop
+#                    installers differ). Whoever changes DRIVER owns this value:
+#                    compute it on a machine with an up-to-date trust store, e.g.
+#                    `curl -fsSL <installer-url> | sha256sum`, and state in the PR
+#                    how it was obtained; reviewers should recompute it. Treat a
+#                    CI hash mismatch as possible tampering -- never copy the
+#                    "got" value from a job log into this file.
 # - DRIVER: 'earliest' does not work with CUDA 12.9.1
 # - DRIVER: a custom version is not supported with FLAVOR=wsl on Linux.
 # - ENV: optional map of extra environment variables to inject into the test
@@ -50,10 +59,10 @@ linux:
     - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest' }
     - { ARCH: 'amd64', PY_VER: '3.13',  CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'v100',       GPU_COUNT: '1', DRIVER: 'latest' }
     - { ARCH: 'amd64', PY_VER: '3.13',  CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest' }
-    - { ARCH: 'amd64', PY_VER: '3.13',  CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02' }
+    - { ARCH: 'amd64', PY_VER: '3.13',  CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02', DRIVER_SHA256: '3034a054bb4cdf7752ff8dc272564cb105513804bff53538945901b16ca77463' }
     - { ARCH: 'amd64', PY_VER: '3.14',  CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 't4',         GPU_COUNT: '1', DRIVER: 'latest' }
     - { ARCH: 'amd64', PY_VER: '3.14',  CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest' }
-    - { ARCH: 'amd64', PY_VER: '3.14',  CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4',         GPU_COUNT: '1', DRIVER: '610.43.02' }
+    - { ARCH: 'amd64', PY_VER: '3.14',  CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4',         GPU_COUNT: '1', DRIVER: '610.43.02', DRIVER_SHA256: '3034a054bb4cdf7752ff8dc272564cb105513804bff53538945901b16ca77463' }
     - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 't4',         GPU_COUNT: '1', DRIVER: 'latest' }
     - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest' }
     - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest' }
@@ -101,7 +110,7 @@ linux:
     - { ARCH: 'arm64', PY_VER: '3.12',  CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest',     ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1',  TORCH_CUDA: 'cu130' } }
     # nightly-numba-cuda
     - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest',     ENV: { MODE: 'nightly-numba-cuda' } }
-    - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: '580.65.06',  ENV: { MODE: 'nightly-numba-cuda' } }
+    - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: '580.65.06', DRIVER_SHA256: '04b10867af585e765cfbfdcf39ed5f4bd112375bebab0172eaa187c6aa5024ff', ENV: { MODE: 'nightly-numba-cuda' } }
     - { ARCH: 'arm64', PY_VER: '3.12',  CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest',     ENV: { MODE: 'nightly-numba-cuda' } }
     - { ARCH: 'arm64', PY_VER: '3.12',  CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest',     ENV: { MODE: 'nightly-numba-cuda' } }
     # nightly-numba-cuda-mlir (MLIR backend, linux-64 only)
@@ -150,7 +159,7 @@ windows:
     - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest',  DRIVER_MODE: 'TCC',  ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1',  TORCH_CUDA: 'cu130' } }
     # nightly-numba-cuda
     - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: 'latest',  DRIVER_MODE: 'TCC',  ENV: { MODE: 'nightly-numba-cuda' } }
-    - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: '596.36',  DRIVER_MODE: 'TCC',  ENV: { MODE: 'nightly-numba-cuda' } }
+    - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4',         GPU_COUNT: '1', DRIVER: '596.36', DRIVER_SHA256: '99182c2fdf3f23c503c74608945d84be99f036b5ec969f2945062069a15acd2e', DRIVER_MODE: 'TCC',  ENV: { MODE: 'nightly-numba-cuda' } }
     # nightly-numba-cuda-mlir (MLIR backend, win-64)
     - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest',  DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } }
     - { ARCH: 'amd64', PY_VER: '3.12',  CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest',  DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } }

ci/tools/install_gpu_driver.ps1

@@ -10,11 +10,13 @@
 # right driver).
 #
 # Inputs (env):
-#   DRIVER    Driver version, e.g. "610.47". Must NOT be 'latest' or
-#             'earliest' -- those are runner-pre-installed and the
-#             workflow is expected to skip this script for them.
-#   GPU_TYPE  Lower-case GPU label from the matrix (e.g. "l4", "rtx4090").
-#             Selects the data-center vs desktop installer variant.
+#   DRIVER         Driver version, e.g. "610.47". Must NOT be 'latest' or
+#                  'earliest' -- those are runner-pre-installed and the
+#                  workflow is expected to skip this script for them.
+#   DRIVER_SHA256  Expected SHA-256 hex digest of the downloaded driver
+#                  installer.
+#   GPU_TYPE       Lower-case GPU label from the matrix (e.g. "l4", "rtx4090").
+#                  Selects the data-center vs desktop installer variant.
 
 # Install the driver
 function Install-Driver {
@@ -26,6 +28,11 @@ function Install-Driver {
         exit 1
     }
 
+    if (-not $env:DRIVER_SHA256) {
+        Write-Error "DRIVER_SHA256 env var is required to verify the driver installer."
+        exit 1
+    }
+
     # Get GPU type from environment variable
     $gpu_type = $env:GPU_TYPE
 
@@ -68,6 +75,13 @@ function Install-Driver {
     $ProgressPreference = $ProgressPreference_tmp
     Write-Output 'Download complete!'
 
+    $actual_sha256 = (Get-FileHash -Algorithm SHA256 -Path $filepath).Hash
+    if ($actual_sha256 -ne $env:DRIVER_SHA256) {
+        Write-Error "SHA256 mismatch for ${filepath}: expected $($env:DRIVER_SHA256), got $actual_sha256"
+        exit 1
+    }
+    Write-Output "SHA256 verified: $actual_sha256"
+
     # Install the file with the specified path from earlier
     Write-Output 'Running the driver installer...'
     Start-Process -FilePath $filepath -ArgumentList $install_args -Wait

ci/tools/install_gpu_driver.sh

@@ -13,13 +13,15 @@
 # bind-mount refresh) and may cause issues.
 #
 # Inputs (env):
-#   DRIVER    Driver version, e.g. "580.65.06". Must NOT be 'latest' or
-#             'earliest' -- those are runner-pre-installed and the
-#             workflow is expected to skip this script for them.
-#   GPU_TYPE  Lower-case GPU label from the matrix (e.g. "v100", "l4",
-#             "h100"). Used only to pick the kernel module flavor
-#             (Volta needs the proprietary/legacy module; everything
-#             newer can use the open module).
+#   DRIVER         Driver version, e.g. "580.65.06". Must NOT be 'latest' or
+#                  'earliest' -- those are runner-pre-installed and the
+#                  workflow is expected to skip this script for them.
+#   DRIVER_SHA256  Expected lowercase SHA-256 hex digest of the downloaded
+#                  driver installer.
+#   GPU_TYPE       Lower-case GPU label from the matrix (e.g. "v100", "l4",
+#                  "h100"). Used only to pick the kernel module flavor
+#                  (Volta needs the proprietary/legacy module; everything
+#                  newer can use the open module).
 #
 # Arch is detected from `uname -m`.
 #
@@ -32,6 +34,7 @@
 set -euo pipefail
 
 : "${DRIVER:?DRIVER env var is required (e.g. 580.65.06)}"
+: "${DRIVER_SHA256:?DRIVER_SHA256 env var is required (64-char sha256 of the installer)}"
 : "${GPU_TYPE:?GPU_TYPE env var is required (e.g. l4)}"
 
 case "$DRIVER" in
@@ -70,7 +73,7 @@ URL="https://us.download.nvidia.com/XFree86/${ARCH_DIR}/${VERSION}/NVIDIA-Linux-
 
 # Re-elevate to root if needed (sudo is preinstalled on the runner image).
 if [ "$(id -u)" != 0 ]; then
-  exec sudo -E DRIVER="$DRIVER" GPU_TYPE="$GPU_TYPE" "$0" "$@"
+  exec sudo -E DRIVER="$DRIVER" DRIVER_SHA256="$DRIVER_SHA256" GPU_TYPE="$GPU_TYPE" "$0" "$@"
 fi
 
 echo "install_gpu_driver.sh is ALPHA -- it performs live modifications to the host driver stack and may cause issues" >&2
@@ -89,6 +92,15 @@ in_container() {
 host_install() {
   apt-get -y install build-essential dkms "linux-headers-$(uname -r)" psmisc kmod
 
+  # Download and verify the installer before tearing down the existing
+  # host driver so a download failure or SHA-256 mismatch leaves the
+  # runner's current driver intact.
+  local d
+  d=$(mktemp -d)
+  ( cd "$d" \
+    && wget -q -O installer.run "$URL" \
+    && echo "${DRIVER_SHA256}  installer.run" | sha256sum --check --strict - )
+
   systemctl stop nvidia-persistenced dcgm-exporter || true
   # if-test instead of `fuser ... || true` so a kill failure surfaces
   # (fuser exits 1 when nothing holds the device, which is the happy path).
@@ -107,10 +119,7 @@ host_install() {
     | awk -v re="$KEEP_RE" '$0 !~ re' \
     | xargs -r apt-get -y remove --purge || true
 
-  local d
-  d=$(mktemp -d)
   ( cd "$d" \
-    && wget -q -O installer.run "$URL" \
     && sh installer.run --silent --dkms --no-questions \
          --accept-license --ui=none --no-cc-version-check --kernel-module-type="$KMT" )
   modprobe nvidia nvidia_uvm nvidia_modeset
@@ -193,9 +202,10 @@ if [ -z "${_NVDRV_NSENTERED:-}" ] && in_container; then
   # ours lives in the GH workspace mount (container-only), so we pipe the
   # script body in via stdin instead -- the `< "$0"` fd is opened before
   # nsenter and stays valid across the namespace switch. Env vars (DRIVER,
-  # GPU_TYPE, _NVDRV_NSENTERED) are inherited by the host-side bash.
+  # DRIVER_SHA256, GPU_TYPE, _NVDRV_NSENTERED) are inherited by the
+  # host-side bash.
   _NVDRV_NSENTERED=1 nsenter -t 1 -m -p -n -i -u -- bash -s < "$0" \
-    || { echo "::error::container needs 'options: --privileged --pid=host'" >&2; exit 1; }
+    || { echo "::error::host-side driver install failed (see output above; also check container has 'options: --privileged --pid=host')" >&2; exit 1; }
   refresh_container_libs
 
   # Re-bind /run/nvidia-persistenced from the host. The container's

ci/tools/tests/test_test_matrix_driver_hashes.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Ensure custom DRIVER entries in ci/test-matrix.yml carry valid, consistent SHA-256 hashes."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+import yaml
+
+MATRIX_PATH = Path(__file__).resolve().parents[2] / "test-matrix.yml"
+SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
+PREINSTALLED_DRIVERS = {"latest", "earliest"}
+
+
+@pytest.mark.agent_authored(model="gemini-4-argon")
+def test_test_matrix_custom_drivers_have_valid_consistent_sha256() -> None:
+    matrix = yaml.safe_load(MATRIX_PATH.read_text(encoding="utf-8"))
+    seen_hashes: dict[tuple[str, str, str, str], str] = {}
+    custom_rows = 0
+
+    for platform in ("linux", "windows"):
+        for section_name, rows in matrix[platform].items():
+            for index, row in enumerate(rows):
+                driver = row["DRIVER"]
+                if driver in PREINSTALLED_DRIVERS:
+                    assert "DRIVER_SHA256" not in row, f"{platform}.{section_name}[{index}] sets DRIVER_SHA256"
+                    continue
+                custom_rows += 1
+                sha256 = row.get("DRIVER_SHA256", "")
+                assert SHA256_RE.fullmatch(sha256), f"{platform}.{section_name}[{index}] has invalid DRIVER_SHA256"
+                key = (platform, str(row["ARCH"]), "" if platform == "linux" else str(row["GPU"]), str(driver))
+                assert seen_hashes.setdefault(key, sha256) == sha256, f"Conflicting DRIVER_SHA256 for {key}"
+
+    assert custom_rows > 0