Skip to content

Commit 64a8a2b

Browse files
juenglinjuenglin
committed
CI: verify GPU driver installer SHA-256 before execution
Signed-off-by: Ralf Juengling <rjuengling@nvidia.com>
1 parent 20a4aee commit 64a8a2b

6 files changed

Lines changed: 153 additions & 18 deletions

File tree

‎.github/workflows/test-wheel-linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@ jobs:
150150
if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
151151
env:
152152
DRIVER: ${{ matrix.DRIVER }}
153+
DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }}
153154
GPU_TYPE: ${{ matrix.GPU }}
154155
run: ./ci/tools/install_gpu_driver.sh
155156

‎.github/workflows/test-wheel-windows.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ jobs:
123123
if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
124124
env:
125125
DRIVER: ${{ matrix.DRIVER }}
126+
DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }}
126127
GPU_TYPE: ${{ matrix.GPU }}
127128
run: |
128129
ci/tools/install_gpu_driver.ps1

‎ci/test-matrix.yml‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
# - install that version via ci/tools/install_gpu_driver.sh (Linux)
2121
# or ci/tools/install_gpu_driver.ps1 (Windows) at the start of the
2222
# job. The matrix row is routed to the 'latest' runner image (the
23-
# install scripts swap the driver themselves).
23+
# install scripts swap the driver themselves). A version-string
24+
# row must also set DRIVER_SHA256 (regenerate with:
25+
# `curl -fsSL <installer-url> | sha256sum`).
2426
# - DRIVER: 'earliest' does not work with CUDA 12.9.1
2527
# - DRIVER: a custom version is not supported with FLAVOR=wsl on Linux.
2628
# - ENV: optional map of extra environment variables to inject into the test
@@ -50,10 +52,10 @@ linux:
5052
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' }
5153
- { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest' }
5254
- { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest' }
53-
- { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02' }
55+
- { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02', DRIVER_SHA256: '3034a054bb4cdf7752ff8dc272564cb105513804bff53538945901b16ca77463' }
5456
- { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest' }
5557
- { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' }
56-
- { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: '610.43.02' }
58+
- { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: '610.43.02', DRIVER_SHA256: '3034a054bb4cdf7752ff8dc272564cb105513804bff53538945901b16ca77463' }
5759
- { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest' }
5860
- { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' }
5961
- { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' }
@@ -101,7 +103,7 @@ linux:
101103
- { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } }
102104
# nightly-numba-cuda
103105
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } }
104-
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '580.65.06', ENV: { MODE: 'nightly-numba-cuda' } }
106+
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '580.65.06', DRIVER_SHA256: '04b10867af585e765cfbfdcf39ed5f4bd112375bebab0172eaa187c6aa5024ff', ENV: { MODE: 'nightly-numba-cuda' } }
105107
- { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } }
106108
- { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } }
107109
# nightly-numba-cuda-mlir (MLIR backend, linux-64 only)
@@ -150,7 +152,7 @@ windows:
150152
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } }
151153
# nightly-numba-cuda
152154
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } }
153-
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '596.36', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } }
155+
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '596.36', DRIVER_SHA256: '99182c2fdf3f23c503c74608945d84be99f036b5ec969f2945062069a15acd2e', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } }
154156
# nightly-numba-cuda-mlir (MLIR backend, win-64)
155157
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } }
156158
- { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } }

‎ci/tools/install_gpu_driver.ps1‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@
1010
# right driver).
1111
#
1212
# Inputs (env):
13-
# DRIVER Driver version, e.g. "610.47". Must NOT be 'latest' or
14-
# 'earliest' -- those are runner-pre-installed and the
15-
# workflow is expected to skip this script for them.
16-
# GPU_TYPE Lower-case GPU label from the matrix (e.g. "l4", "rtx4090").
17-
# Selects the data-center vs desktop installer variant.
13+
# DRIVER Driver version, e.g. "610.47". Must NOT be 'latest' or
14+
# 'earliest' -- those are runner-pre-installed and the
15+
# workflow is expected to skip this script for them.
16+
# DRIVER_SHA256 Expected SHA-256 hex digest of the downloaded driver
17+
# installer.
18+
# GPU_TYPE Lower-case GPU label from the matrix (e.g. "l4", "rtx4090").
19+
# Selects the data-center vs desktop installer variant.
1820

1921
# Install the driver
2022
function Install-Driver {
@@ -68,6 +70,17 @@ function Install-Driver {
6870
$ProgressPreference = $ProgressPreference_tmp
6971
Write-Output 'Download complete!'
7072

73+
if (-not $env:DRIVER_SHA256) {
74+
Write-Error "DRIVER_SHA256 env var is required to verify the driver installer."
75+
exit 1
76+
}
77+
$actual_sha256 = (Get-FileHash -Algorithm SHA256 -Path $filepath).Hash
78+
if ($actual_sha256 -ne $env:DRIVER_SHA256) {
79+
Write-Error "SHA256 mismatch for ${filepath}: expected $($env:DRIVER_SHA256), got $actual_sha256"
80+
exit 1
81+
}
82+
Write-Output "SHA256 verified: $actual_sha256"
83+
7184
# Install the file with the specified path from earlier
7285
Write-Output 'Running the driver installer...'
7386
Start-Process -FilePath $filepath -ArgumentList $install_args -Wait

‎ci/tools/install_gpu_driver.sh‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@
1313
# bind-mount refresh) and may cause issues.
1414
#
1515
# Inputs (env):
16-
# DRIVER Driver version, e.g. "580.65.06". Must NOT be 'latest' or
17-
# 'earliest' -- those are runner-pre-installed and the
18-
# workflow is expected to skip this script for them.
19-
# GPU_TYPE Lower-case GPU label from the matrix (e.g. "v100", "l4",
20-
# "h100"). Used only to pick the kernel module flavor
21-
# (Volta needs the proprietary/legacy module; everything
22-
# newer can use the open module).
16+
# DRIVER Driver version, e.g. "580.65.06". Must NOT be 'latest' or
17+
# 'earliest' -- those are runner-pre-installed and the
18+
# workflow is expected to skip this script for them.
19+
# DRIVER_SHA256 Expected lowercase SHA-256 hex digest of the downloaded
20+
# driver installer.
21+
# GPU_TYPE Lower-case GPU label from the matrix (e.g. "v100", "l4",
22+
# "h100"). Used only to pick the kernel module flavor
23+
# (Volta needs the proprietary/legacy module; everything
24+
# newer can use the open module).
2325
#
2426
# Arch is detected from `uname -m`.
2527
#
@@ -32,6 +34,7 @@
3234
set -euo pipefail
3335

3436
: "${DRIVER:?DRIVER env var is required (e.g. 580.65.06)}"
37+
: "${DRIVER_SHA256:?DRIVER_SHA256 env var is required (64-char sha256 of the installer)}"
3538
: "${GPU_TYPE:?GPU_TYPE env var is required (e.g. l4)}"
3639

3740
case "$DRIVER" in
@@ -70,7 +73,7 @@ URL="https://us.download.nvidia.com/XFree86/${ARCH_DIR}/${VERSION}/NVIDIA-Linux-
7073

7174
# Re-elevate to root if needed (sudo is preinstalled on the runner image).
7275
if [ "$(id -u)" != 0 ]; then
73-
exec sudo -E DRIVER="$DRIVER" GPU_TYPE="$GPU_TYPE" "$0" "$@"
76+
exec sudo -E DRIVER="$DRIVER" DRIVER_SHA256="$DRIVER_SHA256" GPU_TYPE="$GPU_TYPE" "$0" "$@"
7477
fi
7578

7679
echo "install_gpu_driver.sh is ALPHA -- it performs live modifications to the host driver stack and may cause issues" >&2
@@ -111,6 +114,7 @@ host_install() {
111114
d=$(mktemp -d)
112115
( cd "$d" \
113116
&& wget -q -O installer.run "$URL" \
117+
&& echo "${DRIVER_SHA256} installer.run" | sha256sum --check --strict - \
114118
&& sh installer.run --silent --dkms --no-questions \
115119
--accept-license --ui=none --no-cc-version-check --kernel-module-type="$KMT" )
116120
modprobe nvidia nvidia_uvm nvidia_modeset

‎ci/tools/tests/test_test_matrix_driver_hashes.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Ensure custom DRIVER entries in ci/test-matrix.yml carry verified SHA-256 hashes."""
5+
6+
from __future__ import annotations
7+
8+
import re
9+
from pathlib import Path
10+
11+
import pytest
12+
import yaml
13+
14+
REPO_ROOT = Path(__file__).resolve().parents[3]
15+
MATRIX_PATH = REPO_ROOT / "ci" / "test-matrix.yml"
16+
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
17+
PREINSTALLED_DRIVERS = {"latest", "earliest"}
18+
WINDOWS_DATA_CENTER_GPUS = {"a100", "h100", "l4", "t4", "v100", "rtxa6000", "rtx6000ada"}
19+
WINDOWS_DESKTOP_GPUS = {"rtx2080", "rtx4090", "rtxpro6000"}
20+
21+
22+
def _load_matrix() -> dict[str, object]:
23+
return yaml.safe_load(MATRIX_PATH.read_text(encoding="utf-8"))
24+
25+
26+
def _iter_rows(matrix: dict[str, object]):
27+
for platform in ("linux", "windows"):
28+
sections = matrix.get(platform, {})
29+
assert isinstance(sections, dict)
30+
for section_name, rows in sections.items():
31+
assert isinstance(rows, list)
32+
for index, row in enumerate(rows):
33+
assert isinstance(row, dict)
34+
yield platform, section_name, index, row
35+
36+
37+
def _installer_identity(platform: str, row: dict[str, object]) -> tuple[str, str, str]:
38+
arch = str(row["ARCH"])
39+
driver = str(row["DRIVER"])
40+
if platform == "linux":
41+
return (platform, arch, driver)
42+
gpu = str(row["GPU"])
43+
if gpu in WINDOWS_DATA_CENTER_GPUS:
44+
variant = "data-center"
45+
elif gpu in WINDOWS_DESKTOP_GPUS:
46+
variant = "desktop"
47+
else:
48+
raise AssertionError(f"Unknown Windows GPU '{gpu}' in ci/test-matrix.yml")
49+
return (f"{platform}-{variant}", arch, driver)
50+
51+
52+
def validate_matrix_driver_hashes(matrix: dict[str, object]) -> None:
53+
seen_hashes: dict[tuple[str, str, str], str] = {}
54+
custom_rows = 0
55+
56+
for platform, section_name, index, row in _iter_rows(matrix):
57+
driver = row.get("DRIVER")
58+
assert isinstance(driver, str) and driver, f"{platform}.{section_name}[{index}] is missing a non-empty DRIVER"
59+
if driver in PREINSTALLED_DRIVERS:
60+
assert "DRIVER_SHA256" not in row, (
61+
f"{platform}.{section_name}[{index}] sets DRIVER={driver!r} and must not set DRIVER_SHA256"
62+
)
63+
continue
64+
65+
custom_rows += 1
66+
sha256 = row.get("DRIVER_SHA256")
67+
assert isinstance(sha256, str) and SHA256_RE.match(sha256), (
68+
f"{platform}.{section_name}[{index}] (DRIVER={driver!r}) must set "
69+
"DRIVER_SHA256 to a 64-character lowercase hex SHA-256 digest"
70+
)
71+
72+
identity = _installer_identity(platform, row)
73+
previous = seen_hashes.setdefault(identity, sha256)
74+
assert sha256 == previous, f"Inconsistent DRIVER_SHA256 for {identity}: {sha256!r} != {previous!r}"
75+
76+
assert custom_rows > 0, "Expected at least one custom DRIVER row in ci/test-matrix.yml"
77+
78+
79+
@pytest.mark.agent_authored(model="gemini-4-argon")
80+
def test_test_matrix_custom_drivers_have_valid_consistent_sha256() -> None:
81+
validate_matrix_driver_hashes(_load_matrix())
82+
83+
84+
@pytest.mark.agent_authored(model="gemini-4-argon")
85+
def test_missing_or_invalid_driver_sha256_is_rejected() -> None:
86+
matrix = _load_matrix()
87+
for _platform, _section_name, _index, row in _iter_rows(matrix):
88+
if row["DRIVER"] not in PREINSTALLED_DRIVERS:
89+
target_row = row
90+
break
91+
else:
92+
pytest.fail("No custom DRIVER row found in ci/test-matrix.yml")
93+
94+
original = target_row.pop("DRIVER_SHA256")
95+
with pytest.raises(AssertionError, match="must set DRIVER_SHA256"):
96+
validate_matrix_driver_hashes(matrix)
97+
98+
target_row["DRIVER_SHA256"] = "UPPERCASE" + ("0" * 55)
99+
with pytest.raises(AssertionError, match="64-character lowercase hex"):
100+
validate_matrix_driver_hashes(matrix)
101+
102+
target_row["DRIVER_SHA256"] = original
103+
104+
105+
@pytest.mark.agent_authored(model="gemini-4-argon")
106+
def test_conflicting_driver_sha256_for_same_installer_is_rejected() -> None:
107+
matrix = _load_matrix()
108+
linux_pr_rows = matrix["linux"]["pull-request"]
109+
custom_rows = [row for row in linux_pr_rows if row["DRIVER"] == "610.43.02"]
110+
assert len(custom_rows) >= 2
111+
112+
custom_rows[1]["DRIVER_SHA256"] = "0" * 64
113+
with pytest.raises(AssertionError, match="Inconsistent DRIVER_SHA256"):
114+
validate_matrix_driver_hashes(matrix)

0 commit comments

Comments
 (0)