runner-trust-probe.yml

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
# Temporary, report-only probe of OS and bundled Node.js certificate stores
# across hosted and self-hosted runner labels for the eMudhra/emSign CA
# incident. Delete this workflow and ci/tools/runner_trust_probe* once all
# runner labels pass and the rollout is complete.

name: "CI: Runner trust probe"

on:
  schedule:
    # Daily at 06:00 PST (14:00 UTC)
    - cron: "0 14 * * *"
  workflow_dispatch:
    inputs:
      label-filter:
        description: "Optional regex to filter runner labels (e.g. '^(ubuntu-latest|linux-amd64-cpu8)$'; empty probes all)"
        required: false
        default: ""
        type: string

permissions:
  contents: read

defaults:
  run:
    shell: bash --noprofile --norc -xeuo pipefail {0}

jobs:
  compute-matrix:
    name: Compute runner label sets
    # Skip on forks (which cannot reach the self-hosted runners). This is true on
    # both NVIDIA/cuda-python and the private development repo, so the one guard
    # works unchanged on both and the workflow does not diverge between branches.
    if: ${{ !github.event.repository.fork }}
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      host: ${{ steps.compute.outputs.host }}
      linux_gpu: ${{ steps.compute.outputs.linux_gpu }}
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          filter: blob:none
          persist-credentials: false

      - name: Compute runner labels
        id: compute
        env:
          LABEL_FILTER: ${{ inputs.label-filter || '' }}
        run: |
          # GitHub-hosted labels are a small, stable seed. Self-hosted labels are
          # discovered from the runner labels referenced across this branch's own
          # workflows, so the probe self-adapts per branch (for example the
          # internal labels on the private development branch) without this file
          # diverging between branches. GPU labels that are built from
          # ci/test-matrix.yml at runtime are not literals in any workflow, so
          # they are still derived from that file.
          hosted='["ubuntu-latest","windows-latest","windows-2022","windows-11-arm"]'

          # Discover runner-label literals from the workflows. The grammar matches
          # self-hosted labels (-cpuN, -gpu-, -internal, -1xgb10, riscv64) and any
          # GPU labels hard-coded in a workflow; GitHub-hosted names and
          # host-platform strings (e.g. 'linux-64') deliberately do not match.
          scanned="$(grep -rhoE \
            '(linux|windows|wsl)-[a-z0-9-]*(cpu[0-9]+|gpu-[a-z0-9]|internal|1xgb10)[a-z0-9-]*|(linux|windows|wsl)-riscv64-[a-z0-9][a-z0-9-]*[a-z0-9]' \
            .github/workflows/ --exclude=runner-trust-probe.yml \
            | sort -u | jq -Rsc 'split("\n") | map(select(length > 0 and (endswith("-") | not)))' || true)"
          scanned="${scanned:-[]}"

          # GPU labels derived from the test matrix cover labels passed via matrix
          # outputs that never appear as literals in a workflow.
          gpu="$(yq -o json '.' ci/test-matrix.yml | jq -c '
            {
              windows: [ (.windows["pull-request"][]?, .windows["nightly"][]?)
                         | "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" ],
              linux:   [ (.linux["pull-request"][]?, .linux["nightly"][]?)
                         | "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" ]
            }')"

          # A label runs in the Linux GPU container probe when it is a linux/wsl
          # GPU label; every other discovered label runs on the host probe.
          labels_json="$(jq -cn \
            --argjson hosted "${hosted}" \
            --argjson scanned "${scanned}" \
            --argjson gpu "${gpu}" \
            --arg filter "${LABEL_FILTER}" '
            def is_linux_gpu: test("^(linux|wsl)-.*-gpu-");
            (($scanned + $gpu.linux + $gpu.windows)) as $discovered
            | {
                host:      ($hosted + ($discovered | map(select(is_linux_gpu | not)))),
                linux_gpu: ($discovered | map(select(is_linux_gpu)))
              }
            | map_values(unique | if $filter != "" then map(select(test($filter))) else . end)
          ')"

          echo "host=$(jq -c '.host' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}"
          echo "linux_gpu=$(jq -c '.linux_gpu' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}"

  probe-host:
    name: Probe (${{ matrix.label }})
    needs: compute-matrix
    if: ${{ needs.compute-matrix.outputs.host != '[]' }}
    strategy:
      fail-fast: false
      matrix:
        label: ${{ fromJSON(needs.compute-matrix.outputs.host) }}
    runs-on: ${{ matrix.label }}
    timeout-minutes: 15
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          filter: blob:none
          persist-credentials: false

      - name: Probe Linux OS trust store
        if: ${{ runner.os != 'Windows' }}
        env:
          TRUST_PROBE_LABEL: "${{ matrix.label == 'ubuntu-latest' && 'OS trust store (before ca-certificates upgrade)' || 'OS trust store' }}"
        run: ./ci/tools/runner_trust_probe.sh

      - name: Probe Linux OS trust store after ca-certificates upgrade (ubuntu-latest)
        if: ${{ always() && matrix.label == 'ubuntu-latest' }}
        env:
          TRUST_PROBE_LABEL: "OS trust store (after ca-certificates upgrade)"
        run: |
          sudo apt-get update
          sudo apt-get install --only-upgrade -y ca-certificates
          ./ci/tools/runner_trust_probe.sh

      - name: Probe Windows OS trust store
        if: ${{ runner.os == 'Windows' }}
        shell: pwsh
        run: ./ci/tools/runner_trust_probe.ps1

      - name: Probe Node 20 action runtime (informational)
        if: ${{ always() }}
        continue-on-error: true
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1 (declares node20)
        with:
          script: |
            const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
            await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false });

      - name: Probe bundled Node runtime and trust store
        if: ${{ always() }}
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
        with:
          script: |
            const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
            await probe({ core });

  probe-linux-gpu:
    name: Probe (${{ matrix.label }})
    needs: compute-matrix
    if: ${{ needs.compute-matrix.outputs.linux_gpu != '[]' }}
    strategy:
      fail-fast: false
      matrix:
        label: ${{ fromJSON(needs.compute-matrix.outputs.linux_gpu) }}
    runs-on: ${{ matrix.label }}
    timeout-minutes: 15
    # Match test-wheel-linux.yml container setup so the probe inspects the exact
    # container environment that Linux GPU test jobs see.
    container:
      options: -u root --security-opt seccomp=unconfined --shm-size 16g
      image: ubuntu:24.04
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          filter: blob:none
          persist-credentials: false

      - name: Setup proxy cache
        uses: nv-gha-runners/setup-proxy-cache@main
        continue-on-error: true

      - name: Install container CA certificates and probe tools
        uses: ./.github/actions/install_unix_deps
        with:
          dependencies: "ca-certificates curl openssl"
          dependent_exes: "curl openssl"

      - name: Probe container OS trust store
        env:
          TRUST_PROBE_LABEL: "Container OS trust store (ubuntu:24.04)"
        run: ./ci/tools/runner_trust_probe.sh

      - name: Probe Node 20 action runtime (informational)
        if: ${{ always() }}
        continue-on-error: true
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1 (declares node20)
        with:
          script: |
            const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
            await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false });

      - name: Probe bundled Node runtime and trust store
        if: ${{ always() }}
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
        with:
          script: |
            const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
            await probe({ core });