test(tmachine): qualify RPM package upgrades

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

.agents/skills/test-release-canary/SKILL.md

@@ -29,12 +29,13 @@ rolling dev release produced by the triggering workflow. Kubernetes pins the
 matching `0.0.0-dev` chart and `:dev` images.
 
 The host-package jobs exercise fresh installs, not upgrades from persisted
-gateway state. Release Dev and Release Tag additionally run the tmachine Ubuntu
-DEB upgrade lane: it installs the latest retained prerelease source, upgrades
-to the candidate DEB, verifies an existing sandbox survives, and creates a new
-sandbox. That lane is part of the release conformance matrix, not this canary.
-Validate Homebrew and RPM exact-default migration with the release-tooling and
-package lifecycle tests before relying on the canary.
+gateway state. Release Dev and Release Tag additionally run tmachine Ubuntu DEB
+and Fedora RPM upgrade lanes: they install the latest retained prerelease
+packages and matching runtime images, upgrade to the candidate packages, verify
+an existing sandbox survives, and create a new sandbox. Those lanes are part of
+the release conformance matrix, not this canary. Validate Homebrew exact-default
+migration with the release-tooling and package lifecycle tests before relying
+on the canary.
 
 The canary does not install or import `@nvidia/openshell-sdk`. TypeScript SDK
 validation lives in the `TypeScript SDK` branch check, including a publish

.github/workflows/branch-e2e.yml

@@ -95,6 +95,8 @@ jobs:
     outputs:
       cargo: ${{ steps.version.outputs.cargo }}
       deb: ${{ steps.version.outputs.deb }}
+      rpm_version: ${{ steps.version.outputs.rpm_version }}
+      rpm_release: ${{ steps.version.outputs.rpm_release }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -106,9 +108,13 @@ jobs:
         run: |
           cargo="$(python3 tasks/scripts/release.py get-version --cargo)"
           deb="$(python3 tasks/scripts/release.py get-version --dev --deb)"
+          rpm_version="$(python3 tasks/scripts/release.py get-version --dev --rpm-version)"
+          rpm_release="$(python3 tasks/scripts/release.py get-version --dev --rpm-release)"
           {
             echo "cargo=$cargo"
             echo "deb=$deb"
+            echo "rpm_version=$rpm_version"
+            echo "rpm_release=$rpm_release"
           } >> "$GITHUB_OUTPUT"
 
   build-binaries:
@@ -232,9 +238,23 @@ jobs:
       deb-version: ${{ needs.version.outputs.deb }}
       checkout-ref: ${{ github.sha }}
 
+  build-rpm-upgrade:
+    name: Build RPM packages for upgrade qualification
+    needs: [pr_metadata, version, build-binaries]
+    if: needs.pr_metadata.outputs.run_integration_upgrades == 'true'
+    permissions:
+      actions: read
+      contents: read
+    uses: ./.github/workflows/rpm-package.yml
+    with:
+      checkout-ref: ${{ github.sha }}
+      rpm-version: ${{ needs.version.outputs.rpm_version }}
+      rpm-release: ${{ needs.version.outputs.rpm_release }}
+      cargo-version: ${{ needs.version.outputs.cargo }}
+
   prepare-integration-upgrades:
     name: Prepare upgrade qualification inputs
-    needs: [pr_metadata, build-binaries, build-images, build-deb-upgrade]
+    needs: [pr_metadata, build-binaries, build-images, build-deb-upgrade, build-rpm-upgrade]
     if: needs.pr_metadata.outputs.run_integration_upgrades == 'true'
     permissions:
       actions: read
@@ -243,7 +263,9 @@ jobs:
     uses: ./.github/workflows/prepare-integration-inputs.yml
     with:
       deb-artifact-name: deb-linux-amd64
+      rpm-artifact-name: rpm-linux-x86_64
       include-deb-upgrade-source: true
+      include-rpm-upgrade-source: true
 
   integration-upgrades:
     name: Integration upgrades
@@ -259,7 +281,8 @@ jobs:
       integration-inputs-artifact-id: ${{ needs.prepare-integration-upgrades.outputs.integration_inputs_artifact_id }}
       test-matrix: >-
         [
-          {"environment":"ubuntu-docker-rootful","installer":"deb-upgrade-source","testsuite":"deb-upgrade"}
+          {"environment":"ubuntu-docker-rootful","installer":"deb-upgrade-source","testsuite":"deb-upgrade"},
+          {"environment":"fedora-podman-rootless","installer":"rpm-upgrade-source","testsuite":"rpm-upgrade"}
         ]
 
   # Run driver-independent conformance tests.

.github/workflows/prepare-integration-inputs.yml

@@ -16,11 +16,21 @@ on:
         required: false
         type: string
         default: ""
+      rpm-artifact-name:
+        description: RPM package artifact to include in the tmachine inputs
+        required: false
+        type: string
+        default: ""
       include-deb-upgrade-source:
         description: Include the latest retained prerelease DEB and runtime images as upgrade inputs
         required: false
         type: boolean
         default: false
+      include-rpm-upgrade-source:
+        description: Include the latest retained prerelease RPMs and runtime images as upgrade inputs
+        required: false
+        type: boolean
+        default: false
     outputs:
       source_sha:
         description: Source revision of the candidate artifacts
@@ -91,6 +101,30 @@ jobs:
           mv artifacts/packages/download/*.deb artifacts/packages/openshell.deb
           rmdir artifacts/packages/download
 
+      - name: Download RPM package artifact
+        if: inputs['rpm-artifact-name'] != ''
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: ${{ inputs['rpm-artifact-name'] }}
+          path: artifacts/packages/rpm/download
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs['artifact-run-id'] || github.run_id }}
+
+      - name: Stage RPM package inputs
+        if: inputs['rpm-artifact-name'] != ''
+        run: |
+          set -euo pipefail
+          download_dir=artifacts/packages/rpm/download
+          cli_rpm=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell-[0-9]*.x86_64.rpm' -print -quit)
+          gateway_rpm=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell-gateway-[0-9]*.x86_64.rpm' -print -quit)
+          if [[ -z "$cli_rpm" || -z "$gateway_rpm" ]]; then
+            echo "candidate artifact did not contain the expected x86_64 CLI and gateway RPMs" >&2
+            exit 1
+          fi
+          mv "$cli_rpm" artifacts/packages/rpm/openshell.rpm
+          mv "$gateway_rpm" artifacts/packages/rpm/openshell-gateway.rpm
+          rm -rf "$download_dir"
+
       - name: Resolve prerelease Debian upgrade source
         if: inputs['include-deb-upgrade-source']
         id: upgrade-source
@@ -160,22 +194,25 @@ jobs:
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: ${{ steps.upgrade-source.outputs.artifact }}
-          path: artifacts/upgrade/source/download
+          path: artifacts/upgrade/deb/source/download
           github-token: ${{ github.token }}
           run-id: ${{ steps.upgrade-source.outputs['run-id'] }}
 
       - name: Stage prerelease Debian upgrade source
         if: inputs['include-deb-upgrade-source']
         run: |
           set -euo pipefail
-          source_dir=artifacts/upgrade/source
+          source_dir=artifacts/upgrade/deb/source
           download_dir="${source_dir}/download"
           deb_path=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell_*_amd64.deb' -print -quit)
           if [[ -z "$deb_path" || ! -f "${download_dir}/openshell-checksums-sha256.txt" ]]; then
             echo "prerelease source artifact did not contain the expected Debian package and checksums" >&2
             exit 1
           fi
-          grep -F " $(basename "$deb_path")" "${download_dir}/openshell-checksums-sha256.txt" | sha256sum --check --status
+          (
+            cd "$download_dir"
+            grep -F " $(basename "$deb_path")" openshell-checksums-sha256.txt | sha256sum --check --status
+          )
           mv "$deb_path" "${source_dir}/openshell.deb"
           mv "${download_dir}/openshell-checksums-sha256.txt" "${source_dir}/openshell-checksums-sha256.txt"
           rmdir "$download_dir"
@@ -188,15 +225,127 @@ jobs:
         run: |
           set -euo pipefail
           image_tag="${IMAGE_TAG#v}"
-          mkdir -p artifacts/upgrade/source
+          mkdir -p artifacts/upgrade/deb/source
+
+          docker pull "ghcr.io/nvidia/openshell/sandbox:${image_tag}"
+          docker tag "ghcr.io/nvidia/openshell/sandbox:${image_tag}" openshell/sandbox:tmachine
+          docker save --output artifacts/upgrade/deb/source/openshell-sandbox-tmachine.tar openshell/sandbox:tmachine
+
+          docker pull "ghcr.io/nvidia/openshell/supervisor:${image_tag}"
+          docker tag "ghcr.io/nvidia/openshell/supervisor:${image_tag}" openshell/supervisor:tmachine
+          docker save --output artifacts/upgrade/deb/source/openshell-supervisor-tmachine.tar openshell/supervisor:tmachine
+
+      - name: Resolve prerelease RPM upgrade source
+        if: inputs['include-rpm-upgrade-source']
+        id: rpm-upgrade-source
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          successful_run_ids=$(gh api --paginate \
+            "repos/${GITHUB_REPOSITORY}/actions/workflows/release-tag.yml/runs?status=success&per_page=100" \
+            --jq '.workflow_runs[] | select(.status == "completed" and .conclusion == "success") | .id')
+          artifact_records=$(gh api --paginate \
+            "repos/${GITHUB_REPOSITORY}/actions/artifacts?per_page=100" \
+            --jq '.artifacts[] | select(.expired == false) | [.workflow_run.id, .name] | @tsv')
+          source_tag=$(printf '%s\n--ARTIFACTS--\n%s\n' "$successful_run_ids" "$artifact_records" | awk -F '\t' '
+            $0 == "--ARTIFACTS--" {
+              reading_artifacts = 1
+              next
+            }
+            !reading_artifacts {
+              if ($1 ~ /^[0-9]+$/) successful_runs[$1] = 1
+              next
+            }
+            $1 in successful_runs && $2 ~ /^openshell-v[0-9]+\.[0-9]+\.[0-9]+-pre\.[1-9][0-9]*-linux-x86_64-rpm$/ {
+              tag = $2
+              sub(/^openshell-/, "", tag)
+              sub(/-linux-x86_64-rpm$/, "", tag)
+              version = tag
+              sub(/^v/, "", version)
+              split(version, parts, "-pre\\.")
+              split(parts[1], core, "\\.")
+              sequence = parts[2] + 0
+              if (!found || core[1] + 0 > major ||
+                  (core[1] + 0 == major && core[2] + 0 > minor) ||
+                  (core[1] + 0 == major && core[2] + 0 == minor && core[3] + 0 > patch) ||
+                  (core[1] + 0 == major && core[2] + 0 == minor && core[3] + 0 == patch && sequence > prerelease)) {
+                selected = tag
+                major = core[1] + 0
+                minor = core[2] + 0
+                patch = core[3] + 0
+                prerelease = sequence
+                found = 1
+              }
+            }
+            END {
+              if (found) print selected
+            }')
+          if [[ -z "$source_tag" ]]; then
+            echo "no retained prerelease RPM artifact is available" >&2
+            exit 1
+          fi
+          source_artifact="openshell-${source_tag}-linux-x86_64-rpm"
+          source_run_id=$(gh api \
+            "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${source_artifact}&per_page=100" \
+            --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last | .workflow_run.id')
+          if [[ -z "$source_run_id" || "$source_run_id" == "null" ]]; then
+            echo "no retained artifact run is available for ${source_artifact}" >&2
+            exit 1
+          fi
+          {
+            echo "tag=${source_tag}"
+            echo "artifact=${source_artifact}"
+            echo "run-id=${source_run_id}"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Download prerelease RPM upgrade source
+        if: inputs['include-rpm-upgrade-source']
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: ${{ steps.rpm-upgrade-source.outputs.artifact }}
+          path: artifacts/upgrade/rpm/source/download
+          github-token: ${{ github.token }}
+          run-id: ${{ steps.rpm-upgrade-source.outputs['run-id'] }}
+
+      - name: Stage prerelease RPM upgrade source
+        if: inputs['include-rpm-upgrade-source']
+        run: |
+          set -euo pipefail
+          source_dir=artifacts/upgrade/rpm/source
+          download_dir="${source_dir}/download"
+          cli_rpm=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell-[0-9]*.x86_64.rpm' -print -quit)
+          gateway_rpm=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell-gateway-[0-9]*.x86_64.rpm' -print -quit)
+          if [[ -z "$cli_rpm" || -z "$gateway_rpm" || ! -f "${download_dir}/openshell-checksums-sha256.txt" ]]; then
+            echo "prerelease source artifact did not contain the expected RPMs and checksums" >&2
+            exit 1
+          fi
+          (
+            cd "$download_dir"
+            grep -F " $(basename "$cli_rpm")" openshell-checksums-sha256.txt | sha256sum --check --status
+            grep -F " $(basename "$gateway_rpm")" openshell-checksums-sha256.txt | sha256sum --check --status
+          )
+          mv "$cli_rpm" "${source_dir}/openshell.rpm"
+          mv "$gateway_rpm" "${source_dir}/openshell-gateway.rpm"
+          mv "${download_dir}/openshell-checksums-sha256.txt" "${source_dir}/openshell-checksums-sha256.txt"
+          rm -rf "$download_dir"
+
+      - name: Export prerelease runtime images for RPM upgrade source
+        if: inputs['include-rpm-upgrade-source']
+        env:
+          IMAGE_TAG: ${{ steps.rpm-upgrade-source.outputs.tag }}
+        run: |
+          set -euo pipefail
+          image_tag="${IMAGE_TAG#v}"
+          mkdir -p artifacts/upgrade/rpm/source
 
           docker pull "ghcr.io/nvidia/openshell/sandbox:${image_tag}"
           docker tag "ghcr.io/nvidia/openshell/sandbox:${image_tag}" openshell/sandbox:tmachine
-          docker save --output artifacts/upgrade/source/openshell-sandbox-tmachine.tar openshell/sandbox:tmachine
+          docker save --output artifacts/upgrade/rpm/source/openshell-sandbox-tmachine.tar openshell/sandbox:tmachine
 
           docker pull "ghcr.io/nvidia/openshell/supervisor:${image_tag}"
           docker tag "ghcr.io/nvidia/openshell/supervisor:${image_tag}" openshell/supervisor:tmachine
-          docker save --output artifacts/upgrade/source/openshell-supervisor-tmachine.tar openshell/supervisor:tmachine
+          docker save --output artifacts/upgrade/rpm/source/openshell-supervisor-tmachine.tar openshell/supervisor:tmachine
 
       - name: Log in to GHCR
         run: echo "${{ github.token }}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin

.github/workflows/release-dev.yml

@@ -102,15 +102,17 @@ jobs:
       checkout-ref: ${{ github.sha }}
 
   prepare-integration:
-    needs: [build-binaries, build-deb, build-images]
+    needs: [build-binaries, build-deb, build-images, build-rpm]
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/prepare-integration-inputs.yml
     with:
       deb-artifact-name: deb-linux-amd64
+      rpm-artifact-name: rpm-linux-x86_64
       include-deb-upgrade-source: true
+      include-rpm-upgrade-source: true
 
   conformance-integration:
     needs: prepare-integration
@@ -127,6 +129,7 @@ jobs:
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
           {"environment":"ubuntu-docker-rootful","installer":"deb-upgrade-source","testsuite":"deb-upgrade"},
+          {"environment":"fedora-podman-rootless","installer":"rpm-upgrade-source","testsuite":"rpm-upgrade"},
           {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"}
         ]

.github/workflows/release-tag.yml

@@ -130,15 +130,17 @@ jobs:
       checkout-ref: ${{ needs.compute-versions.outputs.source_sha }}
 
   prepare-integration:
-    needs: [compute-versions, build-binaries, build-deb, build-images]
+    needs: [compute-versions, build-binaries, build-deb, build-images, build-rpm]
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/prepare-integration-inputs.yml
     with:
       deb-artifact-name: deb-linux-amd64
+      rpm-artifact-name: rpm-linux-x86_64
       include-deb-upgrade-source: true
+      include-rpm-upgrade-source: true
 
   conformance-integration:
     needs: prepare-integration
@@ -155,6 +157,7 @@ jobs:
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
           {"environment":"ubuntu-docker-rootful","installer":"deb-upgrade-source","testsuite":"deb-upgrade"},
+          {"environment":"fedora-podman-rootless","installer":"rpm-upgrade-source","testsuite":"rpm-upgrade"},
           {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"}
         ]