feat(ci): run plugin-sanity daily for the release branch indexes

.github/workflows/e2e-cluster-free.yaml

@@ -32,8 +32,8 @@ on:
       - "e2e-tests/local-harness/resolve-catalog-index-image.sh"
   # The catalog index is built outside this repo and changes on its own, so the
   # plugin sanity job needs a trigger that is not tied to a commit here. GitHub
-  # only runs `schedule` from the default branch, so this covers the `next`
-  # index; use the dispatch input below to check a release branch's index.
+  # only runs `schedule` from the default branch: it checks the `next` index
+  # here and dispatches the release branches (release-plugin-sanity below).
   schedule:
     - cron: "0 5 * * *"
   workflow_dispatch:
@@ -177,3 +177,24 @@ jobs:
             e2e-tests/playwright-report-plugin-sanity
             e2e-tests/plugin-sanity.log
           retention-days: 7
+
+  release-plugin-sanity:
+    name: release-plugin-sanity
+    runs-on: ubuntu-latest
+    # Each release branch runs plugin-sanity on its own ref, against its own
+    # index tag and backend; this job only dispatches and waits for them.
+    if: ${{ github.event_name == 'schedule' }}
+    timeout-minutes: 60
+    permissions:
+      actions: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+        with:
+          persist-credentials: false
+
+      - name: Run plugin-sanity on the newest release branches
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: ./e2e-tests/local-harness/dispatch-release-plugin-sanity.sh

docs/e2e-tests/local-e2e-harness.md

@@ -190,6 +190,27 @@ cluster-free phase: it installs deps + skopeo, resolves a catalog index image, p
 image is built. It triggers on `e2e-tests/**` and `app-config*.yaml` changes; the scope
 can widen to `packages/app/**` / `packages/backend/**` once it is proven stable.
 
+### Plugin sanity
+
+The `plugin-sanity` job in the same workflow installs every plugin in a catalog index
+(`populate-catalog-index.sh`), boots the backend with `app-config.plugin-sanity.yaml`, and
+fails when an installed plugin does not load. It runs daily:
+
+- on `main`, against `quay.io/rhdh/plugin-catalog-index:next`;
+- on up to two of the newest `release-X.Y` branches that carry the harness, against
+  `plugin-catalog-index:X.Y`. That tag follows the newest index build of the stream, so
+  RC and z-stream builds are checked as they land. GitHub only schedules the default
+  branch, so main's `release-plugin-sanity` job dispatches each branch on its own ref and
+  fails unless all of those runs succeed. Branches older than the harness (1.10 and
+  earlier) are not covered.
+
+To check one specific index build, dispatch the workflow on the release branch with the
+`catalog_index_image` input (for example `quay.io/rhdh/plugin-catalog-index:2.1-110`).
+A manual dispatch and the scheduled one share a concurrency group on that branch, so
+whichever starts second cancels the other. Packages that cannot boot standalone are
+listed in `plugin-sanity-excludes.txt`, each with its tracking ticket and where it is
+covered instead.
+
 ## NFS frontend (`packages/app`)
 
 The harness targets the NFS frontend (`packages/app`). The backend serves standard

e2e-tests/local-harness/dispatch-release-plugin-sanity.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Runs the plugin-sanity job on the newest release branches that carry the
+# harness, each on its own ref, so every release index is checked against the
+# backend it ships with. GitHub only fires `schedule` on the default branch,
+# so main's scheduled run calls this.
+#
+# Waits for every dispatched run and exits non-zero unless all of them succeed.
+#
+# Usage (GH_TOKEN needs actions:write on the repository):
+#   GITHUB_REPOSITORY=owner/repo ./e2e-tests/local-harness/dispatch-release-plugin-sanity.sh
+set -euo pipefail
+
+: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
+RELEASE_COUNT="${RELEASE_COUNT:-2}"
+WORKFLOW="e2e-cluster-free.yaml"
+HARNESS_MARKER="e2e-tests/local-harness/populate-catalog-index.sh"
+SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
+
+if [[ ! "${RELEASE_COUNT}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "RELEASE_COUNT must be a positive integer, got: ${RELEASE_COUNT}" >&2
+  exit 1
+fi
+
+# Only a 404 means "no harness"; any other API error must fail the job rather
+# than read as "nothing to check".
+has_harness() {
+  local out
+  out=$(gh api "repos/${GITHUB_REPOSITORY}/contents/${HARNESS_MARKER}?ref=$1" --silent 2>&1) && return 0
+  [[ "${out}" == *"HTTP 404"* ]] && return 1
+  echo "${out}" >&2
+  exit 1
+}
+
+all_branches=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/branches?per_page=100" --jq '.[].name')
+
+branches=()
+while read -r branch; do
+  # Sorted newest first, so the first branch older than the harness ends the
+  # search.
+  has_harness "${branch}" || break
+  branches+=("${branch}")
+  ((${#branches[@]} < RELEASE_COUNT)) || break
+done < <(grep -E '^release-[0-9]+\.[0-9]+$' <<< "${all_branches}" | sort -rV || true)
+
+if ((${#branches[@]} == 0)); then
+  echo "No release branch carries ${HARNESS_MARKER}; nothing to dispatch."
+  exit 0
+fi
+
+# A failed dispatch is recorded and skipped, so the runs already dispatched are
+# still waited on and reported.
+run_ids=()
+run_urls=()
+for branch in "${branches[@]}"; do
+  response=$(gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW}/dispatches" \
+    -f ref="${branch}" -F return_run_details=true --jq '"\(.workflow_run_id) \(.html_url)"') || response=""
+  if [[ -z "${response}" ]]; then
+    echo "Failed to dispatch plugin-sanity on ${branch}" >&2
+  else
+    echo "Dispatched plugin-sanity on ${branch}: ${response#* }"
+  fi
+  run_ids+=("${response%% *}")
+  run_urls+=("${response#* }")
+done
+
+echo "### Release plugin-sanity" >> "${SUMMARY}"
+failed=0
+for i in "${!branches[@]}"; do
+  branch="${branches[i]}"
+  run_id="${run_ids[i]}"
+  if [[ ! "${run_id}" =~ ^[0-9]+$ ]]; then
+    result="dispatch failed"
+  else
+    gh run watch "${run_id}" --repo "${GITHUB_REPOSITORY}" --interval 60 > /dev/null || true
+    # The real conclusion, so a run cancelled through the concurrency group it
+    # shares with manual dispatches on the same branch does not read as a
+    # plugin failure.
+    result=$(gh run view "${run_id}" --repo "${GITHUB_REPOSITORY}" --json conclusion --jq '.conclusion') \
+      || result="unknown"
+  fi
+  [[ "${result}" == "success" ]] || failed=1
+  echo "${branch}: ${result}"
+  echo "- \`${branch}\`: ${result} ${run_urls[i]:+([run](${run_urls[i]}))}" >> "${SUMMARY}"
+done
+
+exit "${failed}"

e2e-tests/local-harness/plugin-sanity-excludes.txt

@@ -36,26 +36,13 @@
 # Re-evaluate when the orchestrator backend gains a standalone-friendly init
 # path: keeping an entry here means the check is being told not to look at the
 # class of breakage it exists to catch.
+#
+# Coverage meanwhile: the orchestrator E2E in rhdh-plugin-export-overlays
+# (e2e-ocp-helm and its nightly), which loads the overlay build, not this
+# index artifact. No rhdh job loads it.
 red-hat-developer-hub-backstage-plugin-orchestrator-backend(@|:)
 
 # TODO(RHIDP-15961): module on the orchestrator backend plugin - excluded with its host plugin.
 # (Also requires orchestrator.workflowLogProvider.loki.baseUrl at startup.)
+# Loaded by the same overlays orchestrator E2E, but no spec exercises it.
 red-hat-developer-hub-backstage-plugin-orchestrator-backend-module-loki(@|:)
-
-# TODO(RHDHBUGS-3515): the three below are not published to any public registry: `skopeo inspect
-# --no-creds` reports "unauthorized" for the whole repository, on quay.io and on
-# registry.access.redhat.com alike. Note these are tag-style refs
-# (:1.11.0--x.y.z); every digest-pinned ref the index declares does resolve
-# anonymously via the CLI's quay.io fallback, so this is about publication, not
-# about the digest-lookup trap described above.
-#
-# Excluding them costs real coverage - homepage in particular is a core plugin.
-# Re-check periodically with:
-#   skopeo inspect --no-creds docker://quay.io/rhdh/<name>:<tag>
-# and delete the entry the moment it resolves.
-red-hat-developer-hub-backstage-plugin-homepage(@|:)
-red-hat-developer-hub-backstage-plugin-intelligent-assistant(@|:)
-red-hat-developer-hub-backstage-plugin-intelligent-assistant-backend(@|:)
-
-# TODO(RHDHBUGS-3678): https://redhat.atlassian.net/browse/RHDHBUGS-3678
-backstage-community-plugin-auth-backend-module-keycloak-provider(@|:)

e2e-tests/unit/dispatch-release-plugin-sanity.test.ts

@@ -0,0 +1,128 @@
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+const script = fileURLToPath(
+  new URL("../local-harness/dispatch-release-plugin-sanity.sh", import.meta.url),
+);
+let root: string;
+
+// Stub gh: branches, harness presence, dispatch and run conclusions come from
+// the environment; every call is appended to $root/events.
+const ghStub = `#!/bin/bash
+echo "$*" >> "$STUB_ROOT/events"
+has() { [[ " $1 " == *" $2 "* ]]; }
+case "$1 $2" in
+  "api --paginate") printf '%s\\n' $BRANCHES ;;
+  "api -X")
+    ref=$(printf '%s\\n' "$@" | sed -n 's/^ref=//p')
+    has "$DISPATCH_FAIL" "$ref" && exit 1
+    id=\${ref//[^0-9]/}
+    echo "$id https://github.example/runs/$id" ;;
+  "run watch") ;;
+  "run view")
+    var="CONCLUSION_$3"
+    echo "\${!var:-success}" ;;
+  api*)
+    ref=\${2##*ref=}
+    has "$HARNESS" "$ref" && exit 0
+    [[ -n "$CONTENTS_ERROR" ]] && { echo "gh: Server Error (HTTP 502)" >&2; exit 1; }
+    echo "gh: Not Found (HTTP 404)" >&2
+    exit 1 ;;
+esac
+`;
+
+function run(env: Record<string, string>) {
+  const result = spawnSync("/bin/bash", [script], {
+    env: {
+      PATH: `${join(root, "bin")}:/usr/bin:/bin`,
+      STUB_ROOT: root,
+      GITHUB_REPOSITORY: "example/rhdh",
+      GITHUB_STEP_SUMMARY: join(root, "summary"),
+      BRANCHES: "main release-1.10 release-2.1 release-2.2 release-2.3",
+      HARNESS: "release-2.1 release-2.2 release-2.3",
+      ...env,
+    },
+    encoding: "utf8",
+    timeout: 10_000,
+  });
+  const read = (name: string) => {
+    try {
+      return readFileSync(join(root, name), "utf8");
+    } catch {
+      return "";
+    }
+  };
+  const dispatched = read("events")
+    .split("\n")
+    .filter((line) => line.includes("/dispatches"))
+    .map((line) => / ref=(\S+)/u.exec(line)?.[1]);
+  return { ...result, dispatched, summary: read("summary") };
+}
+
+beforeEach(() => {
+  root = mkdtempSync(join(tmpdir(), "rhdh-dispatch-sanity-"));
+  mkdirSync(join(root, "bin"));
+  writeFileSync(join(root, "bin", "gh"), ghStub, { mode: 0o700 });
+});
+
+afterEach(() => {
+  rmSync(root, { recursive: true, force: true });
+});
+
+describe("dispatch-release-plugin-sanity.sh", () => {
+  it("dispatches the two newest release branches by default", () => {
+    const result = run({});
+    expect(result.status).toBe(0);
+    expect(result.dispatched).toEqual(["release-2.3", "release-2.2"]);
+  });
+
+  it("stops at the first branch older than the harness", () => {
+    const result = run({ RELEASE_COUNT: "5", HARNESS: "release-2.1 release-2.3" });
+    expect(result.status).toBe(0);
+    expect(result.dispatched).toEqual(["release-2.3"]);
+  });
+
+  it("exits 0 without dispatching when no release branch has the harness", () => {
+    const result = run({ HARNESS: "" });
+    expect(result.status).toBe(0);
+    expect(result.dispatched).toEqual([]);
+    expect(result.stdout).toContain("nothing to dispatch");
+  });
+
+  it("fails instead of reading an API error as a missing harness", () => {
+    const result = run({ HARNESS: "", CONTENTS_ERROR: "1" });
+    expect(result.status).toBe(1);
+    expect(result.dispatched).toEqual([]);
+    expect(result.stderr).toContain("HTTP 502");
+  });
+
+  it("fails when a run does not succeed and reports each conclusion", () => {
+    const result = run({ CONCLUSION_22: "failure" });
+    expect(result.status).toBe(1);
+    expect(result.summary).toContain(
+      "`release-2.3`: success ([run](https://github.example/runs/23))",
+    );
+    expect(result.summary).toContain(
+      "`release-2.2`: failure ([run](https://github.example/runs/22))",
+    );
+  });
+
+  it("still waits for the other runs when one dispatch fails", () => {
+    const result = run({ DISPATCH_FAIL: "release-2.3" });
+    expect(result.status).toBe(1);
+    expect(result.summary).toContain("`release-2.3`: dispatch failed");
+    expect(result.summary).toContain("`release-2.2`: success");
+  });
+
+  it.each(["0", "two"])("rejects RELEASE_COUNT=%s", (count) => {
+    const result = run({ RELEASE_COUNT: count });
+    expect(result.status).toBe(1);
+    expect(result.dispatched).toEqual([]);
+    expect(result.stderr).toContain("must be a positive integer");
+  });
+});