diff --git a/.ci/pipelines/lib/testing.sh b/.ci/pipelines/lib/testing.sh index 626cd6b6f4..12c4125fec 100644 --- a/.ci/pipelines/lib/testing.sh +++ b/.ci/pipelines/lib/testing.sh @@ -181,9 +181,11 @@ testing::is_main_coverage_nightly() { } # RHIDP-13242: Prints the main commit the RHDH image in a namespace was built from, -# or nothing. Only the main OCP Helm nightly deploying `next` qualifies; the commit +# or nothing. Only the main OCP Helm nightly deploying `next` qualifies. The commit # is the image's org.opencontainers.image.revision label, read from the digest the -# pods actually ran, so a `next` rebuilt mid-job does not change it. +# pods actually ran, so a `next` rebuilt mid-job does not change it. Konflux images +# (quay.io/rhdh/rhdh-hub-*) label the GitLab midstream commit instead; for those the +# upstream commit comes from the app's build metadata. # Args: # $1 - namespace: where the tested RHDH runs # Uses globals: JOB_NAME, TAG_NAME, IMAGE_REGISTRY, IMAGE_REPO, DIR @@ -191,10 +193,17 @@ testing::_coverage_main_commit() { local namespace="${1:-}" testing::is_main_coverage_nightly && [[ "${TAG_NAME:-}" == "next" && -n "${namespace}" ]] || return 0 - local image="${IMAGE_REGISTRY:-quay.io}/${IMAGE_REPO:-rhdh-community/rhdh}" image_ids - image_ids=$(timeout 60 oc get pods -n "${namespace}" \ - -o jsonpath='{range .items[*].status.containerStatuses[*]}{.image}{" "}{.imageID}{"\n"}{end}' 2> /dev/null \ - | awk -v image="${image}" 'index($1, image ":") == 1 || index($1, image "@") == 1 { print $2 }' | sort -u) + local image="${IMAGE_REGISTRY:-quay.io}/${IMAGE_REPO:-rhdh-community/rhdh}" containers image_ids + # One "\t\t\t" line per container that runs the + # image and has pulled it. live is true for a running container in a pod that is + # not being deleted, so `oc exec` never targets a terminating one. + containers=$(timeout 60 oc get pods -n "${namespace}" -o json 2> /dev/null \ + | jq -r --arg image "${image}" '.items[] | .metadata.name as $pod + | (.metadata.deletionTimestamp == null) as $kept | .status.containerStatuses[]? + | select(.imageID != "") + | select((.image | startswith($image + ":")) or (.image | startswith($image + "@"))) + | [$pod, .name, .imageID, ($kept and .state.running != null)] | @tsv' 2> /dev/null) + image_ids=$(cut -f3 <<< "${containers}" | sort -u) if [[ -z "${image_ids}" || "${image_ids}" == *$'\n'* ]]; then log::warn "Expected one ${image} digest in ${namespace}, found: ${image_ids:-none}" return 0 @@ -203,15 +212,74 @@ testing::_coverage_main_commit() { local sha sha=$(timeout 60 skopeo inspect --override-os linux --override-arch amd64 "docker://${image_ids#docker-pullable://}" 2> /dev/null \ | jq -r '.Labels["org.opencontainers.image.revision"] // empty' 2> /dev/null) - if [[ ! "${sha}" =~ ^[0-9a-f]{40}$ ]]; then - log::warn "Could not read a commit revision label from ${image_ids}" + if [[ "${sha}" =~ ^[0-9a-f]{40}$ ]] && testing::_commit_on_main "${sha}"; then + echo "${sha}" return 0 fi - if ! testing::_commit_on_main "${sha}"; then - log::warn "Could not confirm that main contains image revision ${sha}" + + local pod container upstream + read -r pod container < <(awk -F '\t' '$4 == "true" { print $1, $2; exit }' <<< "${containers}") || true + if [[ -z "${pod}" ]]; then + log::warn "Could not find a main commit for ${image_ids}: revision label ${sha:-none} is not confirmed on main, and no running pod has the build metadata" + return 0 + fi + upstream=$(testing::_build_metadata_upstream "${namespace}" "${pod}" "${container}") + if [[ -z "${upstream}" ]]; then + log::warn "Could not find a main commit for ${image_ids}: revision label ${sha:-none} is not confirmed on main, and the build metadata gave no upstream commit" + return 0 + fi + if ! testing::_commit_on_main "${upstream}"; then + log::warn "Could not confirm that main contains upstream commit ${upstream} of ${image_ids}" + return 0 + fi + log::info "Revision label ${sha:-none} is not confirmed on main; using upstream commit ${upstream} from the build metadata" + echo "${upstream}" +} + +# RHIDP-13242: Prints the full upstream main commit named by the app's build metadata +# (".../redhat-developer/rhdh/tree/main @ "), which is minified into the +# bundle the pod ships, or nothing. Logs why when it prints nothing. +# Args: +# $1 - namespace +# $2 - pod +# $3 - container: the one that runs the RHDH image +# Uses globals: DIR +testing::_build_metadata_upstream() { + local namespace="$1" pod="$2" container="$3" + local dist="/opt/app-root/src/packages/app/dist" matches err status=0 short full body + err=$(mktemp) + matches=$(timeout 60 oc exec -n "${namespace}" "${pod}" -c "${container}" -- \ + grep -rhoE 'redhat-developer/rhdh/tree/main ?@ ?[0-9a-f]{7,40}' "${dist}" 2> "${err}") || status=$? + # grep exits 1, silently, when nothing matches; anything else is a failed read. + if [[ ${status} -ne 0 && (${status} -ne 1 || -s "${err}") ]]; then + log::warn "Could not read the build metadata in ${pod}/${container} (exit ${status})" + log::debug "$(< "${err}")" + rm -f "${err}" + return 0 + fi + rm -f "${err}" + short=$(grep -oE '[0-9a-f]{7,40}$' <<< "${matches}" | sort -u | head -1) + if [[ -z "${short}" ]]; then + log::warn "The build metadata in ${pod}/${container} names no redhat-developer/rhdh main commit" + return 0 + fi + if full=$(git -C "${DIR}" rev-parse --verify -q "${short}^{commit}" 2> /dev/null); then + echo "${full}" + return 0 + fi + if ! body=$(curl -fsS --max-time 30 "https://api.github.com/repos/redhat-developer/rhdh/commits/${short}" 2> /dev/null); then + log::warn "GitHub did not return upstream commit ${short} from the build metadata" + return 0 + fi + if ! full=$(jq -r '.sha // empty' <<< "${body}" 2> /dev/null); then + log::warn "GitHub's response for upstream commit ${short} is not JSON" + return 0 + fi + if [[ ! "${full}" =~ ^[0-9a-f]{40}$ ]]; then + log::warn "GitHub's response for upstream commit ${short} names no commit sha" return 0 fi - echo "${sha}" + echo "${full}" } # Succeeds when main contains the commit. Uses the checkout (the periodic clones diff --git a/docs/coverage/e2e-rhdh.md b/docs/coverage/e2e-rhdh.md index 909894f691..a2829e955c 100644 --- a/docs/coverage/e2e-rhdh.md +++ b/docs/coverage/e2e-rhdh.md @@ -163,15 +163,23 @@ the UI have migrated; new specs should import from `@support/coverage/test`. - `main` gets the flag from the `periodic-ci-redhat-developer-rhdh-main-e2e-ocp-helm-nightly` job, which `handle_ocp_nightly` (`.ci/pipelines/jobs/ocp-nightly.sh`) switches to `COLLECT_COVERAGE=true`. PR uploads cannot cover `main`: a squash merge creates a new - commit there. `testing::_coverage_main_commit` takes the image digest the RHDH pods in - the tested namespace ran, reads its `org.opencontainers.image.revision` label with - `skopeo`, checks that `main` contains that commit, and uploads with `--branch main`. - The label, not the checkout, names the commit, because `next` can lag `main`. + commit there. `testing::_coverage_main_commit` takes the digest of the image the job + deploys (`IMAGE_REGISTRY`/`IMAGE_REPO`) from the RHDH pods in the tested namespace, + reads its `org.opencontainers.image.revision` label with `skopeo`, checks that `main` + contains that commit, and uploads with `--branch main`. + A Konflux image such as `quay.io/rhdh/rhdh-hub-rhel10:next` labels the GitLab + midstream commit instead. When the label is not on `main`, the upstream commit comes + from the `.../redhat-developer/rhdh/tree/main @ ` entry of the build metadata + that the downstream build minifies into the bundle the app ships. It is read with + `oc exec` from a running pod and expanded to a full sha through the checkout or GitHub. + The commit always comes from the image, never from the checkout's HEAD, because + `next` can lag `main`. `next` is also rebuilt on every merge, so the first project's commit is kept for the job and a project that ran a newer image skips its upload. Other - nightlies, a tag other than `next`, a missing label, or a commit outside `main` skip - the upload. Codecov carries a flag forward from a commit's parent - (`carryforward: true` in `codecov.yml`), so commits processed before the upload do + nightlies, a tag other than `next`, or an image whose label and build metadata both + fail to name a commit on `main` skip the upload. Codecov + carries a flag forward from a commit's parent (`carryforward: true` in + `codecov.yml`), so commits processed before the upload do not get it: the flag first shows once an upload lands on the newest processed commit, and later unit-only commits keep it from then on. - The repository-wide `project`/`patch` statuses and components in `codecov.yml` diff --git a/e2e-tests/unit/e2e-coverage-upload.test.ts b/e2e-tests/unit/e2e-coverage-upload.test.ts index 603ff697e6..dcab4eb260 100644 --- a/e2e-tests/unit/e2e-coverage-upload.test.ts +++ b/e2e-tests/unit/e2e-coverage-upload.test.ts @@ -182,12 +182,41 @@ describe("testing::_upload_e2e_coverage", () => { const githubSays = (status: string) => `curl() { printf '%s\\n' "$@" > "$ROOT/curl-args"; echo '{"status":"${status}"}'; }`; +// The app build metadata naming an upstream commit, as minified into the bundle. +const upstream = (sha: string) => + `JSON.parse('{"title":"RHDH Metadata","card":{"RHDH Version":"2.2.0","Upstream":"https://github.com/redhat-developer/rhdh/tree/main @ ${sha}","Midstream":"https://gitlab.cee.redhat.com/rhidp/rhdh/-/commits/main @ c4eb3cda"}}')`; + +// A container status: [name, image, imageID, state (default running)]. +type Container = [string, string, string, ("running" | "waiting")?]; +// The `oc get pods -o json` list for pods keyed by name; `terminating` pods carry a +// deletionTimestamp. +const podList = (pods: Record, terminating: string[] = []) => ({ + items: Object.entries(pods).map(([name, containers]) => ({ + metadata: { + name, + ...(terminating.includes(name) && { deletionTimestamp: "2026-10-27T00:00:00Z" }), + }, + status: { + containerStatuses: containers.map(([container, image, imageID, state = "running"]) => ({ + name: container, + image, + imageID, + state: { [state]: {} }, + })), + }, + })), +}); + // The main OCP Helm nightly uploads to the main commit its `next` image was built -// from, named by the image's revision label, not to the checkout's HEAD. +// from, named by the image's revision label or, for Konflux images that label the +// midstream commit, by the app's build metadata. Never the checkout's HEAD. describe("testing::_upload_e2e_coverage from the main nightly", () => { const MAIN_JOB = "periodic-ci-redhat-developer-rhdh-main-e2e-ocp-helm-nightly"; const DIGEST = `quay.io/rhdh-community/rhdh@sha256:${"d".repeat(64)}`; const MAIN_RUN = { CODECOV_TOKEN: "token", JOB_NAME: MAIN_JOB, TAG_NAME: "next" }; + const RHDH: Container = ["backstage-backend", "quay.io/rhdh-community/rhdh:next", DIGEST]; + const SIDECAR: Container = ["sidecar", "registry.redhat.io/sidecar:1", "y@sha256:2"]; + const POSTGRES: Container = ["postgresql", "registry.redhat.io/postgresql:16", "x@sha256:1"]; // A commit this checkout has and HEAD contains. HEAD itself, because CI checks // out with fetch-depth 1 and has no parent commit. const onMain = spawnSync("git", ["-C", repository, "rev-parse", "HEAD"], { @@ -198,21 +227,53 @@ describe("testing::_upload_e2e_coverage from the main nightly", () => { /** * Puts fake `oc` and `skopeo` executables in `$ROOT/bin` (they run under - * `timeout`, so shell functions cannot replace them). `pods` is what - * `oc get pods` prints: one " " line per container. + * `timeout`, so shell functions cannot replace them). `oc get pods -o json` + * prints `pods`, keyed by pod name. `oc exec` runs its command locally, with the + * app's dist directory holding one chunk whose content is `bundle` (no directory + * when `bundle` is null), or prints `execError` and fails. */ function mainUpload( revision: string | null, { - pods = `quay.io/rhdh-community/rhdh:next ${DIGEST}\nregistry.redhat.io/postgresql:16 x@sha256:1`, + // The sidecar comes first, so reading the wrong container shows. + pods = { "rhdh-0": [SIDECAR, RHDH], "pg-0": [POSTGRES] }, + terminating = [], + bundle = "", + execError = "", env = {}, prelude = "", namespace = "showcase-ci-nightly", - }: { pods?: string; env?: Record; prelude?: string; namespace?: string } = {}, + }: { + pods?: Record; + terminating?: string[]; + bundle?: string | null; + execError?: string; + env?: Record; + prelude?: string; + namespace?: string; + } = {}, ) { const bin = join(root, "bin"); mkdirSync(bin, { recursive: true }); - writeFileSync(join(bin, "oc"), `#!/bin/bash\nprintf '%b\\n' ${JSON.stringify(pods)}\n`); + if (bundle !== null) { + mkdirSync(join(root, "dist/static"), { recursive: true }); + writeFileSync(join(root, "dist/static/5032.chunk.js"), bundle); + } + writeFileSync( + join(bin, "oc"), + `#!/bin/bash +if [[ $1 == exec ]]; then + printf '%s\\n' "$@" > "${root}/oc-exec-args" + ${execError ? `echo ${JSON.stringify(execError)} >&2; exit 1` : ""} + while [[ $1 != -- ]]; do shift; done + shift + exec "\${@/#\\/opt\\/app-root\\/src\\/packages\\/app\\/dist/${root}/dist}" +fi +cat <<'JSON' +${JSON.stringify(podList(pods, terminating))} +JSON +`, + ); const labels = revision === null ? {} : { "org.opencontainers.image.revision": revision }; writeFileSync( join(bin, "skopeo"), @@ -259,11 +320,16 @@ describe("testing::_upload_e2e_coverage from the main nightly", () => { ], ["a tag other than next", { env: { TAG_NAME: "2.1-122" } }, ""], ["no namespace", { namespace: "" }, ""], - ["no RHDH pod", { pods: "registry.redhat.io/postgresql:16 x@sha256:1" }, "found: none"], + ["no RHDH pod", { pods: { "pg-0": [POSTGRES] } }, "found: none"], [ "pods on two digests", { - pods: `quay.io/rhdh-community/rhdh:next ${DIGEST}\nquay.io/rhdh-community/rhdh:next ${DIGEST.replace(/d+$/u, "f".repeat(64))}`, + pods: { + "rhdh-0": [RHDH], + "rhdh-1": [ + ["backstage-backend", RHDH[1], DIGEST.replace(/d+$/u, "f".repeat(64))] as Container, + ], + }, }, "Expected one", ], @@ -275,29 +341,177 @@ describe("testing::_upload_e2e_coverage from the main nightly", () => { }); it.each([ - ["the image has no revision label", null, "", "Could not read a commit revision label"], - ["the label is not a commit sha", "v2.1.0", "", "Could not read a commit revision label"], - [ - "main does not contain the commit", - unknown, - githubSays("diverged"), - "Could not confirm that main contains", - ], - [ - "GitHub cannot be reached", - unknown, - "curl() { return 7; }", - "Could not confirm that main contains", - ], - ])("skips the upload, without failing, when %s", (_, revision, prelude, message) => { - const { status, argv, output } = mainUpload(revision, { prelude }); - expect(status).toBe(0); - expect(argv).toEqual([]); - expect(output).toContain(message); + ["the image has no revision label", null, ""], + ["the label is not a commit sha", "v2.1.0", ""], + ["main does not contain the commit", unknown, githubSays("diverged")], + ["GitHub cannot be reached", unknown, "curl() { return 7; }"], + ])( + "skips the upload, without failing, when %s and the build metadata names no commit", + (_, revision, prelude) => { + const { status, argv, output } = mainUpload(revision, { prelude }); + expect(status).toBe(0); + expect(argv).toEqual([]); + expect(output).toContain("Could not find a main commit"); + }, + ); + + // Konflux images (quay.io/rhdh/rhdh-hub-*) label the GitLab midstream commit, + // which GitHub does not know. Their bundle names the upstream commit. + describe("for an image that labels a midstream commit", () => { + const midstream = "c".repeat(40); + const notOnGitHub = `curl() { echo '{"message":"Not Found"}'; return 22; }`; + + it("uploads to the upstream commit the build metadata names", () => { + const { status, argv, output } = mainUpload(midstream, { + bundle: upstream(onMain.slice(0, 8)), + prelude: notOnGitHub, + }); + expect(status).toBe(0); + expect(argv).toContain(onMain); + expect(argv).not.toContain(midstream); + expect(output).toContain(`using upstream commit ${onMain} from the build metadata`); + // Reads the bundle of the container that runs the image, not the sidecar. + expect(readFileSync(join(root, "oc-exec-args"), "utf8")).toMatch( + /^exec\n-n\nshowcase-ci-nightly\nrhdh-0\n-c\nbackstage-backend\n--\ngrep\n/u, + ); + }); + + it("expands a short commit this checkout lacks through GitHub", () => { + const prelude = `curl() { printf '%s\\n' "$@" >> "$ROOT/curl-args"; if [[ $* == */commits/* ]]; then echo '{"sha":"${unknown}"}'; elif [[ $* == */compare/${unknown}...main* ]]; then echo '{"status":"ahead"}'; else return 22; fi; }`; + const { argv } = mainUpload(midstream, { bundle: upstream("eeeeeee"), prelude }); + expect(argv).toContain(unknown); + expect(readFileSync(join(root, "curl-args"), "utf8")).toContain( + "https://api.github.com/repos/redhat-developer/rhdh/commits/eeeeeee\n", + ); + }); + + it("skips the upload when main does not contain the upstream commit", () => { + const prelude = `curl() { if [[ $* == */commits/* ]]; then echo '{"sha":"${unknown}"}'; else echo '{"status":"diverged"}'; fi; }`; + const { status, argv, output } = mainUpload(midstream, { + bundle: upstream("eeeeeee"), + prelude, + }); + expect(status).toBe(0); + expect(argv).toEqual([]); + expect(output).toContain(`Could not confirm that main contains upstream commit ${unknown}`); + }); + + // The metadata comes from a build outside this repository, so its exact shape + // may change. + it.each([ + ["with no space after the colon", (sha: string) => upstream(sha).replace('": "', '":"')], + ["with no spaces around @", (sha: string) => upstream(sha).replace(" @ ", "@")], + [ + "with a suffix after the sha", + (sha: string) => upstream(sha).replace(sha, `${sha} (dirty)`), + ], + ["under another key", (sha: string) => upstream(sha).replace("Upstream", "Source")], + ])("reads the upstream commit %s", (_, bundle) => { + const { argv } = mainUpload(midstream, { + bundle: bundle(onMain.slice(0, 8)), + prelude: notOnGitHub, + }); + expect(argv).toContain(onMain); + }); + + it("picks the same commit whatever order chunks name them in", () => { + // Sorts after the commit on main; GitHub does not know it. + const other = onMain.startsWith("ffffffff") ? "fffffffff" : "ffffffff"; + const { argv } = mainUpload(midstream, { + bundle: upstream(other) + upstream(onMain.slice(0, 8)), + prelude: notOnGitHub, + }); + expect(argv).toContain(onMain); + }); + + it.each([ + ["names another repository", upstream(onMain).replace("redhat-developer", "someone")], + ["names another branch", upstream(onMain).replace("tree/main", "tree/release-2.1")], + ["names a sha too short to identify a commit", upstream("abc")], + ])("skips the upload when the build metadata %s", (_, bundle) => { + const { status, argv, output } = mainUpload(midstream, { bundle, prelude: notOnGitHub }); + expect(status).toBe(0); + expect(argv).toEqual([]); + expect(output).toContain( + "build metadata in rhdh-0/backstage-backend names no redhat-developer/rhdh main commit", + ); + expect(output).toContain("Could not find a main commit"); + }); + + // pods/exec is a permission the job may lack, and the image may move the bundle. + it.each([ + ["oc exec is refused", { execError: 'pods "rhdh-0" is forbidden' }, "is forbidden"], + ["the bundle is not where expected", { bundle: null }, "No such file or directory"], + ])("says the build metadata could not be read when %s", (_, options, reason) => { + const { status, argv, output } = mainUpload(midstream, { + ...options, + env: { LOG_LEVEL: "DEBUG" }, + }); + expect(status).toBe(0); + expect(argv).toEqual([]); + expect(output).toMatch( + /Could not read the build metadata in rhdh-0\/backstage-backend \(exit [12]\)/u, + ); + expect(output).toContain(reason); + expect(output).not.toContain("names no redhat-developer/rhdh main commit"); + }); + + it.each([ + ["fails", "curl() { return 22; }", "GitHub did not return upstream commit eeeeeee"], + [ + "answers with no JSON", + "curl() { echo ''; }", + "for upstream commit eeeeeee is not JSON", + ], + ["names no sha", `curl() { echo '{}'; }`, "for upstream commit eeeeeee names no commit sha"], + ])("says why when GitHub %s for a short sha", (_, prelude, message) => { + const { status, argv, output } = mainUpload(midstream, { + bundle: upstream("eeeeeee"), + prelude, + }); + expect(status).toBe(0); + expect(argv).toEqual([]); + expect(output).toContain(message); + }); + + it("reads the bundle of a running replica, not a terminating or waiting one", () => { + const waiting: Container = [RHDH[0], RHDH[1], DIGEST, "waiting"]; + const { argv } = mainUpload(midstream, { + pods: { "rhdh-3": [waiting], "rhdh-2": [RHDH], "rhdh-0": [SIDECAR, RHDH] }, + terminating: ["rhdh-2"], + bundle: upstream(onMain.slice(0, 8)), + prelude: notOnGitHub, + }); + expect(argv).toContain(onMain); + expect(readFileSync(join(root, "oc-exec-args"), "utf8")).toContain("\nrhdh-0\n"); + }); + + it("skips the upload when no replica is running", () => { + const { status, argv, output } = mainUpload(midstream, { + pods: { "rhdh-0": [RHDH] }, + terminating: ["rhdh-0"], + bundle: upstream(onMain.slice(0, 8)), + }); + expect(status).toBe(0); + expect(argv).toEqual([]); + expect(output).toContain("no running pod has the build metadata"); + }); + }); + + // Mid-rollout, a replica still waiting for its image reports an empty imageID. + it("uploads when another replica has not pulled the image yet", () => { + const pulling: Container = [RHDH[0], RHDH[1], "", "waiting"]; + const { argv, output } = mainUpload(onMain, { + pods: { "rhdh-1": [pulling], "rhdh-0": [SIDECAR, RHDH] }, + }); + expect(output).not.toContain("Expected one"); + expect(argv).toContain(onMain); }); it("reads the commit from pods that run the image by digest", () => { - const { argv } = mainUpload(onMain, { pods: `${DIGEST} ${DIGEST}` }); + const { argv } = mainUpload(onMain, { + pods: { "rhdh-0": [["backstage-backend", DIGEST, DIGEST]] }, + }); expect(argv).toContain(onMain); expect(readFileSync(join(root, "skopeo-args"), "utf8")).toContain(`docker://${DIGEST}\n`); });