Skip to content

Commit eae6441

Browse files
danehansdanehans
committed
fix(e2e): avoid stale Kubernetes reports
Signed-off-by: Daneyon Hansen <daneyon.hansen@solo.io>
1 parent d223943 commit eae6441

3 files changed

Lines changed: 227 additions & 4 deletions

File tree

‎.github/workflows/branch-checks.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,9 @@ jobs:
535535
bash tasks/scripts/test-gateway-pull-policy.sh
536536
bash tasks/scripts/test-gateway-config.sh
537537
538+
- name: Test Kubernetes E2E reporting
539+
run: bash tasks/scripts/test-e2e-kubernetes-reports.sh
540+
538541
- name: Test release qualification summary
539542
run: mise run test:qualification-summary
540543
go:

‎e2e/rust/e2e-kubernetes.sh‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
# cluster where the test host is unreachable from pods? Set
1717
# OPENSHELL_E2E_KUBERNETES_FEATURES=e2e to drop the alias-dependent tests.
1818
#
19-
# Results: `run_suite` writes a JUnit + HTML report under `results/`. Set
19+
# Results: `run_suite` writes a JUnit + HTML report under `results/` when
20+
# Nextest produces XML. Conformance-only runs do not produce these reports. Set
2021
# `OPENSHELL_E2E_REPORT_NAME` to name it per run when invoking this script repeatedly.
2122

2223
set -euo pipefail
@@ -80,12 +81,20 @@ run_suite() {
8081
local name="${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}"
8182
local report="${ROOT}/results/${name}.xml"
8283
local status=0
84+
85+
# Nextest writes to a fixed path. Clear it and this invocation's reports so
86+
# skipped tests or gateway setup failures cannot publish a previous result.
87+
rm -f -- "${JUNIT_XML}" "${report}" "${report%.xml}.html" || return $?
88+
8389
"${ROOT}/e2e/with-kube-gateway.sh" \
8490
bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" || status=$?
85-
if [ "${report}" != "${JUNIT_XML}" ]; then
86-
mv -f "${JUNIT_XML}" "${report}" 2>/dev/null || true
91+
if [ -f "${JUNIT_XML}" ]; then
92+
if [ "${report}" != "${JUNIT_XML}" ] && ! mv -f "${JUNIT_XML}" "${report}"; then
93+
echo "WARNING: failed to move JUnit report to ${report}; skipping HTML report" >&2
94+
return "${status}"
95+
fi
96+
render_html "${report}" "${name}"
8797
fi
88-
render_html "${report}" "${name}"
8998
return "${status}"
9099
}
91100

‎tasks/scripts/test-e2e-kubernetes-reports.sh‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
#!/usr/bin/env bash
2+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -euo pipefail
6+
7+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
8+
WORK="$(mktemp -d)"
9+
trap 'rm -rf "${WORK}"' EXIT
10+
11+
# Exercise the real entrypoint and its recursive invocation without a cluster,
12+
# Rust builds, or writes to the checkout's results directory.
13+
export REPORT_TEST_ROOT="${WORK}/repo"
14+
mkdir -p "${REPORT_TEST_ROOT}/e2e/rust" "${REPORT_TEST_ROOT}/e2e/support" "${WORK}/bin"
15+
cp "${ROOT}/e2e/rust/e2e-kubernetes.sh" "${REPORT_TEST_ROOT}/e2e/rust/"
16+
cp "${ROOT}/e2e/support/conformance.sh" "${REPORT_TEST_ROOT}/e2e/support/"
17+
18+
cat >"${REPORT_TEST_ROOT}/e2e/with-kube-gateway.sh" <<'EOF'
19+
#!/usr/bin/env bash
20+
set -euo pipefail
21+
echo "gateway ${OPENSHELL_E2E_CREDENTIAL_DRIVER:-default}" >>"${REPORT_TEST_ROOT}/calls"
22+
if [ "${REPORT_TEST_GATEWAY_STATUS}" != 0 ]; then
23+
exit "${REPORT_TEST_GATEWAY_STATUS}"
24+
fi
25+
"$@"
26+
EOF
27+
28+
cat >"${WORK}/bin/cargo" <<'EOF'
29+
#!/usr/bin/env bash
30+
set -euo pipefail
31+
echo "$1 ${OPENSHELL_E2E_CREDENTIAL_DRIVER:-default}" >>"${REPORT_TEST_ROOT}/calls"
32+
case "$1" in
33+
test) exit 0 ;;
34+
nextest)
35+
if [ "${OPENSHELL_E2E_CREDENTIAL_DRIVER:-default}" != "${REPORT_TEST_NO_XML_DRIVER}" ]; then
36+
printf '<testsuite name="fresh-%s"/>\n' "${OPENSHELL_E2E_CREDENTIAL_DRIVER:-default}" \
37+
>"${REPORT_TEST_ROOT}/results/e2e-kubernetes.xml"
38+
fi
39+
exit "${REPORT_TEST_NEXTEST_STATUS}"
40+
;;
41+
*) echo "unexpected cargo command: $*" >&2; exit 99 ;;
42+
esac
43+
EOF
44+
45+
cat >"${WORK}/bin/xsltproc" <<'EOF'
46+
#!/usr/bin/env bash
47+
set -euo pipefail
48+
echo "render $3" >>"${REPORT_TEST_ROOT}/calls"
49+
printf '<html><title>%s</title>\n' "$3"
50+
cat "$5"
51+
echo '</html>'
52+
EOF
53+
54+
cat >"${WORK}/bin/mv" <<'EOF'
55+
#!/usr/bin/env bash
56+
set -euo pipefail
57+
if [ "${REPORT_TEST_MOVE_FAIL}" = 1 ]; then
58+
exit 13
59+
fi
60+
exec /bin/mv "$@"
61+
EOF
62+
chmod +x "${REPORT_TEST_ROOT}/e2e/with-kube-gateway.sh" "${WORK}/bin/"*
63+
64+
export PATH="${WORK}/bin:${PATH}"
65+
export OPENSHELL_BIN="${WORK}/bin/openshell"
66+
cp /usr/bin/true "${OPENSHELL_BIN}"
67+
68+
fail() {
69+
echo "FAIL: $*" >&2
70+
cat "${REPORT_TEST_ROOT}/output" >&2
71+
exit 1
72+
}
73+
74+
reset_case() {
75+
rm -rf "${REPORT_TEST_ROOT}/results"
76+
mkdir "${REPORT_TEST_ROOT}/results"
77+
: >"${REPORT_TEST_ROOT}/calls"
78+
: >"${REPORT_TEST_ROOT}/output"
79+
export OPENSHELL_E2E_KUBERNETES_FEATURES=e2e
80+
export OPENSHELL_E2E_REPORT_NAME=e2e-kubernetes
81+
export OPENSHELL_E2E_CREDENTIAL_DRIVERS=0
82+
export OPENSHELL_E2E_CREDENTIAL_DRIVER=""
83+
export REPORT_TEST_GATEWAY_STATUS=0
84+
export REPORT_TEST_NEXTEST_STATUS=0
85+
export REPORT_TEST_NO_XML_DRIVER=""
86+
export REPORT_TEST_MOVE_FAIL=0
87+
}
88+
89+
seed_stale_reports() {
90+
local name
91+
for name in e2e-kubernetes "${OPENSHELL_E2E_REPORT_NAME}"; do
92+
echo '<testsuite name="stale"/>' >"${REPORT_TEST_ROOT}/results/${name}.xml"
93+
echo 'stale HTML' >"${REPORT_TEST_ROOT}/results/${name}.html"
94+
done
95+
echo 'unrelated report' >"${REPORT_TEST_ROOT}/results/unrelated.xml"
96+
echo 'unrelated HTML' >"${REPORT_TEST_ROOT}/results/unrelated.html"
97+
}
98+
99+
run_wrapper() {
100+
local expected=$1 status=0
101+
bash "${REPORT_TEST_ROOT}/e2e/rust/e2e-kubernetes.sh" \
102+
>"${REPORT_TEST_ROOT}/output" 2>&1 || status=$?
103+
[ "${status}" = "${expected}" ] || fail "expected status ${expected}, got ${status}"
104+
[ "$(cat "${REPORT_TEST_ROOT}/results/unrelated.xml")" = 'unrelated report' ] \
105+
|| fail 'modified an unrelated XML report'
106+
[ "$(cat "${REPORT_TEST_ROOT}/results/unrelated.html")" = 'unrelated HTML' ] \
107+
|| fail 'modified an unrelated HTML report'
108+
}
109+
110+
assert_absent() {
111+
[ ! -e "${REPORT_TEST_ROOT}/results/$1" ] || fail "unexpected report: $1"
112+
}
113+
114+
assert_no_render() {
115+
if grep -q '^render ' "${REPORT_TEST_ROOT}/calls"; then
116+
fail 'rendered XML when none was produced'
117+
fi
118+
}
119+
120+
# Reproduction for #3668: conformance-only skips Nextest even with stale default
121+
# XML and a stale destination report. Cover both default and custom names.
122+
for name in e2e-kubernetes custom; do
123+
reset_case
124+
export OPENSHELL_E2E_REPORT_NAME="${name}"
125+
export OPENSHELL_E2E_KUBERNETES_FEATURES=e2e-conformance-only
126+
seed_stale_reports
127+
run_wrapper 0
128+
assert_absent e2e-kubernetes.xml
129+
assert_absent "${name}.xml"
130+
assert_absent "${name}.html"
131+
grep -q '^test ' "${REPORT_TEST_ROOT}/calls" || fail 'did not run conformance'
132+
if grep -q '^nextest ' "${REPORT_TEST_ROOT}/calls"; then
133+
fail 'conformance-only unexpectedly ran Nextest'
134+
fi
135+
assert_no_render
136+
done
137+
138+
# Setup failures must not publish stale reports or lose their exit status.
139+
for name in e2e-kubernetes custom; do
140+
reset_case
141+
export OPENSHELL_E2E_REPORT_NAME="${name}"
142+
export REPORT_TEST_GATEWAY_STATUS=17
143+
seed_stale_reports
144+
run_wrapper 17
145+
assert_absent e2e-kubernetes.xml
146+
assert_absent "${name}.xml"
147+
assert_absent "${name}.html"
148+
assert_no_render
149+
done
150+
151+
# Fresh output is rendered even when Nextest fails, with the selected title.
152+
for name in e2e-kubernetes custom; do
153+
for status in 0 100; do
154+
reset_case
155+
export OPENSHELL_E2E_REPORT_NAME="${name}"
156+
export REPORT_TEST_NEXTEST_STATUS="${status}"
157+
seed_stale_reports
158+
run_wrapper "${status}"
159+
grep -q 'fresh-default' "${REPORT_TEST_ROOT}/results/${name}.xml" || fail 'missing fresh XML'
160+
grep -q 'fresh-default' "${REPORT_TEST_ROOT}/results/${name}.html" || fail 'missing fresh HTML'
161+
grep -Fq "<title>${name}</title>" "${REPORT_TEST_ROOT}/results/${name}.html" || fail 'incorrect title'
162+
if [ "${name}" != e2e-kubernetes ]; then
163+
assert_absent e2e-kubernetes.xml
164+
fi
165+
done
166+
done
167+
168+
# A failed rename must not fall through to rendering the destination.
169+
for status in 0 100; do
170+
reset_case
171+
export OPENSHELL_E2E_REPORT_NAME=custom
172+
export REPORT_TEST_MOVE_FAIL=1
173+
export REPORT_TEST_NEXTEST_STATUS="${status}"
174+
seed_stale_reports
175+
run_wrapper "${status}"
176+
assert_absent custom.xml
177+
assert_absent custom.html
178+
assert_no_render
179+
grep -q 'WARNING:.*report' "${REPORT_TEST_ROOT}/output" || fail 'missing rename warning'
180+
done
181+
182+
# Cleanup errors must stop the run before starting the gateway.
183+
reset_case
184+
seed_stale_reports
185+
rm "${REPORT_TEST_ROOT}/results/e2e-kubernetes.xml"
186+
mkdir "${REPORT_TEST_ROOT}/results/e2e-kubernetes.xml"
187+
run_wrapper 1
188+
[ ! -s "${REPORT_TEST_ROOT}/calls" ] || fail 'started a run after cleanup failed'
189+
190+
# A later credential-driver invocation that emits no XML must not reuse its
191+
# predecessor's report, or an older report under its own name.
192+
reset_case
193+
export OPENSHELL_E2E_CREDENTIAL_DRIVERS=1
194+
export REPORT_TEST_NO_XML_DRIVER=vault
195+
seed_stale_reports
196+
echo '<testsuite name="stale-vault"/>' >"${REPORT_TEST_ROOT}/results/e2e-kubernetes-vault.xml"
197+
echo 'stale vault HTML' >"${REPORT_TEST_ROOT}/results/e2e-kubernetes-vault.html"
198+
run_wrapper 0
199+
grep -q 'fresh-kubernetes-secrets' "${REPORT_TEST_ROOT}/results/e2e-kubernetes-secrets.xml" \
200+
|| fail 'missing first credential-driver XML'
201+
grep -q 'fresh-kubernetes-secrets' "${REPORT_TEST_ROOT}/results/e2e-kubernetes-secrets.html" \
202+
|| fail 'missing first credential-driver HTML'
203+
grep -q '^nextest vault$' "${REPORT_TEST_ROOT}/calls" || fail 'did not run the second driver'
204+
assert_absent e2e-kubernetes.xml
205+
assert_absent e2e-kubernetes-vault.xml
206+
assert_absent e2e-kubernetes-vault.html
207+
if grep -q '^render e2e-kubernetes-vault$' "${REPORT_TEST_ROOT}/calls"; then
208+
fail 'rendered a stale credential-driver report'
209+
fi
210+
211+
echo 'Kubernetes E2E report tests passed'

0 commit comments

Comments
 (0)