Skip to content

Commit e5c3610

Browse files
kkraus14kkraus14
committed
ci: harden prior artifact downloads
1 parent 5c798a7 commit e5c3610

4 files changed

Lines changed: 63 additions & 20 deletions

File tree

‎.github/workflows/build-wheel.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -567,16 +567,20 @@ jobs:
567567
fi
568568
569569
OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml)
570-
OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*"
570+
OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]"
571571
LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI")
572572
PREV_BINDINGS_DIR="cuda_bindings/dist-prev"
573573
574-
gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python
575-
rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts
576-
ls -al $OLD_BASENAME
574+
./ci/tools/retry-gh-run-download \
575+
"${LATEST_PRIOR_RUN_ID}" \
576+
-p "${OLD_ARTIFACT_PATTERN}" \
577+
-R NVIDIA/cuda-python
578+
OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}")
579+
test -d "${OLD_ARTIFACT_DIR}"
580+
ls -al "${OLD_ARTIFACT_DIR}"
577581
mkdir -p "${PREV_BINDINGS_DIR}"
578-
mv $OLD_BASENAME/*.whl "${PREV_BINDINGS_DIR}"
579-
rmdir $OLD_BASENAME
582+
mv "${OLD_ARTIFACT_DIR}"/*.whl "${PREV_BINDINGS_DIR}"
583+
rmdir "${OLD_ARTIFACT_DIR}"
580584
581585
- name: Constrain previous cuda.core to the downloaded cuda.bindings wheel
582586
if: ${{ env.BUILD_CORE == 'true' }}

‎.github/workflows/test-wheel-linux.yml‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -215,18 +215,25 @@ jobs:
215215
&& apt install gh -y
216216
217217
OLD_BRANCH=${{ needs.compute-matrix.outputs.OLD_BRANCH }}
218-
OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*"
218+
OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]"
219219
LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI")
220220
221-
gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python
222-
rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts
223-
ls -al $OLD_BASENAME
221+
./ci/tools/retry-gh-run-download \
222+
"${LATEST_PRIOR_RUN_ID}" \
223+
-p "${OLD_ARTIFACT_PATTERN}" \
224+
-R NVIDIA/cuda-python
225+
OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}")
226+
test -d "${OLD_ARTIFACT_DIR}"
227+
ls -al "${OLD_ARTIFACT_DIR}"
224228
mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
225-
mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/
226-
rmdir $OLD_BASENAME
229+
mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/
230+
rmdir "${OLD_ARTIFACT_DIR}"
227231
228232
if ${{ env.TEST_PYTHON == 'true' }}; then
229-
gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python
233+
./ci/tools/retry-gh-run-download \
234+
"${LATEST_PRIOR_RUN_ID}" \
235+
-p cuda-python-wheel \
236+
-R NVIDIA/cuda-python
230237
ls -al cuda-python-wheel
231238
mv cuda-python-wheel/*.whl .
232239
rmdir cuda-python-wheel

‎.github/workflows/test-wheel-windows.yml‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -195,18 +195,25 @@ jobs:
195195
shell: bash --noprofile --norc -xeuo pipefail {0}
196196
run: |
197197
OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml)
198-
OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*"
198+
OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]"
199199
LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI")
200200
201-
gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python
202-
rm -rf ${OLD_BASENAME}-tests # exclude cython test artifacts
203-
ls -al $OLD_BASENAME
201+
./ci/tools/retry-gh-run-download \
202+
"${LATEST_PRIOR_RUN_ID}" \
203+
-p "${OLD_ARTIFACT_PATTERN}" \
204+
-R NVIDIA/cuda-python
205+
OLD_ARTIFACT_DIR=$(compgen -G "${OLD_ARTIFACT_PATTERN}")
206+
test -d "${OLD_ARTIFACT_DIR}"
207+
ls -al "${OLD_ARTIFACT_DIR}"
204208
mkdir -p "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"
205-
mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/
206-
rmdir $OLD_BASENAME
209+
mv "${OLD_ARTIFACT_DIR}"/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/
210+
rmdir "${OLD_ARTIFACT_DIR}"
207211
208212
if ${{ env.TEST_PYTHON == 'true' }}; then
209-
gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python
213+
./ci/tools/retry-gh-run-download \
214+
"${LATEST_PRIOR_RUN_ID}" \
215+
-p cuda-python-wheel \
216+
-R NVIDIA/cuda-python
210217
ls -al cuda-python-wheel
211218
mv cuda-python-wheel/*.whl .
212219
rmdir cuda-python-wheel

‎ci/tools/retry-gh-run-download‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
3+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4+
#
5+
# SPDX-License-Identifier: Apache-2.0
6+
7+
set -euo pipefail
8+
9+
readonly MAX_ATTEMPTS=5
10+
delay=10
11+
12+
for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do
13+
if gh run download "$@"; then
14+
exit 0
15+
fi
16+
17+
if (( attempt == MAX_ATTEMPTS )); then
18+
echo "Artifact download failed after ${MAX_ATTEMPTS} attempts." >&2
19+
exit 1
20+
fi
21+
22+
echo "Artifact download failed (attempt ${attempt}/${MAX_ATTEMPTS}); retrying in ${delay}s." >&2
23+
sleep "${delay}"
24+
delay=$((delay * 2))
25+
done

0 commit comments

Comments
 (0)