diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a14f80a..e8bef11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -88,7 +88,8 @@ jobs: runs-on: ubuntu-24.04 permissions: contents: write - id-token: write + outputs: + commit_hash: ${{ github.sha }} steps: - uses: actions/checkout@v4 with: @@ -143,14 +144,35 @@ jobs: --notes "Built from commit ${{ github.sha }} on ${{ github.ref_name }}" \ ./release-assets/* + deploy: + needs: publish + if: github.event_name == 'workflow_dispatch' + strategy: + fail-fast: false + matrix: + environment: [staging, juliett, foxtrot] + runs-on: ubuntu-24.04 + environment: ${{ matrix.environment }} + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: google-github-actions/auth@v2 with: - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} + project_id: ${{ vars.GCP_PROJECT_ID }} + workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} - - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: ./builds - destination: ${{ vars.GCP_BUCKET_NAME }}/kernels - gzip: false - parent: false + - uses: google-github-actions/setup-gcloud@v2 + + - name: Upload release to GCS + env: + GH_TOKEN: ${{ github.token }} + GCP_BUCKET_NAME: ${{ vars.GCP_BUCKET_NAME }} + run: | + ./scripts/upload-release-to-gcs.sh \ + --hash "${{ needs.publish.outputs.commit_hash }}" \ + --bucket "${GCP_BUCKET_NAME}/kernels" \ + --repo "${{ github.repository }}" diff --git a/README.md b/README.md index 829be4d..078b776 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Release asset naming for that commit: vmlinux-.bin # legacy (= amd64) for backwards compat ``` -4. The same binaries are uploaded to GCS at `gs://$GCP_BUCKET_NAME/kernels/vmlinux-//vmlinux.bin`. +4. The arch-specific binaries are uploaded to each deploy environment's GCS bucket at `gs://$GCP_BUCKET_NAME/kernels/vmlinux--//vmlinux.bin`. Deploy environments: `staging`, `juliett`, `foxtrot`. To upload an existing release to a bucket manually, run `./scripts/upload-release-to-gcs.sh --hash --bucket /kernels` (add `--dry-run` to preview). Existing objects are never overwritten. ## New kernel in E2B's infra _Note: these steps should give you a new kernel on your self-hosted E2B using https://github.com/e2b-dev/infra_ diff --git a/scripts/upload-release-to-gcs.sh b/scripts/upload-release-to-gcs.sh new file mode 100755 index 0000000..e041937 --- /dev/null +++ b/scripts/upload-release-to-gcs.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Uploads vmlinux-*-{amd64,arm64}.bin assets from a fc-kernels GitHub release +# to GCS at: +# gs:///vmlinux--//vmlinux.bin +# +# Existing objects are never overwritten. The legacy non-arch release asset +# (vmlinux-.bin) is intentionally skipped — under a fresh +# hash-suffixed version name there is no pre-existing flat layout to be +# backwards-compatible with. +# +# Usage: +# ./scripts/upload-release-to-gcs.sh --hash --bucket [--dry-run] [--repo ] +# +# Options: +# --hash Commit hash (full or short prefix) of the build to upload. +# --bucket Target bucket (with optional path prefix), e.g. +# my-bucket +# my-bucket/kernels +# gs://my-bucket/kernels +# --repo GitHub repo (default: e2b-dev/fc-kernels). +# --dry-run Print what would be uploaded without writing. +# -h, --help Show this help. + +set -euo pipefail + +REPO="e2b-dev/fc-kernels" +HASH="" +BUCKET="" +DRY_RUN=false + +usage() { sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --hash) HASH="${2:?--hash needs a value}"; shift 2 ;; + --bucket) BUCKET="${2:?--bucket needs a value}"; shift 2 ;; + --repo) REPO="${2:?--repo needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage 0 ;; + *) echo "Unknown argument: $1" >&2; usage 1 ;; + esac +done + +[[ -n "$HASH" ]] || { echo "ERROR: --hash is required" >&2; usage 1; } +[[ -n "$BUCKET" ]] || { echo "ERROR: --bucket is required" >&2; usage 1; } + +command -v gh >/dev/null || { echo "ERROR: gh CLI not found" >&2; exit 1; } +command -v gcloud >/dev/null || { echo "ERROR: gcloud CLI not found" >&2; exit 1; } + +BUCKET="${BUCKET#gs://}" +BUCKET="${BUCKET%/}" +BUCKET_URI="gs://${BUCKET}" + +if ! FULL_HASH=$(gh api "repos/$REPO/commits/$HASH" --jq '.sha' 2>/dev/null) \ + || [[ -z "$FULL_HASH" || "$FULL_HASH" == "null" ]]; then + echo "ERROR: commit '$HASH' not found in $REPO" >&2 + exit 1 +fi +SHORT_HASH="${FULL_HASH:0:7}" + +# The release workflow writes "Built from commit " into the body, so +# we locate the matching release by scanning bodies. +RELEASE_TAG=$(gh api "repos/$REPO/releases?per_page=100" --paginate \ + --jq ".[] | select((.body // \"\") | contains(\"$FULL_HASH\")) | .tag_name" \ + | head -1) + +if [[ -z "$RELEASE_TAG" ]]; then + echo "ERROR: no release in $REPO references commit $FULL_HASH" >&2 + exit 1 +fi + +echo "Release: $RELEASE_TAG (commit ${SHORT_HASH})" +echo "Target: ${BUCKET_URI}" +$DRY_RUN && echo "Mode: dry-run" + +mapfile -t ASSETS < <(gh release view "$RELEASE_TAG" --repo "$REPO" --json assets \ + --jq '.assets[] | select(.name | test("^vmlinux-.*\\.bin$")) | .name') + +if [[ "${#ASSETS[@]}" -eq 0 ]]; then + echo "ERROR: release $RELEASE_TAG has no vmlinux-*.bin assets" >&2 + exit 1 +fi + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +uploaded=0 +skipped=0 +for asset in "${ASSETS[@]}"; do + if [[ ! "$asset" =~ ^vmlinux-(.+)-(amd64|arm64)\.bin$ ]]; then + # Legacy non-arch release asset or unrecognized name — not uploaded. + continue + fi + version="${BASH_REMATCH[1]}" + arch="${BASH_REMATCH[2]}" + dst="${BUCKET_URI}/vmlinux-${version}-${SHORT_HASH}/${arch}/vmlinux.bin" + + if gcloud storage ls "$dst" >/dev/null 2>&1; then + echo " EXISTS $dst" + skipped=$((skipped + 1)) + continue + fi + + if $DRY_RUN; then + echo " WOULD $asset -> $dst" + continue + fi + + echo " UPLOAD $asset -> $dst" + gh release download "$RELEASE_TAG" --repo "$REPO" \ + --pattern "$asset" --dir "$TMP_DIR" --clobber >/dev/null + gcloud storage cp "$TMP_DIR/$asset" "$dst" + rm -f "$TMP_DIR/$asset" + uploaded=$((uploaded + 1)) +done + +echo "" +if $DRY_RUN; then + echo "Dry run complete. Already in GCS: $skipped." +else + echo "Done. Uploaded: $uploaded, already in GCS: $skipped." +fi