From 95613c4a30d059a674513f25625ced9656b1d9c8 Mon Sep 17 00:00:00 2001 From: Jack Zampolin Date: Mon, 16 Mar 2026 09:04:30 -0700 Subject: [PATCH 1/2] Switch defra/hub.rs binaries to GitHub Actions artifact downloads Replace filesystem-cache lookups with `gh run download` for defra-iroh and hubd binaries. The script now: - Resolves the pinned ref to an exact commit via git ls-remote - Finds the successful CI run for that commit via gh run list --commit - Downloads the release artifact via gh run download - Hard-fails if no successful run exists (same semantics as before) orbis-rs remains a source build since we don't control their CI. Depends on: - sourcenetwork/defradb.rs#586 - sourcenetwork/hub.rs#83 Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/scripts/ensure-binaries.sh | 97 +++++++++++++++++++----------- 1 file changed, 63 insertions(+), 34 deletions(-) diff --git a/.github/scripts/ensure-binaries.sh b/.github/scripts/ensure-binaries.sh index 521373e..637a1ab 100755 --- a/.github/scripts/ensure-binaries.sh +++ b/.github/scripts/ensure-binaries.sh @@ -1,14 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -# Build and cache cross-repo binary dependencies for integration tests. +# Download pre-built binary dependencies for integration tests. # -# defra and hub.rs binaries are built by their own CI and cached on the -# studios. This script just verifies they exist (via symlinks). +# defra and hub.rs binaries are downloaded as GitHub Actions artifacts +# from their CI workflows. The exact commit for each ref must have a +# successful CI run with uploaded artifacts, or this script fails. # -# orbis-rs has no CI on the studios, so this script resolves the commit, -# clones/fetches into a persistent local checkout, and does an incremental -# cargo build --release. +# orbis-rs has no CI artifact pipeline, so this script resolves the +# commit, clones/fetches into a persistent local checkout, and does +# an incremental cargo build --release. # # Required env vars (set in ci.yml): # DEFRA_REF — git ref for defradb.rs (e.g. "main" or a commit SHA) @@ -16,7 +17,7 @@ set -euo pipefail # ORBIS_REF — git ref for orbis-rs (e.g. "jack/integration-testing") # # Optional: -# PRIVATE_REPO_PAT — PAT for private repo access (used in git URLs) +# PRIVATE_REPO_PAT — PAT for private repo access (used for gh CLI and git URLs) # CACHE_DIR — override binary cache root (default: ~/.sourcenetwork/bin) # SRC_DIR — override source clone root (default: ~/.sourcenetwork/src) # MAX_VERSIONS — versions to keep per component (default: 3) @@ -27,6 +28,9 @@ MAX_VERSIONS="${MAX_VERSIONS:-3}" mkdir -p "$CACHE_DIR" "$SRC_DIR" +# gh CLI auth via PAT +export GH_TOKEN="${PRIVATE_REPO_PAT:-}" + repo_url() { local repo=$1 if [[ -n "${PRIVATE_REPO_PAT:-}" ]]; then @@ -54,6 +58,50 @@ resolve_commit() { return 1 } +# Download a binary artifact from a GitHub Actions workflow run. +# Finds the successful CI run for the exact commit, then downloads +# the named artifact. +download_artifact() { + local repo=$1 ref=$2 artifact_name=$3 binary_name=$4 + local commit + commit=$(resolve_commit "$repo" "$ref") + echo "$repo: $ref → ${commit:0:12}" + + # Find successful CI run for this exact commit + local run_id + run_id=$(gh run list -R "sourcenetwork/$repo" \ + --commit "$commit" --status success --workflow ci.yml \ + --limit 1 --json databaseId -q '.[0].databaseId') + + if [[ -z "$run_id" ]]; then + echo " ERROR: no successful CI run found for $repo@${commit:0:12}" >&2 + echo " The CI for $repo must complete successfully before backbone CI can run." >&2 + exit 1 + fi + + echo " Downloading $artifact_name from run $run_id..." + local tmp_dir + tmp_dir=$(mktemp -d) + gh run download "$run_id" -R "sourcenetwork/$repo" \ + --name "$artifact_name" --dir "$tmp_dir" + + # Find the binary in the extracted artifact (may be nested or flat) + local found + found=$(find "$tmp_dir" -type f | head -1) + if [[ -z "$found" ]]; then + echo " ERROR: artifact $artifact_name was empty" >&2 + rm -rf "$tmp_dir" + exit 1 + fi + + cp "$found" "$CACHE_DIR/$binary_name" + chmod +x "$CACHE_DIR/$binary_name" + rm -rf "$tmp_dir" + echo " $binary_name: ready" +} + +# --- orbis-rs helper functions (source build, no CI artifacts) --- + ensure_clone() { local repo=$1 ref=$2 commit=$3 local src="$SRC_DIR/$repo" @@ -156,34 +204,15 @@ prune_old_versions() { fi } -ensure_prebuilt_binary() { - local repo=$1 ref=$2 binary=$3 output=${4:-$3} - local commit cache_path - - commit=$(resolve_commit "$repo" "$ref") - cache_path="$CACHE_DIR/$repo/$commit/$output" - - echo "$repo: $ref → ${commit:0:12}" - if [[ ! -x "$cache_path" ]]; then - echo " ERROR: cached binary missing at $cache_path" >&2 - echo " $repo CI must build and cache commit $commit first." >&2 - exit 1 - fi - - ln -sf "$cache_path" "$CACHE_DIR/$binary" - echo " $binary: $(readlink "$CACHE_DIR/$binary")" -} - echo "=== Ensuring binary dependencies ===" -# defra and hub.rs: binaries built by their own CI, relink to the exact -# commit requested by this workflow so runners do not drift. -echo "--- defra/hub.rs (pre-built by their CI) ---" -ensure_prebuilt_binary "defradb.rs" "$DEFRA_REF" "defra-iroh" -ensure_prebuilt_binary "hub.rs" "$HUBD_REF" "hubd" +# defra and hub.rs: download release artifacts from their CI +echo "--- defra/hub.rs (GitHub Actions artifacts) ---" +download_artifact "defradb.rs" "$DEFRA_REF" "defra-iroh-aarch64-apple-darwin" "defra-iroh" +download_artifact "hub.rs" "$HUBD_REF" "hubd-aarch64-apple-darwin" "hubd" -# orbis-rs: no CI on studios, build here -echo "--- orbis-rs (built by ensure-binaries) ---" +# orbis-rs: no CI artifact pipeline, build from source +echo "--- orbis-rs (built from source) ---" ORBIS_COMMIT=$(resolve_commit "orbis-rs" "$ORBIS_REF") echo "orbis-rs: $ORBIS_REF → ${ORBIS_COMMIT:0:12}" @@ -195,10 +224,10 @@ prune_old_versions "orbis-rs" # Final verification echo "" -echo "=== Binary versions ===" +echo "=== Binary verification ===" for bin in defra-iroh hubd orbis-node cli-tool; do if [[ -x "$CACHE_DIR/$bin" ]]; then - echo " $bin: $(readlink "$CACHE_DIR/$bin")" + echo " $bin: OK" else echo " ERROR: $bin not found in $CACHE_DIR" >&2 exit 1 From a8292885e2b4af4a8b032fda3a43ed1cb766b310 Mon Sep 17 00:00:00 2001 From: Jack Zampolin Date: Mon, 16 Mar 2026 09:28:12 -0700 Subject: [PATCH 2/2] Switch to artifact downloads with refs from backbone.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensure-binaries.sh now reads dependency pins from backbone.toml instead of env vars in ci.yml. This makes each backbone branch/commit declare exactly which defra, hub.rs, and orbis versions it needs — enabling concurrent CI runs with different dependency versions. - Removed DEFRA_REF, HUBD_REF, ORBIS_REF env vars from ci.yml - Script parses backbone.toml for repo URLs and git refs - defra/hub.rs: downloaded as GitHub Actions artifacts (exact commit match) - orbis-rs: still built from source (no CI artifact pipeline) - Hard-fails if no successful CI run exists for the pinned commit Depends on: - sourcenetwork/defradb.rs#586 (merged) - sourcenetwork/hub.rs#83 (merged) Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/scripts/ensure-binaries.sh | 80 +++++++++++++++++++++++------- .github/workflows/ci.yml | 3 -- backbone.toml | 12 ++--- 3 files changed, 64 insertions(+), 31 deletions(-) diff --git a/.github/scripts/ensure-binaries.sh b/.github/scripts/ensure-binaries.sh index 637a1ab..d4fc52c 100755 --- a/.github/scripts/ensure-binaries.sh +++ b/.github/scripts/ensure-binaries.sh @@ -3,6 +3,10 @@ set -euo pipefail # Download pre-built binary dependencies for integration tests. # +# Reads pinned refs from backbone.toml (the single source of truth for +# dependency versions). Each component entry declares a repo and git ref +# (branch, tag, or commit SHA). +# # defra and hub.rs binaries are downloaded as GitHub Actions artifacts # from their CI workflows. The exact commit for each ref must have a # successful CI run with uploaded artifacts, or this script fails. @@ -11,13 +15,11 @@ set -euo pipefail # commit, clones/fetches into a persistent local checkout, and does # an incremental cargo build --release. # -# Required env vars (set in ci.yml): -# DEFRA_REF — git ref for defradb.rs (e.g. "main" or a commit SHA) -# HUBD_REF — git ref for hub.rs (e.g. "main" or a commit SHA) -# ORBIS_REF — git ref for orbis-rs (e.g. "jack/integration-testing") +# Required: +# backbone.toml — in the repo root (or any ancestor directory) # -# Optional: -# PRIVATE_REPO_PAT — PAT for private repo access (used for gh CLI and git URLs) +# Optional env vars: +# PRIVATE_REPO_PAT — PAT for private repo access (gh CLI and git URLs) # CACHE_DIR — override binary cache root (default: ~/.sourcenetwork/bin) # SRC_DIR — override source clone root (default: ~/.sourcenetwork/src) # MAX_VERSIONS — versions to keep per component (default: 3) @@ -25,12 +27,34 @@ set -euo pipefail CACHE_DIR="${CACHE_DIR:-$HOME/.sourcenetwork/bin}" SRC_DIR="${SRC_DIR:-$HOME/.sourcenetwork/src}" MAX_VERSIONS="${MAX_VERSIONS:-3}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +MANIFEST="$REPO_ROOT/backbone.toml" mkdir -p "$CACHE_DIR" "$SRC_DIR" # gh CLI auth via PAT export GH_TOKEN="${PRIVATE_REPO_PAT:-}" +if [[ ! -f "$MANIFEST" ]]; then + echo "ERROR: backbone.toml not found at $MANIFEST" >&2 + exit 1 +fi + +# Parse a field from a [components.] section in backbone.toml. +# Usage: manifest_get +manifest_get() { + local component=$1 field=$2 + # Match the section, then find the field within it (before the next section) + awk -v section="\\[components\\.$component\\]" -v key="$field" ' + $0 ~ section { found=1; next } + found && /^\[/ { found=0 } + found && $0 ~ "^"key"\\s*=" { + gsub(/.*=\s*"/, ""); gsub(/".*/, ""); print; exit + } + ' "$MANIFEST" +} + repo_url() { local repo=$1 if [[ -n "${PRIVATE_REPO_PAT:-}" ]]; then @@ -40,6 +64,13 @@ repo_url() { fi } +# Extract the short repo name from a full URL. +# "https://github.com/sourcenetwork/defradb.rs" → "defradb.rs" +repo_name() { + local url=$1 + basename "$url" .git +} + resolve_commit() { local repo=$1 ref=$2 local commit @@ -59,15 +90,12 @@ resolve_commit() { } # Download a binary artifact from a GitHub Actions workflow run. -# Finds the successful CI run for the exact commit, then downloads -# the named artifact. download_artifact() { local repo=$1 ref=$2 artifact_name=$3 binary_name=$4 local commit commit=$(resolve_commit "$repo" "$ref") echo "$repo: $ref → ${commit:0:12}" - # Find successful CI run for this exact commit local run_id run_id=$(gh run list -R "sourcenetwork/$repo" \ --commit "$commit" --status success --workflow ci.yml \ @@ -85,7 +113,6 @@ download_artifact() { gh run download "$run_id" -R "sourcenetwork/$repo" \ --name "$artifact_name" --dir "$tmp_dir" - # Find the binary in the extracted artifact (may be nested or flat) local found found=$(find "$tmp_dir" -type f | head -1) if [[ -z "$found" ]]; then @@ -124,7 +151,6 @@ build_if_missing() { local cache_path="$CACHE_DIR/$repo/$commit" local src="$SRC_DIR/$repo" - # Build a features fingerprint from all specs local features_fingerprint="" for spec in "$@"; do IFS=: read -r _pkg _binary _output features <<< "$spec" @@ -141,7 +167,6 @@ build_if_missing() { fi done - # Check features match (rebuild if features changed) if $all_present && [[ -f "$cache_path/.features" ]]; then local cached_features cached_features=$(cat "$cache_path/.features") @@ -151,7 +176,6 @@ build_if_missing() { rm -rf "$cache_path" fi elif $all_present && [[ ! -f "$cache_path/.features" ]]; then - # Old cache without features file — rebuild echo "No features record for $repo@${commit:0:12}, rebuilding..." all_present=false rm -rf "$cache_path" @@ -204,23 +228,41 @@ prune_old_versions() { fi } -echo "=== Ensuring binary dependencies ===" +echo "=== Reading dependency pins from backbone.toml ===" + +# Read refs from backbone.toml +DEFRA_REPO=$(repo_name "$(manifest_get defra repo)") +DEFRA_REF=$(manifest_get defra ref) +HUBD_REPO=$(repo_name "$(manifest_get hubd repo)") +HUBD_REF=$(manifest_get hubd ref) +ORBIS_REPO=$(repo_name "$(manifest_get orbis-node repo)") +ORBIS_REF=$(manifest_get orbis-node ref) + +for var in DEFRA_REPO DEFRA_REF HUBD_REPO HUBD_REF ORBIS_REPO ORBIS_REF; do + if [[ -z "${!var}" ]]; then + echo "ERROR: could not read $var from backbone.toml" >&2 + exit 1 + fi + echo " $var=${!var}" +done # defra and hub.rs: download release artifacts from their CI +echo "" echo "--- defra/hub.rs (GitHub Actions artifacts) ---" -download_artifact "defradb.rs" "$DEFRA_REF" "defra-iroh-aarch64-apple-darwin" "defra-iroh" -download_artifact "hub.rs" "$HUBD_REF" "hubd-aarch64-apple-darwin" "hubd" +download_artifact "$DEFRA_REPO" "$DEFRA_REF" "defra-iroh-aarch64-apple-darwin" "defra-iroh" +download_artifact "$HUBD_REPO" "$HUBD_REF" "hubd-aarch64-apple-darwin" "hubd" # orbis-rs: no CI artifact pipeline, build from source +echo "" echo "--- orbis-rs (built from source) ---" -ORBIS_COMMIT=$(resolve_commit "orbis-rs" "$ORBIS_REF") +ORBIS_COMMIT=$(resolve_commit "$ORBIS_REPO" "$ORBIS_REF") echo "orbis-rs: $ORBIS_REF → ${ORBIS_COMMIT:0:12}" -build_if_missing "orbis-rs" "$ORBIS_REF" "$ORBIS_COMMIT" \ +build_if_missing "$ORBIS_REPO" "$ORBIS_REF" "$ORBIS_COMMIT" \ "orbis-node:orbis-node::bls12-381,redb,bulletin-hubrs,iroh,authz-sourcehub" \ "cli-tool:cli-tool" -prune_old_versions "orbis-rs" +prune_old_versions "$ORBIS_REPO" # Final verification echo "" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dba8bb..d9f910a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,6 @@ on: workflow_dispatch: env: - DEFRA_REF: main - HUBD_REF: main - ORBIS_REF: jack/integration-testing CARGO_TERM_COLOR: always PROTOC: /opt/homebrew/bin/protoc diff --git a/backbone.toml b/backbone.toml index 6adbc6c..98c1669 100644 --- a/backbone.toml +++ b/backbone.toml @@ -1,10 +1,8 @@ # Component version manifest. # -# Declares the expected version of each binary dependency. BinaryResolver -# reads this as a fallback when no env var override is set, inserting it -# between "PATH lookup" and "give up" in the resolution chain. -# -# Env vars always win — set DEFRA_BINARY=/path/to/defra to bypass this file. +# Single source of truth for binary dependency versions. The CI script +# ensure-binaries.sh reads refs from this file. To use a different +# version, edit the ref here — branches, tags, and commit SHAs all work. [components.defra] prefix = "DEFRA" @@ -24,10 +22,6 @@ repo = "https://github.com/sourcenetwork/orbis-rs" ref = "jack/integration-testing" cargo_package = "cli-tool" -[components.sourcehubd] -prefix = "SOURCEHUBD" -# Go binary — no cargo_package, must be on PATH or set via env var - [components.hubd] prefix = "HUBD" repo = "https://github.com/sourcenetwork/hub.rs"