Switch to GitHub Actions artifact downloads for CI binaries

.github/scripts/ensure-binaries.sh

@@ -1,32 +1,60 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Build and cache cross-repo binary dependencies for integration tests.
+# Download pre-built binary dependencies for integration tests.
 #
-# defra and hub.rs binaries are built by their own CI and cached on the
-# studios. This script just verifies they exist (via symlinks).
+# Reads pinned refs from backbone.toml (the single source of truth for
+# dependency versions). Each component entry declares a repo and git ref
+# (branch, tag, or commit SHA).
 #
-# orbis-rs has no CI on the studios, so this script resolves the commit,
-# clones/fetches into a persistent local checkout, and does an incremental
-# cargo build --release.
+# defra and hub.rs binaries are downloaded as GitHub Actions artifacts
+# from their CI workflows. The exact commit for each ref must have a
+# successful CI run with uploaded artifacts, or this script fails.
 #
-# Required env vars (set in ci.yml):
-#   DEFRA_REF    — git ref for defradb.rs  (e.g. "main" or a commit SHA)
-#   HUBD_REF     — git ref for hub.rs      (e.g. "main" or a commit SHA)
-#   ORBIS_REF    — git ref for orbis-rs    (e.g. "jack/integration-testing")
+# orbis-rs has no CI artifact pipeline, so this script resolves the
+# commit, clones/fetches into a persistent local checkout, and does
+# an incremental cargo build --release.
 #
-# Optional:
-#   PRIVATE_REPO_PAT — PAT for private repo access (used in git URLs)
+# Required:
+#   backbone.toml    — in the repo root (or any ancestor directory)
+#
+# Optional env vars:
+#   PRIVATE_REPO_PAT — PAT for private repo access (gh CLI and git URLs)
 #   CACHE_DIR        — override binary cache root (default: ~/.sourcenetwork/bin)
 #   SRC_DIR          — override source clone root (default: ~/.sourcenetwork/src)
 #   MAX_VERSIONS     — versions to keep per component (default: 3)
 
 CACHE_DIR="${CACHE_DIR:-$HOME/.sourcenetwork/bin}"
 SRC_DIR="${SRC_DIR:-$HOME/.sourcenetwork/src}"
 MAX_VERSIONS="${MAX_VERSIONS:-3}"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+MANIFEST="$REPO_ROOT/backbone.toml"
 
 mkdir -p "$CACHE_DIR" "$SRC_DIR"
 
+# gh CLI auth via PAT
+export GH_TOKEN="${PRIVATE_REPO_PAT:-}"
+
+if [[ ! -f "$MANIFEST" ]]; then
+    echo "ERROR: backbone.toml not found at $MANIFEST" >&2
+    exit 1
+fi
+
+# Parse a field from a [components.<name>] section in backbone.toml.
+# Usage: manifest_get <component> <field>
+manifest_get() {
+    local component=$1 field=$2
+    # Match the section, then find the field within it (before the next section)
+    awk -v section="\\[components\\.$component\\]" -v key="$field" '
+        $0 ~ section { found=1; next }
+        found && /^\[/ { found=0 }
+        found && $0 ~ "^"key"\\s*=" {
+            gsub(/.*=\s*"/, ""); gsub(/".*/, ""); print; exit
+        }
+    ' "$MANIFEST"
+}
+
 repo_url() {
     local repo=$1
     if [[ -n "${PRIVATE_REPO_PAT:-}" ]]; then
@@ -36,6 +64,13 @@ repo_url() {
     fi
 }
 
+# Extract the short repo name from a full URL.
+# "https://github.com/sourcenetwork/defradb.rs" → "defradb.rs"
+repo_name() {
+    local url=$1
+    basename "$url" .git
+}
+
 resolve_commit() {
     local repo=$1 ref=$2
     local commit
@@ -54,6 +89,46 @@ resolve_commit() {
     return 1
 }
 
+# Download a binary artifact from a GitHub Actions workflow run.
+download_artifact() {
+    local repo=$1 ref=$2 artifact_name=$3 binary_name=$4
+    local commit
+    commit=$(resolve_commit "$repo" "$ref")
+    echo "$repo: $ref → ${commit:0:12}"
+
+    local run_id
+    run_id=$(gh run list -R "sourcenetwork/$repo" \
+        --commit "$commit" --status success --workflow ci.yml \
+        --limit 1 --json databaseId -q '.[0].databaseId')
+
+    if [[ -z "$run_id" ]]; then
+        echo "  ERROR: no successful CI run found for $repo@${commit:0:12}" >&2
+        echo "  The CI for $repo must complete successfully before backbone CI can run." >&2
+        exit 1
+    fi
+
+    echo "  Downloading $artifact_name from run $run_id..."
+    local tmp_dir
+    tmp_dir=$(mktemp -d)
+    gh run download "$run_id" -R "sourcenetwork/$repo" \
+        --name "$artifact_name" --dir "$tmp_dir"
+
+    local found
+    found=$(find "$tmp_dir" -type f | head -1)
+    if [[ -z "$found" ]]; then
+        echo "  ERROR: artifact $artifact_name was empty" >&2
+        rm -rf "$tmp_dir"
+        exit 1
+    fi
+
+    cp "$found" "$CACHE_DIR/$binary_name"
+    chmod +x "$CACHE_DIR/$binary_name"
+    rm -rf "$tmp_dir"
+    echo "  $binary_name: ready"
+}
+
+# --- orbis-rs helper functions (source build, no CI artifacts) ---
+
 ensure_clone() {
     local repo=$1 ref=$2 commit=$3
     local src="$SRC_DIR/$repo"
@@ -76,7 +151,6 @@ build_if_missing() {
     local cache_path="$CACHE_DIR/$repo/$commit"
     local src="$SRC_DIR/$repo"
 
-    # Build a features fingerprint from all specs
     local features_fingerprint=""
     for spec in "$@"; do
         IFS=: read -r _pkg _binary _output features <<< "$spec"
@@ -93,7 +167,6 @@ build_if_missing() {
         fi
     done
 
-    # Check features match (rebuild if features changed)
     if $all_present && [[ -f "$cache_path/.features" ]]; then
         local cached_features
         cached_features=$(cat "$cache_path/.features")
@@ -103,7 +176,6 @@ build_if_missing() {
             rm -rf "$cache_path"
         fi
     elif $all_present && [[ ! -f "$cache_path/.features" ]]; then
-        # Old cache without features file — rebuild
         echo "No features record for $repo@${commit:0:12}, rebuilding..."
         all_present=false
         rm -rf "$cache_path"
@@ -156,49 +228,48 @@ prune_old_versions() {
     fi
 }
 
-ensure_prebuilt_binary() {
-    local repo=$1 ref=$2 binary=$3 output=${4:-$3}
-    local commit cache_path
+echo "=== Reading dependency pins from backbone.toml ==="
 
-    commit=$(resolve_commit "$repo" "$ref")
-    cache_path="$CACHE_DIR/$repo/$commit/$output"
+# Read refs from backbone.toml
+DEFRA_REPO=$(repo_name "$(manifest_get defra repo)")
+DEFRA_REF=$(manifest_get defra ref)
+HUBD_REPO=$(repo_name "$(manifest_get hubd repo)")
+HUBD_REF=$(manifest_get hubd ref)
+ORBIS_REPO=$(repo_name "$(manifest_get orbis-node repo)")
+ORBIS_REF=$(manifest_get orbis-node ref)
 
-    echo "$repo: $ref → ${commit:0:12}"
-    if [[ ! -x "$cache_path" ]]; then
-        echo "  ERROR: cached binary missing at $cache_path" >&2
-        echo "  $repo CI must build and cache commit $commit first." >&2
+for var in DEFRA_REPO DEFRA_REF HUBD_REPO HUBD_REF ORBIS_REPO ORBIS_REF; do
+    if [[ -z "${!var}" ]]; then
+        echo "ERROR: could not read $var from backbone.toml" >&2
         exit 1
     fi
+    echo "  $var=${!var}"
+done
 
-    ln -sf "$cache_path" "$CACHE_DIR/$binary"
-    echo "  $binary: $(readlink "$CACHE_DIR/$binary")"
-}
-
-echo "=== Ensuring binary dependencies ==="
-
-# defra and hub.rs: binaries built by their own CI, relink to the exact
-# commit requested by this workflow so runners do not drift.
-echo "--- defra/hub.rs (pre-built by their CI) ---"
-ensure_prebuilt_binary "defradb.rs" "$DEFRA_REF" "defra-iroh"
-ensure_prebuilt_binary "hub.rs" "$HUBD_REF" "hubd"
+# defra and hub.rs: download release artifacts from their CI
+echo ""
+echo "--- defra/hub.rs (GitHub Actions artifacts) ---"
+download_artifact "$DEFRA_REPO" "$DEFRA_REF" "defra-iroh-aarch64-apple-darwin" "defra-iroh"
+download_artifact "$HUBD_REPO" "$HUBD_REF" "hubd-aarch64-apple-darwin" "hubd"
 
-# orbis-rs: no CI on studios, build here
-echo "--- orbis-rs (built by ensure-binaries) ---"
-ORBIS_COMMIT=$(resolve_commit "orbis-rs" "$ORBIS_REF")
+# orbis-rs: no CI artifact pipeline, build from source
+echo ""
+echo "--- orbis-rs (built from source) ---"
+ORBIS_COMMIT=$(resolve_commit "$ORBIS_REPO" "$ORBIS_REF")
 echo "orbis-rs: $ORBIS_REF → ${ORBIS_COMMIT:0:12}"
 
-build_if_missing "orbis-rs" "$ORBIS_REF" "$ORBIS_COMMIT" \
+build_if_missing "$ORBIS_REPO" "$ORBIS_REF" "$ORBIS_COMMIT" \
     "orbis-node:orbis-node::bls12-381,redb,bulletin-hubrs,iroh,authz-sourcehub" \
     "cli-tool:cli-tool"
 
-prune_old_versions "orbis-rs"
+prune_old_versions "$ORBIS_REPO"
 
 # Final verification
 echo ""
-echo "=== Binary versions ==="
+echo "=== Binary verification ==="
 for bin in defra-iroh hubd orbis-node cli-tool; do
     if [[ -x "$CACHE_DIR/$bin" ]]; then
-        echo "  $bin: $(readlink "$CACHE_DIR/$bin")"
+        echo "  $bin: OK"
     else
         echo "  ERROR: $bin not found in $CACHE_DIR" >&2
         exit 1

.github/workflows/ci.yml

@@ -8,9 +8,6 @@ on:
   workflow_dispatch:
 
 env:
-  DEFRA_REF: main
-  HUBD_REF: main
-  ORBIS_REF: jack/integration-testing
   CARGO_TERM_COLOR: always
   PROTOC: /opt/homebrew/bin/protoc
 

backbone.toml

@@ -1,10 +1,8 @@
 # Component version manifest.
 #
-# Declares the expected version of each binary dependency. BinaryResolver
-# reads this as a fallback when no env var override is set, inserting it
-# between "PATH lookup" and "give up" in the resolution chain.
-#
-# Env vars always win — set DEFRA_BINARY=/path/to/defra to bypass this file.
+# Single source of truth for binary dependency versions. The CI script
+# ensure-binaries.sh reads refs from this file. To use a different
+# version, edit the ref here — branches, tags, and commit SHAs all work.
 
 [components.defra]
 prefix = "DEFRA"
@@ -24,10 +22,6 @@ repo = "https://github.com/sourcenetwork/orbis-rs"
 ref = "jack/integration-testing"
 cargo_package = "cli-tool"
 
-[components.sourcehubd]
-prefix = "SOURCEHUBD"
-# Go binary — no cargo_package, must be on PATH or set via env var
-
 [components.hubd]
 prefix = "HUBD"
 repo = "https://github.com/sourcenetwork/hub.rs"