bench.yml

name: Terminal Bench

on:
  workflow_dispatch:
    inputs:
      dataset:
        description: Harbor dataset or task
        required: false
        default: terminal-bench/terminal-bench-2-1
      attempts:
        description: Attempts per trial (-k)
        required: false
        default: "5"
      concurrency:
        description: Concurrent trials per runner (-n)
        required: false
        default: "16"
      shards:
        description: Number of GitHub runner shards (1-64)
        required: false
        default: "64"
      n_tasks:
        description: Optional max tasks per shard after filtering
        required: false
        default: ""

permissions:
  contents: read

concurrency:
  group: terminal-bench-${{ github.ref }}
  cancel-in-progress: false

jobs:
  bench:
    name: Terminal Bench (${{ matrix.shard }})
    runs-on: ubuntu-latest
    timeout-minutes: 360
    strategy:
      fail-fast: false
      matrix:
        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]
    steps:
      - name: Check benchmark auth
        env:
          LIBRECODE_BENCH_AUTH_JSON_B64: ${{ secrets.LIBRECODE_BENCH_AUTH_JSON_B64 }}
        run: |
          set -euo pipefail
          if [ -n "${LIBRECODE_BENCH_AUTH_JSON_B64:-}" ]; then
            echo "bench_auth_configured=true" >> "$GITHUB_ENV"
            exit 0
          fi
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            echo "LIBRECODE_BENCH_AUTH_JSON_B64 is required for manual benchmark runs." >&2
            exit 1
          fi
          echo "bench_auth_configured=false" >> "$GITHUB_ENV"
          echo "Skipping terminal bench: LIBRECODE_BENCH_AUTH_JSON_B64 is not configured."

      - name: Checkout
        if: env.bench_auth_configured == 'true'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Setup Go
        if: env.bench_auth_configured == 'true'
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
        with:
          go-version-file: go.mod
          cache: true

      - name: Setup Task
        if: env.bench_auth_configured == 'true'
        uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0
        with:
          version: 3.51.1

      - name: Setup Python
        if: env.bench_auth_configured == 'true'
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
        with:
          python-version: "3.14"

      - name: Cache Harbor data
        if: env.bench_auth_configured == 'true'
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: ~/.cache/harbor
          key: harbor-${{ runner.os }}-${{ hashFiles('bench/harbor/librecode_agent.py') }}
          restore-keys: |
            harbor-${{ runner.os }}-

      - name: Install Harbor
        if: env.bench_auth_configured == 'true'
        env:
          HARBOR_VERSION: "0.13.1"
        run: python -m pip install --upgrade "harbor==$HARBOR_VERSION"

      - name: Prepare librecode bench auth
        if: env.bench_auth_configured == 'true'
        env:
          LIBRECODE_BENCH_AUTH_JSON_B64: ${{ secrets.LIBRECODE_BENCH_AUTH_JSON_B64 }}
        run: |
          set -euo pipefail
          mkdir -p "$RUNNER_TEMP/librecode-bench-auth"
          chmod 700 "$RUNNER_TEMP/librecode-bench-auth"
          if [ -z "${LIBRECODE_BENCH_AUTH_JSON_B64:-}" ]; then
            echo "LIBRECODE_BENCH_AUTH_JSON_B64 is required for benchmark runs." >&2
            exit 1
          fi
          printf '%s' "$LIBRECODE_BENCH_AUTH_JSON_B64" | base64 -d > "$RUNNER_TEMP/librecode-bench-auth/auth.json"
          chmod 600 "$RUNNER_TEMP/librecode-bench-auth/auth.json"

      - name: Run Terminal Bench
        if: env.bench_auth_configured == 'true'
        env:
          LIBRECODE_BENCH_AUTH_DIR: ${{ runner.temp }}/librecode-bench-auth
          LIBRECODE_BENCH_DATASET: ${{ github.event.inputs.dataset || 'terminal-bench/terminal-bench-2-1' }}
          LIBRECODE_BENCH_K: ${{ github.event.inputs.attempts || '5' }}
          LIBRECODE_BENCH_N: ${{ github.event.inputs.concurrency || '16' }}
          LIBRECODE_BENCH_SHARD_INDEX: ${{ matrix.shard }}
          LIBRECODE_BENCH_SHARD_TOTAL: ${{ github.event.inputs.shards || '64' }}
          LIBRECODE_BENCH_N_TASKS: ${{ github.event.inputs.n_tasks || '' }}
          LIBRECODE_BENCH_JOBS_DIR: jobs/shard-${{ matrix.shard }}
          LIBRECODE_BENCH_JOB_NAME: ${{ github.run_id }}-${{ github.run_attempt }}-shard-${{ matrix.shard }}
          LIBRECODE_BENCH_DEBUG: "true"
          LIBRECODE_BENCH_LOG_WATCH: "true"
          LIBRECODE_BENCH_LOG_WATCH_INTERVAL: "120"
        run: |
          set -euo pipefail
          SHARDS="${{ github.event.inputs.shards || '64' }}"
          if [ "${{ matrix.shard }}" -ge "$SHARDS" ]; then
            echo "Skipping inactive shard ${{ matrix.shard }} of $SHARDS."
            exit 0
          fi
          task bench

      - name: Upload benchmark jobs
        if: always() && env.bench_auth_configured == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: terminal-bench-jobs-${{ matrix.shard }}
          path: jobs/shard-${{ matrix.shard }}/
          if-no-files-found: warn