From 6d7c98e279172975f1caee8ff990d529d354d4c1 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 14:41:18 -0700 Subject: [PATCH 1/8] ci(repo): skip integration suites that a pull request doesn't affect A planning job runs turbo query affected once per pull request, comparing the merge commit with its base, and each integration job skips its setup and tests when its suite isn't affected. Every job still reports, so required checks pass. Changes under .github/, merge queue runs, and any planning failure run every suite. turbo.json enables affectedUsingTaskInputs so files outside a task's inputs, like changesets, don't mark every root task as affected. --- .changeset/integration-skip-unaffected.md | 2 + .github/workflows/ci.yml | 121 ++++++++++++++++++++-- turbo.json | 3 + 3 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 .changeset/integration-skip-unaffected.md diff --git a/.changeset/integration-skip-unaffected.md b/.changeset/integration-skip-unaffected.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/integration-skip-unaffected.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 650d7e5f916..2821e623156 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -510,9 +510,79 @@ jobs: retention-days: 7 if-no-files-found: warn - integration-tests: - needs: [check-permissions, build-packages] + integration-plan: + needs: [check-permissions] if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + name: Integration Plan + permissions: + contents: read + runs-on: 'blacksmith-8vcpu-ubuntu-2204' + defaults: + run: + shell: bash + timeout-minutes: ${{ vars.TIMEOUT_MINUTES_NORMAL && fromJSON(vars.TIMEOUT_MINUTES_NORMAL) || 10 }} + outputs: + affected: ${{ steps.plan.outputs.affected }} + known: ${{ steps.plan.outputs.known }} + + steps: + - name: Checkout Repo + uses: actions/checkout@v4.4.0 + with: + persist-credentials: false + fetch-depth: 2 + fetch-tags: false + filter: 'blob:none' + show-progress: false + + - name: Setup + uses: $/.github/actions/init-blacksmith + with: + cache-enabled: true + turbo-signature: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} + turbo-team: ${{ vars.TURBO_TEAM }} + turbo-token: ${{ secrets.TURBO_TOKEN }} + + - name: Find affected integration suites + id: plan + env: + EVENT_NAME: ${{ github.event_name }} + run: | + mapfile -t TASKS < <(jq -r '.tasks | keys[] | select(startswith("//#test:integration:")) | ltrimstr("//#")' turbo.json) + if [ "${#TASKS[@]}" -eq 0 ]; then + echo "::error::Found no integration tasks in turbo.json, so every integration suite runs." + exit 1 + fi + echo "known=$(printf '%s\n' "${TASKS[@]#test:integration:}" | jq -Rnc '[inputs]')" >> "$GITHUB_OUTPUT" + + if [ "$EVENT_NAME" != "pull_request" ]; then + echo "Running every integration suite for $EVENT_NAME." + echo "affected=all" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then + echo "This PR changes .github/, so every integration suite runs." + echo "affected=all" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then + echo "::error::turbo query affected failed, so every integration suite runs." + exit 1 + fi + TURBO_JSON=$(printf '%s\n' "$TURBO_RAW" | awk '/^\{/{found=1} found') + if ! AFFECTED=$(jq -ec '[.data.affectedTasks.items[] | select(.package.name == "//" and (.name | startswith("test:integration:"))) | .name | ltrimstr("test:integration:")]' <<< "$TURBO_JSON"); then + echo "::error::turbo query affected returned unexpected output, so every integration suite runs." + printf '%s\n' "$TURBO_RAW" + exit 1 + fi + echo "affected=$AFFECTED" >> "$GITHUB_OUTPUT" + echo "Affected integration suites: $AFFECTED" + + integration-tests: + needs: [check-permissions, build-packages, integration-plan] + if: ${{ !cancelled() && needs.check-permissions.result == 'success' && needs.build-packages.result == 'success' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}${{ matrix.next-version && format(', {0}', matrix.next-version) || '' }}) permissions: contents: read @@ -572,7 +642,28 @@ jobs: next-version: '16' steps: + - name: Check whether this suite is affected + id: affected + env: + AFFECTED: ${{ needs.integration-plan.outputs.affected }} + KNOWN: ${{ needs.integration-plan.outputs.known }} + PLAN_RESULT: ${{ needs.integration-plan.result }} + SUITE: ${{ matrix.test-name }} + run: | + if [ "$PLAN_RESULT" != "success" ] || [ -z "$AFFECTED" ] || [ "$AFFECTED" = "all" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + elif [[ "$KNOWN" != *"\"$SUITE\""* ]]; then + echo "::warning::$SUITE is not an integration task in turbo.json, so it runs and the task validation reports it." + echo "run=true" >> "$GITHUB_OUTPUT" + elif [[ "$AFFECTED" == *"\"$SUITE\""* ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "::notice title=Integration suite skipped::Nothing the $SUITE suite uses changed in this PR." + echo "run=false" >> "$GITHUB_OUTPUT" + fi + - name: Checkout Repo + if: steps.affected.outputs.run == 'true' uses: actions/checkout@v4.4.0 with: persist-credentials: false @@ -582,6 +673,7 @@ jobs: show-progress: false - name: Setup + if: steps.affected.outputs.run == 'true' id: config uses: $/.github/actions/init-blacksmith with: @@ -592,6 +684,7 @@ jobs: playwright-enabled: true - name: Verify jq is installed + if: steps.affected.outputs.run == 'true' shell: bash run: | if ! command -v jq &> /dev/null; then @@ -601,6 +694,7 @@ jobs: jq --version - name: Validate turbo task + if: steps.affected.outputs.run == 'true' env: E2E_APP_CLERK_JS_DIR: ${{runner.temp}} E2E_APP_CLERK_UI_DIR: ${{runner.temp}} @@ -610,10 +704,8 @@ jobs: E2E_PROJECT: ${{ matrix.test-project }} INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }} run: | - # Validate the turbo task exists. Turbo's --affected flag is a no-op for - # root-level tasks (//#) — they are always returned as affected. The previous - # --affected check only served to silently skip tests when the task was missing - # from turbo.json, hiding real configuration errors. + # Validate the turbo task exists, so a missing or misnamed task fails here + # instead of silently running nothing. TASK_NAME="test:integration:${{ matrix.test-name }}" TURBO_STDERR=$(mktemp) if ! TURBO_RAW=$(pnpm turbo run "$TASK_NAME" --dry=json 2>"$TURBO_STDERR"); then @@ -640,15 +732,19 @@ jobs: echo "Task '$TASK_NAME' validated ($TASK_COUNT tasks in graph)" - name: Build packages + if: steps.affected.outputs.run == 'true' run: pnpm turbo build $TURBO_ARGS --only - name: Publish to local registry + if: steps.affected.outputs.run == 'true' run: pkglab pub --force - name: Edit .npmrc [link-workspace-packages=false] + if: steps.affected.outputs.run == 'true' run: sed -i -E 's/link-workspace-packages=(deep|true)/link-workspace-packages=false/' .npmrc - name: Install @clerk/clerk-js in os temp + if: steps.affected.outputs.run == 'true' working-directory: ${{runner.temp}} run: | mkdir clerk-js && cd clerk-js @@ -656,6 +752,7 @@ jobs: pkglab add @clerk/clerk-js - name: Install @clerk/ui in os temp + if: steps.affected.outputs.run == 'true' working-directory: ${{runner.temp}} run: | mkdir clerk-ui && cd clerk-ui @@ -663,10 +760,11 @@ jobs: pkglab add @clerk/ui - name: Copy components @clerk/astro - if: ${{ matrix.test-name == 'astro' }} + if: ${{ steps.affected.outputs.run == 'true' && (matrix.test-name == 'astro') }} run: cd packages/astro && pnpm copy:components - name: Write all ENV certificates to files in integration/certs + if: steps.affected.outputs.run == 'true' uses: actions/github-script@v7.1.0 env: INTEGRATION_CERTS: '${{secrets.INTEGRATION_CERTS}}' @@ -684,11 +782,12 @@ jobs: } - name: LS certs + if: steps.affected.outputs.run == 'true' working-directory: ./integration/certs run: ls -la && pwd - name: Ensure Xvfb is installed - if: ${{ matrix.test-name == 'electron' }} + if: ${{ steps.affected.outputs.run == 'true' && (matrix.test-name == 'electron') }} run: | if ! command -v xvfb-run &> /dev/null; then sudo apt-get update @@ -696,6 +795,7 @@ jobs: fi - name: Configure test user cleanup + if: steps.affected.outputs.run == 'true' run: echo "INTEGRATION_TEST_RUN_KEY=${RUN_ID}-${RUN_ATTEMPT}-${TEST_NAME}-${TEST_PROJECT}-${NEXT_VERSION:-default}" >> "$GITHUB_ENV" env: RUN_ID: ${{ github.run_id }} @@ -705,6 +805,7 @@ jobs: NEXT_VERSION: ${{ matrix.next-version }} - name: Run Integration Tests + if: steps.affected.outputs.run == 'true' id: integration-tests timeout-minutes: 25 run: | @@ -728,7 +829,7 @@ jobs: VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} - name: Sanitize artifact name - if: ${{ cancelled() || failure() }} + if: ${{ steps.affected.outputs.run == 'true' && (cancelled() || failure()) }} id: sanitize run: | SANITIZED="${TEST_NAME//:/-}" @@ -737,7 +838,7 @@ jobs: TEST_NAME: ${{ matrix.test-name }} - name: Upload test-results - if: ${{ cancelled() || failure() }} + if: ${{ steps.affected.outputs.run == 'true' && (cancelled() || failure()) }} uses: actions/upload-artifact@v4.6.2 with: name: playwright-traces-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.sanitize.outputs.artifact-suffix }}${{ matrix.next-version && format('-next{0}', matrix.next-version) || '' }} diff --git a/turbo.json b/turbo.json index 461564fd311..99b3832521a 100644 --- a/turbo.json +++ b/turbo.json @@ -4,6 +4,9 @@ "signature": true }, "ui": "tui", + "futureFlags": { + "affectedUsingTaskInputs": true + }, "globalDependencies": [ ".github/.cache-version", ".npmrc", From 49e682fdb7ec7d950ac80f72b8a1ad8e9629d843 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:34:22 -0700 Subject: [PATCH 2/8] chore(repo): temporarily compare the integration plan with the previous commit --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2821e623156..025f3ee38d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -530,7 +530,7 @@ jobs: uses: actions/checkout@v4.4.0 with: persist-credentials: false - fetch-depth: 2 + fetch-depth: 3 fetch-tags: false filter: 'blob:none' show-progress: false @@ -561,13 +561,13 @@ jobs: exit 0 fi - if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then + if git diff --name-only HEAD^2^1 HEAD^2 | grep -q '^\.github/'; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0 fi - if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^2^1 --head HEAD^2 --tasks "${TASKS[@]}"); then echo "::error::turbo query affected failed, so every integration suite runs." exit 1 fi From 1d9b0f580527810a0caf760ae4f93859d557e5ab Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:34:22 -0700 Subject: [PATCH 3/8] chore(repo): temporary expo-only change to prove skipped integration jobs --- packages/expo/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/expo/src/index.ts b/packages/expo/src/index.ts index aa9223501c0..d62e1a28456 100644 --- a/packages/expo/src/index.ts +++ b/packages/expo/src/index.ts @@ -22,3 +22,5 @@ export * from './trusted-devices'; setErrorThrowerOptions({ packageName: PACKAGE_NAME }); export type { TokenCache } from './cache/types'; + +// CI cache probe, reverted in this PR From 8b8f226e8e261040413237e38c8c8efb42505eb6 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:38:09 -0700 Subject: [PATCH 4/8] chore(repo): revert temporary skip probe changes --- .github/workflows/ci.yml | 6 +++--- packages/expo/src/index.ts | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 025f3ee38d8..2821e623156 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -530,7 +530,7 @@ jobs: uses: actions/checkout@v4.4.0 with: persist-credentials: false - fetch-depth: 3 + fetch-depth: 2 fetch-tags: false filter: 'blob:none' show-progress: false @@ -561,13 +561,13 @@ jobs: exit 0 fi - if git diff --name-only HEAD^2^1 HEAD^2 | grep -q '^\.github/'; then + if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0 fi - if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^2^1 --head HEAD^2 --tasks "${TASKS[@]}"); then + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then echo "::error::turbo query affected failed, so every integration suite runs." exit 1 fi diff --git a/packages/expo/src/index.ts b/packages/expo/src/index.ts index d62e1a28456..aa9223501c0 100644 --- a/packages/expo/src/index.ts +++ b/packages/expo/src/index.ts @@ -22,5 +22,3 @@ export * from './trusted-devices'; setErrorThrowerOptions({ packageName: PACKAGE_NAME }); export type { TokenCache } from './cache/types'; - -// CI cache probe, reverted in this PR From fd80f2f0c3d33b5fa56f605f2d2b3f925e7cc684 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:44:58 -0700 Subject: [PATCH 5/8] chore(repo): temporarily compare the integration plan with the previous commit --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2821e623156..025f3ee38d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -530,7 +530,7 @@ jobs: uses: actions/checkout@v4.4.0 with: persist-credentials: false - fetch-depth: 2 + fetch-depth: 3 fetch-tags: false filter: 'blob:none' show-progress: false @@ -561,13 +561,13 @@ jobs: exit 0 fi - if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then + if git diff --name-only HEAD^2^1 HEAD^2 | grep -q '^\.github/'; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0 fi - if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^2^1 --head HEAD^2 --tasks "${TASKS[@]}"); then echo "::error::turbo query affected failed, so every integration suite runs." exit 1 fi From 38f42f227d8869cda21c6e20804fcd6a2b7ea1bd Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:44:58 -0700 Subject: [PATCH 6/8] chore(repo): temporary mosaic-only change to prove skipped integration jobs --- packages/mosaic/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/mosaic/src/index.ts b/packages/mosaic/src/index.ts index a64f3111ea2..da0e8742943 100644 --- a/packages/mosaic/src/index.ts +++ b/packages/mosaic/src/index.ts @@ -14,3 +14,5 @@ export type { MosaicProfile } from './features/profile-pages/use-mosaic-profile' export { MosaicProvider } from './mosaic-provider'; export type { PortalContainer, PortalProviderProps } from './primitives/portal'; export { PortalProvider } from './primitives/portal'; + +// CI cache probe, reverted in this PR From 2563e2875547166f92f742f88e9ef636c2850ec8 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Wed, 4 Nov 2026 15:52:14 -0700 Subject: [PATCH 7/8] chore(repo): revert temporary skip probe changes --- .github/workflows/ci.yml | 6 +++--- packages/mosaic/src/index.ts | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 025f3ee38d8..2821e623156 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -530,7 +530,7 @@ jobs: uses: actions/checkout@v4.4.0 with: persist-credentials: false - fetch-depth: 3 + fetch-depth: 2 fetch-tags: false filter: 'blob:none' show-progress: false @@ -561,13 +561,13 @@ jobs: exit 0 fi - if git diff --name-only HEAD^2^1 HEAD^2 | grep -q '^\.github/'; then + if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0 fi - if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^2^1 --head HEAD^2 --tasks "${TASKS[@]}"); then + if ! TURBO_RAW=$(pnpm turbo query affected --base HEAD^1 --head HEAD --tasks "${TASKS[@]}"); then echo "::error::turbo query affected failed, so every integration suite runs." exit 1 fi diff --git a/packages/mosaic/src/index.ts b/packages/mosaic/src/index.ts index da0e8742943..a64f3111ea2 100644 --- a/packages/mosaic/src/index.ts +++ b/packages/mosaic/src/index.ts @@ -14,5 +14,3 @@ export type { MosaicProfile } from './features/profile-pages/use-mosaic-profile' export { MosaicProvider } from './mosaic-provider'; export type { PortalContainer, PortalProviderProps } from './primitives/portal'; export { PortalProvider } from './primitives/portal'; - -// CI cache probe, reverted in this PR From 76f4ba709914fa8aedce56ba49c3c9fd6a362ed7 Mon Sep 17 00:00:00 2001 From: austincalvelage Date: Thu, 5 Nov 2026 10:58:03 -0700 Subject: [PATCH 8/8] ci(repo): detect .github changes without a pipe Under pipefail, grep -q exiting on an early .github/ match sent SIGPIPE to git diff, so the check came out false on large PRs and suites were skipped. Replaying the last 60 commits that touched .github/, it missed a 470-file one. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2821e623156..890b2ea0aa1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -561,7 +561,7 @@ jobs: exit 0 fi - if git diff --name-only HEAD^1 HEAD | grep -q '^\.github/'; then + if ! git diff --quiet HEAD^1 HEAD -- .github; then echo "This PR changes .github/, so every integration suite runs." echo "affected=all" >> "$GITHUB_OUTPUT" exit 0