e2e.yml

name: "E2E"

# Deploys the CloudFormation template into a throwaway AWS account the way a customer does: creates an AWS source,
# deploys the stack with that source's ClusterId/SourceToken/SourceId, pastes the stack outputs back into the source,
# checks that the first sync subscribes the run's log group, the static group, aws/spans and the CloudTrail group by
# itself, then produces logs, a custom metric, an X-Ray trace and a CloudTrail management event in AWS and reads all of
# them back through the team's SQL API connection with exact row counts. Pull requests and pushes to main deploy the
# template from the checkout; the daily schedule deploys the published one and fails when it differs from main. Everything the run creates in AWS and Better Stack is named after
# RUN_ID and deleted in the cleanup step, together with leftovers of earlier runs. Within an hour of the previous run's
# stack deletion, the static group, aws/spans and CloudTrail delivering nothing is a warning instead of a failure (see
# the step that measures it). The docs check fails when the docs article stops naming a template toggle or output.
# Pull requests from forks are skipped: they get no OIDC token.
# See the docs guide in https://betterstack.com/docs/logs/aws-cloudformation/
#
# AWS account: better-stack-e2e-aws-stack (251917137055), a member of the Better Stack organization that holds nothing
# but this test. Root sign-in: 1Password, Engineering vault, item "AWS better-stack-e2e-aws-stack". The one standing
# resource is the CloudFormation stack `e2e-bootstrap`: a GitHub OIDC provider and IAM role `github-actions-e2e` with
# AdministratorAccess trusting this repository. The template deploys IAM roles, Lambdas, Firehose streams, CloudTrail
# and S3, and its role and bucket names are fixed per account, so administrator access in an otherwise empty account is
# the honest scope and only one run may deploy at a time (the concurrency group below).
# Secrets: TEST_TEAM_TELEMETRY_API, TEST_TEAM_CONNECTION_USERNAME, TEST_TEAM_CONNECTION_PASSWORD (the E2E testing team),
# LINEAR_* (see notify_linear_failure.yml).

on:
  push:
    branches: [main]
  pull_request:
  schedule:
    - cron: '15 5 * * *'
  workflow_dispatch:
    inputs:
      template_url:
        description: 'Template to deploy instead of the checkout (any URL)'
        type: string

permissions:
  id-token: write
  contents: read
  actions: read

# The template names its IAM roles and buckets globally, so one stack per account at a time.
concurrency:
  group: e2e-aws-stack
  cancel-in-progress: false

env:
  AWS_REGION: us-east-1
  AWS_ROLE_ARN: arn:aws:iam::251917137055:role/github-actions-e2e
  API: https://telemetry.betterstack.com
  DOCS_URL: https://betterstack.com/docs/logs/aws-cloudformation/
  PUBLISHED_TEMPLATE_URL: https://better-stack-cloudformation.s3.amazonaws.com/better-stack-full.yaml
  TEMPLATE_URL_OVERRIDE: ${{ inputs.template_url }}
  RUN_ID: gh${{ github.run_id }}a${{ github.run_attempt }}
  SOURCE_PREFIX: e2e-aws-stack
  STACK_NAME: e2e-gh${{ github.run_id }}a${{ github.run_attempt }}
  LOG_GROUP: /e2e/gh${{ github.run_id }}a${{ github.run_attempt }}
  # Recreated under the same name by every run, so Better Stack sees it with the history of the previous runs.
  STATIC_LOG_GROUP: /e2e/static
  MARKER: e2e run_id=gh${{ github.run_id }}a${{ github.run_attempt }}
  TEST_TEAM_TELEMETRY_API: ${{ secrets.TEST_TEAM_TELEMETRY_API }}
  TEST_TEAM_CONNECTION_USERNAME: ${{ secrets.TEST_TEAM_CONNECTION_USERNAME }}
  TEST_TEAM_CONNECTION_PASSWORD: ${{ secrets.TEST_TEAM_CONNECTION_PASSWORD }}

jobs:
  test:
    name: Stack round trip
    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    runs-on: ubuntu-latest
    timeout-minutes: 90

    steps:
      - name: "[Setup] Checkout code"
        uses: actions/checkout@v7

      - name: "[Check] Pick the template, docs article names every toggle and output, published template vs main"
        run: |
          if [ -n "$TEMPLATE_URL_OVERRIDE" ]; then
            curl -sSfL "$TEMPLATE_URL_OVERRIDE" -o template.yaml; TEMPLATE_FROM="$TEMPLATE_URL_OVERRIDE"
          elif [ "${{ github.event_name }}" = schedule ]; then
            curl -sSfL "$PUBLISHED_TEMPLATE_URL" -o template.yaml; TEMPLATE_FROM="$PUBLISHED_TEMPLATE_URL"
          else
            cp cloudformation/full/better-stack-full.yaml template.yaml; TEMPLATE_FROM="this checkout"
          fi
          echo "TEMPLATE_FROM=$TEMPLATE_FROM ($(git rev-parse --short HEAD))" >> "$GITHUB_ENV"
          curl -sSfL "$PUBLISHED_TEMPLATE_URL" -o template-published.yaml
          if ! diff -q template-published.yaml cloudformation/full/better-stack-full.yaml >/dev/null; then
            if [ "${{ github.event_name }}" = schedule ]; then
              echo "::error::$PUBLISHED_TEMPLATE_URL differs from cloudformation/full/better-stack-full.yaml on main: the published template was not updated after a merge"
              diff template-published.yaml cloudformation/full/better-stack-full.yaml | head -40
              exit 1
            fi
            echo "::warning::$PUBLISHED_TEMPLATE_URL differs from the template in this checkout (expected while a change is unreleased)"
          fi
          TOGGLES=$(awk '/^Parameters:/{p=1;next} /^[A-Z][A-Za-z]*:/{p=0} p && /^  [A-Za-z]+:$/{name=substr($1,1,length($1)-1)} p && /^      - .false.$/{print name}' template.yaml | sort -u)
          OUTPUTS=$(awk '/^Outputs:/{p=1;next} p && /^  [A-Za-z]+:$/{print substr($1,1,length($1)-1)}' template.yaml)
          [ -n "$TOGGLES" ] && [ -n "$OUTPUTS" ] || { echo "::error::found no true/false parameters or no outputs in the template, this check needs updating"; exit 1; }
          curl -sSfL "$DOCS_URL" -o docs.html
          MISSING=""
          for name in $TOGGLES $OUTPUTS https://github.com/BetterStackHQ/aws; do grep -q "$name" docs.html || MISSING="$MISSING $name"; done
          [ -z "$MISSING" ] || { echo "::error::$DOCS_URL does not mention:$MISSING"; exit 1; }
          echo "template from $TEMPLATE_FROM; docs article mentions every toggle ($(echo $TOGGLES)) and output ($(echo $OUTPUTS))"

      - name: "[Setup] AWS credentials (OIDC)"
        uses: aws-actions/configure-aws-credentials@v5
        with:
          role-to-assume: ${{ env.AWS_ROLE_ARN }}
          aws-region: ${{ env.AWS_REGION }}
          # Stack creation and deletion can take 20 min each; the role allows 2 h sessions.
          role-duration-seconds: 7200

      - name: "[Check] Template is valid CloudFormation"
        run: aws cloudformation validate-template --template-body file://template.yaml >/dev/null

      # A cancelled job gets only a short grace period, so its cleanup can be killed while the stack is still being
      # deleted; sweeping first keeps such leftovers from failing this run.
      - name: "[Setup] Sweep leftovers of earlier runs"
        run: bash e2e/cleanup.sh

      # Each run deletes the template's IAM roles and the next run recreates them under the same names, while AWS keeps
      # using sessions of the deleted roles for up to an hour. CloudWatch Logs holds the subscription role's session per
      # log group name: delivery with a deleted role's session fails with a non-retryable error, the filter pauses for
      # 10 minutes, skips whatever arrives meanwhile, and fails again on every retry until the session expires, even
      # when the log group is recreated with a new filter. aws/spans, /aws/cloudtrail/better-stack and the static group
      # have the same names in every run. CloudTrail holds its role's session for writing into
      # /aws/cloudtrail/better-stack the same way and writes nothing until it expires. Customers keep their roles.
      # So within that hour the read-back waits only for the run's own log group and the metric, and the static group,
      # aws/spans or CloudTrail delivering nothing is a warning. 65 min, because the roles go a minute or two after the
      # stack deletion starts.
      - name: "[Setup] Measure how long ago the previous run deleted its stack"
        run: |
          LAST=$(aws cloudformation list-stacks --stack-status-filter DELETE_COMPLETE --query "max(StackSummaries[?starts_with(StackName, 'e2e-gh')].DeletionTime)" --output text)
          [ "$LAST" != None ] || { echo "no earlier stack in this account, everything is checked strictly"; exit 0; }
          AGE=$((($(date +%s) - $(date -d "$LAST" +%s)) / 60))
          [ "$AGE" -lt 65 ] || { echo "the previous stack was deleted $AGE min ago, everything is checked strictly"; exit 0; }
          echo "RECENT_STACK_MIN=$AGE" >> "$GITHUB_ENV"
          echo "the previous stack was deleted $AGE min ago: the static group, aws/spans and CloudTrail delivering nothing will be a warning"

      - name: "[Setup] Create AWS source"
        run: |
          RES=$(curl -sS -X POST "$API/api/v1/sources" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' \
            -d "$(jq -nc --arg name "$SOURCE_PREFIX $RUN_ID" '{name: $name, platform: "aws"}')")
          jq -e '.data.id' <<<"$RES" >/dev/null || { echo "::error::creating the source failed: $RES"; exit 1; }
          SOURCE_TOKEN=$(jq -r '.data.attributes.token' <<<"$RES")
          echo "::add-mask::$SOURCE_TOKEN"
          {
            echo "SOURCE_ID=$(jq -r '.data.id' <<<"$RES")"
            echo "SOURCE_TOKEN=$SOURCE_TOKEN"
            echo "DATA_REGION=$(jq -r '.data.attributes.data_region' <<<"$RES")"
            echo "INGESTING_HOST=$(jq -r '.data.attributes.ingesting_host' <<<"$RES")"
            echo "COLLECTION=t$(jq -r '.data.attributes.team_id' <<<"$RES")_$(jq -r '.data.attributes.table_name' <<<"$RES")"
            echo "CONNECT_HOST=$(jq -r '.data.attributes.data_region' <<<"$RES")-connect.betterstackdata.com"
            echo "ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)"
          } >> "$GITHUB_ENV"
          echo "source $(jq -r '.data.id' <<<"$RES") on $(jq -r '.data.attributes.data_region' <<<"$RES")"

      # A fresh token reaches the ingesters with a delay; until then they answer 401 and Firehose would retry its
      # batches. Send warm-up lines until 20 in a row are accepted, then let it settle.
      - name: "[Setup] Warm up the source until ingest accepts its token"
        run: |
          STREAK=0
          DEADLINE=$((SECONDS + 300))
          while [ "$STREAK" -lt 20 ]; do
            CODE=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "https://$INGESTING_HOST/" -H "Authorization: Bearer $SOURCE_TOKEN" -H 'Content-Type: application/json' -d "{\"message\":\"e2e warm-up run_id=$RUN_ID\"}")
            if [ "$CODE" = 202 ]; then STREAK=$((STREAK + 1)); else echo "ingest answered HTTP $CODE after $STREAK accepted"; STREAK=0; fi
            [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::ingest did not accept the source token 20 times in a row within 5 min"; exit 1; }
            sleep 1
          done
          sleep 15
          echo "ingest accepts the token"

      - name: "[Setup] Create the two log groups before the stack, so the first sync finds them"
        run: |
          for group in "$LOG_GROUP" "$STATIC_LOG_GROUP"; do
            aws logs create-log-group --log-group-name "$group"
            aws logs create-log-stream --log-group-name "$group" --log-stream-name run
          done

      - name: "[Test] Deploy the CloudFormation stack"
        run: |
          aws cloudformation create-stack --stack-name "$STACK_NAME" --template-body file://template.yaml --capabilities CAPABILITY_NAMED_IAM \
            --parameters "ParameterKey=ClusterId,ParameterValue=$DATA_REGION" "ParameterKey=SourceToken,ParameterValue=$SOURCE_TOKEN" "ParameterKey=SourceId,ParameterValue=$SOURCE_ID"
          if ! aws cloudformation wait stack-create-complete --stack-name "$STACK_NAME"; then
            echo "::error::stack creation failed: $(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --query 'Stacks[0].StackStatus' --output text)"
            aws cloudformation describe-stack-events --stack-name "$STACK_NAME" --query "StackEvents[?ends_with(ResourceStatus, 'FAILED')].[LogicalResourceId,ResourceStatus,ResourceStatusReason]" --output table
            exit 1
          fi
          OUTPUTS=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --query 'Stacks[0].Outputs' --output json)
          ROLE_ARN=$(jq -r '.[] | select(.OutputKey == "IntegrationRoleArn") | .OutputValue' <<<"$OUTPUTS")
          EXTERNAL_ID=$(jq -r '.[] | select(.OutputKey == "ExternalId") | .OutputValue' <<<"$OUTPUTS")
          [ -n "$ROLE_ARN" ] && [ -n "$EXTERNAL_ID" ] || { echo "::error::stack outputs lack IntegrationRoleArn or ExternalId: $OUTPUTS"; exit 1; }
          # The external id is half of what Better Stack needs to assume the integration role; this repo is public.
          echo "::add-mask::$EXTERNAL_ID"
          echo "ROLE_ARN=$ROLE_ARN" >> "$GITHUB_ENV"
          echo "EXTERNAL_ID=$EXTERNAL_ID" >> "$GITHUB_ENV"
          echo "stack outputs: IntegrationRoleArn $ROLE_ARN, ExternalId (masked)"

      # X-Ray creates the aws/spans log group when it processes the first span, and the sync subscribes only groups
      # that exist, so a throwaway trace goes in before the source is connected.
      - name: "[Test] Warm-up trace so that aws/spans exists before the first sync"
        run: |
          NOW=$(date +%s)
          TRACE_ID="1-$(printf '%x' "$NOW")-$(openssl rand -hex 12)"
          SEGMENT=$(jq -nc --arg name "e2e-warm-up" --arg id "$(openssl rand -hex 8)" --arg trace "$TRACE_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \
            '{name: $name, id: $id, trace_id: $trace, start_time: $start, end_time: $end}')
          aws xray put-trace-segments --trace-segment-documents "$SEGMENT" --query 'UnprocessedTraceSegments' --output json | jq -e 'length == 0' >/dev/null
          DEADLINE=$((SECONDS + 180))
          until aws logs describe-log-groups --log-group-name-prefix aws/spans --query "logGroups[?logGroupName=='aws/spans'].logGroupName" --output text | grep -q spans; do
            [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::X-Ray did not create aws/spans within 3 min of the warm-up trace"; exit 1; }
            sleep 10
          done
          echo "aws/spans exists, the sync the connect enqueues will find it"

      # aws_account_id would select an already connected account; a new connection is the role ARN + external id only.
      - name: "[Test] Paste the stack outputs into the source (connects the AWS account, enqueues the sync)"
        run: |
          BODY=$(jq -nc --arg r "$ROLE_ARN" --arg e "$EXTERNAL_ID" '{aws_role_arn: $r, aws_external_id: $e}')
          for attempt in 1 2 3; do
            CODE=$(curl -sS -o connect.json -w '%{http_code}' -X PATCH "$API/api/v1/sources/$SOURCE_ID" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API" -H 'Content-Type: application/json' -d "$BODY")
            [ "$CODE" = 200 ] && { echo "source connected to $ROLE_ARN"; exit 0; }
            # IAM propagation occasionally refuses the first AssumeRole on a fresh role; one bounded retry on that answer only.
            if [ "$CODE" = 422 ] && grep -qi assume connect.json && [ "$attempt" -lt 3 ]; then echo "attempt $attempt refused: $(cat connect.json), retrying in 20 s"; sleep 20; continue; fi
            echo "::error::connecting the source failed: HTTP $CODE $(cat connect.json)"; exit 1
          done

      # Whether the sandbox's AwsAccount record was removed since the previous run (8 hours without a source) or is reused
      # with its history, the sync the connect enqueues subscribes all four: the run's and the static group through
      # auto-subscribe or a kept intent, aws/spans and the CloudTrail group by default.
      - name: "[Test] The sync subscribes the run's log group, the static group, aws/spans and the CloudTrail group by itself"
        run: |
          DEADLINE=$((SECONDS + 480))
          while :; do
            PENDING=""
            for group in "$LOG_GROUP" "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do
              aws logs describe-subscription-filters --log-group-name "$group" --query "subscriptionFilters[?filterName=='BetterStackLogs'].destinationArn" --output text 2>/dev/null | grep -q firehose || PENDING="$PENDING $group"
            done
            [ -z "$PENDING" ] && { echo "all four log groups carry the BetterStackLogs subscription filter"; exit 0; }
            [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::the sync did not subscribe within 8 min:$PENDING"; exit 1; }
            echo "waiting for the sync to subscribe:$PENDING"; sleep 15
          done

      - name: "[Test] Produce logs, a custom metric, a two-segment trace and a CloudTrail management event"
        run: |
          echo "SINCE=dt > toDateTime('$(date -u -d '-10 minutes' '+%F %T')')" >> "$GITHUB_ENV"
          for group in "$LOG_GROUP" "$STATIC_LOG_GROUP"; do
            jq -nc --arg m "$MARKER" --arg run "$RUN_ID" --arg g "$group" '[range(1; 21) as $n | {timestamp: ((now * 1000 | floor) + $n), message: ({message: $m, e2e: {run_id: $run, log_group: $g, n: $n}} | tojson)}]' > events.json
            aws logs put-log-events --log-group-name "$group" --log-stream-name run --log-events file://events.json >/dev/null
          done
          # Values 1, 2 and 3 in three consecutive minutes; the read-back checks each minute's value.
          T=$(date +%s)
          jq -nc --arg run "$RUN_ID" --argjson t "$T" \
            '[range(3) | {MetricName: "RunMarker", Value: (. + 1), Unit: "Count", Timestamp: ($t - (2 - .) * 60 | todate), Dimensions: [{Name: "RunId", Value: $run}]}]' > metrics.json
          aws cloudwatch put-metric-data --namespace E2eTest/AwsStack --metric-data file://metrics.json
          echo "METRIC_MINUTES=$(((T - 120) / 60 * 60)) $(((T - 60) / 60 * 60)) $((T / 60 * 60))" >> "$GITHUB_ENV"
          NOW=$(date +%s)
          TRACE_ID="1-$(printf '%x' "$NOW")-$(openssl rand -hex 12)"
          PARENT_ID=$(openssl rand -hex 8)
          PARENT=$(jq -nc --arg name "e2e-parent-$RUN_ID" --arg id "$PARENT_ID" --arg trace "$TRACE_ID" --arg run "$RUN_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \
            '{name: $name, id: $id, trace_id: $trace, start_time: $start, end_time: $end, annotations: {run_id: $run}}')
          CHILD=$(jq -nc --arg name "e2e-child-$RUN_ID" --arg id "$(openssl rand -hex 8)" --arg parent "$PARENT_ID" --arg trace "$TRACE_ID" --arg run "$RUN_ID" --argjson start "$((NOW - 1))" --argjson end "$NOW" \
            '{name: $name, id: $id, parent_id: $parent, trace_id: $trace, start_time: ($start + 0.2), end_time: ($end - 0.2), annotations: {run_id: $run}}')
          aws xray put-trace-segments --trace-segment-documents "$PARENT" "$CHILD" --query 'UnprocessedTraceSegments' --output json | jq -e 'length == 0' >/dev/null
          # The X-Ray trace id 1-<8 hex epoch>-<24 hex> is stored as the 32-hex OpenTelemetry id.
          echo "TRACE_HEX=$(cut -d- -f2,3 <<<"$TRACE_ID" | tr -d -)" >> "$GITHUB_ENV"
          # A management event that happens after the trail exists and names this run.
          aws logs put-retention-policy --log-group-name "$LOG_GROUP" --retention-in-days 1
          echo "20 log events into each of $LOG_GROUP and $STATIC_LOG_GROUP, 3 datapoints of E2eTest/AwsStack RunMarker, trace $TRACE_ID, PutRetentionPolicy on $LOG_GROUP"

      - name: "[Test] Read everything back through the SQL API connection"
        run: |
          set -o pipefail
          mkdir -p out
          fail() { FAILURE="$1"; echo "::error::$1"; exit 1; }
          # Every way this step fails still leaves a summary: what arrived against what was expected, and, after a
          # timeout, what CloudWatch Logs did with each group.
          summarize_failure() {
            local status=$?
            [ "$status" = 0 ] && return
            {
              echo "### AWS CloudFormation stack E2E \`$RUN_ID\`"
              echo
              echo "❌ ${FAILURE:-The read-back stopped on an error, see the log of this step}"
              if [ -n "${LOGS:-}" ]; then
                echo
                echo "Read back through \`$CONNECT_HOST\`:"
                echo
                echo "- logs of \`$LOG_GROUP\`: $LOGS of 20"
                echo "- logs of \`$STATIC_LOG_GROUP\`: $STATIC_LOGS of 20"
                echo "- spans: $SPANS of 2"
                echo "- metric points: $METRICS of 15"
                echo "- CloudTrail event: $CLOUDTRAIL of 1"
              fi
              [ ! -f out/forwarding.md ] || { echo; cat out/forwarding.md; }
            } > out/summary.md
            exit "$status"
          }
          trap summarize_failure EXIT
          sql() {
            CODE=$(curl -sS -o response.json -w '%{http_code}' -u "$TEST_TEAM_CONNECTION_USERNAME:$TEST_TEAM_CONNECTION_PASSWORD" -H 'Content-Type: plain/text' --data-binary "$1 FORMAT JSONEachRow" "https://$CONNECT_HOST/")
            # A source's logs, spans and metrics collections each exist only from their first stored row on;
            # until then the query is "no rows yet".
            [ "$CODE" = 500 ] && grep -q CLUSTER_DOESNT_EXIST response.json && return 0
            [ "$CODE" = 200 ] || { echo "::error::SQL API on $CONNECT_HOST answered HTTP $CODE: $(head -c 300 response.json)" >&2; return 1; }
            cat response.json
          }
          count() { ROWS=$(sql "SELECT count() AS n FROM $1") || return 1; [ -n "$ROWS" ] && jq -r '.n' <<<"$ROWS" || echo 0; }
          METRIC=e2e_test.aws_stack.run_marker
          METRIC_FILTER="tags['run_id'] = '$RUN_ID' AND startsWith(name, '$METRIC')"
          counts() {
            LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$LOG_GROUP\"') > 0")
            STATIC_LOGS=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 AND position(raw, '\"log_group\":\"$STATIC_LOG_GROUP\"') > 0")
            # The mapper stores a CloudTrail event as {id, message: <the event>, context: {logGroup, ...}, dt}; this run's
            # PutRetentionPolicy call names the run's log group.
            CLOUDTRAIL=$(count "remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'context', 'logGroup') = '/aws/cloudtrail/better-stack' AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'")
            # The mapper stores each X-Ray span as {dt, resource, span: {trace_id, span_id, name, ...}}.
            SPANS=$(count "remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID')")
            # The metric stream sends one aggregate per minute; the mapper stores it as the metric plus _max, _min, _sum and
            # _count, so three minutes make 15 points.
            METRICS=$(count "(SELECT name, dt FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND $METRIC_FILTER GROUP BY name, dt)")
            echo "logs $LOGS/20 + static $STATIC_LOGS/20, spans $SPANS/2, metric points $METRICS/15, CloudTrail event $CLOUDTRAIL/1"
          }
          # What CloudWatch Logs did with each group in the last 40 min; DeliveryErrors mean the filter paused for 10 min
          # and skipped what arrived meanwhile.
          forwarding() {
            printf 'CloudWatch Logs in the last 40 min:\n\n' > out/forwarding.md
            for group in "$LOG_GROUP" "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do
              VALUES=()
              for metric in IncomingLogEvents ForwardedLogEvents DeliveryErrors; do
                DIMS="Name=LogGroupName,Value=$group"
                [ "$metric" = IncomingLogEvents ] || DIMS="$DIMS Name=DestinationType,Value=Firehose Name=FilterName,Value=BetterStackLogs"
                VALUE=$(aws cloudwatch get-metric-statistics --namespace AWS/Logs --metric-name "$metric" --dimensions $DIMS \
                  --start-time "$(date -u -d '-40 minutes' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" --period 2400 --statistics Sum --query 'sum(Datapoints[].Sum)' --output text)
                echo "$group $metric $VALUE"
                VALUES+=("${VALUE%.0}")
              done
              echo "- \`$group\`: ${VALUES[0]} incoming, ${VALUES[1]} forwarded, ${VALUES[2]} delivery errors" >> out/forwarding.md
            done
          }
          # Within an hour of the previous stack deletion the static group, aws/spans and CloudTrail may deliver nothing
          # at all, so only the run's own log group and the metric are waited for (see the step measuring that hour).
          # Otherwise everything is: CloudTrail hands events to CloudWatch Logs within about 5 min of the call.
          if [ -n "${RECENT_STACK_MIN:-}" ]; then
            arrived() { [ "$LOGS" -ge 20 ] && [ "$METRICS" -ge 15 ]; }
          else
            arrived() { [ "$LOGS" -ge 20 ] && [ "$STATIC_LOGS" -ge 20 ] && [ "$SPANS" -ge 2 ] && [ "$METRICS" -ge 15 ] && [ "$CLOUDTRAIL" -ge 1 ]; }
          fi
          DEADLINE=$((SECONDS + 1200))
          while :; do
            counts
            arrived && break
            [ "$SECONDS" -lt "$DEADLINE" ] || { forwarding; sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE ORDER BY dt DESC LIMIT 3" > out/sample_spans.json; fail "Not everything arrived within 20 min"; }
            sleep 20
          done
          # A last look 60 s later catches duplicate delivery. Every count must be exact; within that hour a fixed-name
          # source that delivered nothing is a warning instead.
          sleep 60
          counts
          MISSING=()
          for check in "logs:$LOGS:20" "static:$STATIC_LOGS:20" "spans:$SPANS:2" "cloudtrail:$CLOUDTRAIL:1"; do
            IFS=: read -r name got want <<<"$check"
            [ "$got" = "$want" ] && continue
            if [ -n "${RECENT_STACK_MIN:-}" ] && [ "$name" != logs ] && [ "$got" = 0 ]; then MISSING+=("$name"); continue; fi
            fail "Unexpected row counts: logs $LOGS (20), static $STATIC_LOGS (20), spans $SPANS (2), CloudTrail event $CLOUDTRAIL (1)"
          done
          # Each minute must hold exactly the value sent in it: 1, 2 and 3 for the metric, _max, _min and _sum, 1 for _count.
          # The lowest and highest stored value per point catch a re-sent minute with a different value. Points are
          # compared by minute, since AWS does not document whether the stream stamps a minute's start.
          EXPECTED=$(jq -nc --arg n "$METRIC" --arg m "$METRIC_MINUTES" '[$m | split(" ") | to_entries[] | (.value | tonumber) as $t | (.key + 1) as $v
            | ({name: $n, v: $v}, {name: "\($n)_max", v: $v}, {name: "\($n)_min", v: $v}, {name: "\($n)_sum", v: $v}, {name: "\($n)_count", v: 1})
            | {name, minute: $t, min: .v, max: .v}] | sort_by(.name, .minute)')
          ACTUAL=$(sql "SELECT name, toUnixTimestamp(dt) AS t, minMerge(value_min) AS min, maxMerge(value_max) AS max FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND $METRIC_FILTER GROUP BY name, dt" \
            | jq -sc 'map({name, minute: (((.t | tonumber) / 60 | floor) * 60), min: (.min | tonumber), max: (.max | tonumber)}) | sort_by(.name, .minute)')
          [ "$ACTUAL" = "$EXPECTED" ] || fail "Metric points differ from the datapoints sent: expected $EXPECTED, got $ACTUAL"
          if [ "${#MISSING[@]}" -gt 0 ]; then
            NAMES=$(printf '%s\n' "${MISSING[@]}" | sed -e "s#^static\$#the logs of $STATIC_LOG_GROUP#" -e 's#^spans$#the spans#' -e 's#^cloudtrail$#the CloudTrail event#' | paste -sd, - | sed 's/,/, /g')
            WARNING="Not verified: $NAMES. The previous run deleted its stack $RECENT_STACK_MIN min before this one, and for up to an hour AWS keeps using sessions of the IAM roles every run recreates under the same names, so CloudWatch Logs and CloudTrail can skip this data. A run more than an hour after the previous one checks it."
            echo "::warning::$WARNING"
            forwarding
          fi
          sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 1" > out/sample_log.json
          if [ "$SPANS" = 2 ]; then
            sql "SELECT dt, raw FROM remote(${COLLECTION}_spans) WHERE $SINCE AND JSONExtractString(raw, 'span', 'name') IN ('e2e-parent-$RUN_ID', 'e2e-child-$RUN_ID') ORDER BY JSONExtractUInt(raw, 'span', 'start_time_unix_nano')" > out/sample_spans.json
            # The X-Ray trace id 1-<epoch>-<random> should be stored as the 32-hex id the mapper takes from Transaction Search.
            STORED_TRACE_IDS=$(jq -r '.raw | fromjson | .span.trace_id' out/sample_spans.json | sort -u | tr '\n' ' ')
            [ "$STORED_TRACE_IDS" = "$TRACE_HEX " ] || fail "Stored span trace ids '$STORED_TRACE_IDS' differ from the X-Ray trace id $TRACE_HEX"
          fi
          sql "SELECT name, dt, tags, avgMerge(value_avg) AS value FROM remote(${COLLECTION}_metrics) WHERE $SINCE AND tags['run_id'] = '$RUN_ID' AND name = '$METRIC' GROUP BY name, dt, tags ORDER BY dt" > out/sample_metric.json
          [ "$CLOUDTRAIL" = 0 ] || sql "SELECT dt, raw FROM remote(${COLLECTION}_logs) WHERE $SINCE AND JSONExtractString(raw, 'message', 'eventName') = 'PutRetentionPolicy' AND JSONExtractString(raw, 'message', 'requestParameters', 'logGroupName') = '$LOG_GROUP'" > out/sample_cloudtrail.json
          {
            echo "### AWS CloudFormation stack E2E \`$RUN_ID\`"
            echo
            echo "- template from $TEMPLATE_FROM, source $SOURCE_ID on \`$DATA_REGION\`, stack \`$STACK_NAME\`"
            echo "- \`$LOG_GROUP\` and \`$STATIC_LOG_GROUP\` auto-subscribed, \`aws/spans\` and \`/aws/cloudtrail/better-stack\` subscribed by default, all by the first sync"
            echo "- read back through \`$CONNECT_HOST\`: logs $LOGS + static $STATIC_LOGS, spans $SPANS, metric points $METRICS with the values sent, CloudTrail event $CLOUDTRAIL"
            echo
            if [ "${#MISSING[@]}" -gt 0 ]; then
              echo "⚠️ $WARNING"
              echo
              cat out/forwarding.md
              echo
              echo "✅ everything else round-tripped through the stack"
            else
              echo "✅ logs, metrics, spans and CloudTrail events round-tripped through the stack"
            fi
          } > out/summary.md

      - name: "[Cleanup] Delete this run's resources in Better Stack and AWS"
        if: ${{ always() }}
        run: bash e2e/cleanup.sh

      - name: "[Report] Job summary"
        if: ${{ always() }}
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [ -f out/summary.md ]; then
            cat out/summary.md >> "$GITHUB_STEP_SUMMARY"
            # The Linear issue names the job and links the run, so it gets the summary without its heading, and the
            # cleanup only when that left something behind. Without a summary it gets the job's errors instead.
            mkdir -p report
            tail -n +3 out/summary.md > report/summary.md
            grep -q '❌' out/cleanup.md 2>/dev/null && { echo; cat out/cleanup.md; } >> report/summary.md
          else
            # The read-back writes the summary however it ends, so a step before it failed (or the job was cancelled).
            FAILED_STEP=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs" \
              --jq ".jobs[] | select(.runner_name == \"$RUNNER_NAME\") | .steps[] | select(.conclusion == \"failure\") | .name" | head -1) || true
            { echo "### AWS CloudFormation stack E2E \`$RUN_ID\`"; echo; echo "❌ Stopped${FAILED_STEP:+ at **$FAILED_STEP**}, before anything was read back."; } >> "$GITHUB_STEP_SUMMARY"
          fi
          # Rows come back as JSONEachRow with the stored event as a JSON string in raw; render them as a pretty array.
          for f in out/sample_*.json; do [ -f "$f" ] && { echo; echo "<details><summary>$(basename "$f")</summary>"; echo; echo '```json'; jq -s 'map(if has("raw") then .raw |= (fromjson? // .) else . end)' "$f"; echo '```'; echo '</details>'; echo; } >> "$GITHUB_STEP_SUMMARY"; done
          [ -f out/cleanup.md ] && { echo; cat out/cleanup.md; } >> "$GITHUB_STEP_SUMMARY"
          true

      - name: "[Report] Hand the summary to the Linear issue"
        if: ${{ always() }}
        uses: actions/upload-artifact@v7
        with:
          name: e2e-summary-Stack round trip
          path: report/summary.md
          if-no-files-found: ignore
          retention-days: 7
          overwrite: true

  notify-linear:
    if: ${{ failure() && (github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }}
    needs: [test]
    permissions:
      contents: read
      actions: read
      checks: read
    uses: ./.github/workflows/notify_linear_failure.yml
    with:
      test_name: AWS CloudFormation stack
      job_summaries: true
    secrets: inherit