fix: harden GHCR deployment security and logging

.github/workflows/production.yml

@@ -82,6 +82,7 @@ jobs:
         run: >-
           docker run --rm --network none
           --volume "$GITHUB_WORKSPACE/tests:/tmp/codelens-tests:ro"
+          --volume "$GITHUB_WORKSPACE/scripts:/tmp/scripts:ro"
           --volume "$GITHUB_WORKSPACE/pytest.ini:/tmp/pytest.ini:ro"
           --env DEBUG=false
           --env DATABASE_URL=postgresql+asyncpg://test:test@127.0.0.1/test
@@ -132,6 +133,7 @@ jobs:
         run: >-
           docker run --rm --network none
           --volume "$GITHUB_WORKSPACE/tests:/tmp/codelens-tests:ro"
+          --volume "$GITHUB_WORKSPACE/scripts:/tmp/scripts:ro"
           --volume "$GITHUB_WORKSPACE/pytest.ini:/tmp/pytest.ini:ro"
           --env DEBUG=false
           --env DATABASE_URL=postgresql+asyncpg://test:test@127.0.0.1/test

docs/ghcr-deployment.md

@@ -39,13 +39,16 @@ deployment helper.
 ## First controlled deployment
 
 1. Do not proceed while the root filesystem has less than **1536 MiB free**.
-   The latest read-only check showed about **500 MiB free**, so the helper will
-   stop before creating rollback tags or pulling an image. The first GHCR image
-   can require roughly the current 672 MB uncompressed application-image size,
-   plus pull/unpack overhead; the helper also requires at least 300 MiB to remain
-   after pulling. It does not prune images or volumes to create space.
-2. Resolve the exposed local environment credentials noted in the deployment
-   preparation report before starting the workflow.
+   The EC2 EBS volume was expanded to **12 GiB** and the root filesystem was
+   grown online. A subsequent check showed about **4.37 GiB free** (the exact
+   amount changes as Docker images and logs grow). The helper still checks the
+   live free space and inodes before changing anything; it requires 1536 MiB
+   before a pull and at least 300 MiB after it. It does not prune images or
+   volumes to create space.
+2. The previously exposed credential categories have been rotated and the old
+   credentials revoked. Keep the replacement values only in their protected
+   provider settings and production `.env`; never paste or print them in Actions
+   logs or deployment diagnostics.
 3. On the `main` branch, run Actions → **Test, publish, and deploy CodeLens** →
    Run workflow → `deploy`. The image is published only after both test runs
    pass. Approve the `production` environment job only after reviewing the
@@ -64,8 +67,10 @@ deployment helper.
 
 The short-lived GHCR token is transferred over the verified SSH connection,
 stored temporarily with mode `0600`, used through Docker's `--password-stdin`,
-and removed. Docker credentials use a temporary `DOCKER_CONFIG` and are removed
-when the helper exits.
+and removed. Docker credentials use a temporary mode-`0700` `DOCKER_CONFIG` and
+are removed when the helper exits, including failure and handled-signal paths.
+Authentication, pull, task-inspection, and Compose errors are reported using
+fixed sanitized messages; raw command output is not forwarded to Actions logs.
 
 ## Rollback
 

scripts/deploy_ghcr.sh

@@ -8,6 +8,43 @@ STATE_DIR="${CODELENS_STATE_DIR:-/home/ubuntu/.local/state/codelens-ghcr}"
 MIN_FREE_MIB="${CODELENS_MIN_FREE_MIB:-1536}"
 POST_PULL_FREE_MIB="${CODELENS_POST_PULL_FREE_MIB:-300}"
 PUBLIC_HEALTH_URL="${CODELENS_PUBLIC_HEALTH_URL:-https://13-51-158-45.sslip.io/health}"
+AUTH_TMP_ROOT="${TMPDIR:-/tmp}"
+AUTH_TOKEN_FILE=""
+AUTH_DOCKER_CONFIG=""
+
+cleanup_auth() {
+  local cleanup_failed=0 token_name exit_status=$?
+
+  if [[ -n "${AUTH_DOCKER_CONFIG:-}" ]]; then
+    case "$AUTH_DOCKER_CONFIG" in
+      "$AUTH_TMP_ROOT"/codelens-ghcr-auth.*)
+        rm -rf -- "$AUTH_DOCKER_CONFIG" >/dev/null 2>&1 || cleanup_failed=1
+        ;;
+      *) cleanup_failed=1 ;;
+    esac
+  fi
+
+  if [[ -n "${AUTH_TOKEN_FILE:-}" ]]; then
+    token_name="${AUTH_TOKEN_FILE##*/}"
+    if [[ "$AUTH_TOKEN_FILE" == "/tmp/$token_name" && "$token_name" =~ ^codelens-ghcr-token-[A-Za-z0-9._-]+$ ]]; then
+      rm -f -- "$AUTH_TOKEN_FILE" >/dev/null 2>&1 || cleanup_failed=1
+    else
+      cleanup_failed=1
+    fi
+  fi
+
+  if (( cleanup_failed )); then
+    printf 'WARNING: temporary authentication cleanup was incomplete.\n' >&2
+    (( exit_status != 0 )) || exit_status=1
+  fi
+  trap - EXIT
+  exit "$exit_status"
+}
+
+trap cleanup_auth EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
 
 usage() {
   cat <<'USAGE'
@@ -53,20 +90,26 @@ check_preconditions() {
   [[ -f "$COMPOSE_DIR/docker-compose.yml" && -f "$COMPOSE_DIR/docker-compose.prod.yml" ]] || {
     echo "STOP: expected Compose files are missing." >&2; return 1;
   }
-  compose_version="$(docker compose version --short | sed 's/^v//')"
-  if ! python3 -c 'import re, sys; v=tuple(int(x) for x in re.match(r"^(\d+)\.(\d+)\.(\d+)", sys.argv[1]).groups()); raise SystemExit(0 if v >= (2, 24, 4) else 1)' "$compose_version"; then
+  compose_version="$(docker compose version --short 2>/dev/null)" || {
+    echo "STOP: unable to verify Docker Compose version." >&2; return 1;
+  }
+  if ! python3 -c 'import re, sys; m=re.match(r"^v?(\d+)\.(\d+)\.(\d+)", sys.argv[1]); raise SystemExit(0 if m and tuple(map(int, m.groups())) >= (2, 24, 4) else 1)' "$compose_version" >/dev/null 2>&1; then
     echo "STOP: Docker Compose 2.24.4 or newer is required for build reset semantics." >&2
     return 1
   fi
-  services="$(compose_base "$1" config --services 2>/dev/null | sort | tr '\n' ' ')"
+  services="$(compose_base "$1" config --services 2>/dev/null | sort | tr '\n' ' ')" || {
+    echo "STOP: unable to validate the Compose services." >&2; return 1;
+  }
   for service in web worker flower caddy; do
     [[ " $services " == *" $service "* ]] || {
       echo "STOP: expected Compose service '$service' is absent." >&2; return 1;
     }
   done
   for service in web worker flower; do
     local cid
-    cid="$(compose_base "$1" ps -q "$service")"
+    cid="$(compose_base "$1" ps -q "$service" 2>/dev/null)" || {
+      echo "STOP: unable to verify a running application service." >&2; return 1;
+    }
     [[ -n "$cid" ]] || { echo "STOP: running $service container was not found." >&2; return 1; }
   done
 }
@@ -85,7 +128,7 @@ acquire_deployment_lock() {
 }
 
 assert_celery_quiescent() {
-  local output
+  local output active reserved scheduled unacked unacked_index
   if ! output="$(timeout 25s docker exec codelens-worker-1 python -c '
 from app.core.celery_app import celery_app
 connection = celery_app.connection_for_read()
@@ -103,13 +146,25 @@ for method in ("active", "reserved", "scheduled"):
         raise SystemExit("worker inspection unavailable: " + method)
     counts[method] = sum(len(items) for items in response.values())
 print("active={active} reserved={reserved} scheduled={scheduled} unacked={unacked} unacked_index={unacked_index}".format(**counts, unacked=unacked, unacked_index=unacked_index))
-if any(counts.values()) or unacked or unacked_index:
-    raise SystemExit(3)
-' 2>&1)"; then
-    printf 'STOP: could not verify Celery task state safely: %s\n' "$output" >&2
+' 2>/dev/null)"; then
+    echo "STOP: Celery task inspection failed; deployment aborted." >&2
+    return 1
+  fi
+  if [[ ! "$output" =~ ^active=([0-9]+)[[:space:]]reserved=([0-9]+)[[:space:]]scheduled=([0-9]+)[[:space:]]unacked=([0-9]+)[[:space:]]unacked_index=([0-9]+)$ ]]; then
+    echo "STOP: Celery task inspection returned unexpected output; deployment aborted." >&2
+    return 1
+  fi
+  active="${BASH_REMATCH[1]}"
+  reserved="${BASH_REMATCH[2]}"
+  scheduled="${BASH_REMATCH[3]}"
+  unacked="${BASH_REMATCH[4]}"
+  unacked_index="${BASH_REMATCH[5]}"
+  if (( active || reserved || scheduled || unacked || unacked_index )); then
+    printf 'STOP: Celery is not quiescent (active=%s reserved=%s scheduled=%s unacked=%s unacked_index=%s).\n' \
+      "$active" "$reserved" "$scheduled" "$unacked" "$unacked_index" >&2
     return 1
   fi
-  printf 'Celery is quiescent: %s\n' "$output"
+  printf 'Celery task inspection: PASS (active=0 reserved=0 scheduled=0 unacked=0 unacked_index=0).\n'
 }
 
 verify_running_release() {
@@ -122,7 +177,7 @@ verify_running_release() {
       if [[ "$status" == healthy ]] && [[ "$flower_status" =~ ^(200|301|302|401|403)$ ]]; then
         if timeout 15s docker exec codelens-worker-1 python -c \
           'from app.core.celery_app import celery_app; r=celery_app.control.ping(timeout=5); raise SystemExit(0 if r else 1)' >/dev/null 2>&1; then
-          if [[ -n "$PUBLIC_HEALTH_URL" ]] && curl --fail --silent --show-error --max-time 10 "$PUBLIC_HEALTH_URL" >/dev/null; then
+          if [[ -n "$PUBLIC_HEALTH_URL" ]] && curl --fail --silent --max-time 10 "$PUBLIC_HEALTH_URL" >/dev/null 2>&1; then
             return 0
           fi
           # Public HTTPS can fail transiently while Caddy is serving a new web container.
@@ -162,7 +217,7 @@ rollback_state() {
   local state_file="$1" override_file="$2"
   [[ -f "$state_file" ]] || { echo "STOP: rollback state file is missing." >&2; return 1; }
   write_rollback_override "$state_file" "$override_file"
-  if ! compose_base "$override_file" up -d --no-build --pull never --no-deps web worker flower; then
+  if ! compose_base "$override_file" up -d --no-build --pull never --no-deps web worker flower >/dev/null 2>&1; then
     echo "Rollback Compose operation failed; stop for manual inspection." >&2
     return 1
   fi
@@ -174,6 +229,11 @@ deploy() {
   local run_id state_file temp_config web_id worker_id flower_id
   local web_tag worker_tag flower_tag revision platform free_before free_after
 
+  if [[ "$token_file" =~ ^/tmp/codelens-ghcr-token-[A-Za-z0-9._-]+$ ]]; then
+    AUTH_TOKEN_FILE="$token_file"
+  else
+    echo "STOP: invalid short-lived GHCR token file path." >&2; return 1
+  fi
   [[ "$image_ref" =~ ^ghcr\.io/[a-z0-9][a-z0-9._-]*/codelens@sha256:[a-f0-9]{64}$ ]] || {
     echo "STOP: image must be a lowercase GHCR repository reference pinned by sha256 digest." >&2; return 1;
   }
@@ -195,15 +255,27 @@ deploy() {
   [[ "$run_id" =~ ^[A-Za-z0-9._-]+$ ]] || { echo "STOP: invalid deployment id." >&2; return 1; }
   state_file="$STATE_DIR/${run_id}.state"
   [[ ! -e "$state_file" ]] || { echo "STOP: deployment state already exists." >&2; return 1; }
-  web_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q web)")"
-  worker_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q worker)")"
-  flower_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q flower)")"
+  web_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q web 2>/dev/null)" 2>/dev/null)" || {
+    echo "STOP: unable to record the running web image for rollback." >&2; return 1;
+  }
+  worker_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q worker 2>/dev/null)" 2>/dev/null)" || {
+    echo "STOP: unable to record the running worker image for rollback." >&2; return 1;
+  }
+  flower_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q flower 2>/dev/null)" 2>/dev/null)" || {
+    echo "STOP: unable to record the running Flower image for rollback." >&2; return 1;
+  }
   web_tag="codelens-web:rollback-${run_id}"
   worker_tag="codelens-worker:rollback-${run_id}"
   flower_tag="codelens-flower:rollback-${run_id}"
-  docker image tag "$web_id" "$web_tag"
-  docker image tag "$worker_id" "$worker_tag"
-  docker image tag "$flower_id" "$flower_tag"
+  if ! docker image tag "$web_id" "$web_tag" >/dev/null 2>&1; then
+    echo "STOP: unable to preserve the web rollback image." >&2; return 1;
+  fi
+  if ! docker image tag "$worker_id" "$worker_tag" >/dev/null 2>&1; then
+    echo "STOP: unable to preserve the worker rollback image." >&2; return 1;
+  fi
+  if ! docker image tag "$flower_id" "$flower_tag" >/dev/null 2>&1; then
+    echo "STOP: unable to preserve the Flower rollback image." >&2; return 1;
+  fi
   cat > "$state_file" <<STATE
 DEPLOYMENT_ID='$run_id'
 IMAGE_REF='$image_ref'
@@ -217,28 +289,42 @@ FLOWER_ROLLBACK_TAG='$flower_tag'
 STATE
   chmod 600 "$state_file"
 
-  temp_config="$(mktemp -d /tmp/codelens-ghcr-auth.XXXXXX)"
-  export DOCKER_CONFIG="$temp_config"
-  cleanup_auth() {
-    rm -rf -- "$temp_config"
-    rm -f -- "$token_file"
+  temp_config="$(mktemp -d "$AUTH_TMP_ROOT/codelens-ghcr-auth.XXXXXX" 2>/dev/null)" || {
+    echo "STOP: unable to create temporary Docker authentication storage." >&2; return 1;
   }
-  trap cleanup_auth EXIT
-  docker login ghcr.io --username "$ghcr_username" --password-stdin < "$token_file" >/dev/null
-  rm -f -- "$token_file"
-  docker pull "$image_ref"
+  AUTH_DOCKER_CONFIG="$temp_config"
+  chmod 700 "$temp_config" 2>/dev/null || {
+    echo "STOP: unable to secure temporary Docker authentication storage." >&2; return 1;
+  }
+  export DOCKER_CONFIG="$temp_config"
+  if ! docker login ghcr.io --username "$ghcr_username" --password-stdin < "$token_file" >/dev/null 2>&1; then
+    echo "STOP: GHCR authentication failed." >&2; return 1
+  fi
+  if ! rm -f -- "$token_file" >/dev/null 2>&1; then
+    echo "STOP: unable to remove the temporary GHCR token file." >&2; return 1
+  fi
+  AUTH_TOKEN_FILE=""
+  if ! docker pull "$image_ref" >/dev/null 2>&1; then
+    echo "STOP: unable to pull the pinned GHCR image." >&2; return 1
+  fi
   free_after="$(free_kib)"
   if (( free_after < POST_PULL_FREE_MIB * 1024 )); then
     echo "STOP: image pulled but free space fell below the post-pull safety floor; services were not changed." >&2
     return 1
   fi
-  revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image_ref")"
-  platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image_ref")"
+  revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image_ref" 2>/dev/null)" || {
+    echo "STOP: unable to verify the published image revision." >&2; return 1;
+  }
+  platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image_ref" 2>/dev/null)" || {
+    echo "STOP: unable to verify the published image platform." >&2; return 1;
+  }
   [[ "$revision" == "$expected_sha" ]] || { echo "STOP: image revision label does not match the requested commit." >&2; return 1; }
-  [[ "$platform" == linux/amd64 ]] || { echo "STOP: image platform $platform does not match EC2 linux/amd64." >&2; return 1; }
-  compose_base "$override_file" config --quiet
+  [[ "$platform" == linux/amd64 ]] || { echo "STOP: image platform does not match EC2 linux/amd64." >&2; return 1; }
+  if ! compose_base "$override_file" config --quiet >/dev/null 2>&1; then
+    echo "STOP: the deployment Compose configuration is invalid." >&2; return 1;
+  fi
 
-  if ! compose_base "$override_file" up -d --no-build --pull never --no-deps web worker flower; then
+  if ! compose_base "$override_file" up -d --no-build --pull never --no-deps web worker flower >/dev/null 2>&1; then
     echo "Deployment update failed; restoring the recorded image IDs." >&2
     rollback_state "$state_file" "$override_file" || true
     return 1
@@ -249,7 +335,7 @@ STATE
     return 1
   fi
   ln -sfn "$state_file" "$STATE_DIR/latest"
-  printf 'Deployment %s is healthy. Image digest: %s\n' "$run_id" "${image_ref##*@}"
+  printf 'Deployment: PASS. Image digest: %s\n' "${image_ref##*@}"
   printf 'Free space: before=%s MiB after=%s MiB\n' "$((free_before / 1024))" "$((free_after / 1024))"
 }
 
@@ -264,7 +350,7 @@ rollback() {
     state_file="$STATE_DIR/${run_id}.state"
   fi
   rollback_state "$state_file" "$override_file"
-  printf 'Rollback to saved image IDs from %s is healthy.\n' "$state_file"
+  printf 'Rollback: PASS. Saved image IDs are healthy.\n'
 }
 
 main() {