Skip to content

Commit b3c7a3c

Browse files
authored
Merge pull request #36 from sprahasingh/fix/ghcr-deployment-security
fix: harden GHCR deployment security and logging
2 parents ff26963 + 4a13aec commit b3c7a3c

4 files changed

Lines changed: 428 additions & 43 deletions

File tree

‎.github/workflows/production.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ jobs:
8282
run: >-
8383
docker run --rm --network none
8484
--volume "$GITHUB_WORKSPACE/tests:/tmp/codelens-tests:ro"
85+
--volume "$GITHUB_WORKSPACE/scripts:/tmp/scripts:ro"
8586
--volume "$GITHUB_WORKSPACE/pytest.ini:/tmp/pytest.ini:ro"
8687
--env DEBUG=false
8788
--env DATABASE_URL=postgresql+asyncpg://test:test@127.0.0.1/test
@@ -132,6 +133,7 @@ jobs:
132133
run: >-
133134
docker run --rm --network none
134135
--volume "$GITHUB_WORKSPACE/tests:/tmp/codelens-tests:ro"
136+
--volume "$GITHUB_WORKSPACE/scripts:/tmp/scripts:ro"
135137
--volume "$GITHUB_WORKSPACE/pytest.ini:/tmp/pytest.ini:ro"
136138
--env DEBUG=false
137139
--env DATABASE_URL=postgresql+asyncpg://test:test@127.0.0.1/test

‎docs/ghcr-deployment.md‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,16 @@ deployment helper.
3939
## First controlled deployment
4040

4141
1. Do not proceed while the root filesystem has less than **1536 MiB free**.
42-
The latest read-only check showed about **500 MiB free**, so the helper will
43-
stop before creating rollback tags or pulling an image. The first GHCR image
44-
can require roughly the current 672 MB uncompressed application-image size,
45-
plus pull/unpack overhead; the helper also requires at least 300 MiB to remain
46-
after pulling. It does not prune images or volumes to create space.
47-
2. Resolve the exposed local environment credentials noted in the deployment
48-
preparation report before starting the workflow.
42+
The EC2 EBS volume was expanded to **12 GiB** and the root filesystem was
43+
grown online. A subsequent check showed about **4.37 GiB free** (the exact
44+
amount changes as Docker images and logs grow). The helper still checks the
45+
live free space and inodes before changing anything; it requires 1536 MiB
46+
before a pull and at least 300 MiB after it. It does not prune images or
47+
volumes to create space.
48+
2. The previously exposed credential categories have been rotated and the old
49+
credentials revoked. Keep the replacement values only in their protected
50+
provider settings and production `.env`; never paste or print them in Actions
51+
logs or deployment diagnostics.
4952
3. On the `main` branch, run Actions → **Test, publish, and deploy CodeLens** →
5053
Run workflow → `deploy`. The image is published only after both test runs
5154
pass. Approve the `production` environment job only after reviewing the
@@ -64,8 +67,10 @@ deployment helper.
6467

6568
The short-lived GHCR token is transferred over the verified SSH connection,
6669
stored temporarily with mode `0600`, used through Docker's `--password-stdin`,
67-
and removed. Docker credentials use a temporary `DOCKER_CONFIG` and are removed
68-
when the helper exits.
70+
and removed. Docker credentials use a temporary mode-`0700` `DOCKER_CONFIG` and
71+
are removed when the helper exits, including failure and handled-signal paths.
72+
Authentication, pull, task-inspection, and Compose errors are reported using
73+
fixed sanitized messages; raw command output is not forwarded to Actions logs.
6974

7075
## Rollback
7176

‎scripts/deploy_ghcr.sh‎

Lines changed: 120 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,43 @@ STATE_DIR="${CODELENS_STATE_DIR:-/home/ubuntu/.local/state/codelens-ghcr}"
88
MIN_FREE_MIB="${CODELENS_MIN_FREE_MIB:-1536}"
99
POST_PULL_FREE_MIB="${CODELENS_POST_PULL_FREE_MIB:-300}"
1010
PUBLIC_HEALTH_URL="${CODELENS_PUBLIC_HEALTH_URL:-https://13-51-158-45.sslip.io/health}"
11+
AUTH_TMP_ROOT="${TMPDIR:-/tmp}"
12+
AUTH_TOKEN_FILE=""
13+
AUTH_DOCKER_CONFIG=""
14+
15+
cleanup_auth() {
16+
local cleanup_failed=0 token_name exit_status=$?
17+
18+
if [[ -n "${AUTH_DOCKER_CONFIG:-}" ]]; then
19+
case "$AUTH_DOCKER_CONFIG" in
20+
"$AUTH_TMP_ROOT"/codelens-ghcr-auth.*)
21+
rm -rf -- "$AUTH_DOCKER_CONFIG" >/dev/null 2>&1 || cleanup_failed=1
22+
;;
23+
*) cleanup_failed=1 ;;
24+
esac
25+
fi
26+
27+
if [[ -n "${AUTH_TOKEN_FILE:-}" ]]; then
28+
token_name="${AUTH_TOKEN_FILE##*/}"
29+
if [[ "$AUTH_TOKEN_FILE" == "/tmp/$token_name" && "$token_name" =~ ^codelens-ghcr-token-[A-Za-z0-9._-]+$ ]]; then
30+
rm -f -- "$AUTH_TOKEN_FILE" >/dev/null 2>&1 || cleanup_failed=1
31+
else
32+
cleanup_failed=1
33+
fi
34+
fi
35+
36+
if (( cleanup_failed )); then
37+
printf 'WARNING: temporary authentication cleanup was incomplete.\n' >&2
38+
(( exit_status != 0 )) || exit_status=1
39+
fi
40+
trap - EXIT
41+
exit "$exit_status"
42+
}
43+
44+
trap cleanup_auth EXIT
45+
trap 'exit 129' HUP
46+
trap 'exit 130' INT
47+
trap 'exit 143' TERM
1148

1249
usage() {
1350
cat <<'USAGE'
@@ -53,20 +90,26 @@ check_preconditions() {
5390
[[ -f "$COMPOSE_DIR/docker-compose.yml" && -f "$COMPOSE_DIR/docker-compose.prod.yml" ]] || {
5491
echo "STOP: expected Compose files are missing." >&2; return 1;
5592
}
56-
compose_version="$(docker compose version --short | sed 's/^v//')"
57-
if ! python3 -c 'import re, sys; v=tuple(int(x) for x in re.match(r"^(\d+)\.(\d+)\.(\d+)", sys.argv[1]).groups()); raise SystemExit(0 if v >= (2, 24, 4) else 1)' "$compose_version"; then
93+
compose_version="$(docker compose version --short 2>/dev/null)" || {
94+
echo "STOP: unable to verify Docker Compose version." >&2; return 1;
95+
}
96+
if ! python3 -c 'import re, sys; m=re.match(r"^v?(\d+)\.(\d+)\.(\d+)", sys.argv[1]); raise SystemExit(0 if m and tuple(map(int, m.groups())) >= (2, 24, 4) else 1)' "$compose_version" >/dev/null 2>&1; then
5897
echo "STOP: Docker Compose 2.24.4 or newer is required for build reset semantics." >&2
5998
return 1
6099
fi
61-
services="$(compose_base "$1" config --services 2>/dev/null | sort | tr '\n' ' ')"
100+
services="$(compose_base "$1" config --services 2>/dev/null | sort | tr '\n' ' ')" || {
101+
echo "STOP: unable to validate the Compose services." >&2; return 1;
102+
}
62103
for service in web worker flower caddy; do
63104
[[ " $services " == *" $service "* ]] || {
64105
echo "STOP: expected Compose service '$service' is absent." >&2; return 1;
65106
}
66107
done
67108
for service in web worker flower; do
68109
local cid
69-
cid="$(compose_base "$1" ps -q "$service")"
110+
cid="$(compose_base "$1" ps -q "$service" 2>/dev/null)" || {
111+
echo "STOP: unable to verify a running application service." >&2; return 1;
112+
}
70113
[[ -n "$cid" ]] || { echo "STOP: running $service container was not found." >&2; return 1; }
71114
done
72115
}
@@ -85,7 +128,7 @@ acquire_deployment_lock() {
85128
}
86129

87130
assert_celery_quiescent() {
88-
local output
131+
local output active reserved scheduled unacked unacked_index
89132
if ! output="$(timeout 25s docker exec codelens-worker-1 python -c '
90133
from app.core.celery_app import celery_app
91134
connection = celery_app.connection_for_read()
@@ -103,13 +146,25 @@ for method in ("active", "reserved", "scheduled"):
103146
raise SystemExit("worker inspection unavailable: " + method)
104147
counts[method] = sum(len(items) for items in response.values())
105148
print("active={active} reserved={reserved} scheduled={scheduled} unacked={unacked} unacked_index={unacked_index}".format(**counts, unacked=unacked, unacked_index=unacked_index))
106-
if any(counts.values()) or unacked or unacked_index:
107-
raise SystemExit(3)
108-
' 2>&1)"; then
109-
printf 'STOP: could not verify Celery task state safely: %s\n' "$output" >&2
149+
' 2>/dev/null)"; then
150+
echo "STOP: Celery task inspection failed; deployment aborted." >&2
151+
return 1
152+
fi
153+
if [[ ! "$output" =~ ^active=([0-9]+)[[:space:]]reserved=([0-9]+)[[:space:]]scheduled=([0-9]+)[[:space:]]unacked=([0-9]+)[[:space:]]unacked_index=([0-9]+)$ ]]; then
154+
echo "STOP: Celery task inspection returned unexpected output; deployment aborted." >&2
155+
return 1
156+
fi
157+
active="${BASH_REMATCH[1]}"
158+
reserved="${BASH_REMATCH[2]}"
159+
scheduled="${BASH_REMATCH[3]}"
160+
unacked="${BASH_REMATCH[4]}"
161+
unacked_index="${BASH_REMATCH[5]}"
162+
if (( active || reserved || scheduled || unacked || unacked_index )); then
163+
printf 'STOP: Celery is not quiescent (active=%s reserved=%s scheduled=%s unacked=%s unacked_index=%s).\n' \
164+
"$active" "$reserved" "$scheduled" "$unacked" "$unacked_index" >&2
110165
return 1
111166
fi
112-
printf 'Celery is quiescent: %s\n' "$output"
167+
printf 'Celery task inspection: PASS (active=0 reserved=0 scheduled=0 unacked=0 unacked_index=0).\n'
113168
}
114169

115170
verify_running_release() {
@@ -122,7 +177,7 @@ verify_running_release() {
122177
if [[ "$status" == healthy ]] && [[ "$flower_status" =~ ^(200|301|302|401|403)$ ]]; then
123178
if timeout 15s docker exec codelens-worker-1 python -c \
124179
'from app.core.celery_app import celery_app; r=celery_app.control.ping(timeout=5); raise SystemExit(0 if r else 1)' >/dev/null 2>&1; then
125-
if [[ -n "$PUBLIC_HEALTH_URL" ]] && curl --fail --silent --show-error --max-time 10 "$PUBLIC_HEALTH_URL" >/dev/null; then
180+
if [[ -n "$PUBLIC_HEALTH_URL" ]] && curl --fail --silent --max-time 10 "$PUBLIC_HEALTH_URL" >/dev/null 2>&1; then
126181
return 0
127182
fi
128183
# Public HTTPS can fail transiently while Caddy is serving a new web container.
@@ -162,7 +217,7 @@ rollback_state() {
162217
local state_file="$1" override_file="$2"
163218
[[ -f "$state_file" ]] || { echo "STOP: rollback state file is missing." >&2; return 1; }
164219
write_rollback_override "$state_file" "$override_file"
165-
if ! compose_base "$override_file" up -d --no-build --pull never --no-deps web worker flower; then
220+
if ! compose_base "$override_file" up -d --no-build --pull never --no-deps web worker flower >/dev/null 2>&1; then
166221
echo "Rollback Compose operation failed; stop for manual inspection." >&2
167222
return 1
168223
fi
@@ -174,6 +229,11 @@ deploy() {
174229
local run_id state_file temp_config web_id worker_id flower_id
175230
local web_tag worker_tag flower_tag revision platform free_before free_after
176231

232+
if [[ "$token_file" =~ ^/tmp/codelens-ghcr-token-[A-Za-z0-9._-]+$ ]]; then
233+
AUTH_TOKEN_FILE="$token_file"
234+
else
235+
echo "STOP: invalid short-lived GHCR token file path." >&2; return 1
236+
fi
177237
[[ "$image_ref" =~ ^ghcr\.io/[a-z0-9][a-z0-9._-]*/codelens@sha256:[a-f0-9]{64}$ ]] || {
178238
echo "STOP: image must be a lowercase GHCR repository reference pinned by sha256 digest." >&2; return 1;
179239
}
@@ -195,15 +255,27 @@ deploy() {
195255
[[ "$run_id" =~ ^[A-Za-z0-9._-]+$ ]] || { echo "STOP: invalid deployment id." >&2; return 1; }
196256
state_file="$STATE_DIR/${run_id}.state"
197257
[[ ! -e "$state_file" ]] || { echo "STOP: deployment state already exists." >&2; return 1; }
198-
web_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q web)")"
199-
worker_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q worker)")"
200-
flower_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q flower)")"
258+
web_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q web 2>/dev/null)" 2>/dev/null)" || {
259+
echo "STOP: unable to record the running web image for rollback." >&2; return 1;
260+
}
261+
worker_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q worker 2>/dev/null)" 2>/dev/null)" || {
262+
echo "STOP: unable to record the running worker image for rollback." >&2; return 1;
263+
}
264+
flower_id="$(docker inspect --format '{{.Image}}' "$(compose_base "$override_file" ps -q flower 2>/dev/null)" 2>/dev/null)" || {
265+
echo "STOP: unable to record the running Flower image for rollback." >&2; return 1;
266+
}
201267
web_tag="codelens-web:rollback-${run_id}"
202268
worker_tag="codelens-worker:rollback-${run_id}"
203269
flower_tag="codelens-flower:rollback-${run_id}"
204-
docker image tag "$web_id" "$web_tag"
205-
docker image tag "$worker_id" "$worker_tag"
206-
docker image tag "$flower_id" "$flower_tag"
270+
if ! docker image tag "$web_id" "$web_tag" >/dev/null 2>&1; then
271+
echo "STOP: unable to preserve the web rollback image." >&2; return 1;
272+
fi
273+
if ! docker image tag "$worker_id" "$worker_tag" >/dev/null 2>&1; then
274+
echo "STOP: unable to preserve the worker rollback image." >&2; return 1;
275+
fi
276+
if ! docker image tag "$flower_id" "$flower_tag" >/dev/null 2>&1; then
277+
echo "STOP: unable to preserve the Flower rollback image." >&2; return 1;
278+
fi
207279
cat > "$state_file" <<STATE
208280
DEPLOYMENT_ID='$run_id'
209281
IMAGE_REF='$image_ref'
@@ -217,28 +289,42 @@ FLOWER_ROLLBACK_TAG='$flower_tag'
217289
STATE
218290
chmod 600 "$state_file"
219291

220-
temp_config="$(mktemp -d /tmp/codelens-ghcr-auth.XXXXXX)"
221-
export DOCKER_CONFIG="$temp_config"
222-
cleanup_auth() {
223-
rm -rf -- "$temp_config"
224-
rm -f -- "$token_file"
292+
temp_config="$(mktemp -d "$AUTH_TMP_ROOT/codelens-ghcr-auth.XXXXXX" 2>/dev/null)" || {
293+
echo "STOP: unable to create temporary Docker authentication storage." >&2; return 1;
225294
}
226-
trap cleanup_auth EXIT
227-
docker login ghcr.io --username "$ghcr_username" --password-stdin < "$token_file" >/dev/null
228-
rm -f -- "$token_file"
229-
docker pull "$image_ref"
295+
AUTH_DOCKER_CONFIG="$temp_config"
296+
chmod 700 "$temp_config" 2>/dev/null || {
297+
echo "STOP: unable to secure temporary Docker authentication storage." >&2; return 1;
298+
}
299+
export DOCKER_CONFIG="$temp_config"
300+
if ! docker login ghcr.io --username "$ghcr_username" --password-stdin < "$token_file" >/dev/null 2>&1; then
301+
echo "STOP: GHCR authentication failed." >&2; return 1
302+
fi
303+
if ! rm -f -- "$token_file" >/dev/null 2>&1; then
304+
echo "STOP: unable to remove the temporary GHCR token file." >&2; return 1
305+
fi
306+
AUTH_TOKEN_FILE=""
307+
if ! docker pull "$image_ref" >/dev/null 2>&1; then
308+
echo "STOP: unable to pull the pinned GHCR image." >&2; return 1
309+
fi
230310
free_after="$(free_kib)"
231311
if (( free_after < POST_PULL_FREE_MIB * 1024 )); then
232312
echo "STOP: image pulled but free space fell below the post-pull safety floor; services were not changed." >&2
233313
return 1
234314
fi
235-
revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image_ref")"
236-
platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image_ref")"
315+
revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image_ref" 2>/dev/null)" || {
316+
echo "STOP: unable to verify the published image revision." >&2; return 1;
317+
}
318+
platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image_ref" 2>/dev/null)" || {
319+
echo "STOP: unable to verify the published image platform." >&2; return 1;
320+
}
237321
[[ "$revision" == "$expected_sha" ]] || { echo "STOP: image revision label does not match the requested commit." >&2; return 1; }
238-
[[ "$platform" == linux/amd64 ]] || { echo "STOP: image platform $platform does not match EC2 linux/amd64." >&2; return 1; }
239-
compose_base "$override_file" config --quiet
322+
[[ "$platform" == linux/amd64 ]] || { echo "STOP: image platform does not match EC2 linux/amd64." >&2; return 1; }
323+
if ! compose_base "$override_file" config --quiet >/dev/null 2>&1; then
324+
echo "STOP: the deployment Compose configuration is invalid." >&2; return 1;
325+
fi
240326

241-
if ! compose_base "$override_file" up -d --no-build --pull never --no-deps web worker flower; then
327+
if ! compose_base "$override_file" up -d --no-build --pull never --no-deps web worker flower >/dev/null 2>&1; then
242328
echo "Deployment update failed; restoring the recorded image IDs." >&2
243329
rollback_state "$state_file" "$override_file" || true
244330
return 1
@@ -249,7 +335,7 @@ STATE
249335
return 1
250336
fi
251337
ln -sfn "$state_file" "$STATE_DIR/latest"
252-
printf 'Deployment %s is healthy. Image digest: %s\n' "$run_id" "${image_ref##*@}"
338+
printf 'Deployment: PASS. Image digest: %s\n' "${image_ref##*@}"
253339
printf 'Free space: before=%s MiB after=%s MiB\n' "$((free_before / 1024))" "$((free_after / 1024))"
254340
}
255341

@@ -264,7 +350,7 @@ rollback() {
264350
state_file="$STATE_DIR/${run_id}.state"
265351
fi
266352
rollback_state "$state_file" "$override_file"
267-
printf 'Rollback to saved image IDs from %s is healthy.\n' "$state_file"
353+
printf 'Rollback: PASS. Saved image IDs are healthy.\n'
268354
}
269355

270356
main() {

0 commit comments

Comments
 (0)