From 54e8e1018ecb6bd8cf0c2865728744fde4ddb71a Mon Sep 17 00:00:00 2001 From: MDA2AV Date: Fri, 24 Jul 2026 23:45:56 +0100 Subject: [PATCH] Pin every Dockerfile base image to a version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #471. 24 FROM lines across 16 frameworks named a floating tag, so two runs months apart could measure different software with no way to tell from the results. Every pin here resolves to the image the floating tag pointed at when this was written, checked against the registry first, so no build changes and no published result is invalidated: debian:stable-slim -> debian:trixie-slim 15x stable is Debian 13 today oven/bun:latest -> oven/bun:1.3 3x both 1.3.14 (one had no tag) dart:stable -> dart:3.12 2x both 3.12.2 trueasync…:latest-frankenphp -> …:0.8.4-php8.6-frankenphp 2x identical digest buildpack-deps:curl -> buildpack-deps:bookworm-curl 1x gcr.io/distroless/base -> base-debian12@sha256:6273… 1x distroless publishes no version tags at all, so that one is pinned by digest. Left alone: debian:bookworm-slim, debian:trixie, buildpack-deps:bookworm and similar. A Debian codename *is* a release number - bookworm is 12, trixie is 13 - which is the major.minor the issue asks for. validate.sh now rejects floating tags before spending a build on them, the same way it rejects unknown test names. It understands multi-stage builds (a FROM naming an earlier stage is not an image), build args, scratch, digest pins, and treats a distro codename as a version. Checked against all 132 Dockerfiles: 0 unpinned. Checked against a synthetic file: node:latest and a bare python are both rejected, rust:1.83 and a stage reference are not. Documented under Add a Framework, since this is a rule contributors need before they open a PR rather than after. --- frameworks/bun-websocket/Dockerfile | 2 +- frameworks/dart-io/Dockerfile | 2 +- frameworks/elysia/Dockerfile | 5 +- frameworks/fasthttp/Dockerfile | 4 +- frameworks/fletch/Dockerfile | 2 +- frameworks/frankenphp-trueasync/Dockerfile | 2 +- frameworks/hono-bun/Dockerfile | 2 +- frameworks/lute/Dockerfile | 4 +- frameworks/pico/Dockerfile | 4 +- frameworks/symfony-spawn-franken/Dockerfile | 2 +- frameworks/vanilla-epoll/Dockerfile | 4 +- frameworks/vanilla-h2c/Dockerfile | 4 +- frameworks/vanilla-io_uring/Dockerfile | 4 +- frameworks/vanilla-ws/Dockerfile | 4 +- frameworks/veb/Dockerfile | 4 +- scripts/validate.sh | 46 +++++++++++++++++++ .../docs/add-framework/directory-structure.md | 18 ++++++++ 17 files changed, 89 insertions(+), 24 deletions(-) diff --git a/frameworks/bun-websocket/Dockerfile b/frameworks/bun-websocket/Dockerfile index 8c420ab25..9bfa5e9a2 100644 --- a/frameworks/bun-websocket/Dockerfile +++ b/frameworks/bun-websocket/Dockerfile @@ -1,4 +1,4 @@ -FROM oven/bun:latest +FROM oven/bun:1.3 WORKDIR /app COPY server.ts entrypoint.sh ./ RUN chmod +x entrypoint.sh diff --git a/frameworks/dart-io/Dockerfile b/frameworks/dart-io/Dockerfile index 2b9323d00..8d658bc9a 100644 --- a/frameworks/dart-io/Dockerfile +++ b/frameworks/dart-io/Dockerfile @@ -1,5 +1,5 @@ # Stage 1 — AOT compile the Dart server + build SO_REUSEPORT shim -FROM dart:stable AS build +FROM dart:3.12 AS build WORKDIR /app COPY frameworks/dart-io/pubspec.yaml ./ diff --git a/frameworks/elysia/Dockerfile b/frameworks/elysia/Dockerfile index a3413950a..b86fe2280 100644 --- a/frameworks/elysia/Dockerfile +++ b/frameworks/elysia/Dockerfile @@ -1,5 +1,5 @@ # https://elysiajs.com/patterns/deploy.html#deploy-to-production -FROM oven/bun AS build +FROM oven/bun:1.3 AS build WORKDIR /app @@ -20,7 +20,8 @@ RUN bun build \ --outfile server \ server.ts -FROM gcr.io/distroless/base +# distroless publishes no version tags, so pin the digest +FROM gcr.io/distroless/base-debian12@sha256:62730825d3cf03571e0a1b8f014748de94d0404500f063593b614c23da38841d WORKDIR /app diff --git a/frameworks/fasthttp/Dockerfile b/frameworks/fasthttp/Dockerfile index 62305118d..851bdb508 100644 --- a/frameworks/fasthttp/Dockerfile +++ b/frameworks/fasthttp/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable-slim AS build +FROM debian:trixie-slim AS build RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends \ @@ -29,7 +29,7 @@ COPY . . # per-request allocation. Boehm keeps memory bounded; c0624b2's codegen wins still apply. RUN v -prod . -o server -FROM debian:stable-slim +FROM debian:trixie-slim RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends libpq5 && \ rm -rf /var/lib/apt/lists/* diff --git a/frameworks/fletch/Dockerfile b/frameworks/fletch/Dockerfile index 834988618..11f73cb18 100644 --- a/frameworks/fletch/Dockerfile +++ b/frameworks/fletch/Dockerfile @@ -1,5 +1,5 @@ # Stage 1 — AOT compile the Dart server + build SO_REUSEPORT shim -FROM dart:stable AS build +FROM dart:3.12 AS build WORKDIR /app COPY pubspec.yaml ./ diff --git a/frameworks/frankenphp-trueasync/Dockerfile b/frameworks/frankenphp-trueasync/Dockerfile index 82b14ec76..95c904571 100644 --- a/frameworks/frankenphp-trueasync/Dockerfile +++ b/frameworks/frankenphp-trueasync/Dockerfile @@ -1,4 +1,4 @@ -FROM trueasync/php-true-async:latest-frankenphp +FROM trueasync/php-true-async:0.8.4-php8.6-frankenphp WORKDIR /app diff --git a/frameworks/hono-bun/Dockerfile b/frameworks/hono-bun/Dockerfile index bc0da0afb..9160c8013 100644 --- a/frameworks/hono-bun/Dockerfile +++ b/frameworks/hono-bun/Dockerfile @@ -1,4 +1,4 @@ -FROM oven/bun:latest +FROM oven/bun:1.3 WORKDIR /app COPY package.json ./ RUN bun install --production diff --git a/frameworks/lute/Dockerfile b/frameworks/lute/Dockerfile index 4b72cdba9..db4effb38 100644 --- a/frameworks/lute/Dockerfile +++ b/frameworks/lute/Dockerfile @@ -1,6 +1,6 @@ ARG LUTE_VERSION=1.0.1-nightly.20260421 -FROM buildpack-deps:curl AS bin +FROM buildpack-deps:bookworm-curl AS bin RUN apt-get update && apt-get install -y --no-install-recommends unzip \ && rm -rf /var/lib/apt/lists/* @@ -13,7 +13,7 @@ RUN ARCH=$(echo "${TARGETARCH:-amd64}" | sed -e 's/arm64/aarch64/' -e 's/amd64/x && unzip lute.zip \ && chmod +x lute -FROM debian:stable-slim +FROM debian:trixie-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates libssl3 \ diff --git a/frameworks/pico/Dockerfile b/frameworks/pico/Dockerfile index 7e7580ddc..9ae3bbdac 100644 --- a/frameworks/pico/Dockerfile +++ b/frameworks/pico/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable-slim AS build +FROM debian:trixie-slim AS build RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends \ @@ -19,7 +19,7 @@ COPY . . # the default garbage collector is the safe choice here. RUN v -prod . -o server -FROM debian:stable-slim +FROM debian:trixie-slim RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends util-linux && \ rm -rf /var/lib/apt/lists/* diff --git a/frameworks/symfony-spawn-franken/Dockerfile b/frameworks/symfony-spawn-franken/Dockerfile index 6a9729369..bd820b4d2 100644 --- a/frameworks/symfony-spawn-franken/Dockerfile +++ b/frameworks/symfony-spawn-franken/Dockerfile @@ -1,4 +1,4 @@ -FROM trueasync/php-true-async:latest-frankenphp +FROM trueasync/php-true-async:0.8.4-php8.6-frankenphp COPY --from=composer:latest /usr/bin/composer /usr/local/bin/composer diff --git a/frameworks/vanilla-epoll/Dockerfile b/frameworks/vanilla-epoll/Dockerfile index 56d3cc9d7..5d8673c90 100644 --- a/frameworks/vanilla-epoll/Dockerfile +++ b/frameworks/vanilla-epoll/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable-slim AS build +FROM debian:trixie-slim AS build RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends \ @@ -79,7 +79,7 @@ COPY . . # :8081); without it the tls module is a stub and the binary stays mbedtls-free. RUN v -prod -gc none -d vanilla_tls . -o server -FROM debian:stable-slim +FROM debian:trixie-slim RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends libpq5 liburing2 && \ rm -rf /var/lib/apt/lists/* diff --git a/frameworks/vanilla-h2c/Dockerfile b/frameworks/vanilla-h2c/Dockerfile index 8e637bc7c..3ef022bb8 100644 --- a/frameworks/vanilla-h2c/Dockerfile +++ b/frameworks/vanilla-h2c/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable-slim AS build +FROM debian:trixie-slim AS build RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends \ @@ -34,7 +34,7 @@ COPY . . # allocation-free and the read-only dataset lives for the process lifetime. RUN v -prod -gc none . -o server -FROM debian:stable-slim +FROM debian:trixie-slim # The binary links liburing.so.2 (the io_uring backend is compiled in on Linux # even though this entry selects epoll at runtime). RUN apt-get -qq update && \ diff --git a/frameworks/vanilla-io_uring/Dockerfile b/frameworks/vanilla-io_uring/Dockerfile index fbb365625..52409ac15 100644 --- a/frameworks/vanilla-io_uring/Dockerfile +++ b/frameworks/vanilla-io_uring/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable-slim AS build +FROM debian:trixie-slim AS build RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends \ @@ -71,7 +71,7 @@ COPY . . # module is a stub and mbedtls isn't linked). RUN v -prod -d vanilla_tls . -o server -FROM debian:stable-slim +FROM debian:trixie-slim RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends libpq5 liburing2 && \ rm -rf /var/lib/apt/lists/* diff --git a/frameworks/vanilla-ws/Dockerfile b/frameworks/vanilla-ws/Dockerfile index d6d9d5376..55aa2f990 100644 --- a/frameworks/vanilla-ws/Dockerfile +++ b/frameworks/vanilla-ws/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable-slim AS build +FROM debian:trixie-slim AS build RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends \ @@ -31,7 +31,7 @@ WORKDIR /app COPY . . RUN v -prod -gc none . -o server -FROM debian:stable-slim +FROM debian:trixie-slim # The binary links liburing.so.2 (the io_uring backend is compiled in on # Linux even though this entry selects epoll at runtime). RUN apt-get -qq update && \ diff --git a/frameworks/veb/Dockerfile b/frameworks/veb/Dockerfile index c2d29f564..b7ba9a0a7 100644 --- a/frameworks/veb/Dockerfile +++ b/frameworks/veb/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable-slim AS build +FROM debian:trixie-slim AS build RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends \ @@ -28,7 +28,7 @@ COPY . . # per-request allocation. Boehm keeps memory bounded; c0624b2's codegen wins still apply. RUN v -prod . -o server -FROM debian:stable-slim +FROM debian:trixie-slim RUN apt-get -qq update && \ apt-get -qy install --no-install-recommends libpq5 && \ rm -rf /var/lib/apt/lists/* diff --git a/scripts/validate.sh b/scripts/validate.sh index eee711eeb..1cbc6b2a3 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -123,6 +123,52 @@ has_test() { [[ " $TESTS " == *" $1 "* ]] } +# A benchmark has to be reproducible, so every base image must name a version +# (#471). Floating tags - latest, stable, nightly, or no tag at all - mean two +# runs months apart measure different software and the numbers can't be +# compared. major.minor is enough; a Debian/Ubuntu codename counts as one. +FROM_FILE="$ROOT_DIR/frameworks/$FRAMEWORK/Dockerfile" +if [ -f "$FROM_FILE" ]; then + UNPINNED=$(python3 - "$FROM_FILE" <<'PYEOF' +import re, sys + +FLOATING = {"latest", "stable", "nightly", "edge", "main", "master", + "current", "devel", "testing", "rolling", "unstable"} +# distro codenames are a release, i.e. a major version +CODENAMES = {"bookworm", "trixie", "bullseye", "buster", "sid", + "jammy", "noble", "focal", "bionic", "plucky", "oracular"} + +stages, bad = set(), [] +for n, line in enumerate(open(sys.argv[1], encoding="utf-8", errors="replace"), 1): + m = re.match(r"\s*FROM\s+(\S+)(?:\s+AS\s+(\S+))?", line, re.I) + if not m: + continue + img, alias = m.group(1), m.group(2) + if alias: + stages.add(alias.lower()) + if img.lower() in stages or img.startswith("$") or img.lower() == "scratch": + continue # earlier build stage, build arg, or scratch + if "@sha256:" in img: + continue # digest pin is the strongest form + name, _, tag = img.partition(":") + if not tag: + bad.append((n, img, "no tag (same as :latest)")) + elif tag.lower() in FLOATING or tag.lower().split("-")[0] in FLOATING: + bad.append((n, img, f"floating tag ':{tag}'")) + elif not any(c.isdigit() for c in tag) and not any(c in CODENAMES for c in tag.lower().split("-")): + bad.append((n, img, f"no version in ':{tag}'")) +for n, img, why in bad: + print(f"line {n}: {img} - {why}") +PYEOF +) + if [ -n "$UNPINNED" ]; then + echo "FAIL: Dockerfile base images must name a version (see #471)" + printf '%s\n' "$UNPINNED" | sed 's/^/ /' + echo " use e.g. node:22, python:3.13-slim, debian:trixie-slim, or an @sha256 digest" + exit 1 + fi +fi + # Build — skip standalone build if framework only subscribes to compose profiles # (gateway-64, gateway-h3, production-stack) and has no isolated tests. GATEWAY_ONLY=true diff --git a/site/content/docs/add-framework/directory-structure.md b/site/content/docs/add-framework/directory-structure.md index 8bd91a4f9..bfe0fb084 100644 --- a/site/content/docs/add-framework/directory-structure.md +++ b/site/content/docs/add-framework/directory-structure.md @@ -14,6 +14,24 @@ frameworks/ ... (source files) ``` +## Pin your base images + +A benchmark is only meaningful if it can be re-run, so every `FROM` must name a version. `latest`, `stable`, `nightly` or no tag at all mean two runs months apart measure different software, and the numbers stop being comparable. + +```dockerfile +FROM node:22 # good - major +FROM python:3.13-slim # good - major.minor +FROM debian:trixie-slim # good - a Debian release is a version +FROM rust:1.83 AS build # good +FROM gcr.io/distroless/base-debian12@sha256:… # good - digest, for images with no version tags + +FROM node:latest # rejected +FROM python # rejected (same as :latest) +FROM debian:stable-slim # rejected - "stable" moves with each Debian release +``` + +`scripts/validate.sh` enforces this, so a floating tag fails the PR check before a build is spent on it. Patch versions are not required - `3.13` is fine, `3.13.1` is also fine. + ## Dockerfile The Dockerfile should build and run your server. Containers are started with `--network host`, so bind to: