fix(postgres): make pgbackrest backups survive transient GCS 429s

docker-compose.yaml

@@ -120,20 +120,22 @@ services:
     labels:
       # NOTE: the cron format starts with seconds, instead of minutes. https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc
       ofelia.enabled: "true"
+      # NOTE: Ofelia has no retry support, so the commands are wrapped by
+      # pgbackrest-with-retry to survive transient repository errors.
       # XXX: Make sure to align this with the retention policies ./postgres/pgbackrest/pgbackrest.conf
       # Full backup every month (At 10:00 on day-of-month 1)
       ofelia.job-exec.full.schedule: "0 0 10 1 * *"
-      ofelia.job-exec.full.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=full"
+      ofelia.job-exec.full.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest-with-retry --stanza=main backup --type=full"
       ofelia.job-exec.full.user: "postgres"
       ofelia.job-exec.full.no-overlap: 1
       # Differential backup every week (At 02:30 on Monday)
       ofelia.job-exec.diff.schedule: "0 30 2 * * 1"
-      ofelia.job-exec.diff.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=diff"
+      ofelia.job-exec.diff.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest-with-retry --stanza=main backup --type=diff"
       ofelia.job-exec.diff.user: "postgres"
       ofelia.job-exec.diff.no-overlap: 1
       # Expire every day (At 01:30)
       ofelia.job-exec.expire.schedule: "0 30 01 * * *"
-      ofelia.job-exec.expire.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main expire"
+      ofelia.job-exec.expire.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest-with-retry --stanza=main expire"
       ofelia.job-exec.expire.user: "postgres"
       ofelia.job-exec.expire.no-overlap: 1
     secrets:

postgres/Dockerfile

@@ -2,7 +2,7 @@ ARG POSTGIS_VERSION=17-3.5  # postgresqlVersion-postgisVersion
 
 FROM postgis/postgis:${POSTGIS_VERSION} AS pgbackrest-build
 
-ARG PGBACKREST_VERSION=2.55.1
+ARG PGBACKREST_VERSION=2.59.0
 
 WORKDIR /build
 
@@ -24,20 +24,21 @@ RUN apt-get update -y && \
         libssh2-1-dev
 
 RUN wget -q -O - \
-    "https://github.com/pgbackrest/pgbackrest/archive/release/${PGBACKREST_VERSION}.tar.gz" | \
+    "https://github.com/pgbackrest/pgbackrest/releases/download/release/${PGBACKREST_VERSION}/pgbackrest-${PGBACKREST_VERSION}.tar.gz" | \
     tar zx -C ./
 
-RUN meson setup pgbackrest "pgbackrest-release-${PGBACKREST_VERSION}" && \
+RUN meson setup pgbackrest "pgbackrest-${PGBACKREST_VERSION}" && \
     ninja -C pgbackrest
 
 # ---------------- Final image ------
 FROM postgis/postgis:$POSTGIS_VERSION AS postgis
 
 COPY pgbackrest/setup.sh /pgbackrest-setup.sh
+COPY pgbackrest/run-with-retry.sh /usr/local/bin/pgbackrest-with-retry
 
 COPY --from=pgbackrest-build /build/pgbackrest/src/pgbackrest /usr/bin
 
-RUN chmod +x /usr/bin/pgbackrest && \
+RUN chmod +x /usr/bin/pgbackrest /usr/local/bin/pgbackrest-with-retry && \
     pgbackrest version
 
 CMD ["postgres"]

postgres/pgbackrest/run-with-retry.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Run pgbackrest with the given arguments, retrying on failure.
+#
+# Ofelia has no retry support, so a failed scheduled job is simply skipped
+# until the next window -- a missed monthly full or weekly diff backup.
+# Transient repository errors (eg. GCS 429 on object mutation) are common
+# enough that they should not cost us a whole backup cycle.
+
+set -u
+
+RETRY_MAX=${PGBACKREST_RETRY_MAX:-3}
+RETRY_DELAY=${PGBACKREST_RETRY_DELAY:-300}
+
+for attempt in $(seq 1 "$RETRY_MAX"); do
+  if pgbackrest "$@"; then
+    exit 0
+  fi
+
+  if [[ "$attempt" -lt "$RETRY_MAX" ]]; then
+    echo "pgbackrest $* failed (attempt $attempt/$RETRY_MAX), retrying in ${RETRY_DELAY}s" >&2
+    sleep "$RETRY_DELAY"
+  fi
+done
+
+echo "pgbackrest $* failed after $RETRY_MAX attempts" >&2
+exit 1