diff --git a/docker-compose.yaml b/docker-compose.yaml index d254c4c..85bb01b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -120,20 +120,22 @@ services: labels: # NOTE: the cron format starts with seconds, instead of minutes. https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc ofelia.enabled: "true" + # NOTE: Ofelia has no retry support, so the commands are wrapped by + # pgbackrest-with-retry to survive transient repository errors. # XXX: Make sure to align this with the retention policies ./postgres/pgbackrest/pgbackrest.conf # Full backup every month (At 10:00 on day-of-month 1) ofelia.job-exec.full.schedule: "0 0 10 1 * *" - ofelia.job-exec.full.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=full" + ofelia.job-exec.full.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest-with-retry --stanza=main backup --type=full" ofelia.job-exec.full.user: "postgres" ofelia.job-exec.full.no-overlap: 1 # Differential backup every week (At 02:30 on Monday) ofelia.job-exec.diff.schedule: "0 30 2 * * 1" - ofelia.job-exec.diff.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=diff" + ofelia.job-exec.diff.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest-with-retry --stanza=main backup --type=diff" ofelia.job-exec.diff.user: "postgres" ofelia.job-exec.diff.no-overlap: 1 # Expire every day (At 01:30) ofelia.job-exec.expire.schedule: "0 30 01 * * *" - ofelia.job-exec.expire.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main expire" + ofelia.job-exec.expire.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest-with-retry --stanza=main expire" ofelia.job-exec.expire.user: "postgres" ofelia.job-exec.expire.no-overlap: 1 secrets: diff --git a/postgres/Dockerfile b/postgres/Dockerfile index eee8732..367b744 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -2,7 +2,7 @@ ARG POSTGIS_VERSION=17-3.5 # postgresqlVersion-postgisVersion FROM postgis/postgis:${POSTGIS_VERSION} AS pgbackrest-build -ARG PGBACKREST_VERSION=2.55.1 +ARG PGBACKREST_VERSION=2.59.0 WORKDIR /build @@ -24,20 +24,21 @@ RUN apt-get update -y && \ libssh2-1-dev RUN wget -q -O - \ - "https://github.com/pgbackrest/pgbackrest/archive/release/${PGBACKREST_VERSION}.tar.gz" | \ + "https://github.com/pgbackrest/pgbackrest/releases/download/release/${PGBACKREST_VERSION}/pgbackrest-${PGBACKREST_VERSION}.tar.gz" | \ tar zx -C ./ -RUN meson setup pgbackrest "pgbackrest-release-${PGBACKREST_VERSION}" && \ +RUN meson setup pgbackrest "pgbackrest-${PGBACKREST_VERSION}" && \ ninja -C pgbackrest # ---------------- Final image ------ FROM postgis/postgis:$POSTGIS_VERSION AS postgis COPY pgbackrest/setup.sh /pgbackrest-setup.sh +COPY pgbackrest/run-with-retry.sh /usr/local/bin/pgbackrest-with-retry COPY --from=pgbackrest-build /build/pgbackrest/src/pgbackrest /usr/bin -RUN chmod +x /usr/bin/pgbackrest && \ +RUN chmod +x /usr/bin/pgbackrest /usr/local/bin/pgbackrest-with-retry && \ pgbackrest version CMD ["postgres"] diff --git a/postgres/pgbackrest/run-with-retry.sh b/postgres/pgbackrest/run-with-retry.sh new file mode 100755 index 0000000..1501d33 --- /dev/null +++ b/postgres/pgbackrest/run-with-retry.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Run pgbackrest with the given arguments, retrying on failure. +# +# Ofelia has no retry support, so a failed scheduled job is simply skipped +# until the next window -- a missed monthly full or weekly diff backup. +# Transient repository errors (eg. GCS 429 on object mutation) are common +# enough that they should not cost us a whole backup cycle. + +set -u + +RETRY_MAX=${PGBACKREST_RETRY_MAX:-3} +RETRY_DELAY=${PGBACKREST_RETRY_DELAY:-300} + +for attempt in $(seq 1 "$RETRY_MAX"); do + if pgbackrest "$@"; then + exit 0 + fi + + if [[ "$attempt" -lt "$RETRY_MAX" ]]; then + echo "pgbackrest $* failed (attempt $attempt/$RETRY_MAX), retrying in ${RETRY_DELAY}s" >&2 + sleep "$RETRY_DELAY" + fi +done + +echo "pgbackrest $* failed after $RETRY_MAX attempts" >&2 +exit 1