gendesign/ops/backup.sh

#!/usr/bin/env bash
# Daily Postgres backup for the MAIN gendesign DB. Runs from cron on the prod VM.
#
# Modeled on the proven ops/.../backup-tradein-db.sh (#397). Hardened for #71
# after the main-DB backup silently broke (last good dump 2026-05-27): a raw
# deploy-time hard reset of the repo checkout onto origin/main reset this
# file's mode to 644, so a cron entry that invoked the raw path got
# "Permission denied" every run.
#
# Robustness measures here (extended for #2203):
#   - cron should invoke via `bash <path>` so a missing +x bit can't break it
#     (this file is ALSO committed 100755, and deploy.yml re-chmods ops/*.sh);
#   - sanity-check: a suspiciously small dump (< MIN_DUMP_BYTES) is treated as a
#     failed dump — it's deleted and the script exits non-zero, so a good prior
#     dump is never pruned in favour of a truncated one;
#   - integrity-check: gzip -t + a trailer sentinel catch a truncated dump that
#     is neither empty nor undersized (see verify_dump_integrity below);
#   - globals (roles/GRANTs) are dumped separately via `pg_dumpall --globals-only`
#     — `pg_dump` never includes these, so without this a restore has tables
#     but no owning roles/privileges;
#   - retention: keep the KEEP most-recent local dumps, delete the rest — main
#     dumps and globals dumps are tracked as separate series.
#
# Usage (cron — note `bash`, not a bare path, so +x is irrelevant):
#   30 3 * * * bash /opt/gendesign/ops/backup.sh >> /var/log/gendesign-backup.log 2>&1
#
# S3 upload — REQUIRED by default (#3085). Set these in /etc/default/gendesign-backup
# (root-owned, chmod 600, NOT in git). Without all four, the script fails
# LOUDLY before taking a dump — set BACKUP_ALLOW_LOCAL_ONLY=1 to explicitly
# accept a local-only backup instead (e.g. local testing):
#   S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru   # Selectel S3
#   S3_BUCKET=gendsgn-backups
#   S3_ACCESS_KEY=...
#   S3_SECRET_KEY=...
# A redacted template lives at ops/gendesign-backup.default.example.
#
# Restore: see ops/restore.sh (destructive, INTO the live DB).
# Restore drill (safe, throwaway container): see ops/restore-drill.sh.
#
# Missed-run detection (#2203): on a verified-good run this writes a sentinel
# file (SENTINEL_FILE) with the UTC timestamp. A SEPARATE cron entry running
# ops/check-backup-staleness.sh against that file is what actually alerts if
# cron stops firing — see that script's header for the alert channel and
# cron line. This script itself does NOT alert; it only ever marks success.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./lib-backup.sh
source "$SCRIPT_DIR/lib-backup.sh"

# --- config (env-overridable) ---
COMPOSE_DIR="${COMPOSE_DIR:-/opt/gendesign}"
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.prod.yml}"
COMPOSE_PROJECT="${COMPOSE_PROJECT:-gendesign}"
LOCAL_BACKUP_DIR="${LOCAL_BACKUP_DIR:-/opt/gendesign/backups}"
KEEP="${KEEP:-7}"                          # how many recent local dumps to keep
MIN_DUMP_BYTES="${MIN_DUMP_BYTES:-51200}"  # 50 KiB floor; gzip'd schema-only dump
                                           # is already > this, so a healthy dump
                                           # never trips it. Real DB is far larger.
                                           # (No equivalent floor for the globals
                                           # dump — a handful of roles legitimately
                                           # gzips to well under this.)
SENTINEL_FILE="${SENTINEL_FILE:-${LOCAL_BACKUP_DIR}/.last_success}"

# Optional S3 env. Loaded from /etc/default/gendesign-backup if present.
[[ -f /etc/default/gendesign-backup ]] && source /etc/default/gendesign-backup

# --- guard: S3 upload is REQUIRED by default (#3085). A backup that only
#     lives on the same disk as the DB it protects is not a backup once
#     Poincare has no configured fallback host to copy from — before this
#     guard, a missing/incomplete /etc/default/gendesign-backup made the
#     script log "backup stays local only" and exit 0, indistinguishable
#     from a real off-box backup in the cron log. Fail LOUD, before the dump
#     is even taken (no point spending time/disk on a dump that can't leave
#     the box). BACKUP_ALLOW_LOCAL_ONLY=1 is the explicit escape hatch for
#     running on a box without S3 creds on purpose (e.g. local testing) —
#     it still warns loudly so it can't be forgotten silently. ---
if [[ -z "${S3_ENDPOINT:-}" || -z "${S3_BUCKET:-}" || -z "${S3_ACCESS_KEY:-}" || -z "${S3_SECRET_KEY:-}" ]]; then
    if [[ "${BACKUP_ALLOW_LOCAL_ONLY:-0}" == "1" ]]; then
        log "WARNING: S3 vars not fully set — BACKUP_ALLOW_LOCAL_ONLY=1, proceeding with a LOCAL-ONLY backup. This copy will NOT survive loss of this VM." >&2
        BACKUP_LOCAL_ONLY_USED=1
    else
        log "ERROR: S3 upload is not configured — missing one or more of S3_ENDPOINT / S3_BUCKET / S3_ACCESS_KEY / S3_SECRET_KEY." >&2
        log "ERROR: fill them into /etc/default/gendesign-backup (see ops/gendesign-backup.default.example)," >&2
        log "ERROR: or set BACKUP_ALLOW_LOCAL_ONLY=1 to explicitly accept a local-only backup." >&2
        exit 1
    fi
fi

# Integrity check beyond "non-empty"/"big enough": a dump truncated mid-write
# (disk full, OOM-kill, docker exec dropped) can still gzip into a structurally
# valid, non-tiny .gz — neither `-s` nor MIN_DUMP_BYTES catch that reliably.
# Two checks, cheapest first:
#   1. gzip -t   — catches a corrupted/truncated gzip stream itself.
#   2. trailer   — pg_dump/pg_dumpall always write a fixed "I finished writing"
#      comment as literally the last line of the stream; a dump cut off
#      mid-write is missing it even when the gzip framing looks fine.
# Checked via `gunzip -c | tail -N` so we never materialize the full
# decompressed dump on disk just to look at its last few lines.
verify_dump_integrity() {
    local file="$1" trailer="$2" label="$3"
    if ! gzip -t "$file" 2>/dev/null; then
        log "ERROR: ${label} failed gzip integrity check -> $file" >&2
        return 1
    fi
    if ! gunzip -c "$file" 2>/dev/null | tail -5 | grep -qF -- "$trailer"; then
        log "ERROR: ${label} missing trailer '${trailer}' -> $file (truncated dump?)" >&2
        return 1
    fi
    return 0
}

# --- run ---
mkdir -p "$LOCAL_BACKUP_DIR"
# Underscore naming kept (gendesign_YYYYMMDD_HHMMSS.sql.gz) so this matches
# pre-existing prod dumps and restore.sh's example path. UTC for stable ordering.
ts=$(date -u +'%Y%m%d_%H%M%S')
out="${LOCAL_BACKUP_DIR}/gendesign_${ts}.sql.gz"
globals_out="${LOCAL_BACKUP_DIR}/gendesign_globals_${ts}.sql.gz"

cd "$COMPOSE_DIR"

compose() { docker compose -p "$COMPOSE_PROJECT" -f "$COMPOSE_FILE" "$@"; }

# Read DB user/name from the running compose env; fall back to "gendesign".
DB_USER=$(compose exec -T postgres printenv POSTGRES_USER 2>/dev/null | tr -d '\r' || true)
DB_NAME=$(compose exec -T postgres printenv POSTGRES_DB   2>/dev/null | tr -d '\r' || true)
DB_USER=${DB_USER:-gendesign}
DB_NAME=${DB_NAME:-gendesign}

log "Dumping ${DB_NAME} as ${DB_USER} -> ${out}"

# --clean --if-exists -> dump is self-sufficient for restore from scratch.
# --no-owner -> restore does not require identical roles.
# `set -o pipefail` (from set -euo pipefail) makes a pg_dump failure fail the
# whole pipe, so a Postgres error can't yield a "successful" tiny gzip. Stderr
# is NOT redirected to /dev/null (was until #2203) — a pg_dump error needs to
# land in the cron log, not get silently swallowed.
compose exec -T postgres \
    pg_dump -U "$DB_USER" -d "$DB_NAME" --no-owner --clean --if-exists \
    | gzip -9 > "$out"

# --- sanity-check: refuse to keep (and thus never prune good dumps for) a
#     truncated/empty dump. ---
if [[ ! -s "$out" ]]; then
    log "ERROR: dump is empty -> $out — removing, keeping previous good dumps." >&2
    rm -f "$out"
    exit 1
fi

dump_bytes=$(wc -c < "$out" | tr -d ' ')
if (( dump_bytes < MIN_DUMP_BYTES )); then
    log "ERROR: dump only ${dump_bytes} bytes (< ${MIN_DUMP_BYTES} floor) — likely a failed dump." >&2
    log "Removing suspect dump, NOT pruning older good dumps." >&2
    rm -f "$out"
    exit 1
fi

if ! verify_dump_integrity "$out" "-- PostgreSQL database dump complete" "main dump"; then
    log "Removing suspect dump, NOT pruning older good dumps." >&2
    rm -f "$out"
    exit 1
fi

log "Dump OK: ${out} ($(du -h "$out" | cut -f1), ${dump_bytes} bytes)"

# --- globals (roles/GRANTs) — pg_dump never includes these; without them a
#     restored dump has tables and data but no owning roles/privileges. ---
log "Dumping globals (roles/GRANTs) -> ${globals_out}"
compose exec -T postgres \
    pg_dumpall -U "$DB_USER" --globals-only \
    | gzip -9 > "$globals_out"

if [[ ! -s "$globals_out" ]]; then
    log "ERROR: globals dump is empty -> $globals_out — removing." >&2
    rm -f "$globals_out"
    exit 1
fi

if ! verify_dump_integrity "$globals_out" "-- PostgreSQL database cluster dump complete" "globals dump"; then
    rm -f "$globals_out"
    exit 1
fi

log "Globals dump OK: ${globals_out} ($(du -h "$globals_out" | cut -f1))"

# --- S3 upload (only if all four vars present — guaranteed unless the
#     BACKUP_ALLOW_LOCAL_ONLY escape hatch above was used) — both the main dump
#     and the globals dump go up, same bucket, same layout. ---
if [[ -n "${S3_ENDPOINT:-}" && -n "${S3_BUCKET:-}" && -n "${S3_ACCESS_KEY:-}" && -n "${S3_SECRET_KEY:-}" ]]; then
    for f in "$out" "$globals_out"; do
        log "Uploading to s3://${S3_BUCKET}/$(basename "$f")"
        # aws-cli v2 ships its own CA bundle (baked into botocore) instead of
        # trusting the system store, and it's missing the root that Selectel's
        # cert chains up to — so uploads fail with CERTIFICATE_VERIFY_FAILED
        # unless we point it at the container's system store, which has it.
        docker run --rm \
            -e AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
            -e AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
            -e AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
            -v "$LOCAL_BACKUP_DIR":/backup:ro \
            amazon/aws-cli:latest \
            --endpoint-url "$S3_ENDPOINT" \
            s3 cp --no-progress "/backup/$(basename "$f")" "s3://${S3_BUCKET}/"
    done
    log "S3 upload OK"
else
    log "S3 vars not set — backup stays local only"
fi

# --- retention: keep the KEEP most-recent local dumps, delete the rest.
#     Runs only AFTER a verified-good dump above, so a failed run (which exits
#     early) can never delete older good dumps. Main dumps and globals dumps
#     are separate series (the `[0-9]` after the shared `gendesign_` prefix
#     keeps the globals_* files, which share that prefix, out of this glob). ---
# SC2012: ls is fine here — filenames are fully controlled (gendesign_<ts>.sql.gz,
# no spaces/newlines) and we need ls's -t mtime sort for "keep newest N" (same
# pattern as the proven backup-tradein-db.sh).
# shellcheck disable=SC2012
ls -1t "$LOCAL_BACKUP_DIR"/gendesign_[0-9]*.sql.gz 2>/dev/null \
    | tail -n +"$((KEEP + 1))" \
    | xargs -r rm -f
# shellcheck disable=SC2012
ls -1t "$LOCAL_BACKUP_DIR"/gendesign_globals_*.sql.gz 2>/dev/null \
    | tail -n +"$((KEEP + 1))" \
    | xargs -r rm -f

# Count via glob arrays (no ls parsing). nullglob -> empty array if no match.
shopt -s nullglob
remaining=( "$LOCAL_BACKUP_DIR"/gendesign_[0-9]*.sql.gz )
remaining_globals=( "$LOCAL_BACKUP_DIR"/gendesign_globals_*.sql.gz )
shopt -u nullglob
log "Backup done. Local dumps retained: ${#remaining[@]} data + ${#remaining_globals[@]} globals (KEEP=${KEEP})."

# --- missed-run detection (#2203): mark success ONLY here, after every
#     guard above (dump/globals integrity, retention, and — since this whole
#     script runs under `set -euo pipefail` — any S3 upload) has passed. See
#     ops/check-backup-staleness.sh for the piece that actually alerts if
#     this stops being refreshed. A LOCAL-ONLY run (BACKUP_ALLOW_LOCAL_ONLY=1
#     escape hatch above) does NOT count as a verified backup for this
#     purpose — writing the sentinel here would leave the staleness watchdog
#     permanently green while the box has no off-box copy at all (#3085
#     reopened via the escape hatch). Instead: skip the sentinel (watchdog
#     alerts within max_age_hours) and notify() immediately so it isn't only
#     a WARNING buried in a log nobody tails. ---
if [[ "${BACKUP_LOCAL_ONLY_USED:-0}" == "1" ]]; then
    log "Skipping sentinel write — this was a LOCAL-ONLY backup (BACKUP_ALLOW_LOCAL_ONLY=1), not a verified off-box backup." >&2
    notify "⚠️ gendesign main backup: LOCAL-ONLY run (S3 not configured, BACKUP_ALLOW_LOCAL_ONLY=1). Dump ${out} is NOT off-box. Sentinel NOT updated — staleness watchdog will alert if this isn't fixed."
else
    write_sentinel "$SENTINEL_FILE"
fi