gendesign/ops/backup.sh

#!/usr/bin/env bash
# Daily Postgres backup for the MAIN gendesign DB. Runs from cron on the prod VM.
#
# Modeled on the proven ops/.../backup-tradein-db.sh (#397). Hardened for #71
# after the main-DB backup silently broke (last good dump 2026-05-27): a raw
# deploy-time hard reset of the repo checkout onto origin/main reset this
# file's mode to 644, so a cron entry that invoked the raw path got
# "Permission denied" every run.
#
# Robustness measures here (extended for #2203):
#   - cron should invoke via `bash <path>` so a missing +x bit can't break it
#     (this file is ALSO committed 100755, and deploy.yml re-chmods ops/*.sh);
#   - sanity-check: a suspiciously small dump (< MIN_DUMP_BYTES) is treated as a
#     failed dump — it's deleted and the script exits non-zero, so a good prior
#     dump is never pruned in favour of a truncated one;
#   - integrity-check: gzip -t + a trailer sentinel catch a truncated dump that
#     is neither empty nor undersized (see verify_dump_integrity below);
#   - globals (roles/GRANTs) are dumped separately via `pg_dumpall --globals-only`
#     — `pg_dump` never includes these, so without this a restore has tables
#     but no owning roles/privileges;
#   - retention: keep the KEEP most-recent local dumps, delete the rest — main
#     dumps and globals dumps are tracked as separate series.
#
# Usage (cron — note `bash`, not a bare path, so +x is irrelevant):
#   30 3 * * * bash /opt/gendesign/ops/backup.sh >> /var/log/gendesign-backup.log 2>&1
#
# Optional S3 upload — set these in /etc/default/gendesign-backup (root-owned,
# chmod 600, NOT in git). Without them, dumps stay local only:
#   S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru   # Selectel S3
#   S3_BUCKET=gendesign-backups
#   S3_ACCESS_KEY=...
#   S3_SECRET_KEY=...
# A redacted template lives at ops/gendesign-backup.default.example.
#
# Restore: see ops/restore.sh (destructive, INTO the live DB).
# Restore drill (safe, throwaway container): see ops/restore-drill.sh.

set -euo pipefail

# --- config (env-overridable) ---
COMPOSE_DIR="${COMPOSE_DIR:-/opt/gendesign}"
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.prod.yml}"
COMPOSE_PROJECT="${COMPOSE_PROJECT:-gendesign}"
LOCAL_BACKUP_DIR="${LOCAL_BACKUP_DIR:-/opt/gendesign/backups}"
KEEP="${KEEP:-7}"                          # how many recent local dumps to keep
MIN_DUMP_BYTES="${MIN_DUMP_BYTES:-51200}"  # 50 KiB floor; gzip'd schema-only dump
                                           # is already > this, so a healthy dump
                                           # never trips it. Real DB is far larger.
                                           # (No equivalent floor for the globals
                                           # dump — a handful of roles legitimately
                                           # gzips to well under this.)

# Optional S3 env. Loaded from /etc/default/gendesign-backup if present.
[[ -f /etc/default/gendesign-backup ]] && source /etc/default/gendesign-backup

log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }

# Integrity check beyond "non-empty"/"big enough": a dump truncated mid-write
# (disk full, OOM-kill, docker exec dropped) can still gzip into a structurally
# valid, non-tiny .gz — neither `-s` nor MIN_DUMP_BYTES catch that reliably.
# Two checks, cheapest first:
#   1. gzip -t   — catches a corrupted/truncated gzip stream itself.
#   2. trailer   — pg_dump/pg_dumpall always write a fixed "I finished writing"
#      comment as literally the last line of the stream; a dump cut off
#      mid-write is missing it even when the gzip framing looks fine.
# Checked via `gunzip -c | tail -N` so we never materialize the full
# decompressed dump on disk just to look at its last few lines.
verify_dump_integrity() {
    local file="$1" trailer="$2" label="$3"
    if ! gzip -t "$file" 2>/dev/null; then
        log "ERROR: ${label} failed gzip integrity check -> $file" >&2
        return 1
    fi
    if ! gunzip -c "$file" 2>/dev/null | tail -5 | grep -qF -- "$trailer"; then
        log "ERROR: ${label} missing trailer '${trailer}' -> $file (truncated dump?)" >&2
        return 1
    fi
    return 0
}

# --- run ---
mkdir -p "$LOCAL_BACKUP_DIR"
# Underscore naming kept (gendesign_YYYYMMDD_HHMMSS.sql.gz) so this matches
# pre-existing prod dumps and restore.sh's example path. UTC for stable ordering.
ts=$(date -u +'%Y%m%d_%H%M%S')
out="${LOCAL_BACKUP_DIR}/gendesign_${ts}.sql.gz"
globals_out="${LOCAL_BACKUP_DIR}/gendesign_globals_${ts}.sql.gz"

cd "$COMPOSE_DIR"

compose() { docker compose -p "$COMPOSE_PROJECT" -f "$COMPOSE_FILE" "$@"; }

# Read DB user/name from the running compose env; fall back to "gendesign".
DB_USER=$(compose exec -T postgres printenv POSTGRES_USER 2>/dev/null | tr -d '\r' || true)
DB_NAME=$(compose exec -T postgres printenv POSTGRES_DB   2>/dev/null | tr -d '\r' || true)
DB_USER=${DB_USER:-gendesign}
DB_NAME=${DB_NAME:-gendesign}

log "Dumping ${DB_NAME} as ${DB_USER} -> ${out}"

# --clean --if-exists -> dump is self-sufficient for restore from scratch.
# --no-owner -> restore does not require identical roles.
# `set -o pipefail` (from set -euo pipefail) makes a pg_dump failure fail the
# whole pipe, so a Postgres error can't yield a "successful" tiny gzip. Stderr
# is NOT redirected to /dev/null (was until #2203) — a pg_dump error needs to
# land in the cron log, not get silently swallowed.
compose exec -T postgres \
    pg_dump -U "$DB_USER" -d "$DB_NAME" --no-owner --clean --if-exists \
    | gzip -9 > "$out"

# --- sanity-check: refuse to keep (and thus never prune good dumps for) a
#     truncated/empty dump. ---
if [[ ! -s "$out" ]]; then
    log "ERROR: dump is empty -> $out — removing, keeping previous good dumps." >&2
    rm -f "$out"
    exit 1
fi

dump_bytes=$(wc -c < "$out" | tr -d ' ')
if (( dump_bytes < MIN_DUMP_BYTES )); then
    log "ERROR: dump only ${dump_bytes} bytes (< ${MIN_DUMP_BYTES} floor) — likely a failed dump." >&2
    log "Removing suspect dump, NOT pruning older good dumps." >&2
    rm -f "$out"
    exit 1
fi

if ! verify_dump_integrity "$out" "-- PostgreSQL database dump complete" "main dump"; then
    log "Removing suspect dump, NOT pruning older good dumps." >&2
    rm -f "$out"
    exit 1
fi

log "Dump OK: ${out} ($(du -h "$out" | cut -f1), ${dump_bytes} bytes)"

# --- globals (roles/GRANTs) — pg_dump never includes these; without them a
#     restored dump has tables and data but no owning roles/privileges. ---
log "Dumping globals (roles/GRANTs) -> ${globals_out}"
compose exec -T postgres \
    pg_dumpall -U "$DB_USER" --globals-only \
    | gzip -9 > "$globals_out"

if [[ ! -s "$globals_out" ]]; then
    log "ERROR: globals dump is empty -> $globals_out — removing." >&2
    rm -f "$globals_out"
    exit 1
fi

if ! verify_dump_integrity "$globals_out" "-- PostgreSQL database cluster dump complete" "globals dump"; then
    rm -f "$globals_out"
    exit 1
fi

log "Globals dump OK: ${globals_out} ($(du -h "$globals_out" | cut -f1))"

# --- optional S3 upload (only if all four vars present) — both the main dump
#     and the globals dump go up, same bucket, same layout. ---
if [[ -n "${S3_ENDPOINT:-}" && -n "${S3_BUCKET:-}" && -n "${S3_ACCESS_KEY:-}" && -n "${S3_SECRET_KEY:-}" ]]; then
    for f in "$out" "$globals_out"; do
        log "Uploading to s3://${S3_BUCKET}/$(basename "$f")"
        # aws-cli v2 ships its own CA bundle (baked into botocore) instead of
        # trusting the system store, and it's missing the root that Selectel's
        # cert chains up to — so uploads fail with CERTIFICATE_VERIFY_FAILED
        # unless we point it at the container's system store, which has it.
        docker run --rm \
            -e AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
            -e AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
            -e AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
            -v "$LOCAL_BACKUP_DIR":/backup:ro \
            amazon/aws-cli:latest \
            --endpoint-url "$S3_ENDPOINT" \
            s3 cp "/backup/$(basename "$f")" "s3://${S3_BUCKET}/"
    done
    log "S3 upload OK"
else
    log "S3 vars not set — backup stays local only"
fi

# --- retention: keep the KEEP most-recent local dumps, delete the rest.
#     Runs only AFTER a verified-good dump above, so a failed run (which exits
#     early) can never delete older good dumps. Main dumps and globals dumps
#     are separate series (the `[0-9]` after the shared `gendesign_` prefix
#     keeps the globals_* files, which share that prefix, out of this glob). ---
# SC2012: ls is fine here — filenames are fully controlled (gendesign_<ts>.sql.gz,
# no spaces/newlines) and we need ls's -t mtime sort for "keep newest N" (same
# pattern as the proven backup-tradein-db.sh).
# shellcheck disable=SC2012
ls -1t "$LOCAL_BACKUP_DIR"/gendesign_[0-9]*.sql.gz 2>/dev/null \
    | tail -n +"$((KEEP + 1))" \
    | xargs -r rm -f
# shellcheck disable=SC2012
ls -1t "$LOCAL_BACKUP_DIR"/gendesign_globals_*.sql.gz 2>/dev/null \
    | tail -n +"$((KEEP + 1))" \
    | xargs -r rm -f

# Count via glob arrays (no ls parsing). nullglob -> empty array if no match.
shopt -s nullglob
remaining=( "$LOCAL_BACKUP_DIR"/gendesign_[0-9]*.sql.gz )
remaining_globals=( "$LOCAL_BACKUP_DIR"/gendesign_globals_*.sql.gz )
shopt -u nullglob
log "Backup done. Local dumps retained: ${#remaining[@]} data + ${#remaining_globals[@]} globals (KEEP=${KEEP})."