gendesign/ops/restore-drill.sh

#!/usr/bin/env bash
# Restore DRILL for a dump produced by backup.sh / backup-tradein-db.sh (#2203).
#
# THIS SCRIPT NEVER TOUCHES THE PRODUCTION DATABASE. It spins up a throwaway,
# unnamed-volume postgis/postgis container, loads the dump (and its globals
# sibling, if found) into it, prints a few sanity numbers, and always tears
# the container down again — win or lose.
#
# ops/restore.sh IS DIFFERENT AND IS DESTRUCTIVE: it restores INTO THE LIVE
# PRODUCTION DATABASE (--clean --if-exists DROPs existing tables first). Do
# NOT use ops/restore.sh for a drill/rehearsal — use THIS script instead.
#
# Usage:
#   ops/restore-drill.sh /path/to/gendesign_20260820_030000.sql.gz
#   ops/restore-drill.sh /path/to/tradein-20260820-043000.sql.gz [globals.sql.gz]
#
# Globals autodetection: if the 2nd arg is omitted, this script looks next to
# the dump for a sibling file matching the naming convention the backup
# scripts use (gendesign_globals_<ts>.sql.gz / tradein-globals-<ts>.sql.gz).
# Missing globals is not fatal — the dump itself is still restored and checked.
#
# Which tables get row-counted is guessed from the dump's filename prefix
# (gendesign_* vs tradein-*) and can be overridden with RESTORE_DRILL_TABLES
# (comma-separated table names, no schema qualifier — public is assumed).

set -euo pipefail

# notify() on failure (reviewer finding): unlike backup.sh/backup-tradein-db.sh
# this drill has no sentinel/watchdog of its own — its cron entry has no
# MAILTO, so a non-zero exit previously vanished into restore-drill.log with
# nobody looking. Reuse the same Telegram/mail channel as the backups so an
# unrestorable dump is loud, not discovered during an actual incident. Wired
# into cleanup() below (not its own trap) — bash only honours the LAST trap
# registered for a given signal, and cleanup() already owns EXIT.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./lib-backup.sh
source "$SCRIPT_DIR/lib-backup.sh"

IMAGE="${RESTORE_DRILL_IMAGE:-postgis/postgis:16-3.4}"
READY_TIMEOUT="${RESTORE_DRILL_READY_TIMEOUT:-60}"  # seconds to wait for postgres startup
DRILL_DB="drill"

DUMP_FILE="${1:-}"
if [[ -z "$DUMP_FILE" || ! -f "$DUMP_FILE" ]]; then
    echo "Usage: $0 /path/to/dump.sql.gz [globals.sql.gz]" >&2
    exit 2
fi
DUMP_FILE=$(cd "$(dirname "$DUMP_FILE")" && pwd)/$(basename "$DUMP_FILE")

GLOBALS_FILE="${2:-}"
if [[ -n "$GLOBALS_FILE" && ! -f "$GLOBALS_FILE" ]]; then
    echo "Globals file not found: $GLOBALS_FILE" >&2
    exit 2
fi

log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }

# --- autodetect the globals sibling, matching the two naming conventions our
#     backup scripts use:
#       gendesign_<ts>.sql.gz  -> gendesign_globals_<ts>.sql.gz  (ops/backup.sh)
#       tradein-<ts>.sql.gz    -> tradein-globals-<ts>.sql.gz    (backup-tradein-db.sh)
dump_dir=$(dirname "$DUMP_FILE")
dump_base=$(basename "$DUMP_FILE")
project="gendesign"

if [[ -z "$GLOBALS_FILE" ]]; then
    candidate=""
    if [[ "$dump_base" =~ ^([A-Za-z0-9]+)_([0-9]{8}_[0-9]{6})\.sql\.gz$ ]]; then
        candidate="${dump_dir}/${BASH_REMATCH[1]}_globals_${BASH_REMATCH[2]}.sql.gz"
    elif [[ "$dump_base" =~ ^([A-Za-z0-9]+)-([0-9]{8}-[0-9]{6})\.sql\.gz$ ]]; then
        candidate="${dump_dir}/${BASH_REMATCH[1]}-globals-${BASH_REMATCH[2]}.sql.gz"
    fi
    if [[ -n "$candidate" && -f "$candidate" ]]; then
        GLOBALS_FILE="$candidate"
    fi
fi

if [[ "$dump_base" == tradein-* ]]; then
    project="tradein"
fi

if [[ -n "$GLOBALS_FILE" ]]; then
    log "Dump:    $DUMP_FILE"
    log "Globals: $GLOBALS_FILE"
else
    log "Dump:    $DUMP_FILE"
    log "Globals: none found next to the dump — restoring data only, no roles/GRANTs."
fi

# --- default row-count table list, per project; override with
#     RESTORE_DRILL_TABLES=t1,t2,... ---
if [[ -n "${RESTORE_DRILL_TABLES:-}" ]]; then
    tables_csv="$RESTORE_DRILL_TABLES"
elif [[ "$project" == "tradein" ]]; then
    tables_csv="listings,listing_sources,deals,houses,trade_in_estimates"
else
    # Representative core tables across the gendesign schema: the big
    # partitioned deals dataset, cadastral/opportunity overlays, the DomRF
    # snapshot chain, and two smaller user-facing tables.
    tables_csv="rosreestr_deals,cad_opportunity_parcels,domrf_snapshots,trade_in_estimates,parcel_user_status"
fi
IFS=',' read -r -a TABLES <<< "$tables_csv"

# --- throwaway container: random name, docker-assigned host port, no prod
#     volumes mounted (anonymous storage inside the container layer only —
#     gone the instant the container is removed). trust auth is fine here:
#     single-use, bound to 127.0.0.1, destroyed on exit. ---
CONTAINER="restore-drill-$$-$(date -u +%s)"
cleanup() {
    local rc=$?
    log "Cleaning up container ${CONTAINER}"
    docker rm -f -v "$CONTAINER" >/dev/null 2>&1 || true
    if [[ $rc -ne 0 ]]; then
        notify "🔴 restore-drill FAILED (exit ${rc}) for ${DUMP_FILE:-<unset>} — dump may not be restorable. Check restore-drill.log on the VM." || true
    fi
}
trap cleanup EXIT

log "Starting throwaway ${IMAGE} as ${CONTAINER}"
docker run -d --name "$CONTAINER" \
    -e POSTGRES_HOST_AUTH_METHOD=trust \
    -p 127.0.0.1::5432 \
    "$IMAGE" >/dev/null

host_port=$(docker port "$CONTAINER" 5432/tcp | head -1 | cut -d: -f2)
log "Container up, mapped to 127.0.0.1:${host_port} (only reachable while this drill runs)"

log "Waiting for postgres to accept connections (timeout ${READY_TIMEOUT}s)..."
ready=0
for _ in $(seq 1 "$READY_TIMEOUT"); do
    # -h 127.0.0.1 is load-bearing, NOT cosmetic. The postgres image runs a
    # TEMPORARY server during initialisation, and that server already answers
    # "accepting connections" on the unix socket. Probing the socket therefore
    # goes green mid-init; the restore starts against the temporary server and
    # dies with "FATAL: terminating connection due to administrator command"
    # the moment the entrypoint shuts it down to start the real one. The
    # temporary server does NOT listen on TCP, so a TCP probe only goes green
    # on the real server. Same reasoning, same fix as ci-tradein.yml:137-141.
    if docker exec "$CONTAINER" pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1; then
        ready=1
        break
    fi
    sleep 1
done
if [[ "$ready" -ne 1 ]]; then
    log "ERROR: postgres did not become ready within ${READY_TIMEOUT}s" >&2
    exit 1
fi

psql_c() {
    # $1 = target db, $2 = SQL
    docker exec -i "$CONTAINER" psql -v ON_ERROR_STOP=1 -U postgres -d "$1" -c "$2"
}

log "Creating drill database '${DRILL_DB}'"
psql_c postgres "CREATE DATABASE ${DRILL_DB};" >/dev/null

# --- globals: best-effort. A pg_dumpall --globals-only dump against a fresh
#     container almost always trips over the container's own bootstrap
#     'postgres' role (already exists), which is expected and harmless — so
#     this load is NOT ON_ERROR_STOP. The dump restore below is the real
#     integrity check and DOES use ON_ERROR_STOP=1. ---
if [[ -n "$GLOBALS_FILE" ]]; then
    log "Loading globals (best-effort — role-already-exists warnings are expected)"
    gunzip -c "$GLOBALS_FILE" | docker exec -i "$CONTAINER" psql -U postgres -d postgres \
        || log "NOTE: globals load reported errors above — usually just pre-existing default roles, non-fatal for the drill."
fi

log "Restoring dump into '${DRILL_DB}' (ON_ERROR_STOP=1 — any real error aborts the drill)"
gunzip -c "$DUMP_FILE" | docker exec -i "$CONTAINER" psql -v ON_ERROR_STOP=1 -U postgres -d "$DRILL_DB"

log "Restore finished. Running sanity checks."

postgis_version=$(docker exec -i "$CONTAINER" psql -U postgres -d "$DRILL_DB" -Atqc "SELECT postgis_full_version();" 2>/dev/null || echo "N/A (postgis not installed in this dump)")
echo "postgis_full_version(): ${postgis_version}"

table_count=$(docker exec -i "$CONTAINER" psql -U postgres -d "$DRILL_DB" -Atqc \
    "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public';")
echo "Tables in public schema: ${table_count}"

echo
echo "table | rows"
echo "----- | ----"
for t in "${TABLES[@]}"; do
    exists=$(docker exec -i "$CONTAINER" psql -U postgres -d "$DRILL_DB" -Atqc \
        "SELECT to_regclass('public.${t}') IS NOT NULL;")
    if [[ "$exists" == "t" ]]; then
        rows=$(docker exec -i "$CONTAINER" psql -U postgres -d "$DRILL_DB" -Atqc \
            "SELECT count(*) FROM \"${t}\";")
        echo "${t} | ${rows}"
    else
        echo "${t} | n/a (table not found in this dump)"
    fi
done

log "Drill complete. Container will now be removed."