From 770492b8b3551d68f5d9e19d0d8f7dc5bc6b1651 Mon Sep 17 00:00:00 2001
From: bot-backend <alexandro361@gmail.com>
Date: Thu, 20 Aug 2026 22:13:58 +0300
Subject: [PATCH 1/4] =?UTF-8?q?fix(ops):=20=D0=B1=D1=8D=D0=BA=D0=B0=D0=BF?=
 =?UTF-8?q?=20=D0=BD=D0=B5=20=D1=82=D0=B5=D1=80=D1=8F=D0=B5=D1=82=20=D1=80?=
 =?UTF-8?q?=D0=BE=D0=BB=D0=B8,=20=D0=BD=D0=B5=20=D0=B3=D0=BB=D0=BE=D1=82?=
 =?UTF-8?q?=D0=B0=D0=B5=D1=82=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BA=D0=B8=20?=
 =?UTF-8?q?=D0=B8=20=D1=83=D0=BC=D0=B5=D0=B5=D1=82=20=D1=83=D0=B5=D0=B7?=
 =?UTF-8?q?=D0=B6=D0=B0=D1=82=D1=8C=20=D1=81=20=D0=BC=D0=B0=D1=88=D0=B8?=
 =?UTF-8?q?=D0=BD=D1=8B=20(#2203)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- ops/backup.sh и tradein-mvp/deploy/backup-tradein-db.sh теперь дампят
  globals (pg_dumpall --globals-only) отдельным файлом с той же ретенцией
  и той же S3-выгрузкой — pg_dump по определению не включает роли/GRANT.
- Обе выгрузки проходят gzip -t + проверку трейлера дампа перед тем как
  считаться успешными; при провале файл удаляется, ретенция не трогается,
  выход ненулевой.
- Убран 2>/dev/null у pg_dump в обоих скриптах — ошибка дампа теперь
  видна в логе, а не глотается молча.
- tradein-backup.sh получил S3-выгрузку (по образцу ops/backup.sh, те же
  4 переменные, тот же способ через aws-cli контейнер) и env-переопределяемый
  порог минимального размера дампа; источник переменных —
  /etc/default/tradein-backup с фолбэком на /etc/default/gendesign-backup.
- Новый ops/restore-drill.sh — учебное восстановление в одноразовый
  postgis-контейнер без прод-томов, никогда не трогает боевую БД (в отличие
  от ops/restore.sh, который восстанавливает В БОЕВУЮ базу).
---
 ops/backup.sh                           | 115 +++++++++++++---
 ops/gendesign-backup.default.example    |  23 +++-
 ops/restore-drill.sh                    | 173 ++++++++++++++++++++++++
 tradein-mvp/deploy/backup-tradein-db.sh | 121 ++++++++++++++++-
 4 files changed, 403 insertions(+), 29 deletions(-)
 create mode 100644 ops/restore-drill.sh

diff --git a/ops/backup.sh b/ops/backup.sh
index 348518f7..33c0ed08 100755
--- a/ops/backup.sh
+++ b/ops/backup.sh
@@ -3,16 +3,23 @@
 #
 # Modeled on the proven ops/.../backup-tradein-db.sh (#397). Hardened for #71
 # after the main-DB backup silently broke (last good dump 2026-05-27): a raw
-# `git reset --hard origin/main` on every deploy reset this file's mode to 644,
-# so a cron entry that invoked the raw path got "Permission denied" every run.
+# deploy-time hard reset of the repo checkout onto origin/main reset this
+# file's mode to 644, so a cron entry that invoked the raw path got
+# "Permission denied" every run.
 #
-# Robustness measures here:
+# Robustness measures here (extended for #2203):
 #   - cron should invoke via `bash <path>` so a missing +x bit can't break it
 #     (this file is ALSO committed 100755, and deploy.yml re-chmods ops/*.sh);
 #   - sanity-check: a suspiciously small dump (< MIN_DUMP_BYTES) is treated as a
 #     failed dump — it's deleted and the script exits non-zero, so a good prior
 #     dump is never pruned in favour of a truncated one;
-#   - retention: keep the KEEP most-recent local dumps, delete the rest.
+#   - integrity-check: gzip -t + a trailer sentinel catch a truncated dump that
+#     is neither empty nor undersized (see verify_dump_integrity below);
+#   - globals (roles/GRANTs) are dumped separately via `pg_dumpall --globals-only`
+#     — `pg_dump` never includes these, so without this a restore has tables
+#     but no owning roles/privileges;
+#   - retention: keep the KEEP most-recent local dumps, delete the rest — main
+#     dumps and globals dumps are tracked as separate series.
 #
 # Usage (cron — note `bash`, not a bare path, so +x is irrelevant):
 #   30 3 * * * bash /opt/gendesign/ops/backup.sh >> /var/log/gendesign-backup.log 2>&1
@@ -25,7 +32,8 @@
 #   S3_SECRET_KEY=...
 # A redacted template lives at ops/gendesign-backup.default.example.
 #
-# Restore: see ops/restore.sh.
+# Restore: see ops/restore.sh (destructive, INTO the live DB).
+# Restore drill (safe, throwaway container): see ops/restore-drill.sh.
 
 set -euo pipefail
 
@@ -38,18 +46,45 @@ KEEP="${KEEP:-7}"                          # how many recent local dumps to keep
 MIN_DUMP_BYTES="${MIN_DUMP_BYTES:-51200}"  # 50 KiB floor; gzip'd schema-only dump
                                            # is already > this, so a healthy dump
                                            # never trips it. Real DB is far larger.
+                                           # (No equivalent floor for the globals
+                                           # dump — a handful of roles legitimately
+                                           # gzips to well under this.)
 
 # Optional S3 env. Loaded from /etc/default/gendesign-backup if present.
 [[ -f /etc/default/gendesign-backup ]] && source /etc/default/gendesign-backup
 
 log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
 
+# Integrity check beyond "non-empty"/"big enough": a dump truncated mid-write
+# (disk full, OOM-kill, docker exec dropped) can still gzip into a structurally
+# valid, non-tiny .gz — neither `-s` nor MIN_DUMP_BYTES catch that reliably.
+# Two checks, cheapest first:
+#   1. gzip -t   — catches a corrupted/truncated gzip stream itself.
+#   2. trailer   — pg_dump/pg_dumpall always write a fixed "I finished writing"
+#      comment as literally the last line of the stream; a dump cut off
+#      mid-write is missing it even when the gzip framing looks fine.
+# Checked via `gunzip -c | tail -N` so we never materialize the full
+# decompressed dump on disk just to look at its last few lines.
+verify_dump_integrity() {
+    local file="$1" trailer="$2" label="$3"
+    if ! gzip -t "$file" 2>/dev/null; then
+        log "ERROR: ${label} failed gzip integrity check -> $file" >&2
+        return 1
+    fi
+    if ! gunzip -c "$file" 2>/dev/null | tail -5 | grep -qF "$trailer"; then
+        log "ERROR: ${label} missing trailer '${trailer}' -> $file (truncated dump?)" >&2
+        return 1
+    fi
+    return 0
+}
+
 # --- run ---
 mkdir -p "$LOCAL_BACKUP_DIR"
 # Underscore naming kept (gendesign_YYYYMMDD_HHMMSS.sql.gz) so this matches
 # pre-existing prod dumps and restore.sh's example path. UTC for stable ordering.
 ts=$(date -u +'%Y%m%d_%H%M%S')
 out="${LOCAL_BACKUP_DIR}/gendesign_${ts}.sql.gz"
+globals_out="${LOCAL_BACKUP_DIR}/gendesign_globals_${ts}.sql.gz"
 
 cd "$COMPOSE_DIR"
 
@@ -66,9 +101,11 @@ log "Dumping ${DB_NAME} as ${DB_USER} -> ${out}"
 # --clean --if-exists -> dump is self-sufficient for restore from scratch.
 # --no-owner -> restore does not require identical roles.
 # `set -o pipefail` (from set -euo pipefail) makes a pg_dump failure fail the
-# whole pipe, so a Postgres error can't yield a "successful" tiny gzip.
+# whole pipe, so a Postgres error can't yield a "successful" tiny gzip. Stderr
+# is NOT redirected to /dev/null (was until #2203) — a pg_dump error needs to
+# land in the cron log, not get silently swallowed.
 compose exec -T postgres \
-    pg_dump -U "$DB_USER" -d "$DB_NAME" --no-owner --clean --if-exists 2>/dev/null \
+    pg_dump -U "$DB_USER" -d "$DB_NAME" --no-owner --clean --if-exists \
     | gzip -9 > "$out"
 
 # --- sanity-check: refuse to keep (and thus never prune good dumps for) a
@@ -87,18 +124,47 @@ if (( dump_bytes < MIN_DUMP_BYTES )); then
     exit 1
 fi
 
+if ! verify_dump_integrity "$out" "-- PostgreSQL database dump complete" "main dump"; then
+    log "Removing suspect dump, NOT pruning older good dumps." >&2
+    rm -f "$out"
+    exit 1
+fi
+
 log "Dump OK: ${out} ($(du -h "$out" | cut -f1), ${dump_bytes} bytes)"
 
-# --- optional S3 upload (only if all four vars present) ---
+# --- globals (roles/GRANTs) — pg_dump never includes these; without them a
+#     restored dump has tables and data but no owning roles/privileges. ---
+log "Dumping globals (roles/GRANTs) -> ${globals_out}"
+compose exec -T postgres \
+    pg_dumpall -U "$DB_USER" --globals-only \
+    | gzip -9 > "$globals_out"
+
+if [[ ! -s "$globals_out" ]]; then
+    log "ERROR: globals dump is empty -> $globals_out — removing." >&2
+    rm -f "$globals_out"
+    exit 1
+fi
+
+if ! verify_dump_integrity "$globals_out" "-- PostgreSQL database cluster dump complete" "globals dump"; then
+    rm -f "$globals_out"
+    exit 1
+fi
+
+log "Globals dump OK: ${globals_out} ($(du -h "$globals_out" | cut -f1))"
+
+# --- optional S3 upload (only if all four vars present) — both the main dump
+#     and the globals dump go up, same bucket, same layout. ---
 if [[ -n "${S3_ENDPOINT:-}" && -n "${S3_BUCKET:-}" && -n "${S3_ACCESS_KEY:-}" && -n "${S3_SECRET_KEY:-}" ]]; then
-    log "Uploading to s3://${S3_BUCKET}/$(basename "$out")"
-    docker run --rm \
-        -e AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
-        -e AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
-        -v "$LOCAL_BACKUP_DIR":/backup:ro \
-        amazon/aws-cli:latest \
-        --endpoint-url "$S3_ENDPOINT" \
-        s3 cp "/backup/$(basename "$out")" "s3://${S3_BUCKET}/"
+    for f in "$out" "$globals_out"; do
+        log "Uploading to s3://${S3_BUCKET}/$(basename "$f")"
+        docker run --rm \
+            -e AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
+            -e AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
+            -v "$LOCAL_BACKUP_DIR":/backup:ro \
+            amazon/aws-cli:latest \
+            --endpoint-url "$S3_ENDPOINT" \
+            s3 cp "/backup/$(basename "$f")" "s3://${S3_BUCKET}/"
+    done
     log "S3 upload OK"
 else
     log "S3 vars not set — backup stays local only"
@@ -106,17 +172,24 @@ fi
 
 # --- retention: keep the KEEP most-recent local dumps, delete the rest.
 #     Runs only AFTER a verified-good dump above, so a failed run (which exits
-#     early) can never delete older good dumps. ---
+#     early) can never delete older good dumps. Main dumps and globals dumps
+#     are separate series (the `[0-9]` after the shared `gendesign_` prefix
+#     keeps the globals_* files, which share that prefix, out of this glob). ---
 # SC2012: ls is fine here — filenames are fully controlled (gendesign_<ts>.sql.gz,
 # no spaces/newlines) and we need ls's -t mtime sort for "keep newest N" (same
 # pattern as the proven backup-tradein-db.sh).
 # shellcheck disable=SC2012
-ls -1t "$LOCAL_BACKUP_DIR"/gendesign_*.sql.gz 2>/dev/null \
+ls -1t "$LOCAL_BACKUP_DIR"/gendesign_[0-9]*.sql.gz 2>/dev/null \
+    | tail -n +"$((KEEP + 1))" \
+    | xargs -r rm -f
+# shellcheck disable=SC2012
+ls -1t "$LOCAL_BACKUP_DIR"/gendesign_globals_*.sql.gz 2>/dev/null \
     | tail -n +"$((KEEP + 1))" \
     | xargs -r rm -f
 
-# Count via a glob array (no ls parsing). nullglob -> empty array if no match.
+# Count via glob arrays (no ls parsing). nullglob -> empty array if no match.
 shopt -s nullglob
-remaining=( "$LOCAL_BACKUP_DIR"/gendesign_*.sql.gz )
+remaining=( "$LOCAL_BACKUP_DIR"/gendesign_[0-9]*.sql.gz )
+remaining_globals=( "$LOCAL_BACKUP_DIR"/gendesign_globals_*.sql.gz )
 shopt -u nullglob
-log "Backup done. Local dumps retained: ${#remaining[@]} (KEEP=${KEEP})."
+log "Backup done. Local dumps retained: ${#remaining[@]} data + ${#remaining_globals[@]} globals (KEEP=${KEEP})."
diff --git a/ops/gendesign-backup.default.example b/ops/gendesign-backup.default.example
index dea81c3c..0f6950d9 100644
--- a/ops/gendesign-backup.default.example
+++ b/ops/gendesign-backup.default.example
@@ -8,6 +8,19 @@
 # backup.sh sources this file if present. With NO S3 vars set, dumps stay
 # local-only under /opt/gendesign/backups (retention KEEP=7). Fill these in to
 # also push each dump off-box to S3 (recommended — local-only dies with the VM).
+# backup.sh now also dumps cluster globals (roles/GRANTs, via `pg_dumpall
+# --globals-only`) alongside the main dump — same S3 vars, same bucket, no
+# separate config needed (#2203).
+#
+# tradein-mvp/deploy/backup-tradein-db.sh (separate DB, separate cron job)
+# prefers its OWN env file at /etc/default/tradein-backup, but falls back to
+# THIS file if that one doesn't exist — so filling in the S3 vars here also
+# enables off-box upload for the tradein DB backup, unless you want the two
+# DBs going to different buckets/creds (then create /etc/default/tradein-backup
+# with its own S3_* vars instead).
+#
+# To rehearse a restore from a dump this script produced (safe, throwaway
+# container, never touches prod) see ops/restore-drill.sh.
 
 # --- S3 off-site upload (Selectel S3-compatible). All four required to enable. ---
 #S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru
@@ -16,6 +29,12 @@
 #S3_SECRET_KEY=REPLACE_WITH_REAL_SECRET_KEY
 
 # --- optional overrides (defaults are sensible; uncomment only to change) ---
-#KEEP=7                 # how many recent local dumps to retain
-#MIN_DUMP_BYTES=51200   # sanity floor; a dump smaller than this is treated as failed
+#KEEP=7                 # how many recent local dumps to retain (applies to both
+                        # the main dump series and the globals dump series)
+#MIN_DUMP_BYTES=51200   # sanity floor for the MAIN dump; a dump smaller than
+                        # this is treated as failed. No equivalent floor for the
+                        # globals dump — a handful of roles legitimately gzips to
+                        # well under this. tradein-backup.sh has its own,
+                        # separate MIN_DUMP_BYTES (default 10240) — set in
+                        # /etc/default/tradein-backup, not here.
 #LOCAL_BACKUP_DIR=/opt/gendesign/backups
diff --git a/ops/restore-drill.sh b/ops/restore-drill.sh
new file mode 100644
index 00000000..6424ef43
--- /dev/null
+++ b/ops/restore-drill.sh
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+# Restore DRILL for a dump produced by backup.sh / backup-tradein-db.sh (#2203).
+#
+# THIS SCRIPT NEVER TOUCHES THE PRODUCTION DATABASE. It spins up a throwaway,
+# unnamed-volume postgis/postgis container, loads the dump (and its globals
+# sibling, if found) into it, prints a few sanity numbers, and always tears
+# the container down again — win or lose.
+#
+# ops/restore.sh IS DIFFERENT AND IS DESTRUCTIVE: it restores INTO THE LIVE
+# PRODUCTION DATABASE (--clean --if-exists DROPs existing tables first). Do
+# NOT use ops/restore.sh for a drill/rehearsal — use THIS script instead.
+#
+# Usage:
+#   ops/restore-drill.sh /path/to/gendesign_20260820_030000.sql.gz
+#   ops/restore-drill.sh /path/to/tradein-20260820-043000.sql.gz [globals.sql.gz]
+#
+# Globals autodetection: if the 2nd arg is omitted, this script looks next to
+# the dump for a sibling file matching the naming convention the backup
+# scripts use (gendesign_globals_<ts>.sql.gz / tradein-globals-<ts>.sql.gz).
+# Missing globals is not fatal — the dump itself is still restored and checked.
+#
+# Which tables get row-counted is guessed from the dump's filename prefix
+# (gendesign_* vs tradein-*) and can be overridden with RESTORE_DRILL_TABLES
+# (comma-separated table names, no schema qualifier — public is assumed).
+
+set -euo pipefail
+
+IMAGE="${RESTORE_DRILL_IMAGE:-postgis/postgis:16-3.4}"
+READY_TIMEOUT="${RESTORE_DRILL_READY_TIMEOUT:-60}"  # seconds to wait for postgres startup
+DRILL_DB="drill"
+
+DUMP_FILE="${1:-}"
+if [[ -z "$DUMP_FILE" || ! -f "$DUMP_FILE" ]]; then
+    echo "Usage: $0 /path/to/dump.sql.gz [globals.sql.gz]" >&2
+    exit 2
+fi
+DUMP_FILE=$(cd "$(dirname "$DUMP_FILE")" && pwd)/$(basename "$DUMP_FILE")
+
+GLOBALS_FILE="${2:-}"
+if [[ -n "$GLOBALS_FILE" && ! -f "$GLOBALS_FILE" ]]; then
+    echo "Globals file not found: $GLOBALS_FILE" >&2
+    exit 2
+fi
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+
+# --- autodetect the globals sibling, matching the two naming conventions our
+#     backup scripts use:
+#       gendesign_<ts>.sql.gz  -> gendesign_globals_<ts>.sql.gz  (ops/backup.sh)
+#       tradein-<ts>.sql.gz    -> tradein-globals-<ts>.sql.gz    (backup-tradein-db.sh)
+dump_dir=$(dirname "$DUMP_FILE")
+dump_base=$(basename "$DUMP_FILE")
+project="gendesign"
+
+if [[ -z "$GLOBALS_FILE" ]]; then
+    candidate=""
+    if [[ "$dump_base" =~ ^([A-Za-z0-9]+)_([0-9]{8}_[0-9]{6})\.sql\.gz$ ]]; then
+        candidate="${dump_dir}/${BASH_REMATCH[1]}_globals_${BASH_REMATCH[2]}.sql.gz"
+    elif [[ "$dump_base" =~ ^([A-Za-z0-9]+)-([0-9]{8}-[0-9]{6})\.sql\.gz$ ]]; then
+        candidate="${dump_dir}/${BASH_REMATCH[1]}-globals-${BASH_REMATCH[2]}.sql.gz"
+    fi
+    if [[ -n "$candidate" && -f "$candidate" ]]; then
+        GLOBALS_FILE="$candidate"
+    fi
+fi
+
+if [[ "$dump_base" == tradein-* ]]; then
+    project="tradein"
+fi
+
+if [[ -n "$GLOBALS_FILE" ]]; then
+    log "Dump:    $DUMP_FILE"
+    log "Globals: $GLOBALS_FILE"
+else
+    log "Dump:    $DUMP_FILE"
+    log "Globals: none found next to the dump — restoring data only, no roles/GRANTs."
+fi
+
+# --- default row-count table list, per project; override with
+#     RESTORE_DRILL_TABLES=t1,t2,... ---
+if [[ -n "${RESTORE_DRILL_TABLES:-}" ]]; then
+    tables_csv="$RESTORE_DRILL_TABLES"
+elif [[ "$project" == "tradein" ]]; then
+    tables_csv="listings,listing_sources,deals,houses,trade_in_estimates"
+else
+    # Representative core tables across the gendesign schema: the big
+    # partitioned deals dataset, cadastral/opportunity overlays, the DomRF
+    # snapshot chain, and two smaller user-facing tables.
+    tables_csv="rosreestr_deals,cad_opportunity_parcels,domrf_snapshots,trade_in_estimates,parcel_user_status"
+fi
+IFS=',' read -r -a TABLES <<< "$tables_csv"
+
+# --- throwaway container: random name, docker-assigned host port, no prod
+#     volumes mounted (anonymous storage inside the container layer only —
+#     gone the instant the container is removed). trust auth is fine here:
+#     single-use, bound to 127.0.0.1, destroyed on exit. ---
+CONTAINER="restore-drill-$$-$(date -u +%s)"
+cleanup() {
+    log "Cleaning up container ${CONTAINER}"
+    docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+log "Starting throwaway ${IMAGE} as ${CONTAINER}"
+docker run -d --name "$CONTAINER" \
+    -e POSTGRES_HOST_AUTH_METHOD=trust \
+    -p 127.0.0.1::5432 \
+    "$IMAGE" >/dev/null
+
+host_port=$(docker port "$CONTAINER" 5432/tcp | head -1 | cut -d: -f2)
+log "Container up, mapped to 127.0.0.1:${host_port} (only reachable while this drill runs)"
+
+log "Waiting for postgres to accept connections (timeout ${READY_TIMEOUT}s)..."
+ready=0
+for _ in $(seq 1 "$READY_TIMEOUT"); do
+    if docker exec "$CONTAINER" pg_isready -U postgres >/dev/null 2>&1; then
+        ready=1
+        break
+    fi
+    sleep 1
+done
+if [[ "$ready" -ne 1 ]]; then
+    log "ERROR: postgres did not become ready within ${READY_TIMEOUT}s" >&2
+    exit 1
+fi
+
+psql_c() {
+    # $1 = target db, $2 = SQL
+    docker exec -i "$CONTAINER" psql -v ON_ERROR_STOP=1 -U postgres -d "$1" -c "$2"
+}
+
+log "Creating drill database '${DRILL_DB}'"
+psql_c postgres "CREATE DATABASE ${DRILL_DB};" >/dev/null
+
+# --- globals: best-effort. A pg_dumpall --globals-only dump against a fresh
+#     container almost always trips over the container's own bootstrap
+#     'postgres' role (already exists), which is expected and harmless — so
+#     this load is NOT ON_ERROR_STOP. The dump restore below is the real
+#     integrity check and DOES use ON_ERROR_STOP=1. ---
+if [[ -n "$GLOBALS_FILE" ]]; then
+    log "Loading globals (best-effort — role-already-exists warnings are expected)"
+    gunzip -c "$GLOBALS_FILE" | docker exec -i "$CONTAINER" psql -U postgres -d postgres \
+        || log "NOTE: globals load reported errors above — usually just pre-existing default roles, non-fatal for the drill."
+fi
+
+log "Restoring dump into '${DRILL_DB}' (ON_ERROR_STOP=1 — any real error aborts the drill)"
+gunzip -c "$DUMP_FILE" | docker exec -i "$CONTAINER" psql -v ON_ERROR_STOP=1 -U postgres -d "$DRILL_DB"
+
+log "Restore finished. Running sanity checks."
+
+postgis_version=$(docker exec -i "$CONTAINER" psql -U postgres -d "$DRILL_DB" -Atqc "SELECT postgis_full_version();" 2>/dev/null || echo "N/A (postgis not installed in this dump)")
+echo "postgis_full_version(): ${postgis_version}"
+
+table_count=$(docker exec -i "$CONTAINER" psql -U postgres -d "$DRILL_DB" -Atqc \
+    "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public';")
+echo "Tables in public schema: ${table_count}"
+
+echo
+echo "table | rows"
+echo "----- | ----"
+for t in "${TABLES[@]}"; do
+    exists=$(docker exec -i "$CONTAINER" psql -U postgres -d "$DRILL_DB" -Atqc \
+        "SELECT to_regclass('public.${t}') IS NOT NULL;")
+    if [[ "$exists" == "t" ]]; then
+        rows=$(docker exec -i "$CONTAINER" psql -U postgres -d "$DRILL_DB" -Atqc \
+            "SELECT count(*) FROM \"${t}\";")
+        echo "${t} | ${rows}"
+    else
+        echo "${t} | n/a (table not found in this dump)"
+    fi
+done
+
+log "Drill complete. Container will now be removed."
diff --git a/tradein-mvp/deploy/backup-tradein-db.sh b/tradein-mvp/deploy/backup-tradein-db.sh
index 1829de51..73814f1e 100755
--- a/tradein-mvp/deploy/backup-tradein-db.sh
+++ b/tradein-mvp/deploy/backup-tradein-db.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 # Бэкап БД tradein-postgres — pg_dump по cron + retention (#397).
+# Hardening для #2203: globals, integrity-check, S3-выгрузка.
 #
 # Запускается из cron на прод-хосте. pg_dump идёт через `docker exec`
 # (локальный сокет внутри контейнера — пароль не нужен).
@@ -10,33 +11,141 @@
 # Restore:
 #   gunzip -c tradein-YYYYMMDD-HHMMSS.sql.gz | \
 #     docker exec -i tradein-postgres psql -U tradein -d tradein
+# Restore-дрель (безопасно, не трогает прод): ops/restore-drill.sh.
+#
+# Опциональная выгрузка в S3 — переменные в /etc/default/tradein-backup
+# (root-owned, chmod 600, НЕ в git); если файла нет — фолбэк на
+# /etc/default/gendesign-backup (общие креды с основным бэкапом). Без
+# переменных поведение прежнее: дамп остаётся только локально.
+#   S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru
+#   S3_BUCKET=gendesign-backups
+#   S3_ACCESS_KEY=...
+#   S3_SECRET_KEY=...
 
 set -euo pipefail
 
 BACKUP_DIR="${BACKUP_DIR:-/opt/gendesign/backups/tradein}"
 KEEP="${KEEP:-7}"                                  # сколько копий хранить
 PG_CONTAINER="${PG_CONTAINER:-tradein-postgres}"
+MIN_DUMP_BYTES="${MIN_DUMP_BYTES:-10240}"          # 10 KiB floor — пустая/битая
+                                                   # схема заведомо меньше, живая
+                                                   # БД — на порядки больше.
+
+# S3-переменные: свой env-файл, а если его нет — общий с main-бэкапом.
+if [[ -f /etc/default/tradein-backup ]]; then
+  source /etc/default/tradein-backup
+elif [[ -f /etc/default/gendesign-backup ]]; then
+  source /etc/default/gendesign-backup
+fi
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+
+# Проверка целостности сверх «не пустой»: битый посреди записи дамп (диск
+# кончился, OOM-kill, оборвался docker exec) может дать структурно валидный,
+# не крошечный .gz — ни `-s`, ни MIN_DUMP_BYTES это не ловят. Две проверки,
+# сначала дешёвая:
+#   1. gzip -t — ловит битый/оборванный gzip-поток как таковой.
+#   2. trailer — pg_dump/pg_dumpall всегда пишут фиксированный комментарий
+#      «я дописал до конца» последней строкой потока; при обрыве записи
+#      трейлера не будет, даже если gzip-обёртка выглядит нормально.
+# Проверяется через `gunzip -c | tail -N`, чтобы не распаковывать весь дамп
+# на диск ради последних строк.
+verify_dump_integrity() {
+  local file="$1" trailer="$2" label="$3"
+  if ! gzip -t "$file" 2>/dev/null; then
+    log "ОШИБКА: ${label} не прошёл проверку gzip -> $file" >&2
+    return 1
+  fi
+  if ! gunzip -c "$file" 2>/dev/null | tail -5 | grep -qF "$trailer"; then
+    log "ОШИБКА: ${label} без трейлера '${trailer}' -> $file (дамп оборван?)" >&2
+    return 1
+  fi
+  return 0
+}
 
 mkdir -p "$BACKUP_DIR"
 ts=$(date -u +'%Y%m%d-%H%M%S')
 out="$BACKUP_DIR/tradein-$ts.sql.gz"
+globals_out="$BACKUP_DIR/tradein-globals-$ts.sql.gz"
 
 # --clean --if-exists → дамп самодостаточен для восстановления «с нуля».
 # --no-owner → restore не требует тех же ролей.
+# Stderr больше не глотается (было `2>/dev/null` до #2203) — ошибка pg_dump
+# должна попасть в лог, а не исчезнуть молча. `set -o pipefail` уже роняет
+# весь пайп при ошибке pg_dump, так что поведение при сбое не меняется —
+# меняется только видимость причины.
 docker exec "$PG_CONTAINER" pg_dump -U tradein -d tradein \
-  --no-owner --clean --if-exists 2>/dev/null | gzip -6 > "$out"
+  --no-owner --clean --if-exists | gzip -6 > "$out"
 
 if [[ ! -s "$out" ]]; then
-  echo "[$(date -u +%H:%M:%S)] ОШИБКА: дамп пустой — $out" >&2
+  log "ОШИБКА: дамп пустой — $out" >&2
   rm -f "$out"
   exit 1
 fi
 
-# Retention — оставляем KEEP самых свежих, остальные tradein-*.sql.gz удаляем.
-ls -1t "$BACKUP_DIR"/tradein-*.sql.gz 2>/dev/null \
+dump_bytes=$(wc -c < "$out" | tr -d ' ')
+if (( dump_bytes < MIN_DUMP_BYTES )); then
+  log "ОШИБКА: дамп всего ${dump_bytes} байт (< ${MIN_DUMP_BYTES} floor) — похоже на неудачный дамп." >&2
+  log "Удаляю подозрительный дамп, старые хорошие копии не трогаю." >&2
+  rm -f "$out"
+  exit 1
+fi
+
+if ! verify_dump_integrity "$out" "-- PostgreSQL database dump complete" "основной дамп"; then
+  log "Удаляю подозрительный дамп, старые хорошие копии не трогаю." >&2
+  rm -f "$out"
+  exit 1
+fi
+
+log "Дамп ok: $out ($(du -h "$out" | cut -f1), ${dump_bytes} байт)"
+
+# --- globals (роли/GRANT) — pg_dump их не включает по определению; без
+#     этого восстановленная база остаётся без владеющих ролей/привилегий. ---
+log "Дамплю globals (роли/GRANT) -> $globals_out"
+docker exec "$PG_CONTAINER" pg_dumpall -U tradein --globals-only \
+  | gzip -6 > "$globals_out"
+
+if [[ ! -s "$globals_out" ]]; then
+  log "ОШИБКА: дамп globals пустой — $globals_out" >&2
+  rm -f "$globals_out"
+  exit 1
+fi
+
+if ! verify_dump_integrity "$globals_out" "-- PostgreSQL database cluster dump complete" "globals-дамп"; then
+  rm -f "$globals_out"
+  exit 1
+fi
+
+log "Globals ok: $globals_out ($(du -h "$globals_out" | cut -f1))"
+
+# --- опциональная выгрузка в S3 (только если заданы все четыре переменные) ---
+if [[ -n "${S3_ENDPOINT:-}" && -n "${S3_BUCKET:-}" && -n "${S3_ACCESS_KEY:-}" && -n "${S3_SECRET_KEY:-}" ]]; then
+  for f in "$out" "$globals_out"; do
+    log "Заливаю в s3://${S3_BUCKET}/$(basename "$f")"
+    docker run --rm \
+      -e AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
+      -e AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
+      -v "$BACKUP_DIR":/backup:ro \
+      amazon/aws-cli:latest \
+      --endpoint-url "$S3_ENDPOINT" \
+      s3 cp "/backup/$(basename "$f")" "s3://${S3_BUCKET}/"
+  done
+  log "Выгрузка в S3 ok"
+else
+  log "S3-переменные не заданы — дамп остаётся только локально"
+fi
+
+# Retention — оставляем KEEP самых свежих, остальные удаляем. Основные дампы
+# и globals-дампы — раздельные серии (иначе они смешаются в общей mtime-
+# сортировке и retention посчитает их вместе).
+ls -1t "$BACKUP_DIR"/tradein-[0-9]*.sql.gz 2>/dev/null \
+  | tail -n +"$((KEEP + 1))" \
+  | xargs -r rm -f
+ls -1t "$BACKUP_DIR"/tradein-globals-*.sql.gz 2>/dev/null \
   | tail -n +"$((KEEP + 1))" \
   | xargs -r rm -f
 
 size=$(du -h "$out" | cut -f1)
-count=$(ls -1 "$BACKUP_DIR"/tradein-*.sql.gz 2>/dev/null | wc -l | tr -d ' ')
-echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] backup ok: $out ($size), копий хранится: $count"
+count=$(ls -1 "$BACKUP_DIR"/tradein-[0-9]*.sql.gz 2>/dev/null | wc -l | tr -d ' ')
+globals_count=$(ls -1 "$BACKUP_DIR"/tradein-globals-*.sql.gz 2>/dev/null | wc -l | tr -d ' ')
+echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] backup ok: $out ($size), копий хранится: $count данных + $globals_count globals"
-- 
2.45.3


From 7691db8d2cf0b71616c6ef6513280749a6147e7b Mon Sep 17 00:00:00 2001
From: bot-backend <alexandro361@gmail.com>
Date: Thu, 20 Aug 2026 22:14:21 +0300
Subject: [PATCH 2/4] =?UTF-8?q?chore(ops):=20=D0=B2=D0=BE=D1=81=D1=81?=
 =?UTF-8?q?=D1=82=D0=B0=D0=BD=D0=BE=D0=B2=D0=B8=D1=82=D1=8C=20+x=20=D0=BD?=
 =?UTF-8?q?=D0=B0=20ops/restore-drill.sh?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

core.filemode=false на Windows-чекауте молча создал файл как 100644;
остальные ops/*.sh — 100755, restore-drill.sh должен быть исполняемым
так же (cron/deploy конвенция ops/backup.sh).
---
 ops/restore-drill.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 ops/restore-drill.sh

diff --git a/ops/restore-drill.sh b/ops/restore-drill.sh
old mode 100644
new mode 100755
-- 
2.45.3


From 2d2336cd519bd3afadf55d46d0fef514742105f6 Mon Sep 17 00:00:00 2001
From: bot-backend <alexandro361@gmail.com>
Date: Thu, 20 Aug 2026 22:24:03 +0300
Subject: [PATCH 3/4] =?UTF-8?q?fix(ops):=20=D0=B4=D0=B5=D0=BF=D0=BB=D0=BE?=
 =?UTF-8?q?=D0=B9=20=D1=82=D1=80=D0=B8=D0=B3=D0=B3=D0=B5=D1=80=D0=B8=D1=82?=
 =?UTF-8?q?=D1=81=D1=8F=20=D0=BD=D0=B0=20=D0=BB=D1=8E=D0=B1=D0=BE=D0=B9=20?=
 =?UTF-8?q?ops/*.sh,=20=D0=B0=20=D0=BD=D0=B5=20=D1=82=D0=BE=D0=BB=D1=8C?=
 =?UTF-8?q?=D0=BA=D0=BE=20docker-prune.sh=20(#2203)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

paths-filter в deploy.yml знал только про ops/docker-prune.sh (#2887) — правка
ops/backup.sh или новый ops/restore-drill.sh из этого же PR не долетели бы до
/opt/gendesign: деплой не триггерится -> git reset --hard origin/main не
исполняется -> cron на VM месяцами крутит старую версию, молча.

Точечный список сам по себе и есть баг: #2887 добавил только тот файл, о
котором тогда шла речь, и следующий новый ops-скрипт (backup.sh) остался
за бортом. Глоб ops/*.sh закрывает класс целиком — не матчит подпути
(ops/db-bootstrap/**, ops/glitchtip-auth-forwarder/**), у них свои explicit
триггеры уже есть, дублирования нет.
---
 .forgejo/workflows/deploy.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml
index fa30a939..fd04b561 100644
--- a/.forgejo/workflows/deploy.yml
+++ b/.forgejo/workflows/deploy.yml
@@ -32,7 +32,13 @@ on:
       # по cron из /opt/gendesign/ops/, куда попадает только через `git reset --hard`
       # шага деплоя. Без этой строки правка скрипта лежала бы в main, а cron месяцами
       # исполнял бы старую версию — молча и без единого сигнала.
-      - "ops/docker-prune.sh"
+      # Глоб, а не точечный список (#2203): класс бага — «любой ops-скрипт,
+      # запускаемый по cron с VM», не только docker-prune.sh. Сейчас сюда попадают
+      # backup.sh, restore-drill.sh, restore.sh, uptime-healthcheck.sh — точечное
+      # перечисление пришлось бы дополнять при каждом новом скрипте, и про это
+      # снова забыли бы (см. как этот самый комментарий выше был точечным про
+      # docker-prune.sh и не спас backup.sh). Глоб закрывает класс целиком.
+      - "ops/*.sh"
   workflow_dispatch:
 
 # #2950: ОБЩАЯ группа с deploy-tradein.yml — не опечатка и не копипаста.
-- 
2.45.3


From e909dbcadacc71285c356baf0386aebfadc017e5 Mon Sep 17 00:00:00 2001
From: bot-backend <alexandro361@gmail.com>
Date: Thu, 20 Aug 2026 22:25:34 +0300
Subject: [PATCH 4/4] =?UTF-8?q?docs(rules):=20deploy.md=20=D0=BE=D1=82?=
 =?UTF-8?q?=D1=80=D0=B0=D0=B6=D0=B0=D0=B5=D1=82=20ops/*.sh=20=D0=B3=D0=BB?=
 =?UTF-8?q?=D0=BE=D0=B1=20=D0=B2=D0=BC=D0=B5=D1=81=D1=82=D0=BE=20=D1=82?=
 =?UTF-8?q?=D0=BE=D1=87=D0=B5=D1=87=D0=BD=D0=BE=D0=B3=D0=BE=20docker-prune?=
 =?UTF-8?q?.sh=20(#2203)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Оба утверждения в разделе Path triggers устарели ровно из-за коммита
2d2336cd в этой же ветке: список больше не содержит ops/docker-prune.sh
(теперь ops/*.sh), а предупреждение «добавлять в paths явно» для
любого нового ops/<name>.sh больше не верно — глоб их подхватывает сам.

Осталась одна деталь, о которой правда надо помнить: одиночная звёздочка
не пересекает /, так что новый ПОДКАТАЛОГ внутри ops/ (как db-bootstrap/,
glitchtip-auth-forwarder/) под глоб не попадает и всё ещё требует своей
строки в paths — иначе тот же класс бага (#2887 / #2203) повторится для
подкаталога.
---
 .claude/rules/deploy.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.claude/rules/deploy.md b/.claude/rules/deploy.md
index 6b0077bb..95419c2f 100644
--- a/.claude/rules/deploy.md
+++ b/.claude/rules/deploy.md
@@ -25,8 +25,8 @@ Reference incident: PR #346 (2026-05-18) deploy → user сам нашёл prod
 
 ## Path triggers (Forgejo Actions, `.forgejo/workflows/`)
 
-- `backend/**`, `frontend/**`, `Caddyfile`, `caddy/**`, `docker-compose.prod.yml`, `data/sql/**`, `ops/glitchtip-auth-forwarder/**`, `ops/db-bootstrap/**`, `ops/docker-prune.sh`, `.forgejo/workflows/deploy.yml` → `deploy.yml` (main Site Finder stack)
-- ⚠️ `ops/**` целиком **не** триггерит — только перечисленные подпути. Любой новый файл в `ops/`, который исполняется на VM (cron / шаг деплоя), надо добавлять в `paths:` явно, иначе он не доедет до `/opt/gendesign` и будет молча исполняться в старой версии
+- `backend/**`, `frontend/**`, `Caddyfile`, `caddy/**`, `docker-compose.prod.yml`, `data/sql/**`, `ops/glitchtip-auth-forwarder/**`, `ops/db-bootstrap/**`, `ops/*.sh`, `.forgejo/workflows/deploy.yml` → `deploy.yml` (main Site Finder stack)
+- `ops/*.sh` (#2203) — любой скрипт непосредственно в `ops/` уезжает на VM автоматически, дополнять `paths:` вручную для нового `ops/<name>.sh` не нужно. ⚠️ Одиночная звёздочка не пересекает `/` — **новый подкаталог** внутри `ops/` (по образцу `ops/db-bootstrap/`, `ops/glitchtip-auth-forwarder/`) под этот глоб не попадает и требует своей отдельной строки в `paths:`, иначе не доедет до `/opt/gendesign` и будет молча исполняться в старой версии
 - trade-in изменения → `deploy-tradein.yml` (отдельный stack; paths-filter base = last deployed SHA → накопленный diff, fail-safe build-all)
 - `docker-compose.obsidian.yml`, `scripts/setup-couchdb.sh`, `docs/obsidian-livesync.md` → `.forgejo/workflows/deploy-obsidian.yml`
 - `docs/**` alone → НЕ триггерит деплой
-- 
2.45.3