Merge pull request 'ops(backup): missed-run detection + Forgejo code/DB backup to S3' (#3025) from feat/2203-backup-staleness-and-forgejo into main

ops/backup-forgejo.sh

@@ -0,0 +1,205 @@
+#!/usr/bin/env bash
+# Backup for self-hosted Forgejo (git.gendsgn.ru) — repos + DB, off-box to S3
+# under its OWN, more-restricted key (#2203).
+#
+# Forgejo is deployed OUTSIDE this repo's checkout, per
+# infra/Forgejo_Migration_BotServer_To_Beget_2026-05-16.md (vault):
+#   - compose dir on the VM: /home/gendesign/forgejo/
+#   - repo data:             /home/gendesign/forgejo/data/forgejo/git/repositories/
+#   - DB: NOT a dedicated container — a separate `forgejo` user+database
+#     inside the SAME shared postgres container as the main app
+#     (gendesign-postgres-1). `pg_dump -U forgejo forgejo` via `docker exec`.
+#
+# Two artifacts per run:
+#   1. DB dump   — `pg_dump --no-owner --clean --if-exists` (repos/issues/PRs/
+#      users/settings/Actions config — everything except the git data itself).
+#   2. Repo bundle — a tar.gz of the bare-repo tree under FORGEJO_REPOS_DIR.
+#      This is functionally a "bundle of repositories": bare repos already
+#      ARE just refs+objects, byte-identical to what `git bundle` would
+#      capture, and tar-ing the whole tree in one shot is far more robust
+#      than enumerating individual repos (new repos need zero script changes,
+#      nothing to keep in sync). LFS objects/attachments/avatars under
+#      FORGEJO_DATA_DIR are OUT of scope for this pass — repo code + full DB
+#      (which has LFS pointers, not the blobs) covers the "backup the code"
+#      ask; note this gap in the PR if larger asset backup is wanted later.
+#
+# Off-box S3 upload is MANDATORY here, not optional like ops/backup.sh —
+# the whole point of this script is getting Forgejo's code off the VM it
+# lives on, so with no S3 creds configured this refuses to run rather than
+# silently producing a local-only "backup" that doesn't meet that bar.
+#
+# The S3 key here is a SEPARATE, narrower-scoped service user than the one
+# ops/backup.sh and backup-tradein-db.sh use (`gendsgn-backup-writer`, which
+# can write anywhere in the bucket root) — this one can ONLY PutObject under
+# s3://gendsgn-backups/forgejo/*, nothing else, so a compromised Forgejo host
+# can't touch the main/tradein DB backups sitting in the same bucket. See the
+# PR description for the exact bucket policy JSON (mirrors the existing
+# gendsgn-backup-writer policy pattern — vault meta/00_credentials.md
+# "Selectel S3 — бэкап-бакет gendsgn-backups").
+#
+# THE KEY DOES NOT EXIST YET (as of #2203) — a human needs to create the
+# service user + policy in the Selectel panel first. Until then this script
+# is expected to fail loudly and exit non-zero; that is the correct,
+# intentional behaviour, not a bug.
+#
+# Missed-run detection (#2203): same sentinel/staleness-check pattern as
+# ops/backup.sh — see ops/check-backup-staleness.sh for the alerting side.
+#
+# Usage (cron — `bash <path>`, not a bare path, same "+x can go missing on a
+# raw deploy reset" reasoning as the other two backup scripts, #71):
+#   15 4 * * * bash /opt/gendesign/ops/backup-forgejo.sh >> /var/log/gendesign-backup-forgejo.log 2>&1
+#
+# Config file (root-owned, chmod 600, NOT in git) — see
+# ops/gendesign-backup-forgejo.default.example for the full list:
+#   FORGEJO_S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru
+#   FORGEJO_S3_BUCKET=gendsgn-backups
+#   FORGEJO_S3_PREFIX=forgejo/
+#   FORGEJO_S3_ACCESS_KEY=...
+#   FORGEJO_S3_SECRET_KEY=...
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=./lib-backup.sh
+source "$SCRIPT_DIR/lib-backup.sh"
+
+# --- config (env-overridable) ---
+FORGEJO_DIR="${FORGEJO_DIR:-/home/gendesign/forgejo}"
+FORGEJO_DATA_DIR="${FORGEJO_DATA_DIR:-${FORGEJO_DIR}/data/forgejo}"
+FORGEJO_REPOS_DIR="${FORGEJO_REPOS_DIR:-${FORGEJO_DATA_DIR}/git/repositories}"
+
+PG_CONTAINER="${PG_CONTAINER:-gendesign-postgres-1}"
+FORGEJO_DB_USER="${FORGEJO_DB_USER:-forgejo}"
+FORGEJO_DB_NAME="${FORGEJO_DB_NAME:-forgejo}"
+
+LOCAL_BACKUP_DIR="${LOCAL_BACKUP_DIR:-/opt/gendesign/backups/forgejo}"
+KEEP="${KEEP:-7}"
+MIN_DB_DUMP_BYTES="${MIN_DB_DUMP_BYTES:-2048}"        # forgejo DB is small (config/issues/PRs, no git blobs)
+MIN_REPOS_BUNDLE_BYTES="${MIN_REPOS_BUNDLE_BYTES:-10240}"
+SENTINEL_FILE="${SENTINEL_FILE:-${LOCAL_BACKUP_DIR}/.last_success}"
+
+FORGEJO_BACKUP_ENV_FILE="${FORGEJO_BACKUP_ENV_FILE:-/etc/default/gendesign-backup-forgejo}"
+# shellcheck source=/dev/null
+[[ -f "$FORGEJO_BACKUP_ENV_FILE" ]] && source "$FORGEJO_BACKUP_ENV_FILE"
+
+FORGEJO_S3_ENDPOINT="${FORGEJO_S3_ENDPOINT:-}"
+FORGEJO_S3_BUCKET="${FORGEJO_S3_BUCKET:-}"
+FORGEJO_S3_PREFIX="${FORGEJO_S3_PREFIX:-forgejo/}"
+FORGEJO_S3_ACCESS_KEY="${FORGEJO_S3_ACCESS_KEY:-}"
+FORGEJO_S3_SECRET_KEY="${FORGEJO_S3_SECRET_KEY:-}"
+
+# --- guard 0: S3 creds MUST be configured — this backup's whole point is
+#     off-box storage under a narrow-scoped key. No creds = refuse to run,
+#     loudly, rather than silently producing a local-only "backup" (unlike
+#     ops/backup.sh, where local-only is an accepted fallback). ---
+if [[ -z "$FORGEJO_S3_ENDPOINT" || -z "$FORGEJO_S3_BUCKET" || -z "$FORGEJO_S3_ACCESS_KEY" || -z "$FORGEJO_S3_SECRET_KEY" ]]; then
+    log "ERROR: Forgejo S3 backup is NOT CONFIGURED."
+    log "ERROR: missing one or more of FORGEJO_S3_ENDPOINT / FORGEJO_S3_BUCKET / FORGEJO_S3_ACCESS_KEY / FORGEJO_S3_SECRET_KEY."
+    log "ERROR: create the narrow-scoped 'gendsgn-backup-forgejo-writer' S3 service user (see PR for the exact bucket policy)"
+    log "ERROR: and fill them into ${FORGEJO_BACKUP_ENV_FILE} (see ops/gendesign-backup-forgejo.default.example)."
+    log "ERROR: refusing to produce a local-only backup — that does not meet this script's off-box requirement."
+    exit 1
+fi
+
+if [[ ! -d "$FORGEJO_REPOS_DIR" ]]; then
+    log "ERROR: FORGEJO_REPOS_DIR does not exist -> $FORGEJO_REPOS_DIR"
+    log "ERROR: Forgejo's on-disk layout may differ from the documented default — override FORGEJO_REPOS_DIR."
+    exit 1
+fi
+
+mkdir -p "$LOCAL_BACKUP_DIR"
+ts=$(date -u +'%Y%m%d_%H%M%S')
+db_out="${LOCAL_BACKUP_DIR}/forgejo-db_${ts}.sql.gz"
+repos_out="${LOCAL_BACKUP_DIR}/forgejo-repos_${ts}.tar.gz"
+
+# --- 1. DB dump ---
+log "Dumping Forgejo DB (${FORGEJO_DB_NAME} as ${FORGEJO_DB_USER}, container ${PG_CONTAINER}) -> ${db_out}"
+docker exec "$PG_CONTAINER" pg_dump -U "$FORGEJO_DB_USER" -d "$FORGEJO_DB_NAME" --no-owner --clean --if-exists \
+    | gzip -9 > "$db_out"
+
+if [[ ! -s "$db_out" ]]; then
+    log "ERROR: DB dump is empty -> $db_out — removing, keeping previous good backups." >&2
+    rm -f "$db_out"
+    exit 1
+fi
+
+db_bytes=$(wc -c < "$db_out" | tr -d ' ')
+if (( db_bytes < MIN_DB_DUMP_BYTES )); then
+    log "ERROR: DB dump only ${db_bytes} bytes (< ${MIN_DB_DUMP_BYTES} floor) — likely a failed dump." >&2
+    rm -f "$db_out"
+    exit 1
+fi
+
+if ! verify_dump_integrity "$db_out" "-- PostgreSQL database dump complete" "forgejo DB dump"; then
+    log "Removing suspect DB dump, NOT pruning older good backups." >&2
+    rm -f "$db_out"
+    exit 1
+fi
+
+log "DB dump OK: ${db_out} ($(du -h "$db_out" | cut -f1), ${db_bytes} bytes)"
+
+# --- 2. repo bundle (tar.gz of the bare-repo tree) ---
+log "Bundling Forgejo repos from ${FORGEJO_REPOS_DIR} -> ${repos_out}"
+tar -czf "$repos_out" -C "$(dirname "$FORGEJO_REPOS_DIR")" "$(basename "$FORGEJO_REPOS_DIR")"
+
+if [[ ! -s "$repos_out" ]]; then
+    log "ERROR: repo bundle is empty -> $repos_out — removing." >&2
+    rm -f "$repos_out"
+    exit 1
+fi
+
+repos_bytes=$(wc -c < "$repos_out" | tr -d ' ')
+if (( repos_bytes < MIN_REPOS_BUNDLE_BYTES )); then
+    log "ERROR: repo bundle only ${repos_bytes} bytes (< ${MIN_REPOS_BUNDLE_BYTES} floor) — likely empty/failed FORGEJO_REPOS_DIR." >&2
+    rm -f "$repos_out"
+    exit 1
+fi
+
+# tar integrity: a bundle truncated mid-write (disk full, killed process)
+# fails `tar -t` even when the gzip framing looks superficially OK — same
+# "don't trust non-empty + big-enough alone" discipline as verify_dump_integrity.
+if ! tar -tzf "$repos_out" >/dev/null 2>&1; then
+    log "ERROR: repo bundle failed tar integrity check -> $repos_out (truncated archive?)" >&2
+    rm -f "$repos_out"
+    exit 1
+fi
+
+log "Repo bundle OK: ${repos_out} ($(du -h "$repos_out" | cut -f1), ${repos_bytes} bytes)"
+
+# --- 3. upload to S3 (mandatory — see guard 0 above) ---
+for f in "$db_out" "$repos_out"; do
+    key="${FORGEJO_S3_PREFIX%/}/$(basename "$f")"
+    log "Uploading to s3://${FORGEJO_S3_BUCKET}/${key}"
+    # Same AWS_CA_BUNDLE fix as ops/backup.sh / backup-tradein-db.sh — aws-cli
+    # v2's own baked-in CA bundle is missing the root Selectel's cert chains
+    # to; point it at the container's system store instead.
+    docker run --rm \
+        -e AWS_ACCESS_KEY_ID="$FORGEJO_S3_ACCESS_KEY" \
+        -e AWS_SECRET_ACCESS_KEY="$FORGEJO_S3_SECRET_KEY" \
+        -e AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
+        -v "$LOCAL_BACKUP_DIR":/backup:ro \
+        amazon/aws-cli:latest \
+        --endpoint-url "$FORGEJO_S3_ENDPOINT" \
+        s3 cp --no-progress "/backup/$(basename "$f")" "s3://${FORGEJO_S3_BUCKET}/${key}"
+done
+log "S3 upload OK"
+
+# --- 4. local retention ---
+# shellcheck disable=SC2012
+ls -1t "$LOCAL_BACKUP_DIR"/forgejo-db_*.sql.gz 2>/dev/null \
+    | tail -n +"$((KEEP + 1))" \
+    | xargs -r rm -f
+# shellcheck disable=SC2012
+ls -1t "$LOCAL_BACKUP_DIR"/forgejo-repos_*.tar.gz 2>/dev/null \
+    | tail -n +"$((KEEP + 1))" \
+    | xargs -r rm -f
+
+shopt -s nullglob
+remaining_db=( "$LOCAL_BACKUP_DIR"/forgejo-db_*.sql.gz )
+remaining_repos=( "$LOCAL_BACKUP_DIR"/forgejo-repos_*.tar.gz )
+shopt -u nullglob
+log "Forgejo backup done. Local copies retained: ${#remaining_db[@]} db + ${#remaining_repos[@]} repo bundles (KEEP=${KEEP})."
+
+# --- missed-run detection (#2203): mark success ONLY after everything above
+#     (including the mandatory S3 upload) has passed. ---
+write_sentinel "$SENTINEL_FILE"

ops/backup.sh

@@ -34,9 +34,19 @@
 #
 # Restore: see ops/restore.sh (destructive, INTO the live DB).
 # Restore drill (safe, throwaway container): see ops/restore-drill.sh.
+#
+# Missed-run detection (#2203): on a verified-good run this writes a sentinel
+# file (SENTINEL_FILE) with the UTC timestamp. A SEPARATE cron entry running
+# ops/check-backup-staleness.sh against that file is what actually alerts if
+# cron stops firing — see that script's header for the alert channel and
+# cron line. This script itself does NOT alert; it only ever marks success.
 
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=./lib-backup.sh
+source "$SCRIPT_DIR/lib-backup.sh"
+
 # --- config (env-overridable) ---
 COMPOSE_DIR="${COMPOSE_DIR:-/opt/gendesign}"
 COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.prod.yml}"
@@ -49,12 +59,11 @@ MIN_DUMP_BYTES="${MIN_DUMP_BYTES:-51200}"  # 50 KiB floor; gzip'd schema-only du
                                            # (No equivalent floor for the globals
                                            # dump — a handful of roles legitimately
                                            # gzips to well under this.)
+SENTINEL_FILE="${SENTINEL_FILE:-${LOCAL_BACKUP_DIR}/.last_success}"
 
 # Optional S3 env. Loaded from /etc/default/gendesign-backup if present.
 [[ -f /etc/default/gendesign-backup ]] && source /etc/default/gendesign-backup
 
-log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
-
 # Integrity check beyond "non-empty"/"big enough": a dump truncated mid-write
 # (disk full, OOM-kill, docker exec dropped) can still gzip into a structurally
 # valid, non-tiny .gz — neither `-s` nor MIN_DUMP_BYTES catch that reliably.
@@ -198,3 +207,10 @@ remaining=( "$LOCAL_BACKUP_DIR"/gendesign_[0-9]*.sql.gz )
 remaining_globals=( "$LOCAL_BACKUP_DIR"/gendesign_globals_*.sql.gz )
 shopt -u nullglob
 log "Backup done. Local dumps retained: ${#remaining[@]} data + ${#remaining_globals[@]} globals (KEEP=${KEEP})."
+
+# --- missed-run detection (#2203): mark success ONLY here, after every
+#     guard above (dump/globals integrity, retention, and — since this whole
+#     script runs under `set -euo pipefail` — any S3 upload) has passed. See
+#     ops/check-backup-staleness.sh for the piece that actually alerts if
+#     this stops being refreshed. ---
+write_sentinel "$SENTINEL_FILE"

ops/check-backup-staleness.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# Missed-run detector for the backup scripts (#2203).
+#
+# ops/backup.sh, ops/backup-forgejo.sh, and tradein-mvp/deploy/backup-tradein-db.sh
+# each write a sentinel file (via write_sentinel() in ops/lib-backup.sh) the
+# instant a run finishes with a VERIFIED-good dump — same file the older two
+# scripts already treat as "only mark success once every guard passed".
+# This script checks how old that sentinel is. If cron stops firing (or every
+# run fails before reaching the sentinel write), the file goes stale and this
+# script says so loudly instead of the silence that let backups break for
+# weeks undetected before (#71).
+#
+# Alerts only on a STATE TRANSITION (fresh->stale, stale->fresh), same
+# discipline as ops/uptime-healthcheck.sh, so an hourly cron doesn't spam
+# Telegram once a backup is already known to be stale.
+#
+# Usage (cron — one line per sentinel, run more often than the backup itself
+# so a stale state is caught promptly; hourly is a reasonable default for a
+# daily-cron backup with a 26h threshold):
+#   0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \
+#     /opt/gendesign/backups/.last_success 26 "gendesign main backup" \
+#     >> /var/log/gendesign-backup-staleness.log 2>&1
+#   0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \
+#     /opt/gendesign/backups/tradein/.last_success 26 "tradein backup" \
+#     >> /var/log/gendesign-backup-staleness.log 2>&1
+#   0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \
+#     /opt/gendesign/backups/forgejo/.last_success 26 "forgejo backup" \
+#     >> /var/log/gendesign-backup-staleness.log 2>&1
+#
+# Alert channel: reuses the Telegram bot from ops/uptime-healthcheck.sh (see
+# notify() in ops/lib-backup.sh) — creds in /etc/default/gendesign-backup,
+# TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID. Without them, logs only.
+#
+# Exit code: 0 = fresh, 1 = stale or sentinel missing (so this can ALSO be
+# used as a plain healthcheck by anything that just wants the exit code).
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=./lib-backup.sh
+source "$SCRIPT_DIR/lib-backup.sh"
+
+usage() {
+    echo "Usage: $(basename "$0") <sentinel-file> [max-age-hours=26] [label=backup]" >&2
+    exit 2
+}
+
+[[ $# -ge 1 ]] || usage
+
+sentinel_file="$1"
+max_age_hours="${2:-26}"
+label="${3:-backup}"
+state_file="${BACKUP_STALENESS_STATE_FILE:-/var/tmp/gendesign-backup-staleness-state}"
+
+age_hours="$(sentinel_age_hours "$sentinel_file" 2>/dev/null || true)"
+
+if [[ -z "$age_hours" ]]; then
+    now_status="stale"
+    log "ERROR: sentinel missing/unreadable -> $sentinel_file (${label} has never completed successfully, or its state was lost)."
+else
+    if (( age_hours >= max_age_hours )); then
+        now_status="stale"
+        log "ERROR: ${label} sentinel is ${age_hours}h old (threshold ${max_age_hours}h) -> $sentinel_file"
+    else
+        now_status="fresh"
+        log "OK: ${label} sentinel is ${age_hours}h old (< ${max_age_hours}h threshold)."
+    fi
+fi
+
+prev_status="$(backup_prev_status "$state_file" "$label")"
+
+if [[ "$now_status" == "stale" ]]; then
+    if [[ "$prev_status" != "stale" ]]; then
+        notify "🔴 ${label}: backup sentinel is stale (missing or older than ${max_age_hours}h) -> ${sentinel_file}. Check cron / backup logs on the VM."
+    fi
+else
+    if [[ "$prev_status" == "stale" ]]; then
+        notify "✅ ${label}: backup sentinel is fresh again -> ${sentinel_file}."
+    fi
+fi
+
+backup_set_status "$state_file" "$label" "$now_status"
+
+[[ "$now_status" == "fresh" ]]

ops/gendesign-backup-forgejo.default.example

@@ -0,0 +1,44 @@
+# Environment file for ops/backup-forgejo.sh (Forgejo code + DB backup, #2203).
+#
+# Install on the prod VM as a ROOT-OWNED, chmod-600 file that is NOT in git:
+#   sudo cp /opt/gendesign/ops/gendesign-backup-forgejo.default.example /etc/default/gendesign-backup-forgejo
+#   sudo chmod 600 /etc/default/gendesign-backup-forgejo
+#   sudo $EDITOR /etc/default/gendesign-backup-forgejo   # fill in real S3 credentials
+#
+# DELIBERATELY a separate file from /etc/default/gendesign-backup (used by
+# ops/backup.sh and tradein-mvp/deploy/backup-tradein-db.sh) — this backup
+# uses a SEPARATE, narrower-scoped S3 service user that can ONLY PutObject
+# under s3://gendsgn-backups/forgejo/*, nothing else. The prod-host writer
+# key used by the other two backups (`gendsgn-backup-writer`) has root-of-
+# bucket access; that key must NOT be reused here, and this key must NOT be
+# put in /etc/default/gendesign-backup.
+#
+# THE KEY DOES NOT EXIST YET (as of #2203). Until a human creates the
+# `gendsgn-backup-forgejo-writer` service user + bucket policy in the
+# Selectel panel (see the #2203 PR description for the exact policy JSON —
+# PutObject-only on arn:aws:s3:::gendsgn-backups/forgejo/*, explicit Deny on
+# GetObject/ListBucket/Delete*) and fills in the four FORGEJO_S3_* vars
+# below, ops/backup-forgejo.sh refuses to run and exits non-zero loudly. That
+# is expected, not a bug.
+#
+# --- S3 off-site upload (Selectel S3-compatible). All four required. ---
+#FORGEJO_S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru
+#FORGEJO_S3_BUCKET=gendsgn-backups
+#FORGEJO_S3_ACCESS_KEY=REPLACE_WITH_REAL_ACCESS_KEY_ONCE_CREATED
+#FORGEJO_S3_SECRET_KEY=REPLACE_WITH_REAL_SECRET_KEY_ONCE_CREATED
+
+# --- optional overrides (defaults are sensible; uncomment only to change) ---
+#FORGEJO_S3_PREFIX=forgejo/           # key prefix inside the bucket; must match
+                                      # the policy's Resource path exactly
+#KEEP=7                               # how many recent local copies to retain
+                                      # (DB dumps and repo bundles are separate series)
+#MIN_DB_DUMP_BYTES=2048               # sanity floor for the DB dump (small DB —
+                                      # config/issues/PRs/users, no git blobs)
+#MIN_REPOS_BUNDLE_BYTES=10240         # sanity floor for the repo tar.gz
+#LOCAL_BACKUP_DIR=/opt/gendesign/backups/forgejo
+#FORGEJO_DIR=/home/gendesign/forgejo  # Forgejo compose dir on the VM (NOT
+                                      # part of the gendesign git checkout)
+#FORGEJO_REPOS_DIR=/home/gendesign/forgejo/data/forgejo/git/repositories
+#PG_CONTAINER=gendesign-postgres-1    # Forgejo's DB lives in the SAME shared
+                                      # postgres container as the main app,
+                                      # as a separate `forgejo` user+database

ops/gendesign-backup.default.example

@@ -21,6 +21,17 @@
 #
 # To rehearse a restore from a dump this script produced (safe, throwaway
 # container, never touches prod) see ops/restore-drill.sh.
+#
+# Missed-run alerting (#2203): ops/check-backup-staleness.sh (separate cron
+# entry, see its header) alerts when a backup's sentinel goes stale. It reads
+# TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID from THIS file (not from
+# /etc/default/gendesign-uptime — deliberately a separate config so backup
+# alerting doesn't depend on the uptime watchdog's env file existing).
+# SAME variable names as ops/gendesign-uptime.default.example — point both
+# files at the same bot/chat if you want one Telegram destination for
+# everything, or use different bots/chats if you'd rather split the noise.
+# Without these two set, ops/check-backup-staleness.sh still logs, just
+# doesn't send a Telegram alert.
 
 # --- S3 off-site upload (Selectel S3-compatible). All four required to enable. ---
 #S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru
@@ -28,6 +39,10 @@
 #S3_ACCESS_KEY=REPLACE_WITH_REAL_ACCESS_KEY
 #S3_SECRET_KEY=REPLACE_WITH_REAL_SECRET_KEY
 
+# --- missed-run alerting (Telegram, shared bot with uptime watchdog) ---
+#TELEGRAM_BOT_TOKEN=123456789:AA-REPLACE_WITH_REAL_BOT_TOKEN
+#TELEGRAM_CHAT_ID=123456789
+
 # --- optional overrides (defaults are sensible; uncomment only to change) ---
 #KEEP=7                 # how many recent local dumps to retain (applies to both
                         # the main dump series and the globals dump series)

ops/lib-backup.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Shared helpers for ops/backup.sh, ops/backup-forgejo.sh, and
+# tradein-mvp/deploy/backup-tradein-db.sh (#2203, missed-run detection).
+#
+# SOURCED, not executed directly — no shebang execution of its own. Inherits
+# the caller's `set -euo pipefail`. Keep this dependency-free: bash builtins +
+# coreutils (date, stat, mkdir, grep, awk, mktemp) + curl (only used by
+# notify() when Telegram vars are actually set — curl is already a hard
+# requirement of ops/uptime-healthcheck.sh on the same box, so this adds no
+# new dependency).
+#
+# Load with (script computes its own dir so this works regardless of cron's
+# CWD or which repo subdir the caller lives in):
+#   SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+#   source "$SCRIPT_DIR/lib-backup.sh"           # caller is in ops/
+#   source "$SCRIPT_DIR/../../ops/lib-backup.sh" # caller is in tradein-mvp/deploy/
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
+
+# --- notify ------------------------------------------------------------
+# Reuses the SAME Telegram channel/bot as ops/uptime-healthcheck.sh (#75) —
+# this is NOT a second alerting system, just the same TELEGRAM_BOT_TOKEN /
+# TELEGRAM_CHAT_ID variable names read from a DIFFERENT env file
+# (/etc/default/gendesign-backup, not /etc/default/gendesign-uptime) so
+# backup alerting doesn't depend on the uptime watchdog's config being
+# present, and vice versa. Point both files at the same bot/chat if you want
+# one Telegram destination for everything — that's an ops choice, not this
+# script's concern.
+#
+# No-op (logs only) when unset — this is the extension point: to wire a
+# different channel later, edit ONLY this function; every caller in this repo
+# goes through notify(), never curl/telegram directly.
+notify() {
+    local text="$1"
+    local backup_env="${BACKUP_ENV_FILE:-/etc/default/gendesign-backup}"
+
+    if [[ -z "${TELEGRAM_BOT_TOKEN:-}" || -z "${TELEGRAM_CHAT_ID:-}" ]]; then
+        # shellcheck source=/dev/null
+        [[ -f "$backup_env" ]] && source "$backup_env"
+    fi
+
+    if [[ -z "${TELEGRAM_BOT_TOKEN:-}" || -z "${TELEGRAM_CHAT_ID:-}" ]]; then
+        log "NOTIFY (telegram disabled — set TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID in ${backup_env}): $text"
+        return 0
+    fi
+
+    curl -fsS --max-time 15 \
+        -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
+        -d "chat_id=${TELEGRAM_CHAT_ID}" \
+        -d "disable_web_page_preview=true" \
+        --data-urlencode "text=${text}" \
+        >/dev/null 2>&1 \
+        || log "WARN: telegram sendMessage failed"
+}
+
+# --- sentinel (missed-run detection) ------------------------------------
+# write_sentinel <path> — call ONLY after every guard in the caller has
+# already passed (mirrors the existing "never mark success before a dump is
+# verified good" discipline in backup.sh/backup-tradein-db.sh). Content is
+# just an ISO-8601 UTC timestamp for human debugging; the actual staleness
+# check below is based on the file's mtime, not on parsing that content.
+write_sentinel() {
+    local sentinel_file="$1"
+    mkdir -p "$(dirname "$sentinel_file")"
+    date -u +'%Y-%m-%dT%H:%M:%SZ' > "$sentinel_file"
+}
+
+# sentinel_age_hours <path> — echoes the sentinel's age in whole hours on
+# stdout. Returns 1 (nothing echoed) if the file doesn't exist or its mtime
+# can't be read — caller treats that as "stale" (a backup that never
+# succeeded is exactly the case this exists to catch).
+sentinel_age_hours() {
+    local sentinel_file="$1" now_epoch sentinel_epoch
+    [[ -f "$sentinel_file" ]] || return 1
+    now_epoch=$(date -u +%s)
+    # GNU stat (prod VM, Debian) first; BSD stat fallback (local/macOS dev).
+    sentinel_epoch=$(stat -c %Y "$sentinel_file" 2>/dev/null || stat -f %m "$sentinel_file" 2>/dev/null) || return 1
+    echo $(( (now_epoch - sentinel_epoch) / 3600 ))
+}
+
+# --- transition-tracked alert state --------------------------------------
+# Same idiom as ops/uptime-healthcheck.sh's prev_status()/set_status(): a
+# flat "<label> <status>" file, one line per label, so repeated runs alert
+# only on a STATE TRANSITION (fresh->stale, stale->fresh) instead of every
+# single run — avoids Telegram spam from an hourly staleness-check cron.
+# Intentionally a separate, independent implementation (not shared code with
+# uptime-healthcheck.sh) — that script is out of scope for this change.
+backup_prev_status() {
+    local state_file="$1" label="$2" v
+    [[ -f "$state_file" ]] || { echo "unknown"; return; }
+    v="$(grep -E "^${label} " "$state_file" 2>/dev/null | tail -1 | awk '{print $2}')"
+    echo "${v:-unknown}"
+}
+
+backup_set_status() {
+    local state_file="$1" label="$2" status="$3" tmp
+    mkdir -p "$(dirname "$state_file")"
+    tmp="$(mktemp)"
+    if [[ -f "$state_file" ]]; then
+        grep -vE "^${label} " "$state_file" > "$tmp" 2>/dev/null || true
+    fi
+    echo "${label} ${status}" >> "$tmp"
+    mv "$tmp" "$state_file"
+}
+
+# --- dump integrity (opt-in for NEW scripts only) ------------------------
+# Identical logic to the verify_dump_integrity() already duplicated in
+# ops/backup.sh and tradein-mvp/deploy/backup-tradein-db.sh — deliberately
+# NOT deduped there (those two are proven/hardened; don't touch working
+# code for a refactor nobody asked for). This copy exists so NEW scripts
+# (ops/backup-forgejo.sh) get the same guard without a third copy-paste.
+verify_dump_integrity() {
+    local file="$1" trailer="$2" label="$3"
+    if ! gzip -t "$file" 2>/dev/null; then
+        log "ERROR: ${label} failed gzip integrity check -> $file" >&2
+        return 1
+    fi
+    if ! gunzip -c "$file" 2>/dev/null | tail -5 | grep -qF -- "$trailer"; then
+        log "ERROR: ${label} missing trailer '${trailer}' -> $file (truncated dump?)" >&2
+        return 1
+    fi
+    return 0
+}

tradein-mvp/deploy/backup-tradein-db.sh

@@ -23,15 +23,26 @@
 #   S3_BUCKET=gendesign-backups
 #   S3_ACCESS_KEY=...
 #   S3_SECRET_KEY=...
+#
+# Детект пропущенного запуска (#2203): по успеху пишется sentinel-файл
+# (SENTINEL_FILE) — отдельный cron-запуск ops/check-backup-staleness.sh
+# следит за его возрастом и шлёт алерт, если он не свежел (см. заголовок
+# того скрипта — канал алертов и cron-строка). Сам этот скрипт не алертит,
+# только отмечает успех.
 
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=../../ops/lib-backup.sh
+source "$SCRIPT_DIR/../../ops/lib-backup.sh"
+
 BACKUP_DIR="${BACKUP_DIR:-/opt/gendesign/backups/tradein}"
 KEEP="${KEEP:-7}"                                  # сколько копий хранить
 PG_CONTAINER="${PG_CONTAINER:-tradein-postgres}"
 MIN_DUMP_BYTES="${MIN_DUMP_BYTES:-10240}"          # 10 KiB floor — пустая/битая
                                                    # схема заведомо меньше, живая
                                                    # БД — на порядки больше.
+SENTINEL_FILE="${SENTINEL_FILE:-${BACKUP_DIR}/.last_success}"
 
 # S3-переменные: свой env-файл, а если его нет — общий с main-бэкапом.
 if [[ -f /etc/default/tradein-backup ]]; then
@@ -40,8 +51,6 @@ elif [[ -f /etc/default/gendesign-backup ]]; then
   source /etc/default/gendesign-backup
 fi
 
-log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
-
 # Проверка целостности сверх «не пустой»: битый посреди записи дамп (диск
 # кончился, OOM-kill, оборвался docker exec) может дать структурно валидный,
 # не крошечный .gz — ни `-s`, ни MIN_DUMP_BYTES это не ловят. Две проверки,
@@ -155,4 +164,9 @@ ls -1t "$BACKUP_DIR"/tradein-globals-*.sql.gz 2>/dev/null \
 size=$(du -h "$out" | cut -f1)
 count=$(ls -1 "$BACKUP_DIR"/tradein-[0-9]*.sql.gz 2>/dev/null | wc -l | tr -d ' ')
 globals_count=$(ls -1 "$BACKUP_DIR"/tradein-globals-*.sql.gz 2>/dev/null | wc -l | tr -d ' ')
-echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] backup ok: $out ($size), копий хранится: $count данных + $globals_count globals"
+log "backup ok: $out ($size), копий хранится: $count данных + $globals_count globals"
+
+# Детект пропущенного запуска (#2203): отмечаем успех только тут, после всех
+# проверок выше (и — под `set -euo pipefail` — после S3-выгрузки, если она
+# включена). См. ops/check-backup-staleness.sh.
+write_sentinel "$SENTINEL_FILE"