From 83be0882633e7abec659059de36aabbd052e69d5 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Fri, 21 Aug 2026 15:37:19 +0300 Subject: [PATCH] feat(ops): missed-run detection for backups + Forgejo code/DB backup to S3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deliverable 1: ops/backup.sh and tradein-mvp/deploy/backup-tradein-db.sh now write a sentinel file on every verified-good run. A new ops/check-backup- staleness.sh (separate cron entry, hourly) alerts via the existing Telegram channel (same TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID idiom as ops/uptime- healthcheck.sh, transition-tracked so it doesn't spam) if a sentinel goes stale. Shared logic (notify/sentinel/state) factored into ops/lib-backup.sh so it isn't triplicated; the two existing scripts' own hardening (integrity checks, retention, etc.) is untouched. Deliverable 2: ops/backup-forgejo.sh — Forgejo (git.gendsgn.ru) has no backup today. Dumps the shared-postgres `forgejo` DB + tars the bare-repo tree, both off-box to s3://gendsgn-backups/forgejo/ under a SEPARATE, narrower S3 key (root-of-bucket writer key stays out of this). The key doesn't exist yet — the script refuses to run and exits non-zero, loudly, until the four FORGEJO_S3_* vars are filled in (see the example env file and PR description for the exact bucket policy JSON to create it with). Refs #2203, #2989 --- ops/backup-forgejo.sh | 205 +++++++++++++++++++ ops/backup.sh | 20 +- ops/check-backup-staleness.sh | 84 ++++++++ ops/gendesign-backup-forgejo.default.example | 44 ++++ ops/gendesign-backup.default.example | 15 ++ ops/lib-backup.sh | 123 +++++++++++ tradein-mvp/deploy/backup-tradein-db.sh | 20 +- 7 files changed, 506 insertions(+), 5 deletions(-) create mode 100644 ops/backup-forgejo.sh create mode 100644 ops/check-backup-staleness.sh create mode 100644 ops/gendesign-backup-forgejo.default.example create mode 100644 ops/lib-backup.sh diff --git a/ops/backup-forgejo.sh b/ops/backup-forgejo.sh new file mode 100644 index 00000000..11d4ce47 --- /dev/null +++ b/ops/backup-forgejo.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# Backup for self-hosted Forgejo (git.gendsgn.ru) — repos + DB, off-box to S3 +# under its OWN, more-restricted key (#2203). +# +# Forgejo is deployed OUTSIDE this repo's checkout, per +# infra/Forgejo_Migration_BotServer_To_Beget_2026-05-16.md (vault): +# - compose dir on the VM: /home/gendesign/forgejo/ +# - repo data: /home/gendesign/forgejo/data/forgejo/git/repositories/ +# - DB: NOT a dedicated container — a separate `forgejo` user+database +# inside the SAME shared postgres container as the main app +# (gendesign-postgres-1). `pg_dump -U forgejo forgejo` via `docker exec`. +# +# Two artifacts per run: +# 1. DB dump — `pg_dump --no-owner --clean --if-exists` (repos/issues/PRs/ +# users/settings/Actions config — everything except the git data itself). +# 2. Repo bundle — a tar.gz of the bare-repo tree under FORGEJO_REPOS_DIR. +# This is functionally a "bundle of repositories": bare repos already +# ARE just refs+objects, byte-identical to what `git bundle` would +# capture, and tar-ing the whole tree in one shot is far more robust +# than enumerating individual repos (new repos need zero script changes, +# nothing to keep in sync). LFS objects/attachments/avatars under +# FORGEJO_DATA_DIR are OUT of scope for this pass — repo code + full DB +# (which has LFS pointers, not the blobs) covers the "backup the code" +# ask; note this gap in the PR if larger asset backup is wanted later. +# +# Off-box S3 upload is MANDATORY here, not optional like ops/backup.sh — +# the whole point of this script is getting Forgejo's code off the VM it +# lives on, so with no S3 creds configured this refuses to run rather than +# silently producing a local-only "backup" that doesn't meet that bar. +# +# The S3 key here is a SEPARATE, narrower-scoped service user than the one +# ops/backup.sh and backup-tradein-db.sh use (`gendsgn-backup-writer`, which +# can write anywhere in the bucket root) — this one can ONLY PutObject under +# s3://gendsgn-backups/forgejo/*, nothing else, so a compromised Forgejo host +# can't touch the main/tradein DB backups sitting in the same bucket. See the +# PR description for the exact bucket policy JSON (mirrors the existing +# gendsgn-backup-writer policy pattern — vault meta/00_credentials.md +# "Selectel S3 — бэкап-бакет gendsgn-backups"). +# +# THE KEY DOES NOT EXIST YET (as of #2203) — a human needs to create the +# service user + policy in the Selectel panel first. Until then this script +# is expected to fail loudly and exit non-zero; that is the correct, +# intentional behaviour, not a bug. +# +# Missed-run detection (#2203): same sentinel/staleness-check pattern as +# ops/backup.sh — see ops/check-backup-staleness.sh for the alerting side. +# +# Usage (cron — `bash `, not a bare path, same "+x can go missing on a +# raw deploy reset" reasoning as the other two backup scripts, #71): +# 15 4 * * * bash /opt/gendesign/ops/backup-forgejo.sh >> /var/log/gendesign-backup-forgejo.log 2>&1 +# +# Config file (root-owned, chmod 600, NOT in git) — see +# ops/gendesign-backup-forgejo.default.example for the full list: +# FORGEJO_S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru +# FORGEJO_S3_BUCKET=gendsgn-backups +# FORGEJO_S3_PREFIX=forgejo/ +# FORGEJO_S3_ACCESS_KEY=... +# FORGEJO_S3_SECRET_KEY=... + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./lib-backup.sh +source "$SCRIPT_DIR/lib-backup.sh" + +# --- config (env-overridable) --- +FORGEJO_DIR="${FORGEJO_DIR:-/home/gendesign/forgejo}" +FORGEJO_DATA_DIR="${FORGEJO_DATA_DIR:-${FORGEJO_DIR}/data/forgejo}" +FORGEJO_REPOS_DIR="${FORGEJO_REPOS_DIR:-${FORGEJO_DATA_DIR}/git/repositories}" + +PG_CONTAINER="${PG_CONTAINER:-gendesign-postgres-1}" +FORGEJO_DB_USER="${FORGEJO_DB_USER:-forgejo}" +FORGEJO_DB_NAME="${FORGEJO_DB_NAME:-forgejo}" + +LOCAL_BACKUP_DIR="${LOCAL_BACKUP_DIR:-/opt/gendesign/backups/forgejo}" +KEEP="${KEEP:-7}" +MIN_DB_DUMP_BYTES="${MIN_DB_DUMP_BYTES:-2048}" # forgejo DB is small (config/issues/PRs, no git blobs) +MIN_REPOS_BUNDLE_BYTES="${MIN_REPOS_BUNDLE_BYTES:-10240}" +SENTINEL_FILE="${SENTINEL_FILE:-${LOCAL_BACKUP_DIR}/.last_success}" + +FORGEJO_BACKUP_ENV_FILE="${FORGEJO_BACKUP_ENV_FILE:-/etc/default/gendesign-backup-forgejo}" +# shellcheck source=/dev/null +[[ -f "$FORGEJO_BACKUP_ENV_FILE" ]] && source "$FORGEJO_BACKUP_ENV_FILE" + +FORGEJO_S3_ENDPOINT="${FORGEJO_S3_ENDPOINT:-}" +FORGEJO_S3_BUCKET="${FORGEJO_S3_BUCKET:-}" +FORGEJO_S3_PREFIX="${FORGEJO_S3_PREFIX:-forgejo/}" +FORGEJO_S3_ACCESS_KEY="${FORGEJO_S3_ACCESS_KEY:-}" +FORGEJO_S3_SECRET_KEY="${FORGEJO_S3_SECRET_KEY:-}" + +# --- guard 0: S3 creds MUST be configured — this backup's whole point is +# off-box storage under a narrow-scoped key. No creds = refuse to run, +# loudly, rather than silently producing a local-only "backup" (unlike +# ops/backup.sh, where local-only is an accepted fallback). --- +if [[ -z "$FORGEJO_S3_ENDPOINT" || -z "$FORGEJO_S3_BUCKET" || -z "$FORGEJO_S3_ACCESS_KEY" || -z "$FORGEJO_S3_SECRET_KEY" ]]; then + log "ERROR: Forgejo S3 backup is NOT CONFIGURED." + log "ERROR: missing one or more of FORGEJO_S3_ENDPOINT / FORGEJO_S3_BUCKET / FORGEJO_S3_ACCESS_KEY / FORGEJO_S3_SECRET_KEY." + log "ERROR: create the narrow-scoped 'gendsgn-backup-forgejo-writer' S3 service user (see PR for the exact bucket policy)" + log "ERROR: and fill them into ${FORGEJO_BACKUP_ENV_FILE} (see ops/gendesign-backup-forgejo.default.example)." + log "ERROR: refusing to produce a local-only backup — that does not meet this script's off-box requirement." + exit 1 +fi + +if [[ ! -d "$FORGEJO_REPOS_DIR" ]]; then + log "ERROR: FORGEJO_REPOS_DIR does not exist -> $FORGEJO_REPOS_DIR" + log "ERROR: Forgejo's on-disk layout may differ from the documented default — override FORGEJO_REPOS_DIR." + exit 1 +fi + +mkdir -p "$LOCAL_BACKUP_DIR" +ts=$(date -u +'%Y%m%d_%H%M%S') +db_out="${LOCAL_BACKUP_DIR}/forgejo-db_${ts}.sql.gz" +repos_out="${LOCAL_BACKUP_DIR}/forgejo-repos_${ts}.tar.gz" + +# --- 1. DB dump --- +log "Dumping Forgejo DB (${FORGEJO_DB_NAME} as ${FORGEJO_DB_USER}, container ${PG_CONTAINER}) -> ${db_out}" +docker exec "$PG_CONTAINER" pg_dump -U "$FORGEJO_DB_USER" -d "$FORGEJO_DB_NAME" --no-owner --clean --if-exists \ + | gzip -9 > "$db_out" + +if [[ ! -s "$db_out" ]]; then + log "ERROR: DB dump is empty -> $db_out — removing, keeping previous good backups." >&2 + rm -f "$db_out" + exit 1 +fi + +db_bytes=$(wc -c < "$db_out" | tr -d ' ') +if (( db_bytes < MIN_DB_DUMP_BYTES )); then + log "ERROR: DB dump only ${db_bytes} bytes (< ${MIN_DB_DUMP_BYTES} floor) — likely a failed dump." >&2 + rm -f "$db_out" + exit 1 +fi + +if ! verify_dump_integrity "$db_out" "-- PostgreSQL database dump complete" "forgejo DB dump"; then + log "Removing suspect DB dump, NOT pruning older good backups." >&2 + rm -f "$db_out" + exit 1 +fi + +log "DB dump OK: ${db_out} ($(du -h "$db_out" | cut -f1), ${db_bytes} bytes)" + +# --- 2. repo bundle (tar.gz of the bare-repo tree) --- +log "Bundling Forgejo repos from ${FORGEJO_REPOS_DIR} -> ${repos_out}" +tar -czf "$repos_out" -C "$(dirname "$FORGEJO_REPOS_DIR")" "$(basename "$FORGEJO_REPOS_DIR")" + +if [[ ! -s "$repos_out" ]]; then + log "ERROR: repo bundle is empty -> $repos_out — removing." >&2 + rm -f "$repos_out" + exit 1 +fi + +repos_bytes=$(wc -c < "$repos_out" | tr -d ' ') +if (( repos_bytes < MIN_REPOS_BUNDLE_BYTES )); then + log "ERROR: repo bundle only ${repos_bytes} bytes (< ${MIN_REPOS_BUNDLE_BYTES} floor) — likely empty/failed FORGEJO_REPOS_DIR." >&2 + rm -f "$repos_out" + exit 1 +fi + +# tar integrity: a bundle truncated mid-write (disk full, killed process) +# fails `tar -t` even when the gzip framing looks superficially OK — same +# "don't trust non-empty + big-enough alone" discipline as verify_dump_integrity. +if ! tar -tzf "$repos_out" >/dev/null 2>&1; then + log "ERROR: repo bundle failed tar integrity check -> $repos_out (truncated archive?)" >&2 + rm -f "$repos_out" + exit 1 +fi + +log "Repo bundle OK: ${repos_out} ($(du -h "$repos_out" | cut -f1), ${repos_bytes} bytes)" + +# --- 3. upload to S3 (mandatory — see guard 0 above) --- +for f in "$db_out" "$repos_out"; do + key="${FORGEJO_S3_PREFIX%/}/$(basename "$f")" + log "Uploading to s3://${FORGEJO_S3_BUCKET}/${key}" + # Same AWS_CA_BUNDLE fix as ops/backup.sh / backup-tradein-db.sh — aws-cli + # v2's own baked-in CA bundle is missing the root Selectel's cert chains + # to; point it at the container's system store instead. + docker run --rm \ + -e AWS_ACCESS_KEY_ID="$FORGEJO_S3_ACCESS_KEY" \ + -e AWS_SECRET_ACCESS_KEY="$FORGEJO_S3_SECRET_KEY" \ + -e AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + -v "$LOCAL_BACKUP_DIR":/backup:ro \ + amazon/aws-cli:latest \ + --endpoint-url "$FORGEJO_S3_ENDPOINT" \ + s3 cp --no-progress "/backup/$(basename "$f")" "s3://${FORGEJO_S3_BUCKET}/${key}" +done +log "S3 upload OK" + +# --- 4. local retention --- +# shellcheck disable=SC2012 +ls -1t "$LOCAL_BACKUP_DIR"/forgejo-db_*.sql.gz 2>/dev/null \ + | tail -n +"$((KEEP + 1))" \ + | xargs -r rm -f +# shellcheck disable=SC2012 +ls -1t "$LOCAL_BACKUP_DIR"/forgejo-repos_*.tar.gz 2>/dev/null \ + | tail -n +"$((KEEP + 1))" \ + | xargs -r rm -f + +shopt -s nullglob +remaining_db=( "$LOCAL_BACKUP_DIR"/forgejo-db_*.sql.gz ) +remaining_repos=( "$LOCAL_BACKUP_DIR"/forgejo-repos_*.tar.gz ) +shopt -u nullglob +log "Forgejo backup done. Local copies retained: ${#remaining_db[@]} db + ${#remaining_repos[@]} repo bundles (KEEP=${KEEP})." + +# --- missed-run detection (#2203): mark success ONLY after everything above +# (including the mandatory S3 upload) has passed. --- +write_sentinel "$SENTINEL_FILE" diff --git a/ops/backup.sh b/ops/backup.sh index 997c9558..2c429e0c 100755 --- a/ops/backup.sh +++ b/ops/backup.sh @@ -34,9 +34,19 @@ # # Restore: see ops/restore.sh (destructive, INTO the live DB). # Restore drill (safe, throwaway container): see ops/restore-drill.sh. +# +# Missed-run detection (#2203): on a verified-good run this writes a sentinel +# file (SENTINEL_FILE) with the UTC timestamp. A SEPARATE cron entry running +# ops/check-backup-staleness.sh against that file is what actually alerts if +# cron stops firing — see that script's header for the alert channel and +# cron line. This script itself does NOT alert; it only ever marks success. set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./lib-backup.sh +source "$SCRIPT_DIR/lib-backup.sh" + # --- config (env-overridable) --- COMPOSE_DIR="${COMPOSE_DIR:-/opt/gendesign}" COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.prod.yml}" @@ -49,12 +59,11 @@ MIN_DUMP_BYTES="${MIN_DUMP_BYTES:-51200}" # 50 KiB floor; gzip'd schema-only du # (No equivalent floor for the globals # dump — a handful of roles legitimately # gzips to well under this.) +SENTINEL_FILE="${SENTINEL_FILE:-${LOCAL_BACKUP_DIR}/.last_success}" # Optional S3 env. Loaded from /etc/default/gendesign-backup if present. [[ -f /etc/default/gendesign-backup ]] && source /etc/default/gendesign-backup -log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; } - # Integrity check beyond "non-empty"/"big enough": a dump truncated mid-write # (disk full, OOM-kill, docker exec dropped) can still gzip into a structurally # valid, non-tiny .gz — neither `-s` nor MIN_DUMP_BYTES catch that reliably. @@ -198,3 +207,10 @@ remaining=( "$LOCAL_BACKUP_DIR"/gendesign_[0-9]*.sql.gz ) remaining_globals=( "$LOCAL_BACKUP_DIR"/gendesign_globals_*.sql.gz ) shopt -u nullglob log "Backup done. Local dumps retained: ${#remaining[@]} data + ${#remaining_globals[@]} globals (KEEP=${KEEP})." + +# --- missed-run detection (#2203): mark success ONLY here, after every +# guard above (dump/globals integrity, retention, and — since this whole +# script runs under `set -euo pipefail` — any S3 upload) has passed. See +# ops/check-backup-staleness.sh for the piece that actually alerts if +# this stops being refreshed. --- +write_sentinel "$SENTINEL_FILE" diff --git a/ops/check-backup-staleness.sh b/ops/check-backup-staleness.sh new file mode 100644 index 00000000..d21d041a --- /dev/null +++ b/ops/check-backup-staleness.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Missed-run detector for the backup scripts (#2203). +# +# ops/backup.sh, ops/backup-forgejo.sh, and tradein-mvp/deploy/backup-tradein-db.sh +# each write a sentinel file (via write_sentinel() in ops/lib-backup.sh) the +# instant a run finishes with a VERIFIED-good dump — same file the older two +# scripts already treat as "only mark success once every guard passed". +# This script checks how old that sentinel is. If cron stops firing (or every +# run fails before reaching the sentinel write), the file goes stale and this +# script says so loudly instead of the silence that let backups break for +# weeks undetected before (#71). +# +# Alerts only on a STATE TRANSITION (fresh->stale, stale->fresh), same +# discipline as ops/uptime-healthcheck.sh, so an hourly cron doesn't spam +# Telegram once a backup is already known to be stale. +# +# Usage (cron — one line per sentinel, run more often than the backup itself +# so a stale state is caught promptly; hourly is a reasonable default for a +# daily-cron backup with a 26h threshold): +# 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \ +# /opt/gendesign/backups/.last_success 26 "gendesign main backup" \ +# >> /var/log/gendesign-backup-staleness.log 2>&1 +# 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \ +# /opt/gendesign/backups/tradein/.last_success 26 "tradein backup" \ +# >> /var/log/gendesign-backup-staleness.log 2>&1 +# 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \ +# /opt/gendesign/backups/forgejo/.last_success 26 "forgejo backup" \ +# >> /var/log/gendesign-backup-staleness.log 2>&1 +# +# Alert channel: reuses the Telegram bot from ops/uptime-healthcheck.sh (see +# notify() in ops/lib-backup.sh) — creds in /etc/default/gendesign-backup, +# TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID. Without them, logs only. +# +# Exit code: 0 = fresh, 1 = stale or sentinel missing (so this can ALSO be +# used as a plain healthcheck by anything that just wants the exit code). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./lib-backup.sh +source "$SCRIPT_DIR/lib-backup.sh" + +usage() { + echo "Usage: $(basename "$0") [max-age-hours=26] [label=backup]" >&2 + exit 2 +} + +[[ $# -ge 1 ]] || usage + +sentinel_file="$1" +max_age_hours="${2:-26}" +label="${3:-backup}" +state_file="${BACKUP_STALENESS_STATE_FILE:-/var/tmp/gendesign-backup-staleness-state}" + +age_hours="$(sentinel_age_hours "$sentinel_file" 2>/dev/null || true)" + +if [[ -z "$age_hours" ]]; then + now_status="stale" + log "ERROR: sentinel missing/unreadable -> $sentinel_file (${label} has never completed successfully, or its state was lost)." +else + if (( age_hours >= max_age_hours )); then + now_status="stale" + log "ERROR: ${label} sentinel is ${age_hours}h old (threshold ${max_age_hours}h) -> $sentinel_file" + else + now_status="fresh" + log "OK: ${label} sentinel is ${age_hours}h old (< ${max_age_hours}h threshold)." + fi +fi + +prev_status="$(backup_prev_status "$state_file" "$label")" + +if [[ "$now_status" == "stale" ]]; then + if [[ "$prev_status" != "stale" ]]; then + notify "🔴 ${label}: backup sentinel is stale (missing or older than ${max_age_hours}h) -> ${sentinel_file}. Check cron / backup logs on the VM." + fi +else + if [[ "$prev_status" == "stale" ]]; then + notify "✅ ${label}: backup sentinel is fresh again -> ${sentinel_file}." + fi +fi + +backup_set_status "$state_file" "$label" "$now_status" + +[[ "$now_status" == "fresh" ]] diff --git a/ops/gendesign-backup-forgejo.default.example b/ops/gendesign-backup-forgejo.default.example new file mode 100644 index 00000000..945e220b --- /dev/null +++ b/ops/gendesign-backup-forgejo.default.example @@ -0,0 +1,44 @@ +# Environment file for ops/backup-forgejo.sh (Forgejo code + DB backup, #2203). +# +# Install on the prod VM as a ROOT-OWNED, chmod-600 file that is NOT in git: +# sudo cp /opt/gendesign/ops/gendesign-backup-forgejo.default.example /etc/default/gendesign-backup-forgejo +# sudo chmod 600 /etc/default/gendesign-backup-forgejo +# sudo $EDITOR /etc/default/gendesign-backup-forgejo # fill in real S3 credentials +# +# DELIBERATELY a separate file from /etc/default/gendesign-backup (used by +# ops/backup.sh and tradein-mvp/deploy/backup-tradein-db.sh) — this backup +# uses a SEPARATE, narrower-scoped S3 service user that can ONLY PutObject +# under s3://gendsgn-backups/forgejo/*, nothing else. The prod-host writer +# key used by the other two backups (`gendsgn-backup-writer`) has root-of- +# bucket access; that key must NOT be reused here, and this key must NOT be +# put in /etc/default/gendesign-backup. +# +# THE KEY DOES NOT EXIST YET (as of #2203). Until a human creates the +# `gendsgn-backup-forgejo-writer` service user + bucket policy in the +# Selectel panel (see the #2203 PR description for the exact policy JSON — +# PutObject-only on arn:aws:s3:::gendsgn-backups/forgejo/*, explicit Deny on +# GetObject/ListBucket/Delete*) and fills in the four FORGEJO_S3_* vars +# below, ops/backup-forgejo.sh refuses to run and exits non-zero loudly. That +# is expected, not a bug. +# +# --- S3 off-site upload (Selectel S3-compatible). All four required. --- +#FORGEJO_S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru +#FORGEJO_S3_BUCKET=gendsgn-backups +#FORGEJO_S3_ACCESS_KEY=REPLACE_WITH_REAL_ACCESS_KEY_ONCE_CREATED +#FORGEJO_S3_SECRET_KEY=REPLACE_WITH_REAL_SECRET_KEY_ONCE_CREATED + +# --- optional overrides (defaults are sensible; uncomment only to change) --- +#FORGEJO_S3_PREFIX=forgejo/ # key prefix inside the bucket; must match + # the policy's Resource path exactly +#KEEP=7 # how many recent local copies to retain + # (DB dumps and repo bundles are separate series) +#MIN_DB_DUMP_BYTES=2048 # sanity floor for the DB dump (small DB — + # config/issues/PRs/users, no git blobs) +#MIN_REPOS_BUNDLE_BYTES=10240 # sanity floor for the repo tar.gz +#LOCAL_BACKUP_DIR=/opt/gendesign/backups/forgejo +#FORGEJO_DIR=/home/gendesign/forgejo # Forgejo compose dir on the VM (NOT + # part of the gendesign git checkout) +#FORGEJO_REPOS_DIR=/home/gendesign/forgejo/data/forgejo/git/repositories +#PG_CONTAINER=gendesign-postgres-1 # Forgejo's DB lives in the SAME shared + # postgres container as the main app, + # as a separate `forgejo` user+database diff --git a/ops/gendesign-backup.default.example b/ops/gendesign-backup.default.example index 0f6950d9..eb5d13ef 100644 --- a/ops/gendesign-backup.default.example +++ b/ops/gendesign-backup.default.example @@ -21,6 +21,17 @@ # # To rehearse a restore from a dump this script produced (safe, throwaway # container, never touches prod) see ops/restore-drill.sh. +# +# Missed-run alerting (#2203): ops/check-backup-staleness.sh (separate cron +# entry, see its header) alerts when a backup's sentinel goes stale. It reads +# TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID from THIS file (not from +# /etc/default/gendesign-uptime — deliberately a separate config so backup +# alerting doesn't depend on the uptime watchdog's env file existing). +# SAME variable names as ops/gendesign-uptime.default.example — point both +# files at the same bot/chat if you want one Telegram destination for +# everything, or use different bots/chats if you'd rather split the noise. +# Without these two set, ops/check-backup-staleness.sh still logs, just +# doesn't send a Telegram alert. # --- S3 off-site upload (Selectel S3-compatible). All four required to enable. --- #S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru @@ -28,6 +39,10 @@ #S3_ACCESS_KEY=REPLACE_WITH_REAL_ACCESS_KEY #S3_SECRET_KEY=REPLACE_WITH_REAL_SECRET_KEY +# --- missed-run alerting (Telegram, shared bot with uptime watchdog) --- +#TELEGRAM_BOT_TOKEN=123456789:AA-REPLACE_WITH_REAL_BOT_TOKEN +#TELEGRAM_CHAT_ID=123456789 + # --- optional overrides (defaults are sensible; uncomment only to change) --- #KEEP=7 # how many recent local dumps to retain (applies to both # the main dump series and the globals dump series) diff --git a/ops/lib-backup.sh b/ops/lib-backup.sh new file mode 100644 index 00000000..07f3507e --- /dev/null +++ b/ops/lib-backup.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Shared helpers for ops/backup.sh, ops/backup-forgejo.sh, and +# tradein-mvp/deploy/backup-tradein-db.sh (#2203, missed-run detection). +# +# SOURCED, not executed directly — no shebang execution of its own. Inherits +# the caller's `set -euo pipefail`. Keep this dependency-free: bash builtins + +# coreutils (date, stat, mkdir, grep, awk, mktemp) + curl (only used by +# notify() when Telegram vars are actually set — curl is already a hard +# requirement of ops/uptime-healthcheck.sh on the same box, so this adds no +# new dependency). +# +# Load with (script computes its own dir so this works regardless of cron's +# CWD or which repo subdir the caller lives in): +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# source "$SCRIPT_DIR/lib-backup.sh" # caller is in ops/ +# source "$SCRIPT_DIR/../../ops/lib-backup.sh" # caller is in tradein-mvp/deploy/ + +log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; } + +# --- notify ------------------------------------------------------------ +# Reuses the SAME Telegram channel/bot as ops/uptime-healthcheck.sh (#75) — +# this is NOT a second alerting system, just the same TELEGRAM_BOT_TOKEN / +# TELEGRAM_CHAT_ID variable names read from a DIFFERENT env file +# (/etc/default/gendesign-backup, not /etc/default/gendesign-uptime) so +# backup alerting doesn't depend on the uptime watchdog's config being +# present, and vice versa. Point both files at the same bot/chat if you want +# one Telegram destination for everything — that's an ops choice, not this +# script's concern. +# +# No-op (logs only) when unset — this is the extension point: to wire a +# different channel later, edit ONLY this function; every caller in this repo +# goes through notify(), never curl/telegram directly. +notify() { + local text="$1" + local backup_env="${BACKUP_ENV_FILE:-/etc/default/gendesign-backup}" + + if [[ -z "${TELEGRAM_BOT_TOKEN:-}" || -z "${TELEGRAM_CHAT_ID:-}" ]]; then + # shellcheck source=/dev/null + [[ -f "$backup_env" ]] && source "$backup_env" + fi + + if [[ -z "${TELEGRAM_BOT_TOKEN:-}" || -z "${TELEGRAM_CHAT_ID:-}" ]]; then + log "NOTIFY (telegram disabled — set TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID in ${backup_env}): $text" + return 0 + fi + + curl -fsS --max-time 15 \ + -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ + -d "chat_id=${TELEGRAM_CHAT_ID}" \ + -d "disable_web_page_preview=true" \ + --data-urlencode "text=${text}" \ + >/dev/null 2>&1 \ + || log "WARN: telegram sendMessage failed" +} + +# --- sentinel (missed-run detection) ------------------------------------ +# write_sentinel — call ONLY after every guard in the caller has +# already passed (mirrors the existing "never mark success before a dump is +# verified good" discipline in backup.sh/backup-tradein-db.sh). Content is +# just an ISO-8601 UTC timestamp for human debugging; the actual staleness +# check below is based on the file's mtime, not on parsing that content. +write_sentinel() { + local sentinel_file="$1" + mkdir -p "$(dirname "$sentinel_file")" + date -u +'%Y-%m-%dT%H:%M:%SZ' > "$sentinel_file" +} + +# sentinel_age_hours — echoes the sentinel's age in whole hours on +# stdout. Returns 1 (nothing echoed) if the file doesn't exist or its mtime +# can't be read — caller treats that as "stale" (a backup that never +# succeeded is exactly the case this exists to catch). +sentinel_age_hours() { + local sentinel_file="$1" now_epoch sentinel_epoch + [[ -f "$sentinel_file" ]] || return 1 + now_epoch=$(date -u +%s) + # GNU stat (prod VM, Debian) first; BSD stat fallback (local/macOS dev). + sentinel_epoch=$(stat -c %Y "$sentinel_file" 2>/dev/null || stat -f %m "$sentinel_file" 2>/dev/null) || return 1 + echo $(( (now_epoch - sentinel_epoch) / 3600 )) +} + +# --- transition-tracked alert state -------------------------------------- +# Same idiom as ops/uptime-healthcheck.sh's prev_status()/set_status(): a +# flat "