#!/usr/bin/env bash # Missed-run detector for the backup scripts (#2203). # # ops/backup.sh, ops/backup-forgejo.sh, and tradein-mvp/deploy/backup-tradein-db.sh # each write a sentinel file (via write_sentinel() in ops/lib-backup.sh) the # instant a run finishes with a VERIFIED-good dump — same file the older two # scripts already treat as "only mark success once every guard passed". # This script checks how old that sentinel is. If cron stops firing (or every # run fails before reaching the sentinel write), the file goes stale and this # script says so loudly instead of the silence that let backups break for # weeks undetected before (#71). # # Alerts only on a STATE TRANSITION (fresh->stale, stale->fresh), same # discipline as ops/uptime-healthcheck.sh, so an hourly cron doesn't spam # Telegram once a backup is already known to be stale. # # Usage (cron — one line per sentinel, run more often than the backup itself # so a stale state is caught promptly; hourly is a reasonable default for a # daily-cron backup with a 26h threshold): # 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \ # /opt/gendesign/backups/.last_success 26 "gendesign main backup" \ # >> /var/log/gendesign-backup-staleness.log 2>&1 # 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \ # /opt/gendesign/backups/tradein/.last_success 26 "tradein backup" \ # >> /var/log/gendesign-backup-staleness.log 2>&1 # 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \ # /opt/gendesign/backups/forgejo/.last_success 26 "forgejo backup" \ # >> /var/log/gendesign-backup-staleness.log 2>&1 # # Alert channel: reuses the Telegram bot from ops/uptime-healthcheck.sh (see # notify() in ops/lib-backup.sh) — creds in /etc/default/gendesign-backup, # TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID. Without them, logs only. # # Exit code: 0 = fresh, 1 = stale or sentinel missing (so this can ALSO be # used as a plain healthcheck by anything that just wants the exit code). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=./lib-backup.sh source "$SCRIPT_DIR/lib-backup.sh" usage() { echo "Usage: $(basename "$0") [max-age-hours=26] [label=backup]" >&2 exit 2 } [[ $# -ge 1 ]] || usage sentinel_file="$1" max_age_hours="${2:-26}" label="${3:-backup}" state_file="${BACKUP_STALENESS_STATE_FILE:-/var/tmp/gendesign-backup-staleness-state}" age_hours="$(sentinel_age_hours "$sentinel_file" 2>/dev/null || true)" if [[ -z "$age_hours" ]]; then now_status="stale" log "ERROR: sentinel missing/unreadable -> $sentinel_file (${label} has never completed successfully, or its state was lost)." else if (( age_hours >= max_age_hours )); then now_status="stale" log "ERROR: ${label} sentinel is ${age_hours}h old (threshold ${max_age_hours}h) -> $sentinel_file" else now_status="fresh" log "OK: ${label} sentinel is ${age_hours}h old (< ${max_age_hours}h threshold)." fi fi prev_status="$(backup_prev_status "$state_file" "$label")" if [[ "$now_status" == "stale" ]]; then if [[ "$prev_status" != "stale" ]]; then notify "🔴 ${label}: backup sentinel is stale (missing or older than ${max_age_hours}h) -> ${sentinel_file}. Check cron / backup logs on the VM." fi else if [[ "$prev_status" == "stale" ]]; then notify "✅ ${label}: backup sentinel is fresh again -> ${sentinel_file}." fi fi backup_set_status "$state_file" "$label" "$now_status" [[ "$now_status" == "fresh" ]]