c6b408e483
All checks were successful
CI Trade-In / changes (pull_request) Successful in 11s
CI Trade-In / backend-tests (pull_request) Has been skipped
CI / changes (pull_request) Successful in 13s
CI Trade-In / browser-tests (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
84 lines
3.5 KiB
Bash
Executable file
84 lines
3.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Missed-run detector for the backup scripts (#2203).
|
|
#
|
|
# ops/backup.sh, ops/backup-forgejo.sh, and tradein-mvp/deploy/backup-tradein-db.sh
|
|
# each write a sentinel file (via write_sentinel() in ops/lib-backup.sh) the
|
|
# instant a run finishes with a VERIFIED-good dump — same file the older two
|
|
# scripts already treat as "only mark success once every guard passed".
|
|
# This script checks how old that sentinel is. If cron stops firing (or every
|
|
# run fails before reaching the sentinel write), the file goes stale and this
|
|
# script says so loudly instead of the silence that let backups break for
|
|
# weeks undetected before (#71).
|
|
#
|
|
# Alerts only on a STATE TRANSITION (fresh->stale, stale->fresh), same
|
|
# discipline as ops/uptime-healthcheck.sh, so an hourly cron doesn't spam
|
|
# Telegram once a backup is already known to be stale.
|
|
#
|
|
# Usage (cron — one line per sentinel, run more often than the backup itself
|
|
# so a stale state is caught promptly; hourly is a reasonable default for a
|
|
# daily-cron backup with a 26h threshold):
|
|
# 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \
|
|
# /opt/gendesign/backups/.last_success 26 "gendesign main backup" \
|
|
# >> /var/log/gendesign-backup-staleness.log 2>&1
|
|
# 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \
|
|
# /opt/gendesign/backups/tradein/.last_success 26 "tradein backup" \
|
|
# >> /var/log/gendesign-backup-staleness.log 2>&1
|
|
# 0 * * * * bash /opt/gendesign/ops/check-backup-staleness.sh \
|
|
# /opt/gendesign/backups/forgejo/.last_success 26 "forgejo backup" \
|
|
# >> /var/log/gendesign-backup-staleness.log 2>&1
|
|
#
|
|
# Alert channel: reuses the Telegram bot from ops/uptime-healthcheck.sh (see
|
|
# notify() in ops/lib-backup.sh) — creds in /etc/default/gendesign-backup,
|
|
# TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID. Without them, logs only.
|
|
#
|
|
# Exit code: 0 = fresh, 1 = stale or sentinel missing (so this can ALSO be
|
|
# used as a plain healthcheck by anything that just wants the exit code).
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=./lib-backup.sh
|
|
source "$SCRIPT_DIR/lib-backup.sh"
|
|
|
|
usage() {
|
|
echo "Usage: $(basename "$0") <sentinel-file> [max-age-hours=26] [label=backup]" >&2
|
|
exit 2
|
|
}
|
|
|
|
[[ $# -ge 1 ]] || usage
|
|
|
|
sentinel_file="$1"
|
|
max_age_hours="${2:-26}"
|
|
label="${3:-backup}"
|
|
state_file="${BACKUP_STALENESS_STATE_FILE:-/var/tmp/gendesign-backup-staleness-state}"
|
|
|
|
age_hours="$(sentinel_age_hours "$sentinel_file" 2>/dev/null || true)"
|
|
|
|
if [[ -z "$age_hours" ]]; then
|
|
now_status="stale"
|
|
log "ERROR: sentinel missing/unreadable -> $sentinel_file (${label} has never completed successfully, or its state was lost)."
|
|
else
|
|
if (( age_hours >= max_age_hours )); then
|
|
now_status="stale"
|
|
log "ERROR: ${label} sentinel is ${age_hours}h old (threshold ${max_age_hours}h) -> $sentinel_file"
|
|
else
|
|
now_status="fresh"
|
|
log "OK: ${label} sentinel is ${age_hours}h old (< ${max_age_hours}h threshold)."
|
|
fi
|
|
fi
|
|
|
|
prev_status="$(backup_prev_status "$state_file" "$label")"
|
|
|
|
if [[ "$now_status" == "stale" ]]; then
|
|
if [[ "$prev_status" != "stale" ]]; then
|
|
notify "🔴 ${label}: backup sentinel is stale (missing or older than ${max_age_hours}h) -> ${sentinel_file}. Check cron / backup logs on the VM."
|
|
fi
|
|
else
|
|
if [[ "$prev_status" == "stale" ]]; then
|
|
notify "✅ ${label}: backup sentinel is fresh again -> ${sentinel_file}."
|
|
fi
|
|
fi
|
|
|
|
backup_set_status "$state_file" "$label" "$now_status"
|
|
|
|
[[ "$now_status" == "fresh" ]]
|