lekss361
2feb0de446
All checks were successful
Deploy Infra Host / sync-infra-host (push) Successful in 2s
Deploy / changes (push) Successful in 7s
Deploy / build-backend (push) Has been skipped
Deploy / build-worker (push) Has been skipped
Deploy Trade-In / changes (push) Successful in 12s
Deploy / build-frontend (push) Has been skipped
Deploy / deploy-caddy (push) Has been skipped
Deploy Trade-In / build-browser (push) Successful in 45s
Deploy / deploy (push) Successful in 1m45s
Deploy / deploy-status (push) Successful in 1s
Deploy / perimeter-smoke (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Successful in 2m35s
Deploy Trade-In / test (push) Successful in 3m53s
Deploy Trade-In / build-backend (push) Successful in 34s
Deploy Trade-In / deploy (push) Successful in 2m21s
Deploy Trade-In / deploy-status (push) Successful in 1s
Deploy Trade-In / perimeter-smoke (push) Successful in 10s
178 lines
9 KiB
Bash
Executable file
178 lines
9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# Shared helpers for ops/backup.sh, ops/backup-forgejo.sh, and
|
||
# tradein-mvp/deploy/backup-tradein-db.sh (#2203, missed-run detection).
|
||
#
|
||
# SOURCED, not executed directly — no shebang execution of its own. Inherits
|
||
# the caller's `set -euo pipefail`. Keep this dependency-free: bash builtins +
|
||
# coreutils (date, stat, mkdir, grep, awk, mktemp) + curl (only used by
|
||
# notify() when Telegram vars are actually set — curl is already a hard
|
||
# requirement of ops/uptime-healthcheck.sh on the same box, so this adds no
|
||
# new dependency).
|
||
#
|
||
# Load with (script computes its own dir so this works regardless of cron's
|
||
# CWD or which repo subdir the caller lives in):
|
||
# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
# source "$SCRIPT_DIR/lib-backup.sh" # caller is in ops/
|
||
# source "$SCRIPT_DIR/../../ops/lib-backup.sh" # caller is in tradein-mvp/deploy/
|
||
|
||
log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
|
||
|
||
# --- notify ------------------------------------------------------------
|
||
# Reuses the SAME Telegram channel/bot as ops/uptime-healthcheck.sh (#75) —
|
||
# this is NOT a second alerting system, just the same TELEGRAM_BOT_TOKEN /
|
||
# TELEGRAM_CHAT_ID variable names read from a DIFFERENT env file
|
||
# (/etc/default/gendesign-backup, not /etc/default/gendesign-uptime) so
|
||
# backup alerting doesn't depend on the uptime watchdog's config being
|
||
# present, and vice versa. Point both files at the same bot/chat if you want
|
||
# one Telegram destination for everything — that's an ops choice, not this
|
||
# script's concern.
|
||
#
|
||
# No-op (logs only) when unset — this is the extension point: to wire a
|
||
# different channel later, edit ONLY this function; every caller in this repo
|
||
# goes through notify(), never curl/telegram directly.
|
||
notify() {
|
||
local text="$1"
|
||
local backup_env="${BACKUP_ENV_FILE:-/etc/default/gendesign-backup}"
|
||
|
||
if [[ -z "${TELEGRAM_BOT_TOKEN:-}" || -z "${TELEGRAM_CHAT_ID:-}" ]]; then
|
||
# shellcheck source=/dev/null
|
||
[[ -f "$backup_env" ]] && source "$backup_env"
|
||
fi
|
||
|
||
if [[ -z "${TELEGRAM_BOT_TOKEN:-}" || -z "${TELEGRAM_CHAT_ID:-}" ]]; then
|
||
log "NOTIFY (telegram disabled — set TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID in ${backup_env}): $text"
|
||
notify_fallback_mail "$text"
|
||
return 0
|
||
fi
|
||
|
||
curl -fsS --max-time 15 \
|
||
-X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||
-d "chat_id=${TELEGRAM_CHAT_ID}" \
|
||
-d "disable_web_page_preview=true" \
|
||
--data-urlencode "text=${text}" \
|
||
>/dev/null 2>&1 \
|
||
|| { log "WARN: telegram sendMessage failed — пробую запасной канал"; notify_fallback_mail "$text"; }
|
||
}
|
||
|
||
# --- запасной канал оповещения (#3059) ----------------------------------
|
||
# Зачем: раньше единственным каналом был Telegram, и `|| log WARN` означало,
|
||
# что при его недоступности алерт просто ТЕРЯЛСЯ — оставалась строка в логе,
|
||
# который никто не читает, пока не случится беда. Самоскрывающаяся поломка:
|
||
# канал, которым мы узнаём о проблемах, сам и есть проблема.
|
||
#
|
||
# Это не паранойя, а замер (#3059, 2026-08-23): с нового хоста Poincare из семи
|
||
# публикуемых адресов api.telegram.org отвечает РОВНО ОДИН (149.154.167.220),
|
||
# и скан всей подсети 149.154.167.0/24 не нашёл больше ни одного. Через прокси
|
||
# Telegram не проходит вовсе — узлы российские. После переезда весь
|
||
# Telegram-канал висит на одном адресе, без запасного.
|
||
#
|
||
# Почему почта: с Poincare проверено — smtp.beget.com:465 OPEN (587 закрыт).
|
||
# Транспорт — тот же curl, который уже является жёсткой зависимостью файла;
|
||
# новых пакетов не требуется.
|
||
#
|
||
# Настраивается тем же env-файлом, что и Telegram. Не задано — ведёт себя как
|
||
# раньше. Кредов в репозитории нет:
|
||
# ALERT_SMTP_URL=smtps://smtp.beget.com:465
|
||
# ALERT_SMTP_USER= / ALERT_SMTP_PASS= / ALERT_MAIL_FROM= / ALERT_MAIL_TO=
|
||
notify_fallback_mail() {
|
||
local text="$1"
|
||
|
||
if [[ -z "${ALERT_SMTP_URL:-}" || -z "${ALERT_SMTP_USER:-}" \
|
||
|| -z "${ALERT_SMTP_PASS:-}" || -z "${ALERT_MAIL_FROM:-}" \
|
||
|| -z "${ALERT_MAIL_TO:-}" ]]; then
|
||
# ГРОМКО: это последний рубеж, тишина здесь = потерянный алерт.
|
||
log "АЛЕРТ НЕ ДОСТАВЛЕН (telegram недоступен, почта не настроена): $text"
|
||
return 1
|
||
fi
|
||
|
||
local subject body rc=0
|
||
subject=$(printf '%s' "$text" | head -c 120 | tr '\n' ' ')
|
||
body=$(printf 'From: %s\nTo: %s\nSubject: [gendesign] %s\n\n%s\n' \
|
||
"$ALERT_MAIL_FROM" "$ALERT_MAIL_TO" "$subject" "$text")
|
||
|
||
# Пароль идёт через --user, поэтому stderr curl целиком не печатаем — в нём
|
||
# может оказаться строка подключения. Логируем только код возврата.
|
||
printf '%s' "$body" | curl -fsS --max-time 30 --ssl-reqd \
|
||
--url "$ALERT_SMTP_URL" \
|
||
--user "${ALERT_SMTP_USER}:${ALERT_SMTP_PASS}" \
|
||
--mail-from "$ALERT_MAIL_FROM" \
|
||
--mail-rcpt "$ALERT_MAIL_TO" \
|
||
--upload-file - >/dev/null 2>&1 || rc=$?
|
||
|
||
if (( rc == 0 )); then
|
||
log "Алерт доставлен запасным каналом (почта) вместо Telegram"
|
||
return 0
|
||
fi
|
||
|
||
log "АЛЕРТ НЕ ДОСТАВЛЕН НИ ОДНИМ КАНАЛОМ (telegram упал, curl smtp rc=${rc}): $text"
|
||
return 1
|
||
}
|
||
|
||
# --- sentinel (missed-run detection) ------------------------------------
|
||
# write_sentinel <path> — call ONLY after every guard in the caller has
|
||
# already passed (mirrors the existing "never mark success before a dump is
|
||
# verified good" discipline in backup.sh/backup-tradein-db.sh). Content is
|
||
# just an ISO-8601 UTC timestamp for human debugging; the actual staleness
|
||
# check below is based on the file's mtime, not on parsing that content.
|
||
write_sentinel() {
|
||
local sentinel_file="$1"
|
||
mkdir -p "$(dirname "$sentinel_file")"
|
||
date -u +'%Y-%m-%dT%H:%M:%SZ' > "$sentinel_file"
|
||
}
|
||
|
||
# sentinel_age_hours <path> — echoes the sentinel's age in whole hours on
|
||
# stdout. Returns 1 (nothing echoed) if the file doesn't exist or its mtime
|
||
# can't be read — caller treats that as "stale" (a backup that never
|
||
# succeeded is exactly the case this exists to catch).
|
||
sentinel_age_hours() {
|
||
local sentinel_file="$1" now_epoch sentinel_epoch
|
||
[[ -f "$sentinel_file" ]] || return 1
|
||
now_epoch=$(date -u +%s)
|
||
# GNU stat (prod VM, Debian) first; BSD stat fallback (local/macOS dev).
|
||
sentinel_epoch=$(stat -c %Y "$sentinel_file" 2>/dev/null || stat -f %m "$sentinel_file" 2>/dev/null) || return 1
|
||
echo $(( (now_epoch - sentinel_epoch) / 3600 ))
|
||
}
|
||
|
||
# --- transition-tracked alert state --------------------------------------
|
||
# Same idiom as ops/uptime-healthcheck.sh's prev_status()/set_status(): a
|
||
# flat "<label> <status>" file, one line per label, so repeated runs alert
|
||
# only on a STATE TRANSITION (fresh->stale, stale->fresh) instead of every
|
||
# single run — avoids Telegram spam from an hourly staleness-check cron.
|
||
# Intentionally a separate, independent implementation (not shared code with
|
||
# uptime-healthcheck.sh) — that script is out of scope for this change.
|
||
backup_prev_status() {
|
||
local state_file="$1" label="$2" v
|
||
[[ -f "$state_file" ]] || { echo "unknown"; return; }
|
||
v="$(grep -E "^${label} " "$state_file" 2>/dev/null | tail -1 | awk '{print $2}')"
|
||
echo "${v:-unknown}"
|
||
}
|
||
|
||
backup_set_status() {
|
||
local state_file="$1" label="$2" status="$3" tmp
|
||
mkdir -p "$(dirname "$state_file")"
|
||
tmp="$(mktemp)"
|
||
if [[ -f "$state_file" ]]; then
|
||
grep -vE "^${label} " "$state_file" > "$tmp" 2>/dev/null || true
|
||
fi
|
||
echo "${label} ${status}" >> "$tmp"
|
||
mv "$tmp" "$state_file"
|
||
}
|
||
|
||
# --- dump integrity (opt-in for NEW scripts only) ------------------------
|
||
# Identical logic to the verify_dump_integrity() already duplicated in
|
||
# ops/backup.sh and tradein-mvp/deploy/backup-tradein-db.sh — deliberately
|
||
# NOT deduped there (those two are proven/hardened; don't touch working
|
||
# code for a refactor nobody asked for). This copy exists so NEW scripts
|
||
# (ops/backup-forgejo.sh) get the same guard without a third copy-paste.
|
||
verify_dump_integrity() {
|
||
local file="$1" trailer="$2" label="$3"
|
||
if ! gzip -t "$file" 2>/dev/null; then
|
||
log "ERROR: ${label} failed gzip integrity check -> $file" >&2
|
||
return 1
|
||
fi
|
||
if ! gunzip -c "$file" 2>/dev/null | tail -5 | grep -qF -- "$trailer"; then
|
||
log "ERROR: ${label} missing trailer '${trailer}' -> $file (truncated dump?)" >&2
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|