gendesign/ops/uptime-healthcheck.sh

#!/usr/bin/env bash
# External uptime watchdog for gendesign (#75 B6-1, lightweight fallback).
#
# WHY THIS EXISTS alongside Uptime Kuma (docker-compose.uptime.yml): Kuma runs
# ON the prod VM, so if the whole VM dies it can't alert. This script is meant to
# run from cron on a DIFFERENT host (your laptop, a tiny free-tier box, Beget
# shared-host cron) and hit the PUBLIC URLs over the internet — last-resort
# "весь хост лёг" detection. Kuma covers rich per-endpoint/SSL/latency monitoring;
# this covers the case Kuma structurally can't.
#
# Self-contained: only needs `curl` + `bash`. No docker, no repo checkout.
# State (last-known status per check) lives in a file so we alert on TRANSITIONS
# (up→down, down→up) — not every run — to avoid Telegram spam.
#
# Usage (cron — note `bash`, not a bare path, so a missing +x bit can't break it):
#   * * * * * bash /path/to/uptime-healthcheck.sh >> /var/log/gendesign-uptime.log 2>&1
#
# Telegram alerting — set these in an env file (NOT in git, chmod 600):
#   TELEGRAM_BOT_TOKEN=123456:ABC...
#   TELEGRAM_CHAT_ID=123456789
# Default env path: /etc/default/gendesign-uptime (override via UPTIME_ENV_FILE).
# A redacted template lives at ops/gendesign-uptime.default.example.
# Without a token set, the script still logs up/down but sends no alert.

set -euo pipefail

# --- config (env-overridable) ---
UPTIME_ENV_FILE="${UPTIME_ENV_FILE:-/etc/default/gendesign-uptime}"
# shellcheck source=/dev/null
[[ -f "$UPTIME_ENV_FILE" ]] && source "$UPTIME_ENV_FILE"

BASE_URL="${BASE_URL:-https://gendsgn.ru}"
STATE_FILE="${STATE_FILE:-/var/tmp/gendesign-uptime-state}"
CURL_TIMEOUT="${CURL_TIMEOUT:-15}"          # seconds per request (connect+read)
RETRIES="${RETRIES:-2}"                     # extra attempts before declaring DOWN
RETRY_SLEEP="${RETRY_SLEEP:-5}"             # seconds between attempts
TELEGRAM_BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-}"
TELEGRAM_CHAT_ID="${TELEGRAM_CHAT_ID:-}"

# Checks to probe. Format per line: "<label>|<path>|<expected_http_status>".
# Override the whole list via CHECKS env (same newline-separated format).
#
# DEFAULT = /health ONLY. WHY: this watchdog is UNAUTHENTICATED, and in the
# Caddyfile only /health (and /preview/*) are public — ALL /api/* sits behind
# the Basic-Auth gate (import caddy/users.caddy.snippet) and returns 401 to an
# anonymous client. So the issue's /api/v1/analytics/market-pulse and
# /parcels/{cad}/analyze monitors CANNOT be probed anonymously — they belong in
# Uptime Kuma, which can attach the Basic-Auth header (see README "Kuma monitors"
# in docker-compose.uptime.yml). Probing them here would false-alarm forever.
CHECKS="${CHECKS:-health|/health|200}"

log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }

# --- telegram (no-op if token/chat unset) ---
notify() {
  local text="$1"
  if [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$TELEGRAM_CHAT_ID" ]]; then
    log "NOTIFY (telegram disabled — no token/chat): $text"
    return 0
  fi
  # #3059: путь до Telegram теряет соединения. Замер 26.08 с Poincare — 40
  # подключений к ЗАКРЕПЛЁННОМУ (#3093) 149.154.167.220: 3 отказа (7.5%), все
  # таймаутом на установке соединения; успешные при этом стабильны (0.14-0.17 с).
  # Три остальных дата-центра Telegram с Selectel недостижимы вовсе, так что
  # запасного адреса нет — потери на единственном рабочем неустранимы сетью.
  #
  # Раньше здесь был ОДИН curl, и `|| log WARN` означал, что каждый такой отказ
  # ТЕРЯЕТ алерт целиком: уведомление о падении прода не приходит, остаётся
  # строка в логе, который читают уже после аварии. Watchdog, который сам себя
  # не может дозваться, — худший вид самоскрывающейся поломки: чем хуже дела,
  # тем вероятнее, что о них не сообщат.
  #
  # Цикл, а не `curl --retry`: ниже в этом же файле проверки уже повторяются
  # ровно такой конструкцией (см. `for attempt in $(seq 1 ...)`), и семантика
  # `--max-time` при ретраях curl зависит от версии. Здесь таймаут заведомо
  # применяется к КАЖДОЙ попытке.
  #
  # Дубль вместо потери — осознанный размен: sendMessage не идемпотентен, но
  # замер показал, что отказы происходят на СТАДИИ ПОДКЛЮЧЕНИЯ, до отправки
  # запроса, так что повтор почти никогда не дублирует уже доставленное
  # сообщение. А продублированный алерт безвреден, пропущенный — нет.
  local attempt
  for attempt in 1 2 3; do
    if curl -fsS --max-time "$CURL_TIMEOUT" \
      -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
      -d "chat_id=${TELEGRAM_CHAT_ID}" \
      ${TELEGRAM_TOPIC_ID:+-d "message_thread_id=${TELEGRAM_TOPIC_ID}"} \
      -d "disable_web_page_preview=true" \
      --data-urlencode "text=${text}" \
      >/dev/null 2>&1; then
      [[ "$attempt" -gt 1 ]] && log "telegram sendMessage: доставлено с попытки ${attempt}"
      return 0
    fi
    [[ "$attempt" -lt 3 ]] && sleep "${NOTIFY_RETRY_DELAY:-2}"
  done
  log "WARN: telegram sendMessage failed — 3 попытки подряд, алерт НЕ ДОСТАВЛЕН"
}

# --- state helpers (last status per check) ---
prev_status() {
  local label="$1"
  [[ -f "$STATE_FILE" ]] || { echo "unknown"; return; }
  # Line format: "<label> <status>". grep the latest for this label.
  local v
  v="$(grep -E "^${label} " "$STATE_FILE" 2>/dev/null | tail -1 | awk '{print $2}')"
  echo "${v:-unknown}"
}

set_status() {
  local label="$1" status="$2" tmp
  tmp="$(mktemp)"
  # Drop any prior line for this label, then append the fresh one.
  if [[ -f "$STATE_FILE" ]]; then
    grep -vE "^${label} " "$STATE_FILE" > "$tmp" 2>/dev/null || true
  fi
  echo "${label} ${status}" >> "$tmp"
  mv "$tmp" "$STATE_FILE"
}

# --- probe one URL with retries; echoes "up" or "down code=NNN" ---
probe() {
  local url="$1" expect="$2" attempt code
  # NB: NO `-f` here. `-w %{http_code}` always prints exactly a 3-digit status
  # (or 000 on connect/timeout failure), so we judge by the code ourselves and
  # never need a `|| echo` fallback (which previously concatenated → "401000").
  for attempt in $(seq 1 "$((RETRIES + 1))"); do
    code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "$CURL_TIMEOUT" "$url" 2>/dev/null)"
    code="${code:-000}"
    if [[ "$code" == "$expect" ]]; then
      echo "up"
      return 0
    fi
    [[ "$attempt" -le "$RETRIES" ]] && sleep "$RETRY_SLEEP"
  done
  # Report the last code seen for the alert body.
  echo "down code=${code}"
  return 0
}

# --- run ---
overall_rc=0
while IFS= read -r line; do
  [[ -z "$line" ]] && continue
  label="${line%%|*}"
  rest="${line#*|}"
  path="${rest%%|*}"
  expect="${rest##*|}"
  url="${BASE_URL}${path}"

  result="$(probe "$url" "$expect")"
  now="up"
  [[ "$result" == up ]] || now="down"
  prev="$(prev_status "$label")"

  if [[ "$now" == "up" ]]; then
    log "OK   $label ($url)"
    if [[ "$prev" == "down" ]]; then
      notify "✅ RECOVERED: gendesign $label is back UP ($url)"
    fi
  else
    overall_rc=1
    log "DOWN $label ($url) — $result"
    # Alert on transition into down (or first-ever observation that is down).
    if [[ "$prev" != "down" ]]; then
      notify "🔴 DOWN: gendesign $label — ${url} (${result#down }). $(date -u +'%Y-%m-%dT%H:%M:%SZ')"
    fi
  fi
  set_status "$label" "$now"
done <<< "$CHECKS"

exit "$overall_rc"