gendesign/ops/uptime-healthcheck.sh

#!/usr/bin/env bash
# External uptime watchdog for gendesign (#75 B6-1, lightweight fallback).
#
# WHY THIS EXISTS alongside Uptime Kuma (docker-compose.uptime.yml): Kuma runs
# ON the prod VM, so if the whole VM dies it can't alert. This script is meant to
# run from cron on a DIFFERENT host (your laptop, a tiny free-tier box, Beget
# shared-host cron) and hit the PUBLIC URLs over the internet — last-resort
# "весь хост лёг" detection. Kuma covers rich per-endpoint/SSL/latency monitoring;
# this covers the case Kuma structurally can't.
#
# Self-contained: only needs `curl` + `bash`. No docker, no repo checkout.
# State (last-known status per check) lives in a file so we alert on TRANSITIONS
# (up→down, down→up) — not every run — to avoid Telegram spam.
#
# Usage (cron — note `bash`, not a bare path, so a missing +x bit can't break it):
#   * * * * * bash /path/to/uptime-healthcheck.sh >> /var/log/gendesign-uptime.log 2>&1
#
# Telegram alerting — set these in an env file (NOT in git, chmod 600):
#   TELEGRAM_BOT_TOKEN=123456:ABC...
#   TELEGRAM_CHAT_ID=123456789
# Default env path: /etc/default/gendesign-uptime (override via UPTIME_ENV_FILE).
# A redacted template lives at ops/gendesign-uptime.default.example.
# Without a token set, the script still logs up/down but sends no alert.

set -euo pipefail

# --- config (env-overridable) ---
UPTIME_ENV_FILE="${UPTIME_ENV_FILE:-/etc/default/gendesign-uptime}"
# shellcheck source=/dev/null
[[ -f "$UPTIME_ENV_FILE" ]] && source "$UPTIME_ENV_FILE"

BASE_URL="${BASE_URL:-https://gendsgn.ru}"
STATE_FILE="${STATE_FILE:-/var/tmp/gendesign-uptime-state}"
CURL_TIMEOUT="${CURL_TIMEOUT:-15}"          # seconds per request (connect+read)
RETRIES="${RETRIES:-2}"                     # extra attempts before declaring DOWN
RETRY_SLEEP="${RETRY_SLEEP:-5}"             # seconds between attempts
TELEGRAM_BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-}"
TELEGRAM_CHAT_ID="${TELEGRAM_CHAT_ID:-}"

# Checks to probe. Format per line: "<label>|<path>|<expected_http_status>".
# Override the whole list via CHECKS env (same newline-separated format).
#
# DEFAULT = /health ONLY. WHY: this watchdog is UNAUTHENTICATED, and in the
# Caddyfile only /health (and /preview/*) are public — ALL /api/* sits behind
# the Basic-Auth gate (import caddy/users.caddy.snippet) and returns 401 to an
# anonymous client. So the issue's /api/v1/analytics/market-pulse and
# /parcels/{cad}/analyze monitors CANNOT be probed anonymously — they belong in
# Uptime Kuma, which can attach the Basic-Auth header (see README "Kuma monitors"
# in docker-compose.uptime.yml). Probing them here would false-alarm forever.
CHECKS="${CHECKS:-health|/health|200}"

log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }

# --- telegram (no-op if token/chat unset) ---
notify() {
  local text="$1"
  if [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$TELEGRAM_CHAT_ID" ]]; then
    log "NOTIFY (telegram disabled — no token/chat): $text"
    return 0
  fi
  curl -fsS --max-time "$CURL_TIMEOUT" \
    -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
    -d "chat_id=${TELEGRAM_CHAT_ID}" \
    -d "disable_web_page_preview=true" \
    --data-urlencode "text=${text}" \
    >/dev/null 2>&1 \
    || log "WARN: telegram sendMessage failed"
}

# --- state helpers (last status per check) ---
prev_status() {
  local label="$1"
  [[ -f "$STATE_FILE" ]] || { echo "unknown"; return; }
  # Line format: "<label> <status>". grep the latest for this label.
  local v
  v="$(grep -E "^${label} " "$STATE_FILE" 2>/dev/null | tail -1 | awk '{print $2}')"
  echo "${v:-unknown}"
}

set_status() {
  local label="$1" status="$2" tmp
  tmp="$(mktemp)"
  # Drop any prior line for this label, then append the fresh one.
  if [[ -f "$STATE_FILE" ]]; then
    grep -vE "^${label} " "$STATE_FILE" > "$tmp" 2>/dev/null || true
  fi
  echo "${label} ${status}" >> "$tmp"
  mv "$tmp" "$STATE_FILE"
}

# --- probe one URL with retries; echoes "up" or "down code=NNN" ---
probe() {
  local url="$1" expect="$2" attempt code
  # NB: NO `-f` here. `-w %{http_code}` always prints exactly a 3-digit status
  # (or 000 on connect/timeout failure), so we judge by the code ourselves and
  # never need a `|| echo` fallback (which previously concatenated → "401000").
  for attempt in $(seq 1 "$((RETRIES + 1))"); do
    code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "$CURL_TIMEOUT" "$url" 2>/dev/null)"
    code="${code:-000}"
    if [[ "$code" == "$expect" ]]; then
      echo "up"
      return 0
    fi
    [[ "$attempt" -le "$RETRIES" ]] && sleep "$RETRY_SLEEP"
  done
  # Report the last code seen for the alert body.
  echo "down code=${code}"
  return 0
}

# --- run ---
overall_rc=0
while IFS= read -r line; do
  [[ -z "$line" ]] && continue
  label="${line%%|*}"
  rest="${line#*|}"
  path="${rest%%|*}"
  expect="${rest##*|}"
  url="${BASE_URL}${path}"

  result="$(probe "$url" "$expect")"
  now="up"
  [[ "$result" == up ]] || now="down"
  prev="$(prev_status "$label")"

  if [[ "$now" == "up" ]]; then
    log "OK   $label ($url)"
    if [[ "$prev" == "down" ]]; then
      notify "✅ RECOVERED: gendesign $label is back UP ($url)"
    fi
  else
    overall_rc=1
    log "DOWN $label ($url) — $result"
    # Alert on transition into down (or first-ever observation that is down).
    if [[ "$prev" != "down" ]]; then
      notify "🔴 DOWN: gendesign $label — ${url} (${result#down }). $(date -u +'%Y-%m-%dT%H:%M:%SZ')"
    fi
  fi
  set_status "$label" "$now"
done <<< "$CHECKS"

exit "$overall_rc"