71cbdbe238
Добавляет sidecar glitchtip-auth-forwarder который tails Caddy access log, фильтрует 401 события и шлёт их в GlitchTip через Sentry SDK. - ops/glitchtip-auth-forwarder/forwarder.py: tail + throttle + atomic offset - Caddy JSON schema верифицирован по реальному логу: status верхнеуровневое, request.headers["User-Agent"] — list, client_ip vs remote_ip - Monitor UAs (GlitchTip uptime, SentryUptimeBot) фильтруются как individual events - Storm throttle: >10/60s → один digest event типа basic_auth_storm - docker-compose.prod.yml: caddy_logs volume в caddy + новый service + auth_forwarder_state - docs/runbooks/basic_auth_management.md: runbook с GlitchTip секцией
313 lines
11 KiB
Python
313 lines
11 KiB
Python
"""
|
||
Caddy access log -> GlitchTip auth event forwarder.
|
||
|
||
Tails /var/log/caddy/gendsgn.ru.log, фильтрует JSON lines с
|
||
status==401, шлёт в GlitchTip через Sentry SDK как warning event с тэгами.
|
||
|
||
Persistent offset в /state/offset.json — не дублируем при restart.
|
||
Throttle: при >10 401 events за 60s — однократный digest event
|
||
(чтобы не флудить GlitchTip storm'ом); индивидуальные events во время storm пропускаются.
|
||
|
||
Реальный Caddy JSON access log (v2) структура:
|
||
{
|
||
"level": "info",
|
||
"ts": 1779524882.734,
|
||
"logger": "http.log.access.log0",
|
||
"msg": "handled request",
|
||
"request": {
|
||
"remote_ip": "95.165.147.218",
|
||
"remote_port": "50796",
|
||
"client_ip": "95.165.147.218",
|
||
"proto": "HTTP/2.0",
|
||
"method": "GET",
|
||
"host": "gendsgn.ru",
|
||
"uri": "/",
|
||
"headers": {
|
||
"User-Agent": ["Mozilla/5.0 ..."],
|
||
"Authorization": ["REDACTED"],
|
||
...
|
||
},
|
||
"tls": {...}
|
||
},
|
||
"user_id": "",
|
||
"duration": 0.000134,
|
||
"size": 0,
|
||
"status": 401,
|
||
"resp_headers": {...}
|
||
}
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import signal
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from collections import deque
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
import sentry_sdk
|
||
|
||
LOG_FILE = Path(os.getenv("CADDY_LOG_FILE", "/var/log/caddy/gendsgn.ru.log"))
|
||
STATE_FILE = Path(os.getenv("STATE_FILE", "/state/offset.json"))
|
||
DSN = os.environ["GLITCHTIP_DSN"] # required, fail-fast
|
||
ENV = os.getenv("APP_ENV", "production")
|
||
POLL_INTERVAL_S = 2
|
||
THROTTLE_WINDOW_S = 60
|
||
THROTTLE_THRESHOLD = 10 # >10 events за 60s -> digest mode
|
||
|
||
# Известные боты которые получают 401 от Uptime мониторов —
|
||
# логируем их только в digest, не как individual events.
|
||
KNOWN_MONITOR_UAS = frozenset(
|
||
[
|
||
"GlitchTip/",
|
||
"SentryUptimeBot/",
|
||
]
|
||
)
|
||
|
||
_shutdown = False
|
||
|
||
|
||
def _signal_handler(signum: int, frame: object) -> None:
|
||
global _shutdown
|
||
_shutdown = True
|
||
|
||
|
||
def load_offset() -> int:
|
||
if STATE_FILE.exists():
|
||
try:
|
||
data = json.loads(STATE_FILE.read_text())
|
||
return int(data.get("offset", 0))
|
||
except Exception:
|
||
pass
|
||
return 0
|
||
|
||
|
||
def save_offset(offset: int) -> None:
|
||
"""Atomic write через temp file + rename — избегаем corrupt state при kill."""
|
||
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||
payload = json.dumps(
|
||
{"offset": offset, "updated_at": datetime.now(timezone.utc).isoformat()}
|
||
)
|
||
# Пишем в temp рядом с целевым файлом (тот же volume -> атомарный rename)
|
||
fd, tmp_path = tempfile.mkstemp(dir=STATE_FILE.parent, prefix=".offset_tmp_")
|
||
try:
|
||
with os.fdopen(fd, "w") as f:
|
||
f.write(payload)
|
||
os.replace(tmp_path, STATE_FILE)
|
||
except Exception:
|
||
try:
|
||
os.unlink(tmp_path)
|
||
except OSError:
|
||
pass
|
||
raise
|
||
|
||
|
||
def is_monitor_ua(ua: str) -> bool:
|
||
"""Возвращает True если User-Agent принадлежит известному монитору."""
|
||
return any(ua.startswith(prefix) for prefix in KNOWN_MONITOR_UAS)
|
||
|
||
|
||
def emit_event(entry: dict) -> None: # type: ignore[type-arg]
|
||
"""Отправляет одиночный 401 event в GlitchTip."""
|
||
req = entry.get("request", {})
|
||
path = req.get("uri", "?")
|
||
method = req.get("method", "?")
|
||
# client_ip — реальный IP (за прокси); remote_ip — подключающийся (docker network)
|
||
remote_ip = req.get("client_ip") or req.get("remote_ip", "?")
|
||
host = req.get("host", "?")
|
||
# headers.User-Agent — список строк по Caddy JSON schema
|
||
headers = req.get("headers") or {}
|
||
ua_list = headers.get("User-Agent", [])
|
||
ua = ua_list[0] if ua_list else "?"
|
||
ts = entry.get("ts")
|
||
user_id = entry.get("user_id", "")
|
||
|
||
with sentry_sdk.new_scope() as scope:
|
||
scope.set_tag("event_type", "basic_auth_failed")
|
||
scope.set_tag("http_method", method)
|
||
scope.set_tag("remote_ip", remote_ip)
|
||
scope.set_context(
|
||
"caddy_log",
|
||
{
|
||
"uri": path,
|
||
"method": method,
|
||
"host": host,
|
||
"remote_ip": remote_ip,
|
||
"user_agent": ua,
|
||
"caddy_ts": ts,
|
||
"user_id": user_id or "(none)",
|
||
},
|
||
)
|
||
sentry_sdk.capture_message(
|
||
f"basic_auth 401 — {method} {path} from {remote_ip}",
|
||
level="warning",
|
||
)
|
||
|
||
|
||
def emit_digest(count: int, window_s: int, monitor_count: int) -> None:
|
||
"""Отправляет digest event при storm'е."""
|
||
with sentry_sdk.new_scope() as scope:
|
||
scope.set_tag("event_type", "basic_auth_storm")
|
||
sentry_sdk.capture_message(
|
||
f"basic_auth storm — {count} failed attempts in {window_s}s"
|
||
+ (f" (including {monitor_count} monitor probes)" if monitor_count else ""),
|
||
level="error",
|
||
)
|
||
|
||
|
||
def is_401_event(entry: dict) -> bool: # type: ignore[type-arg]
|
||
"""Проверяет что entry — failed auth (401)."""
|
||
return entry.get("status") == 401
|
||
|
||
|
||
def main() -> None:
|
||
signal.signal(signal.SIGTERM, _signal_handler)
|
||
signal.signal(signal.SIGINT, _signal_handler)
|
||
|
||
sentry_sdk.init(
|
||
dsn=DSN,
|
||
environment=ENV,
|
||
release=os.getenv("APP_RELEASE", "auth-forwarder-1"),
|
||
traces_sample_rate=0.0,
|
||
attach_stacktrace=False,
|
||
send_default_pii=False,
|
||
# Отключаем интеграции которые не нужны тонкому sidecar
|
||
default_integrations=False,
|
||
)
|
||
|
||
print(
|
||
f"[forwarder] starting; log={LOG_FILE}; state={STATE_FILE}; env={ENV}",
|
||
flush=True,
|
||
)
|
||
|
||
# Ожидаем появления лог-файла (Caddy пишет его при первом запросе)
|
||
if not LOG_FILE.exists():
|
||
print(
|
||
f"[forwarder] log file {LOG_FILE} does not exist yet, waiting...",
|
||
flush=True,
|
||
)
|
||
while not LOG_FILE.exists() and not _shutdown:
|
||
time.sleep(2)
|
||
if _shutdown:
|
||
return
|
||
|
||
offset = load_offset()
|
||
file_size = LOG_FILE.stat().st_size
|
||
# Если лог был ротирован или обрезан — сбрасываем offset
|
||
if offset > file_size:
|
||
print(
|
||
f"[forwarder] offset={offset} > file_size={file_size}, log rotated, reset to 0",
|
||
flush=True,
|
||
)
|
||
offset = 0
|
||
save_offset(offset)
|
||
|
||
# Sliding window для throttle
|
||
recent_timestamps: deque[float] = deque()
|
||
monitor_timestamps: deque[float] = deque()
|
||
digest_sent = False
|
||
|
||
print(f"[forwarder] starting tail from offset={offset}", flush=True)
|
||
|
||
while not _shutdown:
|
||
try:
|
||
current_size = LOG_FILE.stat().st_size
|
||
# Log rotation detection: файл стал меньше чем наш offset
|
||
if current_size < offset:
|
||
print("[forwarder] log rotation detected, resetting offset to 0", flush=True)
|
||
offset = 0
|
||
save_offset(offset)
|
||
recent_timestamps.clear()
|
||
monitor_timestamps.clear()
|
||
digest_sent = False
|
||
|
||
if current_size > offset:
|
||
with LOG_FILE.open("rb") as f:
|
||
f.seek(offset)
|
||
for raw_line in f:
|
||
if _shutdown:
|
||
break
|
||
line = raw_line.decode("utf-8", errors="replace").strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
entry = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
|
||
if not is_401_event(entry):
|
||
continue
|
||
|
||
now = time.monotonic()
|
||
|
||
# Определяем User-Agent для классификации
|
||
req = entry.get("request", {})
|
||
headers = req.get("headers") or {}
|
||
ua_list = headers.get("User-Agent", [])
|
||
ua = ua_list[0] if ua_list else ""
|
||
is_monitor = is_monitor_ua(ua)
|
||
|
||
# Обновляем скользящее окно
|
||
recent_timestamps.append(now)
|
||
if is_monitor:
|
||
monitor_timestamps.append(now)
|
||
|
||
cutoff = now - THROTTLE_WINDOW_S
|
||
while recent_timestamps and recent_timestamps[0] < cutoff:
|
||
recent_timestamps.popleft()
|
||
while monitor_timestamps and monitor_timestamps[0] < cutoff:
|
||
monitor_timestamps.popleft()
|
||
|
||
window_count = len(recent_timestamps)
|
||
|
||
if window_count > THROTTLE_THRESHOLD:
|
||
# Storm mode: один digest, индивидуальные events пропускаем
|
||
if not digest_sent:
|
||
emit_digest(
|
||
window_count,
|
||
THROTTLE_WINDOW_S,
|
||
len(monitor_timestamps),
|
||
)
|
||
digest_sent = True
|
||
print(
|
||
f"[forwarder] storm digest sent ({window_count} events)",
|
||
flush=True,
|
||
)
|
||
else:
|
||
digest_sent = False
|
||
# Мониторы (GlitchTip uptime, SentryUptimeBot) получают 401 постоянно —
|
||
# не шлём их как individual events, только учитываем в throttle
|
||
if not is_monitor:
|
||
emit_event(entry)
|
||
remote_ip = req.get("client_ip") or req.get("remote_ip", "?")
|
||
path = req.get("uri", "?")
|
||
print(
|
||
f"[forwarder] 401 event sent: {remote_ip} -> {path}",
|
||
flush=True,
|
||
)
|
||
|
||
offset = f.tell()
|
||
|
||
save_offset(offset)
|
||
|
||
time.sleep(POLL_INTERVAL_S)
|
||
|
||
except FileNotFoundError:
|
||
print("[forwarder] log file disappeared, waiting...", flush=True)
|
||
time.sleep(5)
|
||
except Exception as exc:
|
||
print(f"[forwarder] unhandled error: {exc}", file=sys.stderr, flush=True)
|
||
try:
|
||
sentry_sdk.capture_exception(exc)
|
||
except Exception:
|
||
pass
|
||
time.sleep(5)
|
||
|
||
print("[forwarder] shutting down gracefully", flush=True)
|
||
sentry_sdk.flush(timeout=5)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|