Forward-fix после incident May 14 (см. vault events/Incident_Prod_Down_May14_2026). PR #117 был полностью reverted PR #119 (Caddy depends_on frontend: service_healthy + failing wget --spider / → Caddy не стартовал → vsё прод down ~25 мин). Этот PR делает то же что планировал #117, но корректно: ### Frontend healthcheck — TCP probe через node (НЕ HTTP) ```yaml test: ["CMD", "node", "-e", "require('net').createConnection({port:3000,host:'127.0.0.1'}) .once('connect',function(){process.exit(0)}) .once('error',function(){process.exit(1)})"] start_period: 60s retries: 6 interval: 15s ``` - НЕ зависит от HTTP status (200/404/308 — все OK если порт слушает) - `node` гарантированно в alpine image (запускает server.js) - `127.0.0.1` — никакой DNS ambiguity (busybox alpine localhost IPv6 vs IPv4) - 60s + 6×15s = 150s window — с запасом на Next.js standalone cold start ### Caddy depends_on — backend healthy, frontend started (НЕ healthy) ```yaml caddy: depends_on: backend: { condition: service_healthy } # backend имеет /health endpoint frontend: { condition: service_started } # safety net — НЕ блокирует Caddy ``` Lesson from incident: frontend health flaky (Next.js cold start variable). Strict service_healthy для frontend → одна misconfig = full outage обоих доменов (gendsgn.ru + obsidian.gendsgn.ru через тот же Caddy на shared network). ### backend/worker/beat redis: service_started → service_healthy Redis уже имеет `redis-cli ping` healthcheck. Tightening безопасно — был artifact pre-healthcheck Redis. ### Pin rosreestr2coord>=5.0.0 (lock уже 5.3.3) v5 убрала delay параметр; код адаптирован (rate limit на worker уровне). Защита от silent downgrade при `uv lock --upgrade`. ### Что НЕ повторено из #117 - Caddy depends_on frontend: service_healthy — заменено на service_started - wget --spider / — заменено на node TCP probe ### Vault обновлён - `events/Incident_Prod_Down_May14_2026.md` — постмортем + timeline + lessons - Update Changelog в `domains/infra/infra-MOC.md` (если есть) Closes incident. Re-applies #117 goals safely. Co-authored-by: lekss361 <claudestars@proton.me>
This commit is contained in:
parent
6ea2dcd46c
commit
b3f32ae927
3 changed files with 23 additions and 7 deletions
|
|
@ -28,7 +28,7 @@ dependencies = [
|
||||||
"numpy>=2.0.0",
|
"numpy>=2.0.0",
|
||||||
"scikit-learn>=1.5.0",
|
"scikit-learn>=1.5.0",
|
||||||
"sentry-sdk[fastapi]>=2.10.0",
|
"sentry-sdk[fastapi]>=2.10.0",
|
||||||
"rosreestr2coord>=4.0.0",
|
"rosreestr2coord>=5.0.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
|
|
|
||||||
2
backend/uv.lock
generated
2
backend/uv.lock
generated
|
|
@ -616,7 +616,7 @@ requires-dist = [
|
||||||
{ name = "pydantic-settings", specifier = ">=2.3.0" },
|
{ name = "pydantic-settings", specifier = ">=2.3.0" },
|
||||||
{ name = "pyproj", specifier = ">=3.6.0" },
|
{ name = "pyproj", specifier = ">=3.6.0" },
|
||||||
{ name = "redis", specifier = ">=5.0.0" },
|
{ name = "redis", specifier = ">=5.0.0" },
|
||||||
{ name = "rosreestr2coord", specifier = ">=4.0.0" },
|
{ name = "rosreestr2coord", specifier = ">=5.0.0" },
|
||||||
{ name = "scikit-learn", specifier = ">=1.5.0" },
|
{ name = "scikit-learn", specifier = ">=1.5.0" },
|
||||||
{ name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.10.0" },
|
{ name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.10.0" },
|
||||||
{ name = "shapely", specifier = ">=2.0.0" },
|
{ name = "shapely", specifier = ">=2.0.0" },
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,7 @@ services:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
redis:
|
redis:
|
||||||
condition: service_started
|
condition: service_healthy
|
||||||
ports:
|
ports:
|
||||||
- "127.0.0.1:8000:8000"
|
- "127.0.0.1:8000:8000"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
|
|
@ -75,6 +75,16 @@ services:
|
||||||
- "127.0.0.1:3000:3000"
|
- "127.0.0.1:3000:3000"
|
||||||
depends_on:
|
depends_on:
|
||||||
- backend
|
- backend
|
||||||
|
# TCP-only probe через node (есть в alpine image, запускает сам server.js).
|
||||||
|
# НЕ использует HTTP semantics — проверяет только что порт 3000 слушает.
|
||||||
|
# Если Next.js процесс жив и порт открыт → healthy, независимо от HTTP status.
|
||||||
|
# start_period 60s даёт время Next.js standalone bundle bootstrap.
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "node", "-e", "require('net').createConnection({port:3000,host:'127.0.0.1'}).once('connect',function(){process.exit(0)}).once('error',function(){process.exit(1)})"]
|
||||||
|
interval: 15s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 6
|
||||||
|
start_period: 60s
|
||||||
|
|
||||||
worker:
|
worker:
|
||||||
# Отдельный chromium-образ (+200 МБ Playwright). См. backend/Dockerfile target=runner-with-chromium.
|
# Отдельный chromium-образ (+200 МБ Playwright). См. backend/Dockerfile target=runner-with-chromium.
|
||||||
|
|
@ -88,7 +98,7 @@ services:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
redis:
|
redis:
|
||||||
condition: service_started
|
condition: service_healthy
|
||||||
volumes:
|
volumes:
|
||||||
- ./data:/app/data # playwright_state.json + photo binaries
|
- ./data:/app/data # playwright_state.json + photo binaries
|
||||||
# Read-only bind-mount Антоновского /sf/ SQLite — для objective_etl task.
|
# Read-only bind-mount Антоновского /sf/ SQLite — для objective_etl task.
|
||||||
|
|
@ -108,7 +118,7 @@ services:
|
||||||
required: false
|
required: false
|
||||||
depends_on:
|
depends_on:
|
||||||
redis:
|
redis:
|
||||||
condition: service_started
|
condition: service_healthy
|
||||||
command: ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info"]
|
command: ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info"]
|
||||||
|
|
||||||
caddy:
|
caddy:
|
||||||
|
|
@ -121,9 +131,15 @@ services:
|
||||||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||||||
- caddy_data:/data
|
- caddy_data:/data
|
||||||
- caddy_config:/config
|
- caddy_config:/config
|
||||||
|
# backend: service_healthy — backend имеет /health endpoint, готов до Caddy.
|
||||||
|
# frontend: service_started — НЕ service_healthy: даже если frontend healthcheck
|
||||||
|
# дребезжит, Caddy всё равно стартует (можно отдавать 502 для /, но
|
||||||
|
# obsidian.gendsgn.ru, api.gendsgn.ru и т.д. остаются доступны).
|
||||||
depends_on:
|
depends_on:
|
||||||
- backend
|
backend:
|
||||||
- frontend
|
condition: service_healthy
|
||||||
|
frontend:
|
||||||
|
condition: service_started
|
||||||
networks:
|
networks:
|
||||||
- default
|
- default
|
||||||
# shared — для маршрута obsidian.gendsgn.ru → couchdb (отдельный stack).
|
# shared — для маршрута obsidian.gendsgn.ru → couchdb (отдельный stack).
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue