Forward-fix после incident May 14 (см. vault events/Incident_Prod_Down_May14_2026). PR #117 был полностью reverted PR #119 (Caddy depends_on frontend: service_healthy + failing wget --spider / → Caddy не стартовал → vsё прод down ~25 мин). Этот PR делает то же что планировал #117, но корректно: ### Frontend healthcheck — TCP probe через node (НЕ HTTP) ```yaml test: ["CMD", "node", "-e", "require('net').createConnection({port:3000,host:'127.0.0.1'}) .once('connect',function(){process.exit(0)}) .once('error',function(){process.exit(1)})"] start_period: 60s retries: 6 interval: 15s ``` - НЕ зависит от HTTP status (200/404/308 — все OK если порт слушает) - `node` гарантированно в alpine image (запускает server.js) - `127.0.0.1` — никакой DNS ambiguity (busybox alpine localhost IPv6 vs IPv4) - 60s + 6×15s = 150s window — с запасом на Next.js standalone cold start ### Caddy depends_on — backend healthy, frontend started (НЕ healthy) ```yaml caddy: depends_on: backend: { condition: service_healthy } # backend имеет /health endpoint frontend: { condition: service_started } # safety net — НЕ блокирует Caddy ``` Lesson from incident: frontend health flaky (Next.js cold start variable). Strict service_healthy для frontend → одна misconfig = full outage обоих доменов (gendsgn.ru + obsidian.gendsgn.ru через тот же Caddy на shared network). ### backend/worker/beat redis: service_started → service_healthy Redis уже имеет `redis-cli ping` healthcheck. Tightening безопасно — был artifact pre-healthcheck Redis. ### Pin rosreestr2coord>=5.0.0 (lock уже 5.3.3) v5 убрала delay параметр; код адаптирован (rate limit на worker уровне). Защита от silent downgrade при `uv lock --upgrade`. ### Что НЕ повторено из #117 - Caddy depends_on frontend: service_healthy — заменено на service_started - wget --spider / — заменено на node TCP probe ### Vault обновлён - `events/Incident_Prod_Down_May14_2026.md` — постмортем + timeline + lessons - Update Changelog в `domains/infra/infra-MOC.md` (если есть) Closes incident. Re-applies #117 goals safely. Co-authored-by: lekss361 <claudestars@proton.me>
161 lines
5.9 KiB
YAML
161 lines
5.9 KiB
YAML
# Production compose. Pulls pre-built images from GHCR
|
||
# (built and pushed by .github/workflows/deploy.yml).
|
||
#
|
||
# Place at /opt/gendesign/docker-compose.prod.yml on the VM.
|
||
# Required env in /opt/gendesign/.env:
|
||
# POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD
|
||
# IMAGE_TAG (defaults to "latest")
|
||
#
|
||
# In production the browser talks to Caddy on :80/:443.
|
||
# Caddy proxies /api/* and /health straight to backend, the rest to frontend.
|
||
# So the frontend uses same-origin relative URLs — no NEXT_PUBLIC_API_BASE_URL needed.
|
||
#
|
||
# Postgres + Redis run alongside the app on the same VM (Discovery mode).
|
||
# Volumes are shared with docker-compose.yml so switching between files preserves data.
|
||
|
||
services:
|
||
postgres:
|
||
image: postgis/postgis:16-3.4
|
||
restart: unless-stopped
|
||
environment:
|
||
POSTGRES_DB: ${POSTGRES_DB}
|
||
POSTGRES_USER: ${POSTGRES_USER}
|
||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||
# Bind to 127.0.0.1 only — accessible from host (for SSH tunnel) but not from public internet.
|
||
# UFW additionally blocks 5432 from outside.
|
||
ports:
|
||
- "127.0.0.1:5432:5432"
|
||
volumes:
|
||
- postgres_data:/var/lib/postgresql/data
|
||
- ./backend/db/init:/docker-entrypoint-initdb.d:ro
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER}"]
|
||
interval: 5s
|
||
timeout: 5s
|
||
retries: 10
|
||
|
||
redis:
|
||
image: redis:7-alpine
|
||
restart: unless-stopped
|
||
volumes:
|
||
- redis_data:/data
|
||
healthcheck:
|
||
test: ["CMD", "redis-cli", "ping"]
|
||
interval: 10s
|
||
timeout: 3s
|
||
retries: 5
|
||
|
||
backend:
|
||
image: ghcr.io/lekss361/gendesign-backend:${IMAGE_TAG:-latest}
|
||
restart: unless-stopped
|
||
# .env.runtime пишется deploy.yml через SSH (SENTRY_RELEASE=$IMAGE_TAG).
|
||
# required: false — compose не падает если файла нет (первый деплой).
|
||
env_file:
|
||
- path: ./backend/.env
|
||
- path: ./backend/.env.runtime
|
||
required: false
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
redis:
|
||
condition: service_healthy
|
||
ports:
|
||
- "127.0.0.1:8000:8000"
|
||
healthcheck:
|
||
test: ["CMD", "curl", "-fsS", "http://localhost:8000/health"]
|
||
interval: 30s
|
||
timeout: 5s
|
||
retries: 5
|
||
start_period: 20s
|
||
|
||
frontend:
|
||
image: ghcr.io/lekss361/gendesign-frontend:${IMAGE_TAG:-latest}
|
||
restart: unless-stopped
|
||
ports:
|
||
- "127.0.0.1:3000:3000"
|
||
depends_on:
|
||
- backend
|
||
# TCP-only probe через node (есть в alpine image, запускает сам server.js).
|
||
# НЕ использует HTTP semantics — проверяет только что порт 3000 слушает.
|
||
# Если Next.js процесс жив и порт открыт → healthy, независимо от HTTP status.
|
||
# start_period 60s даёт время Next.js standalone bundle bootstrap.
|
||
healthcheck:
|
||
test: ["CMD", "node", "-e", "require('net').createConnection({port:3000,host:'127.0.0.1'}).once('connect',function(){process.exit(0)}).once('error',function(){process.exit(1)})"]
|
||
interval: 15s
|
||
timeout: 5s
|
||
retries: 6
|
||
start_period: 60s
|
||
|
||
worker:
|
||
# Отдельный chromium-образ (+200 МБ Playwright). См. backend/Dockerfile target=runner-with-chromium.
|
||
image: ghcr.io/lekss361/gendesign-worker:${IMAGE_TAG:-latest}
|
||
restart: unless-stopped
|
||
env_file:
|
||
- path: ./backend/.env
|
||
- path: ./backend/.env.runtime
|
||
required: false
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
redis:
|
||
condition: service_healthy
|
||
volumes:
|
||
- ./data:/app/data # playwright_state.json + photo binaries
|
||
# Read-only bind-mount Антоновского /sf/ SQLite — для objective_etl task.
|
||
# На VPS файл лежит в /opt/gendesign/site-finder/analysis.db; путь внутри
|
||
# контейнера задан settings.objective_anton_sqlite_path (default
|
||
# /data/anton-sqlite/analysis.db).
|
||
- /opt/gendesign/site-finder:/data/anton-sqlite:ro
|
||
command: ["celery", "-A", "app.workers.celery_app", "worker", "--loglevel=info", "--concurrency=8", "--queues=celery,scrape_kn,geo"]
|
||
|
||
beat:
|
||
# Lean backend-образ (без Chromium) — beat только триггерит таски в Redis.
|
||
image: ghcr.io/lekss361/gendesign-backend:${IMAGE_TAG:-latest}
|
||
restart: unless-stopped
|
||
env_file:
|
||
- path: ./backend/.env
|
||
- path: ./backend/.env.runtime
|
||
required: false
|
||
depends_on:
|
||
redis:
|
||
condition: service_healthy
|
||
command: ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info"]
|
||
|
||
caddy:
|
||
image: caddy:2
|
||
restart: unless-stopped
|
||
ports:
|
||
- "80:80"
|
||
- "443:443"
|
||
volumes:
|
||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||
- caddy_data:/data
|
||
- caddy_config:/config
|
||
# backend: service_healthy — backend имеет /health endpoint, готов до Caddy.
|
||
# frontend: service_started — НЕ service_healthy: даже если frontend healthcheck
|
||
# дребезжит, Caddy всё равно стартует (можно отдавать 502 для /, но
|
||
# obsidian.gendsgn.ru, api.gendsgn.ru и т.д. остаются доступны).
|
||
depends_on:
|
||
backend:
|
||
condition: service_healthy
|
||
frontend:
|
||
condition: service_started
|
||
networks:
|
||
- default
|
||
# shared — для маршрута obsidian.gendsgn.ru → couchdb (отдельный stack).
|
||
# Если obsidian-стек не задеплоен, Caddy просто получит 502 на этом маршруте,
|
||
# main-приложение не страдает.
|
||
- shared
|
||
|
||
volumes:
|
||
postgres_data:
|
||
redis_data:
|
||
caddy_data:
|
||
caddy_config:
|
||
|
||
networks:
|
||
# Внешняя сеть, создаётся вне compose (см. docs/obsidian-livesync.md).
|
||
# Связывает main-stack (Caddy) и obsidian-stack (CouchDB).
|
||
shared:
|
||
external: true
|
||
name: gendesign_shared
|