From b3f32ae927ad092988e24f138a275ac1e35165f3 Mon Sep 17 00:00:00 2001
From: lekss361 <47113017+lekss361@users.noreply.github.com>
Date: Thu, 14 May 2026 23:24:47 +0300
Subject: [PATCH] chore(infra): correct frontend healthcheck + tighten
 depends_on (re-apply #117) (#120)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Forward-fix после incident May 14 (см. vault events/Incident_Prod_Down_May14_2026).
PR #117 был полностью reverted PR #119 (Caddy depends_on frontend: service_healthy
+ failing wget --spider /  → Caddy не стартовал → vsё прод down ~25 мин).

Этот PR делает то же что планировал #117, но корректно:

### Frontend healthcheck — TCP probe через node (НЕ HTTP)

```yaml
test: ["CMD", "node", "-e",
  "require('net').createConnection({port:3000,host:'127.0.0.1'})
   .once('connect',function(){process.exit(0)})
   .once('error',function(){process.exit(1)})"]
start_period: 60s
retries: 6
interval: 15s
```

- НЕ зависит от HTTP status (200/404/308 — все OK если порт слушает)
- `node` гарантированно в alpine image (запускает server.js)
- `127.0.0.1` — никакой DNS ambiguity (busybox alpine localhost IPv6 vs IPv4)
- 60s + 6×15s = 150s window — с запасом на Next.js standalone cold start

### Caddy depends_on — backend healthy, frontend started (НЕ healthy)

```yaml
caddy:
  depends_on:
    backend: { condition: service_healthy }    # backend имеет /health endpoint
    frontend: { condition: service_started }   # safety net — НЕ блокирует Caddy
```

Lesson from incident: frontend health flaky (Next.js cold start variable).
Strict service_healthy для frontend → одна misconfig = full outage обоих доменов
(gendsgn.ru + obsidian.gendsgn.ru через тот же Caddy на shared network).

### backend/worker/beat redis: service_started → service_healthy

Redis уже имеет `redis-cli ping` healthcheck. Tightening безопасно — был
artifact pre-healthcheck Redis.

### Pin rosreestr2coord>=5.0.0 (lock уже 5.3.3)

v5 убрала delay параметр; код адаптирован (rate limit на worker уровне).
Защита от silent downgrade при `uv lock --upgrade`.

### Что НЕ повторено из #117

- Caddy depends_on frontend: service_healthy — заменено на service_started
- wget --spider / — заменено на node TCP probe

### Vault обновлён

- `events/Incident_Prod_Down_May14_2026.md` — постмортем + timeline + lessons
- Update Changelog в `domains/infra/infra-MOC.md` (если есть)

Closes incident. Re-applies #117 goals safely.

Co-authored-by: lekss361 <claudestars@proton.me>
---
 backend/pyproject.toml  |  2 +-
 backend/uv.lock         |  2 +-
 docker-compose.prod.yml | 26 +++++++++++++++++++++-----
 3 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index afabe12a..8df3fa61 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -28,7 +28,7 @@ dependencies = [
     "numpy>=2.0.0",
     "scikit-learn>=1.5.0",
     "sentry-sdk[fastapi]>=2.10.0",
-    "rosreestr2coord>=4.0.0",
+    "rosreestr2coord>=5.0.0",
 ]
 
 [dependency-groups]
diff --git a/backend/uv.lock b/backend/uv.lock
index 8bd2a644..1422c133 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -616,7 +616,7 @@ requires-dist = [
     { name = "pydantic-settings", specifier = ">=2.3.0" },
     { name = "pyproj", specifier = ">=3.6.0" },
     { name = "redis", specifier = ">=5.0.0" },
-    { name = "rosreestr2coord", specifier = ">=4.0.0" },
+    { name = "rosreestr2coord", specifier = ">=5.0.0" },
     { name = "scikit-learn", specifier = ">=1.5.0" },
     { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.10.0" },
     { name = "shapely", specifier = ">=2.0.0" },
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 54fc3668..90632581 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -58,7 +58,7 @@ services:
       postgres:
         condition: service_healthy
       redis:
-        condition: service_started
+        condition: service_healthy
     ports:
       - "127.0.0.1:8000:8000"
     healthcheck:
@@ -75,6 +75,16 @@ services:
       - "127.0.0.1:3000:3000"
     depends_on:
       - backend
+    # TCP-only probe через node (есть в alpine image, запускает сам server.js).
+    # НЕ использует HTTP semantics — проверяет только что порт 3000 слушает.
+    # Если Next.js процесс жив и порт открыт → healthy, независимо от HTTP status.
+    # start_period 60s даёт время Next.js standalone bundle bootstrap.
+    healthcheck:
+      test: ["CMD", "node", "-e", "require('net').createConnection({port:3000,host:'127.0.0.1'}).once('connect',function(){process.exit(0)}).once('error',function(){process.exit(1)})"]
+      interval: 15s
+      timeout: 5s
+      retries: 6
+      start_period: 60s
 
   worker:
     # Отдельный chromium-образ (+200 МБ Playwright). См. backend/Dockerfile target=runner-with-chromium.
@@ -88,7 +98,7 @@ services:
       postgres:
         condition: service_healthy
       redis:
-        condition: service_started
+        condition: service_healthy
     volumes:
       - ./data:/app/data    # playwright_state.json + photo binaries
       # Read-only bind-mount Антоновского /sf/ SQLite — для objective_etl task.
@@ -108,7 +118,7 @@ services:
         required: false
     depends_on:
       redis:
-        condition: service_started
+        condition: service_healthy
     command: ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info"]
 
   caddy:
@@ -121,9 +131,15 @@ services:
       - ./Caddyfile:/etc/caddy/Caddyfile:ro
       - caddy_data:/data
       - caddy_config:/config
+    # backend: service_healthy — backend имеет /health endpoint, готов до Caddy.
+    # frontend: service_started — НЕ service_healthy: даже если frontend healthcheck
+    # дребезжит, Caddy всё равно стартует (можно отдавать 502 для /, но
+    # obsidian.gendsgn.ru, api.gendsgn.ru и т.д. остаются доступны).
     depends_on:
-      - backend
-      - frontend
+      backend:
+        condition: service_healthy
+      frontend:
+        condition: service_started
     networks:
       - default
       # shared — для маршрута obsidian.gendsgn.ru → couchdb (отдельный stack).