From 7c43bc54960c47d268cbd2cb48d7a76c96668e2a Mon Sep 17 00:00:00 2001
From: lekss361 <47113017+lekss361@users.noreply.github.com>
Date: Fri, 15 May 2026 08:06:54 +0300
Subject: [PATCH] feat(infra): auto-apply data/sql/*.sql migrations in deploy
 pipeline (#150) (#151)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(frontend): apiFetchWithStatus body stream double-read crash

Per user report 2026-05-14: 'Failed to execute text on Response: body stream
already read' при analyze с non-JSON error response.

apiFetchWithStatus делал .json() (consumes stream), потом .text() в catch
(FAILS — stream already consumed). Fetch API: body — ReadableStream,
consumable один раз.

Fix: read text() once → JSON.parse on string. На fail (HTML / non-JSON)
оборачиваем в {detail: rawText}. Никаких double-reads.

Site-finder cad search больше не крашится на error pages.

* feat(infra): auto-apply data/sql/*.sql migrations in deploy pipeline (#150)

Закрывает root cause production 500 на /api/v1/admin/site-finder/weight-profiles
(а также 2 unapplied migrations: #89/#91).

## Why это нужно

После audit обнаружили что:
- backend/alembic/versions/ пустой (Alembic configured но не используется)
- deploy.yml не имеет migration step
- 47+ raw SQL files накопилось без auto-apply
- 3 файла unapplied: 87 (engineering), 89 (drop brin), 90 (weight profiles)

User report: GET weight-profiles → 500 потому что user_weight_profiles
table не существует на prod.

## Changes

### deploy.yml +39 lines
- Path trigger расширен на data/sql/*.sql
- New step 'Apply DB migrations' между compose pull и compose up -d:
  - Idempotent CREATE TABLE _schema_migrations
  - Loop по data/sql/*.sql sorted → skip applied → psql with ON_ERROR_STOP=on
  - Record applied filename в tracking table
  - Exit 1 на failure → containers НЕ обновляются (no partial state)

### NN collision fix
- 87_engineering_networks_191fz.sql → 91_engineering_networks_191fz.sql
  (collision с 87_on_demand_indexes.sql, мой renumber 85→87 был неудачным)

### CLAUDE.md +32 lines
- Subsection 'Auto-apply' под Migration pattern
- Bootstrap procedure docs
- Idempotency requirement
- Failure recovery

### scripts/bootstrap_schema_migrations.sh +100 lines
- One-time seed _schema_migrations с 48 already-applied files
- 3 файла intentionally NOT seeded: 89/90/91 (auto-apply на first deploy)
- Usage docs + verification query

## Vault

domains/infra/Runbook_Auto_Apply_Migrations.md NEW
infra-MOC.md updated

## Required manual steps ДО merge

1. SSH gendesign (tunnel)
2. POSTGRES creds export
3. bash scripts/bootstrap_schema_migrations.sh — seed 48 already-applied files
4. Verify: SELECT COUNT(*) FROM _schema_migrations → 48
5. После merge → first deploy auto-apply #89/#90/#91 → 500 closes

Closes #150

---------

Co-authored-by: lekss361 <claudestars@proton.me>
---
 .github/workflows/deploy.yml                  |  39 +++++++
 CLAUDE.md                                     |  32 +++++-
 ....sql => 91_engineering_networks_191fz.sql} |   6 +-
 scripts/bootstrap_schema_migrations.sh        | 100 ++++++++++++++++++
 4 files changed, 171 insertions(+), 6 deletions(-)
 rename data/sql/{87_engineering_networks_191fz.sql => 91_engineering_networks_191fz.sql} (99%)
 create mode 100644 scripts/bootstrap_schema_migrations.sh

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 0ce81e9f..440bab34 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -11,6 +11,7 @@ on:
       - "docker-compose.prod.yml"
       - "Caddyfile"
       - ".github/workflows/deploy.yml"
+      - "data/sql/*.sql"
       # NB: shared compose-fragments (network) — тоже триггерят main
       # потому что Caddy шарит сеть с obsidian-stack
   workflow_dispatch:
@@ -210,6 +211,44 @@ jobs:
             # Project name явно — `gendesign` (по имени папки auto), но
             # фиксируем для consistency с obsidian-stack (-p gendesign-obsidian).
             docker compose -p gendesign -f docker-compose.prod.yml pull
+
+            # ── Apply pending SQL migrations ──────────────────────────────────
+            # Tracking table: _schema_migrations (filename TEXT PRIMARY KEY).
+            # Each NN_*.sql file is applied exactly once, in sort order.
+            # ON_ERROR_STOP=on ensures a failing migration aborts the deploy.
+            set -a; source backend/.env; set +a
+
+            # Ensure tracking table exists (idempotent).
+            docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
+              psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=on -c "
+                CREATE TABLE IF NOT EXISTS _schema_migrations (
+                  filename TEXT PRIMARY KEY,
+                  applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+                );
+              "
+
+            # Apply each pending .sql file in NN order.
+            for sql_file in $(ls -1 data/sql/*.sql 2>/dev/null | sort); do
+              fname=$(basename "$sql_file")
+              applied=$(docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
+                psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc \
+                "SELECT COUNT(*) FROM _schema_migrations WHERE filename='$fname'")
+              if [ "$applied" = "0" ]; then
+                echo "→ Applying migration: $fname"
+                docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
+                  psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=on \
+                  < "$sql_file" \
+                  || { echo "FAILED on migration: $fname"; exit 1; }
+                docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
+                  psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
+                  "INSERT INTO _schema_migrations (filename) VALUES ('$fname') ON CONFLICT DO NOTHING;"
+              else
+                echo "✓ Already applied: $fname"
+              fi
+            done
+            echo "All migrations applied."
+            # ─────────────────────────────────────────────────────────────────
+
             docker compose -p gendesign -f docker-compose.prod.yml up -d
 
             # Caddyfile is bind-mounted; up -d won't re-read it. Reload explicitly.
diff --git a/CLAUDE.md b/CLAUDE.md
index 4eddb7f1..c879d3cd 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -306,10 +306,36 @@ Credentials — в Obsidian vault `meta/00_credentials.md` (НЕ в этом ф
    - Material partitions / GIST индексы
    - Удалить колонку с CASCADE-зависимостями
 
+### Auto-apply (data/sql/*.sql → prod)
+
+`data/sql/NN_*.sql` миграции применяются **автоматически** через `deploy.yml`
+после `compose pull` и **до** `compose up -d`. Tracking через `_schema_migrations`
+table в prod DB — каждый файл применяется **ровно один раз**, в порядке `sort` (NN order).
+
+При добавлении новой миграции:
+1. Создаёшь `data/sql/NN_xxx.sql` (NN — следующий sequential номер)
+2. Открываешь PR — merge автоматически триггерит deploy
+3. Deploy auto-apply этого файла + любых других pending
+
+**Bootstrap (one-time):** до первого auto-deploy после merge PR #150 —
+запустить `./scripts/bootstrap_schema_migrations.sh` на prod вручную
+(через SSH tunnel) чтобы seed tracking table с уже-applied файлами.
+Иначе auto-deploy попытается применить все 88 исторических файлов, многие не idempotent.
+
+**Idempotency:** каждый новый .sql файл должен быть idempotent
+(`IF NOT EXISTS`, `ON CONFLICT DO NOTHING`, `CREATE OR REPLACE VIEW`) —
+на случай частичного применения или ручного ре-apply.
+
+**Если auto-apply падает:** deploy завершится с ошибкой (exit 1). Containers
+не обновятся. Диагностика: посмотреть GitHub Actions log шага "Apply DB migrations".
+Ручной fix: `psql -f data/sql/NN_xxx.sql` через SSH tunnel, затем вручную
+`INSERT INTO _schema_migrations (filename) VALUES ('NN_xxx.sql') ON CONFLICT DO NOTHING;`,
+затем rerun workflow.
+
 ### Applying migrations к prod
 
 **Order matters:**
-1. Apply migration first (SQL → prod)
+1. Apply migration first (SQL → prod) — теперь auto через deploy.yml
 2. Deploy backend code that matches new schema
 3. Никогда наоборот — иначе deployed code напорется на старую/новую несовместимую схему
 
@@ -326,9 +352,9 @@ GitHub Actions:
 - `.github/workflows/deploy-obsidian.yml` — obsidian: SSH `compose up -d` для couchdb + idempotent `scripts/setup-couchdb.sh` bootstrap
 
 **Триггеры по path:**
-- Push в `backend/**`, `frontend/**`, `Caddyfile`, `docker-compose.prod.yml` → `deploy.yml`
+- Push в `backend/**`, `frontend/**`, `Caddyfile`, `docker-compose.prod.yml`, `data/sql/*.sql` → `deploy.yml`
 - Push в `docker-compose.obsidian.yml`, `scripts/setup-couchdb.sh`, `docs/obsidian-livesync.md` → `deploy-obsidian.yml`
-- Push только в `data/sql/**` или `docs/**` (не из триггеров выше) — деплоев не вызывает
+- Push только в `docs/**` (не из триггеров выше) — деплоев не вызывает
 
 Sentry release tracking: `SENTRY_RELEASE=$IMAGE_TAG` пишется в `backend/.env.runtime` на VPS через `sed` (НЕ полная перезапись файла — там user-managed `COUCHDB_PASSWORD` и пр.).
 
diff --git a/data/sql/87_engineering_networks_191fz.sql b/data/sql/91_engineering_networks_191fz.sql
similarity index 99%
rename from data/sql/87_engineering_networks_191fz.sql
rename to data/sql/91_engineering_networks_191fz.sql
index 8ddb3e1d..ed519161 100644
--- a/data/sql/87_engineering_networks_191fz.sql
+++ b/data/sql/91_engineering_networks_191fz.sql
@@ -1,11 +1,11 @@
--- 87_engineering_networks_191fz.sql
+-- 91_engineering_networks_191fz.sql
 -- Engineering Networks schema по 191-ФЗ (Сводный план наземных и подземных коммуникаций)
 -- Source: QGIS.zip Минстроя Свердл (https://old-minstroy.midural.ru/uploads/2021/05/QGIS.zip)
 -- Extracted: 2026-05-12 (via Communication.qgz reverse-engineering)
 -- See vault/research/Engineering_Networks_Schema_191FZ_May12.md
 --
--- Deploy order: apply after 86_v_bucket_success_score.sql (no dependencies on 85_pzz_zones_ekb.sql)
--- Renamed from 85_engineering_networks_191fz.sql to avoid NN collision with existing 85_pzz_zones_ekb.sql
+-- Deploy order: apply after 90_user_weight_profiles.sql
+-- Renumbered 87→91 due to NN collision with existing 87_on_demand_indexes.sql
 -- Dependencies: PostGIS extension, spatial_ref_sys table
 
 BEGIN;
diff --git a/scripts/bootstrap_schema_migrations.sh b/scripts/bootstrap_schema_migrations.sh
new file mode 100644
index 00000000..c9b10cf3
--- /dev/null
+++ b/scripts/bootstrap_schema_migrations.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# bootstrap_schema_migrations.sh
+# Seed _schema_migrations tracking table with all SQL files already applied
+# to production BEFORE auto-apply was introduced (PR #150).
+#
+# Run ONCE manually on prod via SSH tunnel before the first auto-deploy after
+# merging feat/auto-apply-migrations-150. After that, deploy.yml handles
+# everything automatically.
+#
+# Usage:
+#   ssh gendesign           # open SSH tunnel (LocalForward 15432 → 5432)
+#   export POSTGRES_USER=gendesign
+#   export POSTGRES_PASSWORD=<from vault meta/00_credentials.md>
+#   export POSTGRES_DB=gendesign
+#   bash scripts/bootstrap_schema_migrations.sh
+#
+# The script connects on localhost:15432 (tunnel).
+# Files NOT in this seed list will be applied automatically on next deploy.
+# Currently pending (will be applied by first auto-deploy):
+#   89_drop_dead_brin_rosreestr_deals.sql
+#   90_user_weight_profiles.sql
+#   91_engineering_networks_191fz.sql
+
+set -euo pipefail
+
+DB_URL="${DB_URL:-postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:15432/${POSTGRES_DB}}"
+
+echo "Connecting to: postgresql://${POSTGRES_USER}:***@localhost:15432/${POSTGRES_DB}"
+
+psql "$DB_URL" -v ON_ERROR_STOP=on <<'SQL'
+CREATE TABLE IF NOT EXISTS _schema_migrations (
+  filename TEXT PRIMARY KEY,
+  applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Seed: all NN_*.sql files known to be applied on prod before PR #150.
+-- Excludes 89, 90, 91 — those are pending and will auto-apply on next deploy.
+-- Non-SQL files (*.sh, *.py) are never tracked — only .sql applies.
+INSERT INTO _schema_migrations (filename) VALUES
+  ('01_schema_rosreestr_deals.sql'),
+  ('03_materialized_views.sql'),
+  ('10_schema_domrf.sql'),
+  ('20_schema_domrf_extras.sql'),
+  ('31_schema_domrf_full.sql'),
+  ('31_schema_domrf_raw.sql'),
+  ('33_schema_domrf_lazy.sql'),
+  ('35_schema_domrf_stats.sql'),
+  ('37_cleanup_redundant.sql'),
+  ('38_relations_fix.sql'),
+  ('39_territory_aliases.sql'),
+  ('40_relations.sql'),
+  ('42_schema_breakdown_widen.sql'),
+  ('43_anton_import.sql'),
+  ('45_relations_v2.sql'),
+  ('50_schema_kn_extensions.sql'),
+  ('51_schema_kn_extras.sql'),
+  ('52_schema_prinzip_crm.sql'),
+  ('53_schema_kn_thumbs.sql'),
+  ('54_schema_scrape_log.sql'),
+  ('55_schema_scrape_resume.sql'),
+  ('56_schema_ekb_districts_geom.sql'),
+  ('57_kn_objects_district.sql'),
+  ('58_schema_cad_quarters.sql'),
+  ('59_backfill_obj_cad_quarter.sql'),
+  ('60_v_zk_rosreestr_velocity.sql'),
+  ('63_schema_nspd_runs.sql'),
+  ('64_v_zk_rosreestr_velocity.sql'),
+  ('65_partitions_rosreestr_2024_h1.sql'),
+  ('66_indexes_recommend.sql'),
+  ('67_refresh_ekb_districts_median.sql'),
+  ('68_schema_objective.sql'),
+  ('72_objective_sync_config.sql'),
+  ('73_complexes_master.sql'),
+  ('74_dedupe_unified_views.sql'),
+  ('75_unification_naming.sql'),
+  ('76_complex_connectivity.sql'),
+  ('77_nspd_geo_jobs.sql'),
+  ('78_drop_use_rosreestr2coord.sql'),
+  ('80_complexes_backfill_nulls.sql'),
+  ('81_job_settings.sql'),
+  ('82_osm_poi_ekb.sql'),
+  ('83_cad_parcels_geom.sql'),
+  ('84_osm_noise_sources_ekb.sql'),
+  ('85_pzz_zones_ekb.sql'),
+  ('86_v_bucket_success_score.sql'),
+  ('87_on_demand_indexes.sql'),
+  ('88_nspd_quarter_dumps.sql')
+ON CONFLICT (filename) DO NOTHING;
+
+SELECT COUNT(*) AS seeded_count FROM _schema_migrations;
+SQL
+
+echo ""
+echo "Bootstrap complete. Pending migrations (will apply on next deploy):"
+echo "  89_drop_dead_brin_rosreestr_deals.sql"
+echo "  90_user_weight_profiles.sql"
+echo "  91_engineering_networks_191fz.sql"
+echo ""
+echo "Verify with:"
+echo "  psql \$DB_URL -c \"SELECT filename, applied_at FROM _schema_migrations ORDER BY filename;\""