2026-05-15 05:06:54 +00:00
5 changed files with 181 additions and 12 deletions
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@ -11,6 +11,7 @@ on:
      - "docker-compose.prod.yml"
      - "Caddyfile"
      - ".github/workflows/deploy.yml"
+      - "data/sql/*.sql"
      # NB: shared compose-fragments (network) — тоже триггерят main
      # потому что Caddy шарит сеть с obsidian-stack
  workflow_dispatch:
@ -210,6 +211,44 @@ jobs:
            # Project name явно — `gendesign` (по имени папки auto), но
            # фиксируем для consistency с obsidian-stack (-p gendesign-obsidian).
            docker compose -p gendesign -f docker-compose.prod.yml pull
+
+            # ── Apply pending SQL migrations ──────────────────────────────────
+            # Tracking table: _schema_migrations (filename TEXT PRIMARY KEY).
+            # Each NN_*.sql file is applied exactly once, in sort order.
+            # ON_ERROR_STOP=on ensures a failing migration aborts the deploy.
+            set -a; source backend/.env; set +a
+
+            # Ensure tracking table exists (idempotent).
+            docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
+              psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=on -c "
+                CREATE TABLE IF NOT EXISTS _schema_migrations (
+                  filename TEXT PRIMARY KEY,
+                  applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+                );
+              "
+
+            # Apply each pending .sql file in NN order.
+            for sql_file in $(ls -1 data/sql/*.sql 2>/dev/null | sort); do
+              fname=$(basename "$sql_file")
+              applied=$(docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
+                psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc \
+                "SELECT COUNT(*) FROM _schema_migrations WHERE filename='$fname'")
+              if [ "$applied" = "0" ]; then
+                echo "→ Applying migration: $fname"
+                docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
+                  psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=on \
+                  < "$sql_file" \
+                  || { echo "FAILED on migration: $fname"; exit 1; }
+                docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \
+                  psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
+                  "INSERT INTO _schema_migrations (filename) VALUES ('$fname') ON CONFLICT DO NOTHING;"
+              else
+                echo "✓ Already applied: $fname"
+              fi
+            done
+            echo "All migrations applied."
+            # ─────────────────────────────────────────────────────────────────
+
            docker compose -p gendesign -f docker-compose.prod.yml up -d

            # Caddyfile is bind-mounted; up -d won't re-read it. Reload explicitly.
--- a/CLAUDE.md
+++ b/CLAUDE.md
@ -306,10 +306,36 @@ Credentials — в Obsidian vault `meta/00_credentials.md` (НЕ в этом ф
   - Material partitions / GIST индексы
   - Удалить колонку с CASCADE-зависимостями

+### Auto-apply (data/sql/*.sql → prod)
+
+`data/sql/NN_*.sql` миграции применяются **автоматически** через `deploy.yml`
+после `compose pull` и **до** `compose up -d`. Tracking через `_schema_migrations`
+table в prod DB — каждый файл применяется **ровно один раз**, в порядке `sort` (NN order).
+
+При добавлении новой миграции:
+1. Создаёшь `data/sql/NN_xxx.sql` (NN — следующий sequential номер)
+2. Открываешь PR — merge автоматически триггерит deploy
+3. Deploy auto-apply этого файла + любых других pending
+
+**Bootstrap (one-time):** до первого auto-deploy после merge PR #150 —
+запустить `./scripts/bootstrap_schema_migrations.sh` на prod вручную
+(через SSH tunnel) чтобы seed tracking table с уже-applied файлами.
+Иначе auto-deploy попытается применить все 88 исторических файлов, многие не idempotent.
+
+**Idempotency:** каждый новый .sql файл должен быть idempotent
+(`IF NOT EXISTS`, `ON CONFLICT DO NOTHING`, `CREATE OR REPLACE VIEW`) —
+на случай частичного применения или ручного ре-apply.
+
+**Если auto-apply падает:** deploy завершится с ошибкой (exit 1). Containers
+не обновятся. Диагностика: посмотреть GitHub Actions log шага "Apply DB migrations".
+Ручной fix: `psql -f data/sql/NN_xxx.sql` через SSH tunnel, затем вручную
+`INSERT INTO _schema_migrations (filename) VALUES ('NN_xxx.sql') ON CONFLICT DO NOTHING;`,
+затем rerun workflow.
+
 ### Applying migrations к prod

 **Order matters:**
-1. Apply migration first (SQL → prod)
+1. Apply migration first (SQL → prod) — теперь auto через deploy.yml
 2. Deploy backend code that matches new schema
 3. Никогда наоборот — иначе deployed code напорется на старую/новую несовместимую схему

@ -326,9 +352,9 @@ GitHub Actions:
 - `.github/workflows/deploy-obsidian.yml` — obsidian: SSH `compose up -d` для couchdb + idempotent `scripts/setup-couchdb.sh` bootstrap

 **Триггеры по path:**
- Push в `backend/**`, `frontend/**`, `Caddyfile`, `docker-compose.prod.yml` → `deploy.yml`
+- Push в `backend/**`, `frontend/**`, `Caddyfile`, `docker-compose.prod.yml`, `data/sql/*.sql` → `deploy.yml`
 - Push в `docker-compose.obsidian.yml`, `scripts/setup-couchdb.sh`, `docs/obsidian-livesync.md` → `deploy-obsidian.yml`
- Push только в `data/sql/**` или `docs/**` (не из триггеров выше) — деплоев не вызывает
+- Push только в `docs/**` (не из триггеров выше) — деплоев не вызывает

 Sentry release tracking: `SENTRY_RELEASE=$IMAGE_TAG` пишется в `backend/.env.runtime` на VPS через `sed` (НЕ полная перезапись файла — там user-managed `COUCHDB_PASSWORD` и пр.).

--- a/data/sql/91_engineering_networks_191fz.sql
+++ b/data/sql/91_engineering_networks_191fz.sql
@ -1,11 +1,11 @@
-- 87_engineering_networks_191fz.sql
+-- 91_engineering_networks_191fz.sql
 -- Engineering Networks schema по 191-ФЗ (Сводный план наземных и подземных коммуникаций)
 -- Source: QGIS.zip Минстроя Свердл (https://old-minstroy.midural.ru/uploads/2021/05/QGIS.zip)
 -- Extracted: 2026-05-12 (via Communication.qgz reverse-engineering)
 -- See vault/research/Engineering_Networks_Schema_191FZ_May12.md
 --
-- Deploy order: apply after 86_v_bucket_success_score.sql (no dependencies on 85_pzz_zones_ekb.sql)
-- Renamed from 85_engineering_networks_191fz.sql to avoid NN collision with existing 85_pzz_zones_ekb.sql
+-- Deploy order: apply after 90_user_weight_profiles.sql
+-- Renumbered 87→91 due to NN collision with existing 87_on_demand_indexes.sql
 -- Dependencies: PostGIS extension, spatial_ref_sys table

 BEGIN;
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@ -47,13 +47,17 @@ export async function apiFetchWithStatus<T>(
    },
  });
  let body: unknown = null;
-  // 204 No Content имеет пустое тело; для всех остальных пытаемся JSON
+  // 204 No Content имеет пустое тело; для всех остальных читаем как text один раз
+  // (fetch body stream consumable только один раз — нельзя сначала .json() потом .text()).
  if (response.status !== 204) {
-    try {
-      body = await response.json();
-    } catch {
-      // fallback на text() — может прилететь HTML error page от Caddy
-      body = { detail: await response.text() };
+    const rawText = await response.text();
+    if (rawText) {
+      try {
+        body = JSON.parse(rawText);
+      } catch {
+        // Не JSON (HTML error page от Caddy / 502 / etc) — оборачиваем в detail
+        body = { detail: rawText };
+      }
    }
  }
  if (!response.ok) {
--- a/scripts/bootstrap_schema_migrations.sh
+++ b/scripts/bootstrap_schema_migrations.sh
@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# bootstrap_schema_migrations.sh
+# Seed _schema_migrations tracking table with all SQL files already applied
+# to production BEFORE auto-apply was introduced (PR #150).
+#
+# Run ONCE manually on prod via SSH tunnel before the first auto-deploy after
+# merging feat/auto-apply-migrations-150. After that, deploy.yml handles
+# everything automatically.
+#
+# Usage:
+#   ssh gendesign           # open SSH tunnel (LocalForward 15432 → 5432)
+#   export POSTGRES_USER=gendesign
+#   export POSTGRES_PASSWORD=<from vault meta/00_credentials.md>
+#   export POSTGRES_DB=gendesign
+#   bash scripts/bootstrap_schema_migrations.sh
+#
+# The script connects on localhost:15432 (tunnel).
+# Files NOT in this seed list will be applied automatically on next deploy.
+# Currently pending (will be applied by first auto-deploy):
+#   89_drop_dead_brin_rosreestr_deals.sql
+#   90_user_weight_profiles.sql
+#   91_engineering_networks_191fz.sql
+
+set -euo pipefail
+
+DB_URL="${DB_URL:-postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:15432/${POSTGRES_DB}}"
+
+echo "Connecting to: postgresql://${POSTGRES_USER}:***@localhost:15432/${POSTGRES_DB}"
+
+psql "$DB_URL" -v ON_ERROR_STOP=on <<'SQL'
+CREATE TABLE IF NOT EXISTS _schema_migrations (
+  filename TEXT PRIMARY KEY,
+  applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Seed: all NN_*.sql files known to be applied on prod before PR #150.
+-- Excludes 89, 90, 91 — those are pending and will auto-apply on next deploy.
+-- Non-SQL files (*.sh, *.py) are never tracked — only .sql applies.
+INSERT INTO _schema_migrations (filename) VALUES
+  ('01_schema_rosreestr_deals.sql'),
+  ('03_materialized_views.sql'),
+  ('10_schema_domrf.sql'),
+  ('20_schema_domrf_extras.sql'),
+  ('31_schema_domrf_full.sql'),
+  ('31_schema_domrf_raw.sql'),
+  ('33_schema_domrf_lazy.sql'),
+  ('35_schema_domrf_stats.sql'),
+  ('37_cleanup_redundant.sql'),
+  ('38_relations_fix.sql'),
+  ('39_territory_aliases.sql'),
+  ('40_relations.sql'),
+  ('42_schema_breakdown_widen.sql'),
+  ('43_anton_import.sql'),
+  ('45_relations_v2.sql'),
+  ('50_schema_kn_extensions.sql'),
+  ('51_schema_kn_extras.sql'),
+  ('52_schema_prinzip_crm.sql'),
+  ('53_schema_kn_thumbs.sql'),
+  ('54_schema_scrape_log.sql'),
+  ('55_schema_scrape_resume.sql'),
+  ('56_schema_ekb_districts_geom.sql'),
+  ('57_kn_objects_district.sql'),
+  ('58_schema_cad_quarters.sql'),
+  ('59_backfill_obj_cad_quarter.sql'),
+  ('60_v_zk_rosreestr_velocity.sql'),
+  ('63_schema_nspd_runs.sql'),
+  ('64_v_zk_rosreestr_velocity.sql'),
+  ('65_partitions_rosreestr_2024_h1.sql'),
+  ('66_indexes_recommend.sql'),
+  ('67_refresh_ekb_districts_median.sql'),
+  ('68_schema_objective.sql'),
+  ('72_objective_sync_config.sql'),
+  ('73_complexes_master.sql'),
+  ('74_dedupe_unified_views.sql'),
+  ('75_unification_naming.sql'),
+  ('76_complex_connectivity.sql'),
+  ('77_nspd_geo_jobs.sql'),
+  ('78_drop_use_rosreestr2coord.sql'),
+  ('80_complexes_backfill_nulls.sql'),
+  ('81_job_settings.sql'),
+  ('82_osm_poi_ekb.sql'),
+  ('83_cad_parcels_geom.sql'),
+  ('84_osm_noise_sources_ekb.sql'),
+  ('85_pzz_zones_ekb.sql'),
+  ('86_v_bucket_success_score.sql'),
+  ('87_on_demand_indexes.sql'),
+  ('88_nspd_quarter_dumps.sql')
+ON CONFLICT (filename) DO NOTHING;
+
+SELECT COUNT(*) AS seeded_count FROM _schema_migrations;
+SQL
+
+echo ""
+echo "Bootstrap complete. Pending migrations (will apply on next deploy):"
+echo "  89_drop_dead_brin_rosreestr_deals.sql"
+echo "  90_user_weight_profiles.sql"
+echo "  91_engineering_networks_191fz.sql"
+echo ""
+echo "Verify with:"
+echo "  psql \$DB_URL -c \"SELECT filename, applied_at FROM _schema_migrations ORDER BY filename;\""