From 9197a539e92749d485c090a2cce3cdd5498d14be Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 15 May 2026 14:50:22 +0300 Subject: [PATCH 1/4] fix(parcels): _parse_floors handle int (post-migration #169 schema change) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After PR #169 cad_buildings schema migration, `floors` column is INT (was TEXT in legacy schema). Existing `_parse_floors(r.get("floors"))` call in analyze_parcel → _neighbors_summary crashes with: AttributeError: 'int' object has no attribute 'strip' Fix: type union str | int | None. If int → return directly (no strip). Preserve TEXT range parsing ("5-7" → 7) for backwards-compat with any legacy data still in cad_buildings_old_apr26. --- backend/app/api/v1/parcels.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/app/api/v1/parcels.py b/backend/app/api/v1/parcels.py index a1697a9a..63a3cfba 100644 --- a/backend/app/api/v1/parcels.py +++ b/backend/app/api/v1/parcels.py @@ -639,16 +639,23 @@ _COST_PER_M2_MIN = 1000 # ₽/м² — ниже скорее всего оши _COST_PER_M2_MAX = 500_000 # ₽/м² — выше скорее всего outlier -def _parse_floors(raw: str | None) -> int | None: - """cad_buildings.floors хранится TEXT (могут быть диапазоны '1-2', '5-7'). +def _parse_floors(raw: str | int | None) -> int | None: + """cad_buildings.floors — после schema migration #169 теперь INT, + но historically мог быть TEXT с диапазоном '5-7'. Поддерживаем оба + для backwards-compat с legacy data + tests. Возвращаем верхнюю границу (более консервативный сосед-высотка). NB: `isdigit()` намеренно фильтрует malformed parts типа "5а-7"; для multi-range "1-2-3" возвращается max(1,2,3)=3 (acceptable degradation). """ + if raw is None or raw == "": + return None + # Post-migration: INT column → fast path + if isinstance(raw, int): + return raw if raw > 0 else None + raw = raw.strip() if not raw: return None - raw = raw.strip() # range like "5-7" → 7 if "-" in raw: parts = raw.split("-") -- 2.45.3 From 7c4d6678b14355facebe07794430921173b929d7 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 15 May 2026 14:52:16 +0300 Subject: [PATCH 2/4] test(smoke): production smoke tests for post-deploy regressions (#168) Add tests/smoke/test_prod_smoke.py covering known regression surfaces. Marks: prod_smoke + slow. Env: PROD_SMOKE_BASE_URL, PROD_SMOKE_ADMIN_TOKEN. Run manually: cd backend && uv run pytest tests/smoke/ -m prod_smoke -v --- backend/tests/smoke/__init__.py | 0 backend/tests/smoke/test_prod_smoke.py | 237 +++++++++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 backend/tests/smoke/__init__.py create mode 100644 backend/tests/smoke/test_prod_smoke.py diff --git a/backend/tests/smoke/__init__.py b/backend/tests/smoke/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/smoke/test_prod_smoke.py b/backend/tests/smoke/test_prod_smoke.py new file mode 100644 index 00000000..501ac967 --- /dev/null +++ b/backend/tests/smoke/test_prod_smoke.py @@ -0,0 +1,237 @@ +"""Production smoke tests — must pass after every merge to main. + +Run after deploy completes (~2-3min after merge). CI workflow: + cd backend && uv run pytest tests/smoke/test_prod_smoke.py -m prod_smoke -v + +Each test makes a real HTTP call to https://gendsgn.ru, validates response +structure (not exact data) и crash regressions, and reports timing. + +Marks: +- @pytest.mark.prod_smoke — skip by default in CI, run only on-demand +- @pytest.mark.slow — long-running (≥5s) + +Tests cover the regression surface that bot review window cannot catch: +- Schema migrations break field types → AttributeError на user actions +- Frontend bundle ↔ backend contract drift +- Auth/admin endpoints за токеном +- Background workers + Celery queue +""" + +from __future__ import annotations + +import os + +import httpx +import pytest + +BASE_URL = os.getenv("PROD_SMOKE_BASE_URL", "https://gendsgn.ru") +ADMIN_TOKEN = os.getenv("PROD_SMOKE_ADMIN_TOKEN") +TIMEOUT = httpx.Timeout(30.0, connect=10.0) + +# Test cads — known-good в prod, в quarter 66:41:0303161 (Крауля) +TEST_CAD = "66:41:0204016:10" # Билимбаевская, 32 ЖК velocity baseline +TEST_QUARTER = "66:41:0303161" # 502 объекта (170 ЗУ + 256 зд + 75 соор + 1 ЕНК) + + +pytestmark = pytest.mark.prod_smoke + + +# ───────────────────────────────────────────────────────────────────────────── +# Public endpoints (no auth) +# ───────────────────────────────────────────────────────────────────────────── + + +def test_health_returns_ok() -> None: + """Smoke: /health должен возвращать {"status":"ok"}.""" + r = httpx.get(f"{BASE_URL}/health", timeout=TIMEOUT) + assert r.status_code == 200 + assert r.json().get("status") == "ok" + + +def test_site_finder_page_renders() -> None: + """SSR returns HTML с заголовком GenDesign.""" + r = httpx.get(f"{BASE_URL}/site-finder", timeout=TIMEOUT) + assert r.status_code == 200 + assert "GenDesign" in r.text or "Site Finder" in r.text + + +def test_parcels_analyze_returns_200() -> None: + """REGRESSION SHIELD: ловит crashes в analyze pipeline. + + Раньше ломали: + - PR #149: body stream double-read (api.ts) + - PR #152: SQLAlchemy :weights::jsonb cast + - PR #160: :window_interval::interval cast + - PR #169 → 'int has no attribute strip' (cad_buildings.floors INT) + + Test: POST /api/v1/parcels/{cad}/analyze → 200 + required fields. + """ + r = httpx.post( + f"{BASE_URL}/api/v1/parcels/{TEST_CAD}/analyze", + json={}, + timeout=httpx.Timeout(60.0, connect=10.0), # analyze может быть медленным + ) + assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text[:500]}" + body = r.json() + # Required top-level keys (validate схема не дрейфит): + required = ["cad_num", "score", "pipeline_24mo", "velocity", "gate_verdict"] + for key in required: + assert key in body, f"Missing field '{key}' in analyze response. Keys: {list(body.keys())}" + # neighbors_summary must NOT crash even when floors=int (regression #169): + assert "neighbors_summary" in body + assert body["neighbors_summary"].get("data_available") is not False or ( + body["neighbors_summary"].get("note", "").startswith("neighbors query failed") is False + ) + + +def test_parcels_analyze_velocity_block_present() -> None: + """REGRESSION SHIELD: velocity migrated to objective_corpus_room_month (#158, #163). + + После migration ожидаем что для testovogo cad есть velocity-блок: + - score ≥ 0 + - period.start/end не None (значит objective fresh data есть) + - by_room_bucket список (per-room breakdown — PR #163) + """ + r = httpx.post( + f"{BASE_URL}/api/v1/parcels/{TEST_CAD}/analyze", + json={}, + timeout=httpx.Timeout(60.0, connect=10.0), + ) + assert r.status_code == 200 + velocity = r.json().get("velocity") + if velocity is None: + pytest.skip("velocity null — possibly no data in cadastre, not regression") + assert "score" in velocity + assert "period" in velocity + # PR #163 enrichment: + assert "by_room_bucket" in velocity, "Missing by_room_bucket per PR #163" + + +def test_parcels_analyze_pipeline_by_class_enriched() -> None: + """REGRESSION SHIELD: pipeline_by_class obj_class enrichment (#165). + + Раньше показывало "Не указан: 4075". После #165 — стандарт/комфорт/etc + через JOIN на objective_complex_mapping. + """ + r = httpx.post( + f"{BASE_URL}/api/v1/parcels/{TEST_CAD}/analyze", + json={}, + timeout=httpx.Timeout(60.0, connect=10.0), + ) + assert r.status_code == 200 + pipeline = r.json().get("pipeline_24mo", {}) + by_class = pipeline.get("by_class", {}) + # Не должно быть так что unknown == 100% (это означало бы regression #165) + if by_class: + total = sum(by_class.values()) + unknown_share = by_class.get("unknown", 0) / total if total > 0 else 0 + assert unknown_share < 0.9, ( + f"pipeline_by_class.unknown == {unknown_share:.1%} — " + "regression of #165? Should JOIN on objective_lots." + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Admin endpoints (auth required — skip if no token) +# ───────────────────────────────────────────────────────────────────────────── + + +@pytest.fixture +def admin_headers() -> dict[str, str]: + if not ADMIN_TOKEN: + pytest.skip("PROD_SMOKE_ADMIN_TOKEN env var not set") + return {"X-Admin-Token": ADMIN_TOKEN} + + +def test_admin_jobs_settings_returns_200(admin_headers: dict[str, str]) -> None: + r = httpx.get( + f"{BASE_URL}/api/v1/admin/jobs/settings", + headers=admin_headers, + timeout=TIMEOUT, + ) + assert r.status_code == 200 + assert isinstance(r.json(), list | dict) + + +def test_admin_cadastre_jobs_endpoint(admin_headers: dict[str, str]) -> None: + """PR #171 admin cadastre endpoints.""" + r = httpx.get( + f"{BASE_URL}/api/v1/admin/cadastre/jobs?limit=5", + headers=admin_headers, + timeout=TIMEOUT, + ) + assert r.status_code == 200 + body = r.json() + assert isinstance(body, list) + if body: + # Validate схема jobs row (PR #171) + required = ["job_id", "status", "targets_total", "targets_done"] + for key in required: + assert key in body[0], f"Missing field '{key}' in job row" + + +def test_admin_weight_profiles_endpoint(admin_headers: dict[str, str]) -> None: + """PR #138 weight profiles list.""" + r = httpx.get( + f"{BASE_URL}/api/v1/admin/site-finder/weight-profiles?user_id=admin", + headers=admin_headers, + timeout=TIMEOUT, + ) + assert r.status_code == 200 + + +# ───────────────────────────────────────────────────────────────────────────── +# Frontend bundle (catches build regressions) +# ───────────────────────────────────────────────────────────────────────────── + + +def test_admin_scrape_cadastre_page_renders() -> None: + """PR #172 — new admin UI page must SSR ok.""" + r = httpx.get(f"{BASE_URL}/admin/scrape/cadastre", timeout=TIMEOUT) + assert r.status_code == 200 + assert "Cadastre" in r.text or "кадастр" in r.text.lower() or "Bulk" in r.text + + +def test_admin_scrape_geo_page_renders() -> None: + """Existing pattern — должен работать (PR #172 не должен сломать).""" + r = httpx.get(f"{BASE_URL}/admin/scrape/geo", timeout=TIMEOUT) + assert r.status_code == 200 + + +# ───────────────────────────────────────────────────────────────────────────── +# Schema invariants (через DB read через admin endpoints) +# ───────────────────────────────────────────────────────────────────────────── + + +def test_health_db_connection_works() -> None: + """`/health` returns 200 если DB reachable + migrations applied.""" + r = httpx.get(f"{BASE_URL}/health?check_db=1", timeout=TIMEOUT) + if r.status_code == 200 and "db" in r.json(): + assert r.json()["db"] == "ok", f"DB check failed: {r.json()}" + + +# ───────────────────────────────────────────────────────────────────────────── +# How to run after merge (CI integration) +# ───────────────────────────────────────────────────────────────────────────── + +# Add to .github/workflows/post-deploy-smoke.yml: +# +# on: +# workflow_run: +# workflows: [deploy.yml] +# types: [completed] +# jobs: +# smoke: +# if: ${{ github.event.workflow_run.conclusion == 'success' }} +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - run: uv sync +# - run: | +# cd backend && uv run pytest tests/smoke/ -m prod_smoke -v +# env: +# PROD_SMOKE_BASE_URL: https://gendsgn.ru +# PROD_SMOKE_ADMIN_TOKEN: ${{ secrets.SCRAPE_ADMIN_TOKEN }} +# - name: Post slack/sentry alert if failed +# if: failure() +# run: echo "smoke tests failed — investigate" -- 2.45.3 From 4ace87af1443246a64834e463fe09e94b394712c Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 15 May 2026 14:59:41 +0300 Subject: [PATCH 3/4] fix(cadastre): exact-match headers to nspd_lite to bypass NSPD WAF (#168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pilot v2 (job_id=2) failed 50/50 with HTTP 403 WAF block. Comparing nspd_bulk_client.DEFAULT_HEADERS vs legacy nspd_lite.HEADERS (which works on VPS IP since April 2026): PascalCase keys → lowercase keys Chrome/148 UA → Chrome/144 UA No cache-control / pragma → "no-cache" both accept-language ru first → en first No origin → "https://nspd.gov.ru" referer "/map" → "/map?thematic=PKK" NSPD WAF (BotShield-class) likely fingerprints на header order + values combined with TLS fingerprint. Matching legacy exactly minimizes deltas. Test plan: retry pilot job after deploy, expect 0 WAF blocks for first 5 quarters. --- backend/app/scrapers/nspd_bulk_client.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/backend/app/scrapers/nspd_bulk_client.py b/backend/app/scrapers/nspd_bulk_client.py index b51ba3e3..f45c98b9 100644 --- a/backend/app/scrapers/nspd_bulk_client.py +++ b/backend/app/scrapers/nspd_bulk_client.py @@ -48,17 +48,21 @@ _SEMAPHORE = asyncio.Semaphore(3) # Таймаут подключения и чтения DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0) -# Headers из HAR audit 2026-05-15 +# Headers — exact match to legacy nspd_lite.py (proven to bypass NSPD WAF +# on VPS IP since April 2026). All-lowercase keys + cache-control + Chrome 144 +# UA. Initial HAR-extracted headers (Pascal-Case, Chrome 148, no cache-control) +# were blocked 50/50 в pilot run v2 (job_id=2, all 403 WAF). DEFAULT_HEADERS: dict[str, str] = { - "User-Agent": ( + "accept": "*/*", + "accept-language": "en-US,en;q=0.9,ru-RU;q=0.8,ru;q=0.7,es;q=0.6", + "cache-control": "no-cache", + "pragma": "no-cache", + "referer": "https://nspd.gov.ru/map?thematic=PKK", + "origin": "https://nspd.gov.ru", + "user-agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + "(KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36" ), - "Referer": "https://nspd.gov.ru/map", - "Accept": "*/*", - "Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7", - "x-timezone": "Europe/Moscow", - # Origin только для POST endpoints — не включаем для GET (лишний header) } # Retry settings -- 2.45.3 From d83fb9cfa86f2379285e3437a7532c2e0e6514f4 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 15 May 2026 15:04:10 +0300 Subject: [PATCH 4/4] fix(test): exclude prod_smoke tests by default (#168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI ran tests/smoke/test_prod_smoke.py and they hit production https://gendsgn.ru/api/v1/parcels/.../analyze which currently returns 500 (parse_floors regression — exactly what this PR fixes). Catch-22: PR can't merge because smoke tests fail against pre-merge prod. Fix: add `addopts = ["-m", "not prod_smoke"]` so default pytest excludes them. Run manually post-deploy with: pytest -m prod_smoke -v --- backend/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 89525714..dbbbb3b0 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -79,6 +79,8 @@ strict = true [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" +addopts = ["-m", "not prod_smoke"] markers = [ "slow: marks tests as slow (need real network, deselect with -m 'not slow')", + "prod_smoke: production smoke tests against live https://gendsgn.ru (run only post-deploy with -m prod_smoke)", ] -- 2.45.3