* fix(parcels): _parse_floors handle int (post-migration #169 schema change) After PR #169 cad_buildings schema migration, `floors` column is INT (was TEXT in legacy schema). Existing `_parse_floors(r.get("floors"))` call in analyze_parcel → _neighbors_summary crashes with: AttributeError: 'int' object has no attribute 'strip' Fix: type union str | int | None. If int → return directly (no strip). Preserve TEXT range parsing ("5-7" → 7) for backwards-compat with any legacy data still in cad_buildings_old_apr26. * test(smoke): production smoke tests for post-deploy regressions (#168) Add tests/smoke/test_prod_smoke.py covering known regression surfaces. Marks: prod_smoke + slow. Env: PROD_SMOKE_BASE_URL, PROD_SMOKE_ADMIN_TOKEN. Run manually: cd backend && uv run pytest tests/smoke/ -m prod_smoke -v * fix(cadastre): exact-match headers to nspd_lite to bypass NSPD WAF (#168) Pilot v2 (job_id=2) failed 50/50 with HTTP 403 WAF block. Comparing nspd_bulk_client.DEFAULT_HEADERS vs legacy nspd_lite.HEADERS (which works on VPS IP since April 2026): PascalCase keys → lowercase keys Chrome/148 UA → Chrome/144 UA No cache-control / pragma → "no-cache" both accept-language ru first → en first No origin → "https://nspd.gov.ru" referer "/map" → "/map?thematic=PKK" NSPD WAF (BotShield-class) likely fingerprints на header order + values combined with TLS fingerprint. Matching legacy exactly minimizes deltas. Test plan: retry pilot job after deploy, expect 0 WAF blocks for first 5 quarters. * fix(test): exclude prod_smoke tests by default (#168) CI ran tests/smoke/test_prod_smoke.py and they hit production https://gendsgn.ru/api/v1/parcels/.../analyze which currently returns 500 (parse_floors regression — exactly what this PR fixes). Catch-22: PR can't merge because smoke tests fail against pre-merge prod. Fix: add `addopts = ["-m", "not prod_smoke"]` so default pytest excludes them. Run manually post-deploy with: pytest -m prod_smoke -v --------- Co-authored-by: lekss361 <claudestars@proton.me>
This commit is contained in:
lekss361
GitHub
parent
8dee9546ae
commit
608490fbba
5 changed files with 261 additions and 11 deletions
|
|
@ -639,16 +639,23 @@ _COST_PER_M2_MIN = 1000 # ₽/м² — ниже скорее всего оши
|
|||
_COST_PER_M2_MAX = 500_000 # ₽/м² — выше скорее всего outlier
|
||||
|
||||
|
||||
def _parse_floors(raw: str | None) -> int | None:
|
||||
"""cad_buildings.floors хранится TEXT (могут быть диапазоны '1-2', '5-7').
|
||||
def _parse_floors(raw: str | int | None) -> int | None:
|
||||
"""cad_buildings.floors — после schema migration #169 теперь INT,
|
||||
но historically мог быть TEXT с диапазоном '5-7'. Поддерживаем оба
|
||||
для backwards-compat с legacy data + tests.
|
||||
|
||||
Возвращаем верхнюю границу (более консервативный сосед-высотка).
|
||||
NB: `isdigit()` намеренно фильтрует malformed parts типа "5а-7"; для
|
||||
multi-range "1-2-3" возвращается max(1,2,3)=3 (acceptable degradation).
|
||||
"""
|
||||
if raw is None or raw == "":
|
||||
return None
|
||||
# Post-migration: INT column → fast path
|
||||
if isinstance(raw, int):
|
||||
return raw if raw > 0 else None
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
return None
|
||||
raw = raw.strip()
|
||||
# range like "5-7" → 7
|
||||
if "-" in raw:
|
||||
parts = raw.split("-")
|
||||
|
|
|
|||
|
|
@ -48,17 +48,21 @@ _SEMAPHORE = asyncio.Semaphore(3)
|
|||
# Таймаут подключения и чтения
|
||||
DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0)
|
||||
|
||||
# Headers из HAR audit 2026-05-15
|
||||
# Headers — exact match to legacy nspd_lite.py (proven to bypass NSPD WAF
|
||||
# on VPS IP since April 2026). All-lowercase keys + cache-control + Chrome 144
|
||||
# UA. Initial HAR-extracted headers (Pascal-Case, Chrome 148, no cache-control)
|
||||
# were blocked 50/50 в pilot run v2 (job_id=2, all 403 WAF).
|
||||
DEFAULT_HEADERS: dict[str, str] = {
|
||||
"User-Agent": (
|
||||
"accept": "*/*",
|
||||
"accept-language": "en-US,en;q=0.9,ru-RU;q=0.8,ru;q=0.7,es;q=0.6",
|
||||
"cache-control": "no-cache",
|
||||
"pragma": "no-cache",
|
||||
"referer": "https://nspd.gov.ru/map?thematic=PKK",
|
||||
"origin": "https://nspd.gov.ru",
|
||||
"user-agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36"
|
||||
"(KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Referer": "https://nspd.gov.ru/map",
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"x-timezone": "Europe/Moscow",
|
||||
# Origin только для POST endpoints — не включаем для GET (лишний header)
|
||||
}
|
||||
|
||||
# Retry settings
|
||||
|
|
|
|||
|
|
@ -79,6 +79,8 @@ strict = true
|
|||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
asyncio_mode = "auto"
|
||||
addopts = ["-m", "not prod_smoke"]
|
||||
markers = [
|
||||
"slow: marks tests as slow (need real network, deselect with -m 'not slow')",
|
||||
"prod_smoke: production smoke tests against live https://gendsgn.ru (run only post-deploy with -m prod_smoke)",
|
||||
]
|
||||
|
|
|
|||
0
backend/tests/smoke/__init__.py
Normal file
0
backend/tests/smoke/__init__.py
Normal file
237
backend/tests/smoke/test_prod_smoke.py
Normal file
237
backend/tests/smoke/test_prod_smoke.py
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
"""Production smoke tests — must pass after every merge to main.
|
||||
|
||||
Run after deploy completes (~2-3min after merge). CI workflow:
|
||||
cd backend && uv run pytest tests/smoke/test_prod_smoke.py -m prod_smoke -v
|
||||
|
||||
Each test makes a real HTTP call to https://gendsgn.ru, validates response
|
||||
structure (not exact data) и crash regressions, and reports timing.
|
||||
|
||||
Marks:
|
||||
- @pytest.mark.prod_smoke — skip by default in CI, run only on-demand
|
||||
- @pytest.mark.slow — long-running (≥5s)
|
||||
|
||||
Tests cover the regression surface that bot review window cannot catch:
|
||||
- Schema migrations break field types → AttributeError на user actions
|
||||
- Frontend bundle ↔ backend contract drift
|
||||
- Auth/admin endpoints за токеном
|
||||
- Background workers + Celery queue
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
BASE_URL = os.getenv("PROD_SMOKE_BASE_URL", "https://gendsgn.ru")
|
||||
ADMIN_TOKEN = os.getenv("PROD_SMOKE_ADMIN_TOKEN")
|
||||
TIMEOUT = httpx.Timeout(30.0, connect=10.0)
|
||||
|
||||
# Test cads — known-good в prod, в quarter 66:41:0303161 (Крауля)
|
||||
TEST_CAD = "66:41:0204016:10" # Билимбаевская, 32 ЖК velocity baseline
|
||||
TEST_QUARTER = "66:41:0303161" # 502 объекта (170 ЗУ + 256 зд + 75 соор + 1 ЕНК)
|
||||
|
||||
|
||||
pytestmark = pytest.mark.prod_smoke
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Public endpoints (no auth)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_health_returns_ok() -> None:
|
||||
"""Smoke: /health должен возвращать {"status":"ok"}."""
|
||||
r = httpx.get(f"{BASE_URL}/health", timeout=TIMEOUT)
|
||||
assert r.status_code == 200
|
||||
assert r.json().get("status") == "ok"
|
||||
|
||||
|
||||
def test_site_finder_page_renders() -> None:
|
||||
"""SSR returns HTML с заголовком GenDesign."""
|
||||
r = httpx.get(f"{BASE_URL}/site-finder", timeout=TIMEOUT)
|
||||
assert r.status_code == 200
|
||||
assert "GenDesign" in r.text or "Site Finder" in r.text
|
||||
|
||||
|
||||
def test_parcels_analyze_returns_200() -> None:
|
||||
"""REGRESSION SHIELD: ловит crashes в analyze pipeline.
|
||||
|
||||
Раньше ломали:
|
||||
- PR #149: body stream double-read (api.ts)
|
||||
- PR #152: SQLAlchemy :weights::jsonb cast
|
||||
- PR #160: :window_interval::interval cast
|
||||
- PR #169 → 'int has no attribute strip' (cad_buildings.floors INT)
|
||||
|
||||
Test: POST /api/v1/parcels/{cad}/analyze → 200 + required fields.
|
||||
"""
|
||||
r = httpx.post(
|
||||
f"{BASE_URL}/api/v1/parcels/{TEST_CAD}/analyze",
|
||||
json={},
|
||||
timeout=httpx.Timeout(60.0, connect=10.0), # analyze может быть медленным
|
||||
)
|
||||
assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text[:500]}"
|
||||
body = r.json()
|
||||
# Required top-level keys (validate схема не дрейфит):
|
||||
required = ["cad_num", "score", "pipeline_24mo", "velocity", "gate_verdict"]
|
||||
for key in required:
|
||||
assert key in body, f"Missing field '{key}' in analyze response. Keys: {list(body.keys())}"
|
||||
# neighbors_summary must NOT crash even when floors=int (regression #169):
|
||||
assert "neighbors_summary" in body
|
||||
assert body["neighbors_summary"].get("data_available") is not False or (
|
||||
body["neighbors_summary"].get("note", "").startswith("neighbors query failed") is False
|
||||
)
|
||||
|
||||
|
||||
def test_parcels_analyze_velocity_block_present() -> None:
|
||||
"""REGRESSION SHIELD: velocity migrated to objective_corpus_room_month (#158, #163).
|
||||
|
||||
После migration ожидаем что для testovogo cad есть velocity-блок:
|
||||
- score ≥ 0
|
||||
- period.start/end не None (значит objective fresh data есть)
|
||||
- by_room_bucket список (per-room breakdown — PR #163)
|
||||
"""
|
||||
r = httpx.post(
|
||||
f"{BASE_URL}/api/v1/parcels/{TEST_CAD}/analyze",
|
||||
json={},
|
||||
timeout=httpx.Timeout(60.0, connect=10.0),
|
||||
)
|
||||
assert r.status_code == 200
|
||||
velocity = r.json().get("velocity")
|
||||
if velocity is None:
|
||||
pytest.skip("velocity null — possibly no data in cadastre, not regression")
|
||||
assert "score" in velocity
|
||||
assert "period" in velocity
|
||||
# PR #163 enrichment:
|
||||
assert "by_room_bucket" in velocity, "Missing by_room_bucket per PR #163"
|
||||
|
||||
|
||||
def test_parcels_analyze_pipeline_by_class_enriched() -> None:
|
||||
"""REGRESSION SHIELD: pipeline_by_class obj_class enrichment (#165).
|
||||
|
||||
Раньше показывало "Не указан: 4075". После #165 — стандарт/комфорт/etc
|
||||
через JOIN на objective_complex_mapping.
|
||||
"""
|
||||
r = httpx.post(
|
||||
f"{BASE_URL}/api/v1/parcels/{TEST_CAD}/analyze",
|
||||
json={},
|
||||
timeout=httpx.Timeout(60.0, connect=10.0),
|
||||
)
|
||||
assert r.status_code == 200
|
||||
pipeline = r.json().get("pipeline_24mo", {})
|
||||
by_class = pipeline.get("by_class", {})
|
||||
# Не должно быть так что unknown == 100% (это означало бы regression #165)
|
||||
if by_class:
|
||||
total = sum(by_class.values())
|
||||
unknown_share = by_class.get("unknown", 0) / total if total > 0 else 0
|
||||
assert unknown_share < 0.9, (
|
||||
f"pipeline_by_class.unknown == {unknown_share:.1%} — "
|
||||
"regression of #165? Should JOIN on objective_lots."
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Admin endpoints (auth required — skip if no token)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_headers() -> dict[str, str]:
|
||||
if not ADMIN_TOKEN:
|
||||
pytest.skip("PROD_SMOKE_ADMIN_TOKEN env var not set")
|
||||
return {"X-Admin-Token": ADMIN_TOKEN}
|
||||
|
||||
|
||||
def test_admin_jobs_settings_returns_200(admin_headers: dict[str, str]) -> None:
|
||||
r = httpx.get(
|
||||
f"{BASE_URL}/api/v1/admin/jobs/settings",
|
||||
headers=admin_headers,
|
||||
timeout=TIMEOUT,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert isinstance(r.json(), list | dict)
|
||||
|
||||
|
||||
def test_admin_cadastre_jobs_endpoint(admin_headers: dict[str, str]) -> None:
|
||||
"""PR #171 admin cadastre endpoints."""
|
||||
r = httpx.get(
|
||||
f"{BASE_URL}/api/v1/admin/cadastre/jobs?limit=5",
|
||||
headers=admin_headers,
|
||||
timeout=TIMEOUT,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert isinstance(body, list)
|
||||
if body:
|
||||
# Validate схема jobs row (PR #171)
|
||||
required = ["job_id", "status", "targets_total", "targets_done"]
|
||||
for key in required:
|
||||
assert key in body[0], f"Missing field '{key}' in job row"
|
||||
|
||||
|
||||
def test_admin_weight_profiles_endpoint(admin_headers: dict[str, str]) -> None:
|
||||
"""PR #138 weight profiles list."""
|
||||
r = httpx.get(
|
||||
f"{BASE_URL}/api/v1/admin/site-finder/weight-profiles?user_id=admin",
|
||||
headers=admin_headers,
|
||||
timeout=TIMEOUT,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Frontend bundle (catches build regressions)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_admin_scrape_cadastre_page_renders() -> None:
|
||||
"""PR #172 — new admin UI page must SSR ok."""
|
||||
r = httpx.get(f"{BASE_URL}/admin/scrape/cadastre", timeout=TIMEOUT)
|
||||
assert r.status_code == 200
|
||||
assert "Cadastre" in r.text or "кадастр" in r.text.lower() or "Bulk" in r.text
|
||||
|
||||
|
||||
def test_admin_scrape_geo_page_renders() -> None:
|
||||
"""Existing pattern — должен работать (PR #172 не должен сломать)."""
|
||||
r = httpx.get(f"{BASE_URL}/admin/scrape/geo", timeout=TIMEOUT)
|
||||
assert r.status_code == 200
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Schema invariants (через DB read через admin endpoints)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_health_db_connection_works() -> None:
|
||||
"""`/health` returns 200 если DB reachable + migrations applied."""
|
||||
r = httpx.get(f"{BASE_URL}/health?check_db=1", timeout=TIMEOUT)
|
||||
if r.status_code == 200 and "db" in r.json():
|
||||
assert r.json()["db"] == "ok", f"DB check failed: {r.json()}"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# How to run after merge (CI integration)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Add to .github/workflows/post-deploy-smoke.yml:
|
||||
#
|
||||
# on:
|
||||
# workflow_run:
|
||||
# workflows: [deploy.yml]
|
||||
# types: [completed]
|
||||
# jobs:
|
||||
# smoke:
|
||||
# if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
# runs-on: ubuntu-latest
|
||||
# steps:
|
||||
# - uses: actions/checkout@v4
|
||||
# - run: uv sync
|
||||
# - run: |
|
||||
# cd backend && uv run pytest tests/smoke/ -m prod_smoke -v
|
||||
# env:
|
||||
# PROD_SMOKE_BASE_URL: https://gendsgn.ru
|
||||
# PROD_SMOKE_ADMIN_TOKEN: ${{ secrets.SCRAPE_ADMIN_TOKEN }}
|
||||
# - name: Post slack/sentry alert if failed
|
||||
# if: failure()
|
||||
# run: echo "smoke tests failed — investigate"
|
||||
Loading…
Add table
Reference in a new issue