chore(#3051): снять новые тесты до живой проверки; валидация REGION_CODE в deploy-скрипте
All checks were successful
CI Trade-In / changes (pull_request) Successful in 12s
CI Trade-In / browser-tests (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / changes (pull_request) Successful in 14s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 5m13s
All checks were successful
CI Trade-In / changes (pull_request) Successful in 12s
CI Trade-In / browser-tests (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / changes (pull_request) Successful in 14s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 5m13s
Тесты трека «Москва» пишутся отдельным заходом после живого прогона импорта по 77 — правило проекта с 2026-09-08. Правки существующих тестов (bind-параметр вместо литерала 66, канонический source rosreestr_dkp_import_77) остаются. REGION_CODE в import-rosreestr.sh подставляется в SQL текстом — допускаем только целое.
This commit is contained in:
bot-backend
parent
84ee8e5990
commit
1f24f12af9
2 changed files with 2 additions and 247 deletions
|
|
@ -1,247 +0,0 @@
|
|||
"""#3051 п.3: параметризация import_rosreestr_dkp по региону + deals.doc_type.
|
||||
|
||||
Трек 2 подготовки Mera к Москве (region_code=77). Чисто-юнит: SQL-текст
|
||||
(inspect.getsource), _FakeDb-двойники для чекпоинта, миграция 288 и реестр
|
||||
регионов — без живого FDW/Postgres (тот же стиль, что test_rosreestr_dedup_key.py
|
||||
и test_3168_backfill_cursor_resume.py).
|
||||
|
||||
Покрывает пункты задачи:
|
||||
(a) region_code больше не литерал 66 в SQL — bind-параметр (см. также
|
||||
test_rosreestr_dedup_key.py::test_live_import_region_code_is_bind_param).
|
||||
(b) маппинг 77: city='Москва', address с префиксом, raw_payload с src_city/okato/
|
||||
quarter_cad_number/district.
|
||||
(c) маппинг 66 не изменился (canonical_city=None → старые SQL-выражения нетронуты).
|
||||
(d) чекпоинт per-region: _resume_dkp_cursor(source=...) изолирует регионы.
|
||||
(e) реестр хендлеров резолвит rosreestr_dkp_import_77 через wildcard.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
from app.services import scheduler as sched
|
||||
from app.services.regions import REGIONS
|
||||
|
||||
_IMPORT_SRC = inspect.getsource(sched.import_rosreestr_dkp)
|
||||
_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
|
||||
_MIGRATION_288 = _SQL_DIR / "288_deals_doc_type.sql"
|
||||
|
||||
|
||||
# ── регион 66/77 в реестре (canonical_city) ──────────────────────────────────
|
||||
|
||||
|
||||
def test_region_66_has_no_canonical_city_override() -> None:
|
||||
"""66 — источник несёт свой city как есть, поведение byte-for-byte прежнее."""
|
||||
assert REGIONS[66].canonical_city is None
|
||||
|
||||
|
||||
def test_region_77_canonical_city_is_moskva() -> None:
|
||||
"""77 — Росреестр отдаёт округ/поселение вместо города, нужен override."""
|
||||
assert REGIONS[77].canonical_city == "Москва"
|
||||
|
||||
|
||||
# ── _dkp_source_for_region — имя чекпоинта per-region ────────────────────────
|
||||
|
||||
|
||||
def test_dkp_source_for_region_66_keeps_legacy_name() -> None:
|
||||
"""66 — byte-for-byte прежнее имя, под которым годами писались scrape_runs."""
|
||||
assert sched._dkp_source_for_region(66) == "rosreestr_dkp_import"
|
||||
|
||||
|
||||
def test_dkp_source_for_region_77_gets_suffix() -> None:
|
||||
"""77 — суффикс кода, тот же формат, что у строки scrape_schedules (миграция 288)."""
|
||||
assert sched._dkp_source_for_region(77) == "rosreestr_dkp_import_77"
|
||||
|
||||
|
||||
# ── валидация неизвестного региона — падает ДО любого обращения к БД ────────
|
||||
|
||||
|
||||
def test_import_rejects_unknown_region_code_before_touching_db() -> None:
|
||||
"""Неизвестный region_code — явный ValueError, БД не трогается вовсе."""
|
||||
db = MagicMock()
|
||||
try:
|
||||
sched.import_rosreestr_dkp(db, run_id=1, params={"region_code": 404})
|
||||
raised = False
|
||||
except ValueError as exc:
|
||||
raised = True
|
||||
assert "404" in str(exc)
|
||||
assert raised, "expected ValueError for unknown region_code"
|
||||
db.execute.assert_not_called()
|
||||
|
||||
|
||||
# ── SQL: region_code — bind-параметр, не литерал (см. также test_rosreestr_dedup_key) ─
|
||||
|
||||
|
||||
def test_sql_selects_region_code_via_bind_param() -> None:
|
||||
assert "WHERE region_code = CAST(:region_code AS int)" in _IMPORT_SRC
|
||||
|
||||
|
||||
def test_no_python_branch_on_region_77() -> None:
|
||||
"""Маппинг city/address/raw_payload идёт ОДНОЙ SQL-веткой на bind-параметре
|
||||
:canonical_city (CASE WHEN), а не Python if/else на конкретный код региона."""
|
||||
assert "== 77" not in _IMPORT_SRC
|
||||
assert "region_code == 77" not in _IMPORT_SRC
|
||||
|
||||
|
||||
# ── SQL: маппинг city/address через canonical_city (регион 77) ──────────────
|
||||
|
||||
|
||||
def test_sql_address_uses_canonical_city_case() -> None:
|
||||
assert "CAST(:canonical_city AS text) || ', ' || trim(street)" in _IMPORT_SRC, (
|
||||
"address для canonical_city-региона обязан быть 'Москва, <street>'"
|
||||
)
|
||||
assert "trim(city) || ', ' || trim(street)" in _IMPORT_SRC, (
|
||||
"ELSE-ветка (регион 66) обязана остаться прежней"
|
||||
)
|
||||
|
||||
|
||||
def test_sql_city_uses_canonical_city_case() -> None:
|
||||
# WHEN CAST(:canonical_city AS text) IS NOT NULL THEN CAST(:canonical_city AS text)
|
||||
assert "THEN CAST(:canonical_city AS text)\n" in _IMPORT_SRC
|
||||
assert "ELSE trim(city)\n" in _IMPORT_SRC
|
||||
|
||||
|
||||
def test_sql_city_not_null_filter_skipped_when_canonical_city_set() -> None:
|
||||
"""Фильтр city IS NOT NULL применяется ТОЛЬКО когда canonical_city не задан —
|
||||
иначе на 77 теряется ~10% строк с пустым city источника."""
|
||||
assert (
|
||||
"CAST(:canonical_city AS text) IS NOT NULL\n"
|
||||
" OR (city IS NOT NULL AND trim(city) <> '')" in _IMPORT_SRC
|
||||
)
|
||||
|
||||
|
||||
def test_sql_raw_payload_carries_source_columns() -> None:
|
||||
"""raw_payload (только когда canonical_city задан) несёт src_city/okato/
|
||||
quarter_cad_number/district — исходные значения источника, не потерянные."""
|
||||
for key in (
|
||||
"'src_city', city",
|
||||
"'okato', okato",
|
||||
"'quarter_cad_number', quarter_cad_number",
|
||||
"'district', district",
|
||||
):
|
||||
assert key in _IMPORT_SRC, f"raw_payload missing key expression: {key!r}"
|
||||
|
||||
|
||||
def test_params_dict_binds_region_code_and_canonical_city() -> None:
|
||||
assert '"region_code": region_code' in _IMPORT_SRC
|
||||
assert '"canonical_city": region.canonical_city' in _IMPORT_SRC
|
||||
|
||||
|
||||
def test_insert_carries_doc_type_and_raw_payload() -> None:
|
||||
assert "doc_type, raw_payload" in _IMPORT_SRC
|
||||
assert "doc_type = EXCLUDED.doc_type" in _IMPORT_SRC
|
||||
assert "raw_payload = EXCLUDED.raw_payload" in _IMPORT_SRC
|
||||
|
||||
|
||||
# ── чекпоинт per-region: 77 не подхватывает last_id региона 66 ──────────────
|
||||
|
||||
|
||||
class _FakeRow:
|
||||
def __init__(self, **kw: Any) -> None:
|
||||
self.__dict__.update(kw)
|
||||
|
||||
|
||||
class _SourceKeyedFakeDb:
|
||||
"""Двойник сессии: отдаёт кандидата ТОЛЬКО для своего source, иначе None.
|
||||
|
||||
Эмулирует реальный `WHERE source = :source` — единственный способ честно
|
||||
проверить изоляцию чекпоинтов между регионами без живого Postgres.
|
||||
"""
|
||||
|
||||
def __init__(self, rows_by_source: dict[str, Any]) -> None:
|
||||
self.rows_by_source = rows_by_source
|
||||
self.seen_sources: list[str] = []
|
||||
|
||||
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> Any:
|
||||
if params is not None and "counters" in params:
|
||||
return MagicMock()
|
||||
source = (params or {}).get("source")
|
||||
self.seen_sources.append(source)
|
||||
return MagicMock(fetchone=lambda: self.rows_by_source.get(source))
|
||||
|
||||
def commit(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def test_resume_cursor_region_77_does_not_see_region_66_checkpoint() -> None:
|
||||
"""Прогон source='rosreestr_dkp_import' (66) не резюмится под source региона 77."""
|
||||
row_66 = _FakeRow(
|
||||
prev_id=777,
|
||||
prev_status="zombie",
|
||||
prev_counters={"last_id": 999999},
|
||||
age_h=1.0,
|
||||
)
|
||||
db = _SourceKeyedFakeDb({"rosreestr_dkp_import": row_66})
|
||||
|
||||
last_id_77, verdict_77 = sched._resume_dkp_cursor(
|
||||
db, run_id=1, source="rosreestr_dkp_import_77"
|
||||
)
|
||||
assert last_id_77 == 0
|
||||
assert verdict_77["resume_reason"] == "no_prev_run"
|
||||
|
||||
# Регион 66 по-прежнему видит СВОЙ чекпоинт.
|
||||
last_id_66, verdict_66 = sched._resume_dkp_cursor(db, run_id=2, source="rosreestr_dkp_import")
|
||||
assert last_id_66 == 999999
|
||||
assert verdict_66["resume_reason"] == "ok"
|
||||
|
||||
|
||||
def test_resume_cursor_default_source_is_backward_compatible() -> None:
|
||||
"""Вызов без явного source (как в старых тестах/коде) — прежнее поведение (66)."""
|
||||
row = _FakeRow(prev_id=1, prev_status="zombie", prev_counters={"last_id": 42}, age_h=1.0)
|
||||
db = _SourceKeyedFakeDb({"rosreestr_dkp_import": row})
|
||||
last_id, _verdict = sched._resume_dkp_cursor(db, run_id=1)
|
||||
assert last_id == 42
|
||||
assert db.seen_sources == ["rosreestr_dkp_import"]
|
||||
|
||||
|
||||
# ── реестр хендлеров: rosreestr_dkp_import_77 резолвится через wildcard ──────
|
||||
|
||||
|
||||
def test_handler_registry_region_77_uses_same_job_as_region_66() -> None:
|
||||
from scraper_kit.orchestration.scheduler import build_registry, resolve_handler
|
||||
|
||||
from app.services.product_handlers import _job_rosreestr_dkp, build_product_handlers
|
||||
|
||||
registry = build_registry(build_product_handlers(ctx=None)) # type: ignore[arg-type]
|
||||
h66 = resolve_handler("rosreestr_dkp_import", registry)
|
||||
h77 = resolve_handler("rosreestr_dkp_import_77", registry)
|
||||
assert h66 is not None and h77 is not None
|
||||
assert h66.job is _job_rosreestr_dkp
|
||||
assert h77.job is _job_rosreestr_dkp
|
||||
|
||||
|
||||
# ── миграция 288 ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_migration_288_exists() -> None:
|
||||
assert _MIGRATION_288.is_file(), f"missing migration: {_MIGRATION_288}"
|
||||
|
||||
|
||||
def test_migration_288_adds_doc_type_idempotently() -> None:
|
||||
sql = _MIGRATION_288.read_text("utf-8")
|
||||
assert "ADD COLUMN IF NOT EXISTS doc_type text" in sql
|
||||
assert "SET doc_type = 'ДКП'" in sql
|
||||
assert "WHERE source = 'rosreestr'" in sql
|
||||
assert "AND doc_type IS NULL" in sql
|
||||
|
||||
|
||||
def test_migration_288_seeds_disabled_region_77_schedule() -> None:
|
||||
sql = _MIGRATION_288.read_text("utf-8")
|
||||
assert "'rosreestr_dkp_import_77'" in sql
|
||||
assert "false" in sql
|
||||
assert '"region_code": 77' in sql
|
||||
assert "ON CONFLICT (source) DO NOTHING" in sql
|
||||
|
||||
|
||||
def test_migration_288_has_lock_timeout_before_alter_table() -> None:
|
||||
sql = _MIGRATION_288.read_text("utf-8")
|
||||
lt_pos = sql.find("SET LOCAL lock_timeout")
|
||||
alter_pos = sql.find("ALTER TABLE deals")
|
||||
assert lt_pos != -1 and alter_pos != -1
|
||||
assert lt_pos < alter_pos
|
||||
|
|
@ -29,6 +29,8 @@ SRC_DB="${SRC_DB:-gendesign}"
|
|||
SRC_USER="${SRC_USER:-gendesign}"
|
||||
SINCE="${SINCE:-2024-01-01}"
|
||||
REGION_CODE="${REGION_CODE:-66}"
|
||||
# Значение подставляется в SQL текстом — допускаем только целое число.
|
||||
[[ "$REGION_CODE" =~ ^[0-9]+$ ]] || { echo "REGION_CODE должен быть целым числом, получено: '$REGION_CODE'" >&2; exit 1; }
|
||||
|
||||
echo "[$(date -u +%H:%M:%S)] import-rosreestr: регион $REGION_CODE, квартиры с $SINCE"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue