Light1YT
e332052d0e
Migration 070 — добавляет houses.house_fias_id + dadata_qc_geo / qc_house / enriched_at для tracking. Скрипт scripts/backfill_houses_dadata.py проходит по ~3.8k rows с NULL coords + ~8.6k с NULL cadnum daily 100/day (DaData demo quota → ~124 дня walltime для полного прохода). Per-row SAVEPOINT, resume-safe (фильтр dadata_enriched_at IS NULL), qc_geo IN (0,1) → overwrite coord-payload, иначе только tracking. --priority coords|cadnum|both, --dry-run, --limit N (default 100). 20 тестов покрывают happy path qc_geo=0, qc_geo=3 city-level skip, DaData None response, per-row exception isolation, --dry-run, --limit, --priority filter, и missing-creds fail-fast.
464 lines
18 KiB
Python
464 lines
18 KiB
Python
"""Unit tests for PR Q3 — backfill_houses_dadata.py.
|
||
|
||
Coverage:
|
||
- Happy path: row с NULL coords + qc_geo=0 → UPDATE с полным набором полей.
|
||
- qc_geo=3 (city-level) → tracking only, lat/lon не overwrite'аются.
|
||
- DaData returns None → tracking only (qc_geo / qc_house = NULL).
|
||
- Per-row SAVEPOINT: exception от clean_address не валит batch.
|
||
- --dry-run: nothing written в DB.
|
||
- --limit N: батч обрезается до N даже если кандидатов больше.
|
||
- --priority coords / cadnum / both — фильтр source-rows работает.
|
||
- _is_enriched_result — accept / reject правила.
|
||
- main() с отсутствующими DADATA_API_TOKEN / SECRET → SystemExit.
|
||
|
||
No real Postgres. Session — MagicMock что записывает UPDATE и роутит SELECT
|
||
side-effects (тот же pattern что в test_backfill_house_coords.py /
|
||
test_backfill_listing_sources.py).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
# Settings требует DSN на import — set dummy DSN до любого app.* import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
import pytest
|
||
|
||
from app.services.dadata import DadataAddressResult
|
||
from scripts.backfill_houses_dadata import (
|
||
HouseRow,
|
||
Stats,
|
||
_is_enriched_result,
|
||
_run_backfill,
|
||
_select_candidates,
|
||
_update_house_attempt,
|
||
_update_house_enriched,
|
||
main,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _make_dadata_result(
|
||
*,
|
||
qc_geo: int | None = 0,
|
||
qc_house: int | None = 2,
|
||
lat: float | None = 56.838,
|
||
lon: float | None = 60.586,
|
||
cadnum: str | None = "66:41:0704045:350",
|
||
fias: str | None = "abcdef00-aaaa-bbbb-cccc-1234567890ab",
|
||
canonical: str | None = "г Екатеринбург, ул Малышева, д 51",
|
||
) -> DadataAddressResult:
|
||
"""DadataAddressResult fixture с sensible defaults для tests."""
|
||
return DadataAddressResult(
|
||
canonical_address=canonical,
|
||
house_cadnum=cadnum,
|
||
house_fias_id=fias,
|
||
lat=lat,
|
||
lon=lon,
|
||
qc_geo=qc_geo,
|
||
qc_house=qc_house,
|
||
kladr_id=None,
|
||
okato=None,
|
||
oktmo=None,
|
||
metro=[],
|
||
raw={"test": True},
|
||
)
|
||
|
||
|
||
def _make_db_mock(
|
||
coords_sample: list[dict] | None = None,
|
||
cadnum_sample: list[dict] | None = None,
|
||
) -> tuple[MagicMock, list[dict]]:
|
||
"""MagicMock Session что:
|
||
- возвращает coords_sample для SELECT с `lat IS NULL OR lon IS NULL`
|
||
- возвращает cadnum_sample для SELECT с `cadastral_number IS NULL`
|
||
- записывает UPDATE houses в `updated` список (полные binds dict'ы)
|
||
- корректно работает с `db.begin_nested()` контекст-менеджером
|
||
"""
|
||
coords_sample = coords_sample or []
|
||
cadnum_sample = cadnum_sample or []
|
||
updated: list[dict] = []
|
||
|
||
db = MagicMock()
|
||
db.begin_nested.return_value.__enter__ = lambda self: self
|
||
db.begin_nested.return_value.__exit__ = lambda self, *a: False
|
||
|
||
def execute_side_effect(sql, params=None):
|
||
sql_str = str(sql)
|
||
result = MagicMock()
|
||
if "FROM houses" in sql_str and "lat IS NULL OR lon IS NULL" in sql_str:
|
||
lim = params.get("lim", len(coords_sample)) if params else len(coords_sample)
|
||
result.mappings.return_value.all.return_value = coords_sample[:lim]
|
||
elif "FROM houses" in sql_str and "cadastral_number IS NULL" in sql_str:
|
||
lim = params.get("lim", len(cadnum_sample)) if params else len(cadnum_sample)
|
||
result.mappings.return_value.all.return_value = cadnum_sample[:lim]
|
||
elif "UPDATE houses" in sql_str:
|
||
updated.append(dict(params) if params else {})
|
||
return result
|
||
|
||
db.execute.side_effect = execute_side_effect
|
||
db.commit = MagicMock()
|
||
db.rollback = MagicMock()
|
||
db.close = MagicMock()
|
||
return db, updated
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _is_enriched_result — accept / reject правила
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_is_enriched_result_qc_geo_0_lat_lon_present_accepts():
|
||
res = _make_dadata_result(qc_geo=0, lat=56.838, lon=60.586)
|
||
assert _is_enriched_result(res) is True
|
||
|
||
|
||
def test_is_enriched_result_qc_geo_1_accepts():
|
||
"""qc_geo=1 (street-level) — также accept'аем для overwrite."""
|
||
res = _make_dadata_result(qc_geo=1, lat=56.838, lon=60.586)
|
||
assert _is_enriched_result(res) is True
|
||
|
||
|
||
def test_is_enriched_result_qc_geo_2_rejects():
|
||
"""qc_geo=2 (settlement) — слишком грубо для дома."""
|
||
res = _make_dadata_result(qc_geo=2)
|
||
assert _is_enriched_result(res) is False
|
||
|
||
|
||
def test_is_enriched_result_qc_geo_3_rejects():
|
||
"""qc_geo=3 (city) — точно не дом."""
|
||
res = _make_dadata_result(qc_geo=3)
|
||
assert _is_enriched_result(res) is False
|
||
|
||
|
||
def test_is_enriched_result_none_input_rejects():
|
||
assert _is_enriched_result(None) is False
|
||
|
||
|
||
def test_is_enriched_result_qc_geo_ok_but_no_coords_rejects():
|
||
"""Defensive: qc_geo=0 но lat/lon отсутствуют → не overwrite."""
|
||
res = _make_dadata_result(qc_geo=0, lat=None, lon=None)
|
||
assert _is_enriched_result(res) is False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _update_house_enriched — bind shape
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_update_house_enriched_passes_full_payload():
|
||
"""UPDATE houses содержит lat/lon/cadnum/fias/qc-codes и dadata_enriched_at = NOW()."""
|
||
db = MagicMock()
|
||
res = _make_dadata_result(
|
||
qc_geo=0, qc_house=2, lat=56.838, lon=60.586,
|
||
cadnum="66:41:0704045:350", fias="fias-uuid-here",
|
||
)
|
||
_update_house_enriched(db, house_id=42, result=res)
|
||
|
||
args, _kw = db.execute.call_args
|
||
sql_str = str(args[0])
|
||
binds = args[1]
|
||
assert "UPDATE houses" in sql_str
|
||
assert "dadata_enriched_at = NOW()" in sql_str
|
||
assert "cadastral_number" in sql_str
|
||
assert "house_fias_id" in sql_str
|
||
assert binds["id"] == 42
|
||
assert binds["lat"] == 56.838
|
||
assert binds["lon"] == 60.586
|
||
assert binds["cadnum"] == "66:41:0704045:350"
|
||
assert binds["fias"] == "fias-uuid-here"
|
||
assert binds["qc_geo"] == 0
|
||
assert binds["qc_house"] == 2
|
||
|
||
|
||
def test_update_house_attempt_writes_only_tracking_fields():
|
||
"""attempt-write не должен затрагивать lat/lon/cadnum/fias columns."""
|
||
db = MagicMock()
|
||
_update_house_attempt(db, house_id=7, qc_geo=3, qc_house=10)
|
||
args, _kw = db.execute.call_args
|
||
sql_str = str(args[0])
|
||
binds = args[1]
|
||
assert "UPDATE houses" in sql_str
|
||
assert "dadata_enriched_at = NOW()" in sql_str
|
||
# Не должно быть UPDATE на coord-payload поля
|
||
assert "lat =" not in sql_str
|
||
assert "lon =" not in sql_str
|
||
assert "cadastral_number =" not in sql_str
|
||
assert "house_fias_id =" not in sql_str
|
||
assert binds == {"id": 7, "qc_geo": 3, "qc_house": 10}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _run_backfill — happy path qc_geo=0
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_run_backfill_qc_geo_0_writes_enriched_update():
|
||
"""Happy path — qc_geo=0 → UPDATE с полным payload, stats.enriched=1."""
|
||
row = HouseRow(id=1, address="ул Малышева 51", lat=None, lon=None, cadastral_number=None)
|
||
db, updated = _make_db_mock()
|
||
fake = _make_dadata_result(qc_geo=0)
|
||
|
||
with patch(
|
||
"scripts.backfill_houses_dadata.clean_address",
|
||
new=AsyncMock(return_value=fake),
|
||
):
|
||
stats = await _run_backfill(
|
||
db, [(row, "coords")], batch="b1", dry_run=False
|
||
)
|
||
|
||
assert stats.enriched == 1
|
||
assert stats.no_change == 0
|
||
assert stats.failed == 0
|
||
assert stats.processed == 1
|
||
assert len(updated) == 1
|
||
assert updated[0]["lat"] == 56.838
|
||
assert updated[0]["lon"] == 60.586
|
||
assert updated[0]["cadnum"] == "66:41:0704045:350"
|
||
assert updated[0]["qc_geo"] == 0
|
||
assert db.commit.call_count == 1
|
||
assert stats.by_priority["coords"]["enriched"] == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _run_backfill — qc_geo=3 city-level → tracking-only
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_run_backfill_qc_geo_3_records_attempt_only():
|
||
"""qc_geo=3 → НЕ overwrite coords, только tracking UPDATE."""
|
||
row = HouseRow(
|
||
id=2, address="Екатеринбург (без улицы)", lat=None, lon=None, cadastral_number=None
|
||
)
|
||
db, updated = _make_db_mock()
|
||
fake = _make_dadata_result(qc_geo=3, qc_house=None, lat=None, lon=None, cadnum=None, fias=None)
|
||
|
||
with patch(
|
||
"scripts.backfill_houses_dadata.clean_address",
|
||
new=AsyncMock(return_value=fake),
|
||
):
|
||
stats = await _run_backfill(
|
||
db, [(row, "coords")], batch="b2", dry_run=False
|
||
)
|
||
|
||
assert stats.enriched == 0
|
||
assert stats.no_change == 1
|
||
assert len(updated) == 1
|
||
# Tracking-only UPDATE — нет lat/lon/cadnum binds
|
||
assert "lat" not in updated[0]
|
||
assert "lon" not in updated[0]
|
||
assert updated[0]["qc_geo"] == 3
|
||
assert stats.by_priority["coords"]["no_change"] == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _run_backfill — DaData returns None (graceful)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_run_backfill_dadata_returns_none_records_attempt():
|
||
"""clean_address возвращает None (HTTP error / unrecognized) → tracking only с NULL qc."""
|
||
row = HouseRow(id=3, address="несуществующая 99", lat=None, lon=None, cadastral_number=None)
|
||
db, updated = _make_db_mock()
|
||
|
||
with patch(
|
||
"scripts.backfill_houses_dadata.clean_address",
|
||
new=AsyncMock(return_value=None),
|
||
):
|
||
stats = await _run_backfill(
|
||
db, [(row, "coords")], batch="b3", dry_run=False
|
||
)
|
||
|
||
assert stats.no_change == 1
|
||
assert stats.enriched == 0
|
||
assert len(updated) == 1
|
||
assert updated[0]["qc_geo"] is None
|
||
assert updated[0]["qc_house"] is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Per-row SAVEPOINT — exception в clean_address не валит batch
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_run_backfill_clean_address_exception_isolated_to_row():
|
||
"""Один clean_address raise → этот row failed, остальные processed normally."""
|
||
rows = [
|
||
(HouseRow(id=10, address="ул Bad", lat=None, lon=None, cadastral_number=None), "coords"),
|
||
(HouseRow(id=11, address="ул Good 5", lat=None, lon=None, cadastral_number=None), "coords"),
|
||
]
|
||
db, updated = _make_db_mock()
|
||
good_result = _make_dadata_result(qc_geo=0)
|
||
|
||
call_count = {"n": 0}
|
||
|
||
async def fake_clean(addr):
|
||
call_count["n"] += 1
|
||
if call_count["n"] == 1:
|
||
raise RuntimeError("dadata blew up")
|
||
return good_result
|
||
|
||
with patch("scripts.backfill_houses_dadata.clean_address", side_effect=fake_clean):
|
||
stats = await _run_backfill(db, rows, batch="b4", dry_run=False)
|
||
|
||
assert stats.processed == 1 # только 2-й row дошёл до DB-write success
|
||
assert stats.failed == 1
|
||
assert stats.enriched == 1
|
||
# Только один UPDATE — для второго row
|
||
assert len(updated) == 1
|
||
assert updated[0]["id"] == 11
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# --dry-run — no DB writes
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_run_backfill_dry_run_skips_db_writes():
|
||
"""dry_run=True → counters обновляются, но db.execute UPDATE не вызывается."""
|
||
row = HouseRow(id=20, address="ул Test 1", lat=None, lon=None, cadastral_number=None)
|
||
db, updated = _make_db_mock()
|
||
fake = _make_dadata_result(qc_geo=0)
|
||
|
||
with patch(
|
||
"scripts.backfill_houses_dadata.clean_address",
|
||
new=AsyncMock(return_value=fake),
|
||
):
|
||
stats = await _run_backfill(
|
||
db, [(row, "coords")], batch="dry", dry_run=True
|
||
)
|
||
|
||
assert stats.processed == 1
|
||
assert stats.enriched == 1
|
||
assert updated == [] # никаких UPDATE'ов
|
||
assert db.commit.call_count == 0 # никаких commit'ов
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _select_candidates — priority filter работает
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_select_candidates_coords_priority_only_fetches_null_coords():
|
||
"""priority=coords → запрашивается только `lat IS NULL OR lon IS NULL` SELECT."""
|
||
coords_rows = [
|
||
{"id": 1, "address": "ул A 1", "lat": None, "lon": None, "cadastral_number": None},
|
||
{"id": 2, "address": "ул B 2", "lat": None, "lon": None, "cadastral_number": None},
|
||
]
|
||
db, _updated = _make_db_mock(coords_sample=coords_rows)
|
||
|
||
out = _select_candidates(db, priority="coords", limit=10)
|
||
|
||
assert len(out) == 2
|
||
assert all(p == "coords" for _row, p in out)
|
||
assert out[0][0].id == 1
|
||
assert out[1][0].id == 2
|
||
|
||
|
||
def test_select_candidates_cadnum_priority_only_fetches_null_cadnum():
|
||
"""priority=cadnum → только NOT NULL coords + NULL cadastral_number rows."""
|
||
cadnum_rows = [
|
||
{"id": 50, "address": "ул C 3", "lat": 56.8, "lon": 60.6, "cadastral_number": None},
|
||
]
|
||
db, _updated = _make_db_mock(cadnum_sample=cadnum_rows)
|
||
|
||
out = _select_candidates(db, priority="cadnum", limit=10)
|
||
|
||
assert len(out) == 1
|
||
assert out[0][1] == "cadnum"
|
||
assert out[0][0].id == 50
|
||
|
||
|
||
def test_select_candidates_both_priority_fetches_coords_first_then_cadnum():
|
||
"""priority=both → сначала NULL coords, потом NULL cadnum (в пределах limit)."""
|
||
coords_rows = [
|
||
{"id": 1, "address": "ул A 1", "lat": None, "lon": None, "cadastral_number": None},
|
||
]
|
||
cadnum_rows = [
|
||
{"id": 100, "address": "ул C 3", "lat": 56.8, "lon": 60.6, "cadastral_number": None},
|
||
]
|
||
db, _updated = _make_db_mock(coords_sample=coords_rows, cadnum_sample=cadnum_rows)
|
||
|
||
out = _select_candidates(db, priority="both", limit=10)
|
||
|
||
assert len(out) == 2
|
||
# coords bucket первый
|
||
assert out[0][1] == "coords"
|
||
assert out[0][0].id == 1
|
||
assert out[1][1] == "cadnum"
|
||
assert out[1][0].id == 100
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# --limit caps batch size
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_main_respects_limit(monkeypatch):
|
||
"""main(--limit 1) при 3 кандидатах → только 1 row processed."""
|
||
coords_rows = [
|
||
{"id": 1, "address": "ул A 1", "lat": None, "lon": None, "cadastral_number": None},
|
||
{"id": 2, "address": "ул B 2", "lat": None, "lon": None, "cadastral_number": None},
|
||
{"id": 3, "address": "ул C 3", "lat": None, "lon": None, "cadastral_number": None},
|
||
]
|
||
db, updated = _make_db_mock(coords_sample=coords_rows)
|
||
monkeypatch.setenv("DADATA_API_TOKEN", "TEST_TOKEN")
|
||
monkeypatch.setenv("DADATA_API_SECRET", "TEST_SECRET")
|
||
fake = _make_dadata_result(qc_geo=0)
|
||
|
||
with (
|
||
patch("scripts.backfill_houses_dadata.SessionLocal", return_value=db),
|
||
patch(
|
||
"scripts.backfill_houses_dadata.clean_address",
|
||
new=AsyncMock(return_value=fake),
|
||
),
|
||
# Уберём sleep чтобы test не висел 0.5s
|
||
patch("scripts.backfill_houses_dadata.asyncio.sleep", new=AsyncMock()),
|
||
):
|
||
n = await main(["--limit", "1", "--batch", "test_limit"])
|
||
|
||
assert n == 1
|
||
assert len(updated) == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Missing creds → SystemExit
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_main_requires_dadata_creds(monkeypatch):
|
||
"""Без DADATA_API_TOKEN + DADATA_API_SECRET → SystemExit ещё до DB connect."""
|
||
monkeypatch.delenv("DADATA_API_TOKEN", raising=False)
|
||
monkeypatch.delenv("DADATA_API_SECRET", raising=False)
|
||
with pytest.raises(SystemExit):
|
||
await main(["--limit", "1"])
|
||
|
||
|
||
async def test_main_requires_secret_not_just_token(monkeypatch):
|
||
"""Только TOKEN (без SECRET) → /clean/address не сработает → fail fast."""
|
||
monkeypatch.setenv("DADATA_API_TOKEN", "TEST_TOKEN")
|
||
monkeypatch.delenv("DADATA_API_SECRET", raising=False)
|
||
with pytest.raises(SystemExit):
|
||
await main(["--limit", "1"])
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Stats summary contains by_priority breakdown
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_stats_bump_buckets_by_priority():
|
||
"""Stats.bump поддерживает несколько priority bucket'ов одновременно."""
|
||
s = Stats()
|
||
s.bump("coords", "enriched")
|
||
s.bump("coords", "enriched")
|
||
s.bump("cadnum", "no_change")
|
||
assert s.by_priority["coords"]["enriched"] == 2
|
||
assert s.by_priority["cadnum"]["no_change"] == 1
|
||
# Не существующие outcome'ы должны быть 0
|
||
assert s.by_priority["coords"]["failed"] == 0
|