510 lines
18 KiB
Python
510 lines
18 KiB
Python
"""Unit tests for the Phase 2-3 backfill/audit script (issue #582).
|
|
|
|
Coverage:
|
|
- `forward_via_api` request shape — verifies geocode/format/locality bias.
|
|
- `_parse_api_payload` precision + kind extraction.
|
|
- `_classify_backfill_status` precision filter rules.
|
|
- `_update_house_coords` — UPDATE shape + raw_payload merge.
|
|
- `_run_backfill_mode` — happy path UPDATE + audit row, plus imprecise-skip.
|
|
- `_run_audit_mode` — ok / mismatch / no_match distinction.
|
|
- `main()` resumability — second pass on same batch inserts 0.
|
|
|
|
No real Postgres in unit tests (same convention as test_audit_address_mismatch).
|
|
DB is a MagicMock that records INSERT/UPDATE calls and routes SELECT side-effects.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
# Same dance as test_audit_address_mismatch — settings needs a DSN at import.
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from scripts._yandex_reverse import (
|
|
YandexReverseResult,
|
|
_parse_api_payload,
|
|
forward_via_api,
|
|
)
|
|
from scripts.backfill_house_coords import (
|
|
HouseRow,
|
|
_classify_backfill_status,
|
|
_run_audit_mode,
|
|
_run_backfill_mode,
|
|
_update_house_coords,
|
|
main,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# forward_via_api — request shape
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_forward_api_request_shape():
|
|
"""Verify the GET param dict — address as `geocode`, kind=house, EKB bias."""
|
|
fixture = {
|
|
"response": {
|
|
"GeoObjectCollection": {
|
|
"featureMember": [
|
|
{
|
|
"GeoObject": {
|
|
"metaDataProperty": {
|
|
"GeocoderMetaData": {
|
|
"text": "Россия, Свердловская область, Екатеринбург, "
|
|
"улица Малышева, 51",
|
|
"precision": "exact",
|
|
"kind": "house",
|
|
}
|
|
},
|
|
"name": "улица Малышева, 51",
|
|
"Point": {"pos": "60.586155 56.838004"},
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
captured: dict[str, httpx.Request] = {}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured["req"] = request
|
|
return httpx.Response(200, json=fixture)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as client:
|
|
res = await forward_via_api("ул Малышева 51", "DUMMY_KEY", client=client)
|
|
|
|
assert res.address is not None and "Малышева" in res.address
|
|
assert res.precision == "exact"
|
|
assert res.kind == "house"
|
|
assert res.snapped_lon == pytest.approx(60.586155, abs=1e-6)
|
|
assert res.snapped_lat == pytest.approx(56.838004, abs=1e-6)
|
|
|
|
qs = dict(httpx.QueryParams(captured["req"].url.query))
|
|
assert qs["apikey"] == "DUMMY_KEY"
|
|
assert qs["geocode"] == "ул Малышева 51"
|
|
assert qs["format"] == "json"
|
|
assert qs["kind"] == "house"
|
|
# EKB locality bias for forward geocode — important so addresses without
|
|
# the city resolve to the correct Малышева (there's one in Moscow too).
|
|
assert "ll" in qs
|
|
assert "spn" in qs
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Precision / kind passthrough in _parse_api_payload
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_parse_api_payload_propagates_precision_and_kind():
|
|
data = {
|
|
"response": {
|
|
"GeoObjectCollection": {
|
|
"featureMember": [
|
|
{
|
|
"GeoObject": {
|
|
"metaDataProperty": {
|
|
"GeocoderMetaData": {
|
|
"text": "ул Ленина 5",
|
|
"precision": "exact",
|
|
"kind": "house",
|
|
}
|
|
},
|
|
"name": "ул Ленина 5",
|
|
"Point": {"pos": "60.6 56.8"},
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
res = _parse_api_payload(data)
|
|
assert res.precision == "exact"
|
|
assert res.kind == "house"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _classify_backfill_status — precision filter rules
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_classify_backfill_status_exact_match():
|
|
res = YandexReverseResult(
|
|
address="ул Малышева 51",
|
|
snapped_lat=56.838,
|
|
snapped_lon=60.586,
|
|
precision="exact",
|
|
kind="house",
|
|
)
|
|
assert _classify_backfill_status(res) == "backfill"
|
|
|
|
|
|
def test_classify_backfill_status_number_match():
|
|
res = YandexReverseResult(
|
|
address="ул Ленина 5",
|
|
snapped_lat=56.840,
|
|
snapped_lon=60.600,
|
|
precision="number",
|
|
kind="house",
|
|
)
|
|
assert _classify_backfill_status(res) == "backfill"
|
|
|
|
|
|
def test_classify_backfill_status_street_is_imprecise():
|
|
res = YandexReverseResult(
|
|
address="ул Ленина",
|
|
snapped_lat=56.840,
|
|
snapped_lon=60.600,
|
|
precision="street",
|
|
kind="street",
|
|
)
|
|
assert _classify_backfill_status(res) == "imprecise"
|
|
|
|
|
|
def test_classify_backfill_status_other_is_imprecise():
|
|
res = YandexReverseResult(
|
|
address="Свердловская область",
|
|
snapped_lat=56.8,
|
|
snapped_lon=60.6,
|
|
precision="other",
|
|
kind="locality",
|
|
)
|
|
assert _classify_backfill_status(res) == "imprecise"
|
|
|
|
|
|
def test_classify_backfill_status_no_match():
|
|
res = YandexReverseResult(address=None, snapped_lat=None, snapped_lon=None)
|
|
assert _classify_backfill_status(res) == "no_match"
|
|
|
|
|
|
def test_classify_backfill_status_none():
|
|
assert _classify_backfill_status(None) == "no_match"
|
|
|
|
|
|
def test_classify_backfill_status_precision_ok_but_no_coords():
|
|
"""Defensive: precision=exact but snapped point missing → no_match, not backfill."""
|
|
res = YandexReverseResult(
|
|
address="ул Малышева 51",
|
|
snapped_lat=None,
|
|
snapped_lon=None,
|
|
precision="exact",
|
|
kind="house",
|
|
)
|
|
assert _classify_backfill_status(res) == "no_match"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _update_house_coords — UPDATE shape verification
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_update_house_coords_passes_bindings():
|
|
db = MagicMock()
|
|
_update_house_coords(
|
|
db,
|
|
house_id=42,
|
|
lat=56.838,
|
|
lon=60.586,
|
|
payload={"address": "ул Малышева 51", "precision": "exact"},
|
|
)
|
|
args, _kw = db.execute.call_args
|
|
sql_str = str(args[0])
|
|
binds = args[1]
|
|
assert "UPDATE houses" in sql_str
|
|
assert "raw_payload" in sql_str
|
|
assert "yandex_geocode" in sql_str
|
|
assert binds["id"] == 42
|
|
assert binds["lat"] == 56.838
|
|
assert binds["lon"] == 60.586
|
|
# payload bound as JSON string for CAST(:payload AS jsonb)
|
|
decoded = json.loads(binds["payload"])
|
|
assert decoded["address"] == "ул Малышева 51"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DB mock helper — same approach as test_audit_address_mismatch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_db_mock(
|
|
backfill_sample: list[dict] | None = None,
|
|
audit_sample: list[dict] | None = None,
|
|
processed_ids: set[int] | None = None,
|
|
distance_value: float = 12.5,
|
|
):
|
|
"""MagicMock DB that:
|
|
- returns `backfill_sample` for `lat IS NULL OR lon IS NULL` SELECT
|
|
- returns `audit_sample` for `lat IS NOT NULL` SELECT
|
|
- returns `processed_ids` for the resume SELECT
|
|
- records INSERTs and UPDATEs
|
|
- returns `distance_value` for ST_Distance calls
|
|
"""
|
|
backfill_sample = backfill_sample or []
|
|
audit_sample = audit_sample or []
|
|
processed_ids = processed_ids if processed_ids is not None else set()
|
|
|
|
inserted: list[dict] = []
|
|
updated: list[dict] = []
|
|
|
|
db = MagicMock()
|
|
db.begin_nested.return_value.__enter__ = lambda self: self
|
|
db.begin_nested.return_value.__exit__ = lambda self, *a: False
|
|
|
|
def execute_side_effect(sql, params=None):
|
|
sql_str = str(sql)
|
|
result = MagicMock()
|
|
if "FROM houses" in sql_str and "lat IS NULL OR lon IS NULL" in sql_str:
|
|
result.mappings.return_value.all.return_value = backfill_sample
|
|
elif "FROM houses" in sql_str and "lat IS NOT NULL" in sql_str:
|
|
result.mappings.return_value.all.return_value = audit_sample
|
|
elif "FROM address_mismatch_audit" in sql_str and "house_id" in sql_str:
|
|
result.all.return_value = [(hid,) for hid in processed_ids]
|
|
elif "INSERT INTO address_mismatch_audit" in sql_str:
|
|
inserted.append(dict(params))
|
|
processed_ids.add(params["house_id"])
|
|
elif "UPDATE houses" in sql_str:
|
|
updated.append(dict(params))
|
|
elif "ST_Distance" in sql_str:
|
|
result.first.return_value = (distance_value,)
|
|
return result
|
|
|
|
db.execute.side_effect = execute_side_effect
|
|
db.commit = MagicMock()
|
|
db.rollback = MagicMock()
|
|
db.close = MagicMock()
|
|
return db, inserted, updated
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _run_backfill_mode — happy path + imprecise-skip
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_run_backfill_mode_writes_update_and_audit():
|
|
sample = [
|
|
HouseRow(id=1, address="ул Малышева 51", lat=None, lon=None),
|
|
]
|
|
db, inserted, updated = _make_db_mock()
|
|
res = YandexReverseResult(
|
|
address="Россия, Екатеринбург, улица Малышева, 51",
|
|
snapped_lat=56.838,
|
|
snapped_lon=60.586,
|
|
precision="exact",
|
|
kind="house",
|
|
raw={"ok": True},
|
|
)
|
|
with patch(
|
|
"scripts.backfill_house_coords.forward_via_api",
|
|
new=AsyncMock(return_value=res),
|
|
):
|
|
n = await _run_backfill_mode(db, sample, "b1", "KEY")
|
|
assert n == 1
|
|
assert len(updated) == 1
|
|
assert updated[0]["id"] == 1
|
|
assert updated[0]["lat"] == 56.838
|
|
assert updated[0]["lon"] == 60.586
|
|
assert len(inserted) == 1
|
|
assert inserted[0]["audit_status"] == "backfill"
|
|
assert inserted[0]["snapped_address"] == "Россия, Екатеринбург, улица Малышева, 51"
|
|
|
|
|
|
async def test_run_backfill_mode_imprecise_skips_update():
|
|
"""precision='street' → audit row written with status=imprecise, no UPDATE."""
|
|
sample = [HouseRow(id=2, address="ул Ленина", lat=None, lon=None)]
|
|
db, inserted, updated = _make_db_mock()
|
|
res = YandexReverseResult(
|
|
address="ул Ленина",
|
|
snapped_lat=56.840,
|
|
snapped_lon=60.600,
|
|
precision="street",
|
|
kind="street",
|
|
raw={"oh_well": True},
|
|
)
|
|
with patch(
|
|
"scripts.backfill_house_coords.forward_via_api",
|
|
new=AsyncMock(return_value=res),
|
|
):
|
|
n = await _run_backfill_mode(db, sample, "b2", "KEY")
|
|
assert n == 1
|
|
assert updated == []
|
|
assert len(inserted) == 1
|
|
assert inserted[0]["audit_status"] == "imprecise"
|
|
|
|
|
|
async def test_run_backfill_mode_no_match():
|
|
"""Yandex returns empty result → status=no_match, no UPDATE."""
|
|
sample = [HouseRow(id=3, address="несуществующая улица 99", lat=None, lon=None)]
|
|
db, inserted, updated = _make_db_mock()
|
|
res = YandexReverseResult(
|
|
address=None, snapped_lat=None, snapped_lon=None, raw={"empty": True}
|
|
)
|
|
with patch(
|
|
"scripts.backfill_house_coords.forward_via_api",
|
|
new=AsyncMock(return_value=res),
|
|
):
|
|
n = await _run_backfill_mode(db, sample, "b3", "KEY")
|
|
assert n == 1
|
|
assert updated == []
|
|
assert inserted[0]["audit_status"] == "no_match"
|
|
|
|
|
|
async def test_run_backfill_mode_http_error_marks_error():
|
|
sample = [HouseRow(id=4, address="ул X 1", lat=None, lon=None)]
|
|
db, inserted, updated = _make_db_mock()
|
|
with patch(
|
|
"scripts.backfill_house_coords.forward_via_api",
|
|
new=AsyncMock(side_effect=httpx.HTTPError("boom")),
|
|
):
|
|
n = await _run_backfill_mode(db, sample, "b4", "KEY")
|
|
assert n == 1
|
|
assert updated == []
|
|
assert inserted[0]["audit_status"] == "error"
|
|
assert "boom" in (inserted[0]["error_message"] or "")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _run_audit_mode — ok / mismatch / no_match
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_run_audit_mode_ok_within_50m():
|
|
sample = [HouseRow(id=10, address="ул Малышева 51", lat=56.838, lon=60.586)]
|
|
db, inserted, _updated = _make_db_mock(distance_value=12.5)
|
|
res = YandexReverseResult(
|
|
address="Россия, Екатеринбург, улица Малышева, 51",
|
|
snapped_lat=56.838004,
|
|
snapped_lon=60.586155,
|
|
precision="exact",
|
|
kind="house",
|
|
raw={"r": 1},
|
|
)
|
|
with patch(
|
|
"scripts.backfill_house_coords.reverse_via_api",
|
|
new=AsyncMock(return_value=res),
|
|
):
|
|
n = await _run_audit_mode(db, sample, "ba1", "KEY")
|
|
assert n == 1
|
|
assert inserted[0]["audit_status"] == "ok"
|
|
assert inserted[0]["distance_m"] == 12.5
|
|
|
|
|
|
async def test_run_audit_mode_mismatch_above_50m():
|
|
sample = [HouseRow(id=11, address="ул Ленина 5", lat=56.840, lon=60.600)]
|
|
db, inserted, _updated = _make_db_mock(distance_value=312.0)
|
|
res = YandexReverseResult(
|
|
address="Россия, Екатеринбург, улица Ленина, 7",
|
|
snapped_lat=56.841,
|
|
snapped_lon=60.601,
|
|
precision="exact",
|
|
kind="house",
|
|
raw={"r": 2},
|
|
)
|
|
with patch(
|
|
"scripts.backfill_house_coords.reverse_via_api",
|
|
new=AsyncMock(return_value=res),
|
|
):
|
|
n = await _run_audit_mode(db, sample, "ba2", "KEY")
|
|
assert n == 1
|
|
assert inserted[0]["audit_status"] == "mismatch"
|
|
assert inserted[0]["distance_m"] == 312.0
|
|
|
|
|
|
async def test_run_audit_mode_no_match():
|
|
sample = [HouseRow(id=12, address="ул X 99", lat=56.0, lon=60.0)]
|
|
db, inserted, _updated = _make_db_mock()
|
|
res = YandexReverseResult(
|
|
address=None, snapped_lat=None, snapped_lon=None, raw={"empty": True}
|
|
)
|
|
with patch(
|
|
"scripts.backfill_house_coords.reverse_via_api",
|
|
new=AsyncMock(return_value=res),
|
|
):
|
|
n = await _run_audit_mode(db, sample, "ba3", "KEY")
|
|
assert n == 1
|
|
assert inserted[0]["audit_status"] == "no_match"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Resumability — second pass on same batch inserts 0
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_main_resumable_skips_processed(monkeypatch):
|
|
"""Run main() twice with same batch — second pass processes nothing."""
|
|
backfill_sample = [
|
|
{"id": 1, "address": "ул Малышева 51", "lat": None, "lon": None},
|
|
{"id": 2, "address": "ул Ленина 5", "lat": None, "lon": None},
|
|
]
|
|
processed_ids: set[int] = set()
|
|
db, inserted, updated = _make_db_mock(
|
|
backfill_sample=backfill_sample, processed_ids=processed_ids
|
|
)
|
|
|
|
monkeypatch.setenv("YANDEX_GEOCODER_API_KEY", "TEST_KEY")
|
|
fake = YandexReverseResult(
|
|
address="ул Малышева 51",
|
|
snapped_lat=56.838,
|
|
snapped_lon=60.586,
|
|
precision="exact",
|
|
kind="house",
|
|
raw={"ok": True},
|
|
)
|
|
|
|
with (
|
|
patch("scripts.backfill_house_coords.SessionLocal", return_value=db),
|
|
patch(
|
|
"scripts.backfill_house_coords.forward_via_api",
|
|
new=AsyncMock(return_value=fake),
|
|
),
|
|
):
|
|
n1 = await main(["--batch", "resume_test"])
|
|
assert n1 == 2
|
|
assert len(inserted) == 2
|
|
assert len(updated) == 2
|
|
|
|
inserted.clear()
|
|
updated.clear()
|
|
n2 = await main(["--batch", "resume_test"])
|
|
assert n2 == 0
|
|
assert inserted == []
|
|
assert updated == []
|
|
|
|
|
|
async def test_main_requires_api_key(monkeypatch):
|
|
"""Without YANDEX_GEOCODER_API_KEY the script exits cleanly."""
|
|
monkeypatch.delenv("YANDEX_GEOCODER_API_KEY", raising=False)
|
|
with pytest.raises(SystemExit):
|
|
await main(["--batch", "no_key"])
|
|
|
|
|
|
async def test_main_audit_only_flag_routes_to_audit_loop(monkeypatch):
|
|
"""--audit-only switches sample query + loop, no UPDATE expected."""
|
|
audit_sample = [
|
|
{"id": 50, "address": "ул Малышева 51", "lat": 56.838, "lon": 60.586},
|
|
]
|
|
db, inserted, updated = _make_db_mock(audit_sample=audit_sample, distance_value=8.0)
|
|
monkeypatch.setenv("YANDEX_GEOCODER_API_KEY", "TEST_KEY")
|
|
fake = YandexReverseResult(
|
|
address="Россия, Екатеринбург, улица Малышева, 51",
|
|
snapped_lat=56.838004,
|
|
snapped_lon=60.586155,
|
|
precision="exact",
|
|
kind="house",
|
|
raw={"r": 1},
|
|
)
|
|
with (
|
|
patch("scripts.backfill_house_coords.SessionLocal", return_value=db),
|
|
patch(
|
|
"scripts.backfill_house_coords.reverse_via_api",
|
|
new=AsyncMock(return_value=fake),
|
|
),
|
|
):
|
|
n = await main(["--batch", "audit_run", "--audit-only"])
|
|
assert n == 1
|
|
assert updated == [] # audit mode never updates houses
|
|
assert inserted[0]["audit_status"] == "ok"
|
|
assert inserted[0]["distance_m"] == 8.0
|