Light1YT
1c6d59e3ee
deals (49,791 ДКП-сделок Росреестра) на 100% NULL geom → estimator _fetch_deals (ST_DWithin) никогда не матчит, actual_deals всегда пустой (корень #571 «deals disabled»). Добавляет scripts/geocode_deals_from_houses.py: строит per-street центроиды из ~8.6k геокодированных houses и проставляет lat/lon каждой сделке по нормализованному ключу улицы (geom авто-заполняется триггером deals_set_geom_trg). Zero внешних geocoder-вызовов; --dry-run проецирует coverage и топ-10 unmatched улиц. Pre-push review поймал data-corruption баг: _APT_SUFFIX_RE без правого якоря жрал реальные улицы на кв/корп/оф/пом/стр («Строителей»→'ул.' garbage key → неверный центроид). Fix: требовать цифру после суффикс-токена + регресс-тест. Tests: 26 (street-key нормализация вкл. regression, centroid AVG, matched UPDATE, no_street_match, --dry-run no-op, SAVEPOINT isolation, geom-not-manual). Ruff clean.
440 lines
18 KiB
Python
440 lines
18 KiB
Python
"""Unit tests for issue #569 Step 2 — geocode_deals_from_houses.py.
|
||
|
||
Coverage:
|
||
- _street_key normalization: the three spec examples collapse to one key,
|
||
plus boundary cases (numeric streets '8 марта', names that start with a
|
||
type-token substring like 'алмазная', district/apt/region stripping).
|
||
- _build_centroid_map: AVG(lat)/AVG(lon) over multiple houses on one street.
|
||
- _run_backfill happy path: a deal whose street matches → UPDATE with the
|
||
centroid lat/lon, geocoded counter bumps.
|
||
- _run_backfill miss: a deal whose street has no centroid → no UPDATE,
|
||
counted as no_street_match + tracked for the dry-run report.
|
||
- --dry-run: no UPDATE / no commit, counters still move.
|
||
- per-row SAVEPOINT isolates a failing UPDATE.
|
||
- main() wires SessionLocal, respects --limit, and returns geocoded count.
|
||
|
||
No real Postgres. Session is a MagicMock that routes SELECT side-effects by
|
||
SQL substring and records UPDATE binds (same pattern as
|
||
test_backfill_houses_dadata.py).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
# Settings needs a DSN at import time — set a dummy before any app.* import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from scripts.geocode_deals_from_houses import (
|
||
Centroid,
|
||
DealRow,
|
||
Stats,
|
||
_build_centroid_map,
|
||
_run_backfill,
|
||
_street_key,
|
||
_update_deal_coords,
|
||
main,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Mock-DB helper
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _make_db_mock(
|
||
*,
|
||
house_rows: list[dict] | None = None,
|
||
deal_rows: list[dict] | None = None,
|
||
null_count: int = 0,
|
||
) -> tuple[MagicMock, list[dict]]:
|
||
"""MagicMock Session that:
|
||
|
||
- returns `house_rows` for the centroid SELECT (`FROM houses`)
|
||
- returns `deal_rows` for the candidate SELECT (`FROM deals ... lat IS NULL`)
|
||
- returns `null_count` for the `count(*)` SELECT
|
||
- records UPDATE binds into the returned `updated` list
|
||
- supports `db.begin_nested()` as a context manager
|
||
"""
|
||
house_rows = house_rows or []
|
||
deal_rows = deal_rows or []
|
||
updated: list[dict] = []
|
||
|
||
db = MagicMock()
|
||
db.begin_nested.return_value.__enter__ = lambda self: self
|
||
db.begin_nested.return_value.__exit__ = lambda self, *a: False
|
||
|
||
def execute_side_effect(sql, params=None):
|
||
sql_str = str(sql)
|
||
result = MagicMock()
|
||
if "UPDATE deals" in sql_str:
|
||
updated.append(dict(params) if params else {})
|
||
return result
|
||
if "count(*)" in sql_str:
|
||
result.first.return_value = (null_count,)
|
||
return result
|
||
if "FROM houses" in sql_str:
|
||
result.mappings.return_value.all.return_value = house_rows
|
||
return result
|
||
if "FROM deals" in sql_str:
|
||
lim = params.get("lim", len(deal_rows)) if params else len(deal_rows)
|
||
result.mappings.return_value.all.return_value = deal_rows[:lim]
|
||
return result
|
||
return result
|
||
|
||
db.execute.side_effect = execute_side_effect
|
||
db.commit = MagicMock()
|
||
db.rollback = MagicMock()
|
||
db.close = MagicMock()
|
||
return db, updated
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _street_key — normalization (the crux)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_street_key_three_spec_variants_collapse_to_same_key():
|
||
"""The spec's three Малышева variants must produce one identical key."""
|
||
a = _street_key("Екатеринбург, ул. Малышева, 125")
|
||
b = _street_key("г Екатеринбург, улица Малышева")
|
||
c = _street_key("Екатеринбург, Малышева")
|
||
assert a == b == c == "малышева"
|
||
|
||
|
||
def test_street_key_strips_region_and_house_number():
|
||
assert (
|
||
_street_key("Свердловская обл., Екатеринбург, ул. Большакова, 17")
|
||
== "большакова"
|
||
)
|
||
|
||
|
||
def test_street_key_strips_district_marker():
|
||
assert _street_key("улица Яскина, 12 · р-н Октябрьский") == "яскина"
|
||
|
||
|
||
def test_street_key_handles_prospekt():
|
||
assert _street_key("г. Екатеринбург, проспект Ленина, 50") == "ленина"
|
||
|
||
|
||
def test_street_key_preserves_numeric_street_name():
|
||
"""'8 марта' is a real street — the leading number must survive while the
|
||
trailing house number is stripped."""
|
||
assert _street_key("Екатеринбург, ул. 8 Марта, 100") == "8 марта"
|
||
assert _street_key("Екатеринбург, 8 Марта") == "8 марта"
|
||
|
||
|
||
def test_street_key_preserves_multiword_numeric_street():
|
||
assert _street_key("Екатеринбург, ул. 40 лет Октября") == "40 лет октября"
|
||
|
||
|
||
def test_street_key_does_not_eat_name_starting_like_type_token():
|
||
"""'алмазная' must NOT lose 'ал', 'пришвина' must NOT lose 'пр' — the
|
||
street-type strip is \\b-bounded."""
|
||
assert _street_key("ул. Алмазная, 7") == "алмазная"
|
||
assert _street_key("ул. Пришвина, 3") == "пришвина"
|
||
|
||
|
||
def test_street_key_strips_korpus_and_kv_suffix():
|
||
assert _street_key("Екатеринбург, ул. Крауля, 48, корп. 2") == "крауля"
|
||
assert (
|
||
_street_key("РФ, Свердловская обл., Екатеринбург, ул. Ленина, 5, кв. 12")
|
||
== "ленина"
|
||
)
|
||
|
||
|
||
def test_street_key_does_not_eat_names_starting_like_apt_suffix():
|
||
"""Regression (pre-push review): un-anchored _APT_SUFFIX_RE collapsed real
|
||
streets starting with кв/корп/оф/пом/стр («Строителей», «Офицеров»,
|
||
«Корпусная», «Помолова») to the garbage key 'ул.' → wrong centroid → wrong
|
||
coords. Suffix strip now requires a DIGIT after the token."""
|
||
assert _street_key("Екатеринбург, ул. Строителей, 5") == "строителей"
|
||
assert _street_key("Екатеринбург, ул. Офицеров") == "офицеров"
|
||
assert _street_key("Екатеринбург, ул. Корпусная, 14") == "корпусная"
|
||
assert _street_key("Екатеринбург, ул. Помолова, 3") == "помолова"
|
||
# deals-form (no house number) must still resolve to the street, not 'ул.'
|
||
assert _street_key("Екатеринбург, Стрелочников") == "стрелочников"
|
||
|
||
|
||
def test_street_key_normalizes_yo_to_e():
|
||
"""Рабочей Молодёжи (ё) and Молодежи (е) must collapse — sources differ."""
|
||
assert _street_key("Екатеринбург, наб. Рабочей Молодёжи, 2") == "рабочей молодежи"
|
||
assert _street_key("Екатеринбург, Рабочей Молодежи") == "рабочей молодежи"
|
||
|
||
|
||
def test_street_key_house_prefix_dom():
|
||
assert _street_key("Екатеринбург, пр. Ленина, д. 5") == "ленина"
|
||
|
||
|
||
def test_street_key_none_and_empty_and_city_only():
|
||
assert _street_key(None) == ""
|
||
assert _street_key("") == ""
|
||
# City alone has no street segment → empty key (caller treats as no-match).
|
||
assert _street_key("Екатеринбург") == ""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _build_centroid_map — AVG over multiple houses on a street
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_build_centroid_map_averages_houses_on_one_street():
|
||
"""Two houses on Малышева → centroid is the mean of their coords."""
|
||
house_rows = [
|
||
{"address": "Екатеринбург, ул. Малышева, 10", "lat": 56.80, "lon": 60.50},
|
||
{"address": "Екатеринбург, ул. Малышева, 20", "lat": 56.90, "lon": 60.70},
|
||
]
|
||
db, _ = _make_db_mock(house_rows=house_rows)
|
||
|
||
centroids = _build_centroid_map(db)
|
||
|
||
assert set(centroids) == {"малышева"}
|
||
c = centroids["малышева"]
|
||
assert c.lat == pytest.approx(56.85)
|
||
assert c.lon == pytest.approx(60.60)
|
||
assert c.house_count == 2
|
||
|
||
|
||
def test_build_centroid_map_groups_distinct_streets():
|
||
house_rows = [
|
||
{"address": "Екатеринбург, ул. Малышева, 10", "lat": 56.80, "lon": 60.50},
|
||
{"address": "г Екатеринбург, Малышева", "lat": 56.82, "lon": 60.54},
|
||
{"address": "Екатеринбург, проспект Ленина, 5", "lat": 56.84, "lon": 60.60},
|
||
]
|
||
db, _ = _make_db_mock(house_rows=house_rows)
|
||
|
||
centroids = _build_centroid_map(db)
|
||
|
||
assert set(centroids) == {"малышева", "ленина"}
|
||
assert centroids["малышева"].house_count == 2
|
||
assert centroids["ленина"].house_count == 1
|
||
# Малышева centroid = mean of the two Малышева rows.
|
||
assert centroids["малышева"].lat == pytest.approx(56.81)
|
||
assert centroids["ленина"].lat == pytest.approx(56.84)
|
||
|
||
|
||
def test_build_centroid_map_skips_unparseable_address():
|
||
"""A house whose address yields too-short a key is dropped, not crashed."""
|
||
house_rows = [
|
||
{"address": "Екатеринбург", "lat": 56.8, "lon": 60.6}, # city only → ''
|
||
{"address": "Екатеринбург, ул. Малышева, 1", "lat": 56.84, "lon": 60.6},
|
||
]
|
||
db, _ = _make_db_mock(house_rows=house_rows)
|
||
|
||
centroids = _build_centroid_map(db)
|
||
assert set(centroids) == {"малышева"}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _run_backfill — happy path: street match → UPDATE with centroid coords
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_run_backfill_matched_street_issues_update_with_centroid():
|
||
deal = DealRow(id=42, address="Екатеринбург, ул. Малышева, 125")
|
||
centroids = {"малышева": Centroid(lat=56.838, lon=60.586, house_count=3)}
|
||
db, updated = _make_db_mock()
|
||
|
||
stats = _run_backfill(db, [deal], centroids, batch="b1", dry_run=False)
|
||
|
||
assert stats.geocoded == 1
|
||
assert stats.no_street_match == 0
|
||
assert stats.failed == 0
|
||
assert stats.processed == 1
|
||
assert len(updated) == 1
|
||
assert updated[0]["id"] == 42
|
||
assert updated[0]["lat"] == 56.838
|
||
assert updated[0]["lon"] == 60.586
|
||
assert db.commit.call_count == 1
|
||
|
||
|
||
def test_run_backfill_deal_with_only_street_name_matches():
|
||
"""Deal address with no house number / no type word still matches."""
|
||
deal = DealRow(id=7, address="Екатеринбург, Малышева")
|
||
centroids = {"малышева": Centroid(lat=56.8, lon=60.5, house_count=1)}
|
||
db, updated = _make_db_mock()
|
||
|
||
stats = _run_backfill(db, [deal], centroids, batch="b", dry_run=False)
|
||
|
||
assert stats.geocoded == 1
|
||
assert len(updated) == 1
|
||
assert updated[0]["lat"] == 56.8
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _run_backfill — miss: no centroid → no UPDATE, counted as no_street_match
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_run_backfill_unmatched_street_no_update():
|
||
deal = DealRow(id=99, address="Екатеринбург, ул. Несуществующая, 1")
|
||
centroids = {"малышева": Centroid(lat=56.8, lon=60.5, house_count=1)}
|
||
db, updated = _make_db_mock()
|
||
|
||
stats = _run_backfill(db, [deal], centroids, batch="b", dry_run=False)
|
||
|
||
assert stats.geocoded == 0
|
||
assert stats.no_street_match == 1
|
||
assert stats.processed == 1
|
||
assert updated == []
|
||
assert db.commit.call_count == 0
|
||
# Unmatched street tracked for the dry-run report.
|
||
assert stats.unmatched_streets["несуществующая"] == 1
|
||
|
||
|
||
def test_run_backfill_unparseable_deal_tracked_as_sentinel():
|
||
"""A deal whose address yields an empty key is a no-match under a sentinel."""
|
||
deal = DealRow(id=5, address="Екатеринбург")
|
||
centroids = {"малышева": Centroid(lat=56.8, lon=60.5, house_count=1)}
|
||
db, updated = _make_db_mock()
|
||
|
||
stats = _run_backfill(db, [deal], centroids, batch="b", dry_run=False)
|
||
|
||
assert stats.no_street_match == 1
|
||
assert updated == []
|
||
assert stats.unmatched_streets["<no-street-parsed>"] == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# --dry-run — no DB writes
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_run_backfill_dry_run_issues_no_update():
|
||
deal = DealRow(id=1, address="Екатеринбург, ул. Малышева, 1")
|
||
centroids = {"малышева": Centroid(lat=56.8, lon=60.5, house_count=1)}
|
||
db, updated = _make_db_mock()
|
||
|
||
stats = _run_backfill(db, [deal], centroids, batch="dry", dry_run=True)
|
||
|
||
assert stats.geocoded == 1
|
||
assert stats.processed == 1
|
||
assert updated == [] # no UPDATE
|
||
assert db.commit.call_count == 0 # no commit
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# per-row SAVEPOINT — one bad UPDATE doesn't abort the batch
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_run_backfill_db_write_failure_isolated_to_row():
|
||
deals = [
|
||
DealRow(id=10, address="Екатеринбург, ул. Малышева, 1"),
|
||
DealRow(id=11, address="Екатеринбург, ул. Малышева, 2"),
|
||
]
|
||
centroids = {"малышева": Centroid(lat=56.8, lon=60.5, house_count=1)}
|
||
db, updated = _make_db_mock()
|
||
|
||
# Make the FIRST UPDATE raise, the rest succeed.
|
||
call = {"n": 0}
|
||
real_side_effect = db.execute.side_effect
|
||
|
||
def failing_execute(sql, params=None):
|
||
if "UPDATE deals" in str(sql):
|
||
call["n"] += 1
|
||
if call["n"] == 1:
|
||
raise RuntimeError("constraint blew up")
|
||
return real_side_effect(sql, params)
|
||
|
||
db.execute.side_effect = failing_execute
|
||
|
||
stats = _run_backfill(db, deals, centroids, batch="b", dry_run=False)
|
||
|
||
assert stats.failed == 1
|
||
assert stats.geocoded == 1
|
||
assert db.rollback.call_count == 1
|
||
# Only the second deal's UPDATE was recorded.
|
||
assert [u["id"] for u in updated] == [11]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _update_deal_coords — bind shape + geom NOT set manually
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_update_deal_coords_sets_lat_lon_tried_at_not_geom():
|
||
db = MagicMock()
|
||
_update_deal_coords(db, deal_id=3, lat=56.1, lon=60.2)
|
||
args, _kw = db.execute.call_args
|
||
sql_str = str(args[0])
|
||
binds = args[1]
|
||
assert "UPDATE deals" in sql_str
|
||
assert "geocode_tried_at = NOW()" in sql_str
|
||
# geom must NOT be set manually — the deals_set_geom_trg trigger fills it.
|
||
assert "geom" not in sql_str
|
||
assert binds == {"id": 3, "lat": 56.1, "lon": 60.2}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# main() — wiring, --limit, return value
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_main_respects_limit_and_returns_geocoded():
|
||
house_rows = [
|
||
{"address": "Екатеринбург, ул. Малышева, 1", "lat": 56.80, "lon": 60.50},
|
||
{"address": "Екатеринбург, ул. Малышева, 2", "lat": 56.90, "lon": 60.70},
|
||
]
|
||
deal_rows = [
|
||
{"id": 1, "address": "Екатеринбург, Малышева"},
|
||
{"id": 2, "address": "Екатеринбург, ул. Малышева, 9"},
|
||
{"id": 3, "address": "Екатеринбург, ул. Малышева, 99"},
|
||
]
|
||
db, updated = _make_db_mock(house_rows=house_rows, deal_rows=deal_rows)
|
||
|
||
with patch("scripts.geocode_deals_from_houses.SessionLocal", return_value=db):
|
||
n = main(["--limit", "1", "--batch", "test_limit"])
|
||
|
||
assert n == 1
|
||
assert len(updated) == 1
|
||
assert updated[0]["id"] == 1
|
||
# Centroid used = mean of the two house rows.
|
||
assert updated[0]["lat"] == pytest.approx(56.85)
|
||
assert updated[0]["lon"] == pytest.approx(60.60)
|
||
|
||
|
||
def test_main_dry_run_writes_nothing():
|
||
house_rows = [
|
||
{"address": "Екатеринбург, ул. Малышева, 1", "lat": 56.80, "lon": 60.50},
|
||
]
|
||
deal_rows = [
|
||
{"id": 1, "address": "Екатеринбург, Малышева"},
|
||
{"id": 2, "address": "Екатеринбург, ул. Неизвестная, 5"},
|
||
]
|
||
db, updated = _make_db_mock(house_rows=house_rows, deal_rows=deal_rows, null_count=2)
|
||
|
||
with patch("scripts.geocode_deals_from_houses.SessionLocal", return_value=db):
|
||
n = main(["--dry-run"])
|
||
|
||
# dry-run returns geocoded count (would-match), writes nothing.
|
||
assert n == 1
|
||
assert updated == []
|
||
assert db.commit.call_count == 0
|
||
|
||
|
||
def test_main_no_centroids_returns_zero():
|
||
"""Empty houses table → nothing to join against → graceful 0."""
|
||
db, updated = _make_db_mock(house_rows=[], deal_rows=[{"id": 1, "address": "x"}])
|
||
|
||
with patch("scripts.geocode_deals_from_houses.SessionLocal", return_value=db):
|
||
n = main([])
|
||
|
||
assert n == 0
|
||
assert updated == []
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Stats dataclass
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_stats_unmatched_streets_counter_defaults_empty():
|
||
s = Stats()
|
||
assert s.processed == 0
|
||
assert s.geocoded == 0
|
||
assert s.no_street_match == 0
|
||
assert s.failed == 0
|
||
assert s.unmatched_streets.most_common(3) == []
|