bf4ce1ba36
All checks were successful
CI Trade-In / changes (pull_request) Successful in 12s
CI / changes (pull_request) Successful in 13s
CI / frontend-tests (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m3s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
#2487 avito oblast per-city sweep был dead-on-arrival: _parse_html дропал все карточки без /ekaterinburg/ в source_url, поэтому oblast-sweep по городу вне ЕКБ терял 100% выдачи. Kept-slug теперь параметризуется через AvitoScraper.target_city_slug (дефолт "ekaterinburg" — ЕКБ-поведение неизменно), прокинут scheduler → run_avito_city_sweep → AvitoScraper. avito_serp_ekb_only НЕ отключается. Bug #2 (oblast bare-street mis-bucket): голые адреса 'ул. Ленина 100' в Н.Тагиле мис-баккетились в одноимённые ЕКБ-дома через Tier-2b (match по глобально-уникальному normalized_address без geo/city guard). Добавлен coarse-guard: coords present → ST_DWithin ≤3км до geom дома; coords absent → требуется city-token (self- disambiguating); иначе skip → консервативно New house вместо мис-матча. _insert_alias при коллизии normalized_address больше не перебивает fingerprint чужого дома (иначе guard деградировал бы в Tier-2a-дыру). Оба дефекта латентные (oblast per-city schedules отключены) — правка делает capability безопасной к включению, без влияния на прод сегодня.
930 lines
32 KiB
Python
930 lines
32 KiB
Python
"""Unit tests for cross-source matching service.
|
||
|
||
Covers:
|
||
- normalize_address(): abbreviation expansion, punctuation stripping, edge cases
|
||
- address_fingerprint(): stability on case variation, sensitivity to coords, None handling
|
||
- match_or_create_house() / match_or_create_listing(): mock-DB tier routing
|
||
|
||
Reference: decisions/Cross_Source_Matching_Strategy.md
|
||
"""
|
||
|
||
from unittest.mock import MagicMock
|
||
|
||
from app.services.matching.conflict_resolution import (
|
||
HOUSE_FIELD_PRIORITY,
|
||
LISTING_FIELD_PRIORITY,
|
||
update_canonical_fields,
|
||
)
|
||
from app.services.matching.normalize import (
|
||
address_fingerprint,
|
||
has_city_token,
|
||
has_house_number,
|
||
house_number_token,
|
||
normalize_address,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# normalize_address
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_normalize_strips_punctuation_and_collapses_whitespace():
|
||
assert normalize_address(" Test, Address! ") == "test address"
|
||
|
||
|
||
def test_normalize_expands_ul_abbreviation():
|
||
result = normalize_address("ул Ленина, 5")
|
||
assert "улица" in result
|
||
assert "5" in result
|
||
|
||
|
||
def test_normalize_expands_pr_abbreviation():
|
||
result = normalize_address("пр Мира 100")
|
||
assert "проспект" in result
|
||
assert "100" in result
|
||
|
||
|
||
def test_normalize_expands_prkt_abbreviation():
|
||
result = normalize_address("пр-кт Ленина 10")
|
||
assert "проспект" in result
|
||
# 'кт' must NOT remain as a stray token after expansion
|
||
assert "кт" not in result.split()
|
||
|
||
|
||
def test_normalize_prkt_no_leftover_kt():
|
||
"""пр-кт must expand to 'проспект' cleanly — no leftover 'кт' token."""
|
||
result = normalize_address("пр-кт Ленина 10")
|
||
tokens = result.split()
|
||
assert "проспект" in tokens
|
||
assert "пр" not in tokens
|
||
assert "кт" not in tokens
|
||
|
||
|
||
def test_normalize_handles_none():
|
||
assert normalize_address(None) == ""
|
||
|
||
|
||
def test_normalize_handles_empty_string():
|
||
assert normalize_address("") == ""
|
||
|
||
|
||
def test_normalize_lowercases():
|
||
assert normalize_address("ТЕСТ") == "тест"
|
||
|
||
|
||
def test_normalize_unicode_nfc():
|
||
# Composed vs decomposed е (U+0435 vs U+0435)
|
||
s1 = normalize_address("улица")
|
||
s2 = normalize_address("улица")
|
||
assert s1 == s2
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# has_house_number / house_number_token (mis-bucketing hardening)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_has_house_number_true_when_trailing_number():
|
||
assert has_house_number("улица мамина сибиряка 126а") is True
|
||
|
||
|
||
def test_has_house_number_false_for_street_name_digit_only():
|
||
# 'улица 8 марта' — the digit is part of the street NAME, not a house number
|
||
assert has_house_number("улица 8 марта") is False
|
||
|
||
|
||
def test_has_house_number_true_for_street_name_digit_plus_house():
|
||
# 'улица 8 марта 5' — '5' is the house number, '8' is in the name
|
||
assert has_house_number("улица 8 марта 5") is True
|
||
|
||
|
||
def test_has_house_number_false_for_bare_street():
|
||
assert has_house_number("улица мамина сибиряка") is False
|
||
|
||
|
||
def test_has_house_number_false_for_none_and_empty():
|
||
assert has_house_number(None) is False
|
||
assert has_house_number("") is False
|
||
|
||
|
||
def test_house_number_token_extracts_trailing_token():
|
||
assert house_number_token("улица мамина сибиряка 126а") == "126а"
|
||
|
||
|
||
def test_house_number_token_for_street_name_digit_plus_house():
|
||
assert house_number_token("улица 8 марта 5") == "5"
|
||
|
||
|
||
def test_house_number_token_none_for_bare_street():
|
||
assert house_number_token("мамина сибиряка") is None
|
||
|
||
|
||
def test_house_number_token_none_for_none():
|
||
assert house_number_token(None) is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# address_fingerprint
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_fingerprint_stable_on_case_variation():
|
||
f1 = address_fingerprint("Ленина 5", 56.8378, 60.5946)
|
||
f2 = address_fingerprint("ЛЕНИНА 5", 56.8378, 60.5946)
|
||
assert f1 == f2
|
||
|
||
|
||
def test_fingerprint_distinct_on_different_lat():
|
||
f1 = address_fingerprint("Same Address", 56.8, 60.5)
|
||
f2 = address_fingerprint("Same Address", 56.9, 60.5)
|
||
assert f1 != f2
|
||
|
||
|
||
def test_fingerprint_distinct_on_different_lon():
|
||
f1 = address_fingerprint("Same Address", 56.8, 60.5)
|
||
f2 = address_fingerprint("Same Address", 56.8, 60.6)
|
||
assert f1 != f2
|
||
|
||
|
||
def test_fingerprint_stable_within_11m(monkeypatch):
|
||
# 4 decimal places → ~11 m resolution; coords differing in 5th decimal should match
|
||
f1 = address_fingerprint("Addr", 56.83780, 60.59460)
|
||
f2 = address_fingerprint("Addr", 56.83781, 60.59461)
|
||
# 4dp rounding: 56.8378 == 56.8378, 60.5946 == 60.5946 → same fingerprint
|
||
assert f1 == f2
|
||
|
||
|
||
def test_fingerprint_handles_none_coords():
|
||
fp = address_fingerprint("Some Address", None, None)
|
||
assert len(fp) == 32 # truncated sha256 hex
|
||
|
||
|
||
def test_fingerprint_handles_none_address():
|
||
fp = address_fingerprint(None, 56.8, 60.5)
|
||
assert len(fp) == 32
|
||
|
||
|
||
def test_fingerprint_all_none():
|
||
fp = address_fingerprint(None, None, None)
|
||
assert len(fp) == 32
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# has_city_token — обл.66 city-token detection (Tier-2b oblast guard)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_has_city_token_detects_ekb():
|
||
assert has_city_token("екатеринбург улица ленина 5") is True
|
||
|
||
|
||
def test_has_city_token_detects_multiword_oblast_city():
|
||
assert has_city_token("нижний тагил улица ленина 100") is True
|
||
assert has_city_token("каменск уральский улица мира 7") is True
|
||
|
||
|
||
def test_has_city_token_bare_street_is_false():
|
||
# The collision-prone case: a bare street+number carries no city token.
|
||
assert has_city_token("улица ленина 100") is False
|
||
assert has_city_token("уральская улица 62к1") is False
|
||
|
||
|
||
def test_has_city_token_no_false_positive_on_street_named_after_city():
|
||
# 'улица серова' must NOT match the city token 'серов' (word-bounded, not substring).
|
||
assert has_city_token("улица серова 5") is False
|
||
assert has_city_token("первоуральская улица 3") is False
|
||
|
||
|
||
def test_has_city_token_handles_empty():
|
||
assert has_city_token("") is False
|
||
assert has_city_token(None) is False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# match_or_create_house — mock DB tier routing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _make_db(rows: list[dict | None]) -> MagicMock:
|
||
"""Build a Session mock that returns rows sequentially per execute() call."""
|
||
db = MagicMock()
|
||
side_effects = []
|
||
for row in rows:
|
||
result = MagicMock()
|
||
mappings = MagicMock()
|
||
if row is None:
|
||
mappings.first.return_value = None
|
||
mappings.one.return_value = None
|
||
else:
|
||
mappings.first.return_value = row
|
||
mappings.one.return_value = row
|
||
result.mappings.return_value = mappings
|
||
side_effects.append(result)
|
||
db.execute.side_effect = side_effects
|
||
return db
|
||
|
||
|
||
def test_match_house_tier0_cadastr():
|
||
"""Tier 0: cadastral_number match → returns cadastr_exact without geo query."""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
{"id": 42}, # cadastral match
|
||
None, # _insert_alias (added by #1543: register fp/addr alias after cadastr_exact)
|
||
None, # _upsert_house_source (INSERT ... ON CONFLICT)
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db,
|
||
"cian",
|
||
"ext-001",
|
||
address="ул Ленина 5",
|
||
lat=56.8,
|
||
lon=60.5,
|
||
cadastral_number="66:41:0000000:999",
|
||
)
|
||
assert house_id == 42
|
||
assert conf == 1.0
|
||
assert method == "cadastr_exact"
|
||
|
||
|
||
def test_match_house_tier05_fias_exact():
|
||
"""Tier 0.5: house_fias_id match → fias_exact (conf 0.95) before source/fp/geo.
|
||
|
||
No cadastral_number supplied → Tier 0 skipped; the fias SELECT is the first
|
||
lookup after the advisory lock. On hit, _upsert_house_source + _insert_alias
|
||
fire (address carries a house number so the alias is registered).
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
{"id": 55}, # fias_exact hit (Tier 0.5)
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db,
|
||
"avito",
|
||
"ext-fias-1",
|
||
address="ул Ленина 5",
|
||
lat=56.8,
|
||
lon=60.5,
|
||
house_fias_id="0a1b2c3d-0000-4000-8000-000000000001",
|
||
)
|
||
assert house_id == 55
|
||
assert conf == 0.95
|
||
assert method == "fias_exact"
|
||
|
||
|
||
def test_match_house_fias_skipped_when_absent():
|
||
"""No house_fias_id → Tier 0.5 does not run; first lookup is house_sources."""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
{"house_id": 8}, # house_sources hit (fias tier skipped, no fias SELECT)
|
||
]
|
||
)
|
||
house_id, _conf, method = match_or_create_house(
|
||
db,
|
||
"avito",
|
||
"ext-nofias",
|
||
address="пр Мира 10",
|
||
lat=56.8,
|
||
lon=60.5,
|
||
)
|
||
assert house_id == 8
|
||
assert method == "source_exact"
|
||
|
||
|
||
def test_match_house_tier1_source_exact():
|
||
"""Tier 1: ext_source+ext_id already in house_sources.
|
||
|
||
No cadastral_number supplied → Tier 0 skipped entirely.
|
||
First db.execute call goes to house_sources query.
|
||
source_exact returns immediately without calling _upsert_house_source.
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
{"house_id": 7}, # house_sources hit
|
||
]
|
||
)
|
||
house_id, _conf, method = match_or_create_house(
|
||
db,
|
||
"avito",
|
||
"ext-999",
|
||
address="пр Мира 10",
|
||
lat=56.8,
|
||
lon=60.5,
|
||
)
|
||
assert house_id == 7
|
||
assert method == "source_exact"
|
||
|
||
|
||
def test_match_house_tier2_fingerprint():
|
||
"""Tier 2a: fingerprint match in house_address_aliases (exact fp hit).
|
||
|
||
After Tier 2a hit, _insert_alias is called to ensure fp is kept in sync
|
||
(#849 Part 2 — alias sync).
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
{"house_id": 15}, # fingerprint hit (Tier 2a)
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias (sync fingerprint — Part 2)
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db,
|
||
"yandex",
|
||
"ext-123",
|
||
address="Тестовая 1",
|
||
lat=56.84,
|
||
lon=60.60,
|
||
)
|
||
assert house_id == 15
|
||
assert conf == 0.9
|
||
assert method == "fingerprint"
|
||
|
||
|
||
def test_match_house_tier3_geo():
|
||
"""Tier 3: geo-proximity match.
|
||
|
||
Tier 2b (normalized_address fallback) is tried after Tier 2a miss.
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a)
|
||
None, # normalized_address miss (Tier 2b — Part 2)
|
||
{"id": 22, "dist": 15.5}, # geo hit (Tier 3)
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db,
|
||
"n1",
|
||
"ext-456",
|
||
address="Новая 3",
|
||
lat=56.83,
|
||
lon=60.59,
|
||
)
|
||
assert house_id == 22
|
||
assert conf == 0.7
|
||
assert method == "geo_proximity"
|
||
|
||
|
||
def test_match_house_new():
|
||
"""All tiers miss → new house created.
|
||
|
||
Tier 2b (normalized_address fallback) is tried after Tier 2a miss.
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a)
|
||
None, # normalized_address miss (Tier 2b — Part 2)
|
||
None, # geo miss
|
||
{"id": 99}, # INSERT RETURNING id
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db,
|
||
"avito",
|
||
"new-ext",
|
||
address="Новостройка 1",
|
||
lat=56.85,
|
||
lon=60.61,
|
||
)
|
||
assert house_id == 99
|
||
assert conf == 1.0
|
||
assert method == "new"
|
||
|
||
|
||
def test_match_house_advisory_lock_called_first():
|
||
"""Regression: pg_advisory_xact_lock must be the first execute() call
|
||
on match_or_create_house. Re-ordering would re-introduce the race
|
||
(finding #1 from 2026-05-24 audit). PR #501."""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
{"house_id": 7}, # house_sources Tier 1 hit (short path)
|
||
]
|
||
)
|
||
match_or_create_house(
|
||
db,
|
||
"avito",
|
||
"regress-001",
|
||
address="Тестовая 1",
|
||
lat=56.8,
|
||
lon=60.5,
|
||
)
|
||
|
||
# First execute MUST be the lock acquisition, with bind {'fp': <hex>}.
|
||
first_call = db.execute.call_args_list[0]
|
||
sql_obj = first_call[0][0] # TextClause
|
||
bind = first_call[0][1] # dict
|
||
|
||
assert "pg_advisory_xact_lock" in str(
|
||
sql_obj
|
||
), f"first execute must be advisory lock, got: {sql_obj}"
|
||
assert "fp" in bind, f"lock bind must include fp, got: {bind}"
|
||
assert (
|
||
isinstance(bind["fp"], str) and len(bind["fp"]) == 32
|
||
), f"fp must be 32-char sha256 hex, got: {bind.get('fp')!r}"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Mis-bucketing hardening — bare-street + geo address-consistency guards
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _executed_sqls(db: MagicMock) -> list[str]:
|
||
"""Collapsed SQL text of every db.execute() call (whitespace-normalized)."""
|
||
out = []
|
||
for call in db.execute.call_args_list:
|
||
if call.args:
|
||
out.append(" ".join(str(call.args[0]).split()))
|
||
return out
|
||
|
||
|
||
def test_geo_match_does_not_register_alias():
|
||
"""P3: a geo-proximity match must NOT INSERT a house_address_aliases row.
|
||
|
||
A 0.7-confidence proximity hit is too weak to cement as a building key.
|
||
Execute order: lock, source miss, fp miss, norm_addr miss, geo hit, upsert.
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a)
|
||
None, # normalized_address miss (Tier 2b)
|
||
{"id": 22, "dist": 12.0, "h_addr": "улица новая 3"}, # geo hit, same number
|
||
None, # _upsert_house_source
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db, "n1", "ext-geo", address="улица Новая 3", lat=56.83, lon=60.59
|
||
)
|
||
assert (house_id, conf, method) == (22, 0.7, "geo_proximity")
|
||
assert not any(
|
||
"INSERT INTO house_address_aliases" in s for s in _executed_sqls(db)
|
||
), "geo match must not write an alias (P3)"
|
||
|
||
|
||
def test_geo_match_rejected_when_house_number_differs():
|
||
"""P2: geo candidate with a DIFFERENT house number is rejected → New house.
|
||
|
||
Listing 'Мамина Сибиряка 126' must not bucket into nearby house '... 52'.
|
||
On rejection the flow falls through to the New-house INSERT.
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a)
|
||
None, # normalized_address miss (Tier 2b)
|
||
{"id": 99, "dist": 5.0, "h_addr": "улица мамина сибиряка 52"}, # geo hit, WRONG number
|
||
{"id": 500}, # INSERT RETURNING id (New house)
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias (address has number → executes)
|
||
]
|
||
)
|
||
house_id, _conf, method = match_or_create_house(
|
||
db, "yandex", "ext-rej", address="улица Мамина-Сибиряка 126", lat=56.84, lon=60.61
|
||
)
|
||
assert method == "new", "geo candidate with different house number must be rejected"
|
||
assert house_id == 500
|
||
|
||
|
||
def test_bare_street_numberless_no_house_created():
|
||
"""P1 (extended to INSERT-new): a numberless address with no cadastral → NO house.
|
||
|
||
'екатеринбург улица мамина сибиряка' has no house number and no cadastral number,
|
||
so match_or_create_house refuses to MINT a canonical row (which becomes a mega-bucket
|
||
that later numberless listings collapse into). Returns (None, 0.0, 'no_house_number')
|
||
and writes nothing: no Tier 2b lookup, no INSERT INTO houses, no alias, no house_sources.
|
||
Execute order: lock, source miss, fp miss — then the gate returns.
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a)
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db, "yandex", "ext-bare", address="екатеринбург улица мамина сибиряка"
|
||
)
|
||
assert (house_id, conf, method) == (None, 0.0, "no_house_number")
|
||
sqls = _executed_sqls(db)
|
||
assert not any(
|
||
"normalized_address = :na" in s for s in sqls
|
||
), "bare-street address must not run Tier 2b normalized_address lookup (P1)"
|
||
assert not any(
|
||
"INSERT INTO houses" in s for s in sqls
|
||
), "numberless address must not create a house (P1 extended)"
|
||
assert not any(
|
||
"INSERT INTO house_address_aliases" in s for s in sqls
|
||
), "numberless address must not register an alias (P1)"
|
||
assert not any(
|
||
"INSERT INTO house_sources" in s for s in sqls
|
||
), "numberless refusal must not upsert house_sources"
|
||
|
||
|
||
def test_numberless_none_address_with_coords_no_house_created():
|
||
"""P1 edge: address=None but lat/lon present → still gated (no address-NULL house).
|
||
|
||
Previously the New-house INSERT would mint a house with address NULL keyed only by
|
||
coords — exactly the mis-bucket the gate refuses. has_num is False for an empty
|
||
normalized address, so with no cadastral the flow returns 'no_house_number'.
|
||
Execute order: lock, source miss, fp miss, geo miss — then the gate returns.
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a); Tier 2b skipped (empty norm_addr)
|
||
None, # geo miss (Tier 3 — lat/lon present)
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db, "yandex", "ext-none-addr", address=None, lat=56.83, lon=60.59
|
||
)
|
||
assert (house_id, conf, method) == (None, 0.0, "no_house_number")
|
||
assert not any(
|
||
"INSERT INTO houses" in s for s in _executed_sqls(db)
|
||
), "address-NULL coords-only listing must not create a house (P1 extended)"
|
||
|
||
|
||
def test_numberless_address_with_cadastral_creates_house():
|
||
"""P1 exemption: a numberless address WITH a cadastral number still creates a house.
|
||
|
||
The cadastral number is a precise building identity, so the numberless gate does not
|
||
apply. Tier 0 cadastr lookup misses (new cad), remaining tiers miss, and the New-house
|
||
INSERT proceeds because `cad` is truthy.
|
||
Execute order: lock, cadastr miss, source miss, fp miss, INSERT, upsert
|
||
(_insert_alias is a no-op — bare-street normalized_address has no number).
|
||
"""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # Tier 0 cadastr miss
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a; no lat/lon → no Tier 2b/geo)
|
||
{"id": 321}, # INSERT RETURNING id (New house — cad exempts the gate)
|
||
None, # _upsert_house_source
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db,
|
||
"cian",
|
||
"ext-cad",
|
||
address="екатеринбург улица мамина сибиряка",
|
||
cadastral_number="66:41:0000000:12345",
|
||
)
|
||
assert (house_id, conf, method) == (321, 1.0, "new")
|
||
assert any(
|
||
"INSERT INTO houses" in s for s in _executed_sqls(db)
|
||
), "numberless + cadastral must still create a house (cadastral = identity)"
|
||
|
||
|
||
def test_insert_alias_noop_for_bare_street():
|
||
"""_insert_alias is a no-op when the normalized address has no house number."""
|
||
from app.services.matching.houses import _insert_alias
|
||
|
||
db = MagicMock()
|
||
_insert_alias(
|
||
db,
|
||
house_id=1,
|
||
address="екатеринбург улица мамина сибиряка",
|
||
fp="a" * 32,
|
||
source="yandex",
|
||
)
|
||
db.execute.assert_not_called()
|
||
|
||
|
||
def test_insert_alias_executes_when_house_number_present():
|
||
"""_insert_alias still writes when the normalized address carries a house number."""
|
||
from app.services.matching.houses import _insert_alias
|
||
|
||
db = MagicMock()
|
||
_insert_alias(
|
||
db,
|
||
house_id=1,
|
||
address="улица Мамина-Сибиряка 126",
|
||
fp="b" * 32,
|
||
source="yandex",
|
||
)
|
||
db.execute.assert_called_once()
|
||
|
||
|
||
def test_insert_alias_conflict_only_rebinds_same_house():
|
||
"""Oblast guard (#2): ON CONFLICT refreshes fingerprint/source ONLY for the same
|
||
house. A different house colliding on a bare common-street normalized_address must
|
||
not rewrite the owner's fingerprint (that would downgrade a coord-bearing alias to a
|
||
coord-less one and let the next no-coord card mis-bucket via Tier 2a)."""
|
||
from app.services.matching.houses import _insert_alias
|
||
|
||
db = MagicMock()
|
||
_insert_alias(db, house_id=1, address="улица Ленина 100", fp="c" * 32, source="avito")
|
||
sql = " ".join(str(db.execute.call_args.args[0]).split())
|
||
assert "ON CONFLICT (normalized_address) DO UPDATE" in sql
|
||
# fingerprint/source updated only when the existing owner == the incoming house.
|
||
assert "house_address_aliases.house_id = EXCLUDED.house_id" in sql
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# match_or_create_house — Tier-2b oblast geo/city guard (bug #2)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_tier2b_coord_present_uses_geo_guard_and_matches_within_radius():
|
||
"""Coords present → Tier-2b runs the ST_DWithin JOIN guard; a house within the
|
||
radius is accepted as a fingerprint match (0.9)."""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss (Tier 1)
|
||
None, # fingerprint miss (Tier 2a)
|
||
{"house_id": 77}, # Tier 2b JOIN+ST_DWithin hit (same place, within radius)
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db, "avito", "ext-2b-geo", address="улица Ленина 100", lat=57.910, lon=59.980
|
||
)
|
||
assert (house_id, conf, method) == (77, 0.9, "fingerprint")
|
||
# The Tier-2b lookup must be the geo-guarded JOIN, not a bare normalized_address SELECT.
|
||
tier2b_sql = _executed_sqls(db)[3]
|
||
assert "ST_DWithin" in tier2b_sql
|
||
assert "JOIN houses" in tier2b_sql
|
||
|
||
|
||
def test_tier2b_coord_present_cross_city_falls_through_to_new():
|
||
"""A bare street+number oblast card WITH coords whose only same-street alias is a
|
||
far-away (cross-city) house: the ST_DWithin guard filters it out (JOIN miss) → the
|
||
card falls through to a New house instead of mis-bucketing into the ЕКБ house."""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a)
|
||
None, # Tier 2b JOIN+ST_DWithin miss (candidate house is >3km away)
|
||
None, # geo miss (Tier 3 — coords present)
|
||
{"id": 909}, # INSERT RETURNING id (New house)
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db, "avito", "ext-2b-xcity", address="улица Ленина 100", lat=57.910, lon=59.980
|
||
)
|
||
assert (house_id, conf, method) == (909, 1.0, "new")
|
||
|
||
|
||
def test_tier2b_no_coords_no_city_token_skips_and_creates_new():
|
||
"""No coords AND a bare common-street (no city token) → Tier-2b is SKIPPED entirely
|
||
(no normalized_address lookup). The card falls through to a New house — conservative:
|
||
do not mis-bucket a bare 'улица ленина 100' into a same-named ЕКБ alias."""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a); Tier 2b SKIPPED (no coords, no city token)
|
||
{"id": 910}, # INSERT RETURNING id (New house) — no Tier 3 (no coords)
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db, "avito", "ext-2b-bare", address="улица Ленина 100"
|
||
)
|
||
assert (house_id, conf, method) == (910, 1.0, "new")
|
||
assert not any(
|
||
"normalized_address = :na" in s for s in _executed_sqls(db)
|
||
), "bare common-street with no coords/city must not run a Tier-2b lookup"
|
||
|
||
|
||
def test_tier2b_no_coords_with_city_token_matches():
|
||
"""No coords but the normalized_address carries a city token → self-disambiguating,
|
||
so Tier-2b matches on normalized_address alone (0.9). ЕКБ cross-source dedup for
|
||
city-qualified addresses is preserved."""
|
||
from app.services.matching.houses import match_or_create_house
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # pg_advisory_xact_lock
|
||
None, # house_sources miss
|
||
None, # fingerprint miss (Tier 2a)
|
||
{"house_id": 33}, # Tier 2b normalized_address hit (city token present)
|
||
None, # _upsert_house_source
|
||
None, # _insert_alias
|
||
]
|
||
)
|
||
house_id, conf, method = match_or_create_house(
|
||
db, "avito", "ext-2b-city", address="Екатеринбург, улица Ленина, 100"
|
||
)
|
||
assert (house_id, conf, method) == (33, 0.9, "fingerprint")
|
||
tier2b_sql = _executed_sqls(db)[3]
|
||
assert "normalized_address = :na" in tier2b_sql
|
||
assert "ST_DWithin" not in tier2b_sql
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# match_or_create_listing — mock DB tier routing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_match_listing_tier0_cadastr():
|
||
from app.services.matching.listings import match_or_create_listing
|
||
|
||
db = _make_db(
|
||
[
|
||
{"id": 200}, # cadastral hit
|
||
None, # _upsert_listing_source
|
||
]
|
||
)
|
||
listing_id, _conf, method = match_or_create_listing(
|
||
db,
|
||
"cian",
|
||
"cian-555",
|
||
house_id=10,
|
||
floor=5,
|
||
rooms_count=2,
|
||
area_m2=48.5,
|
||
price_rub=5_000_000.0,
|
||
cadastral_number="66:41:0204016:1234",
|
||
)
|
||
assert listing_id == 200
|
||
assert method == "cadastr_exact"
|
||
|
||
|
||
def test_match_listing_tier1_source_exact():
|
||
"""Tier 1: no cadastral_number → Tier 0 skipped; first execute is listing_sources hit."""
|
||
from app.services.matching.listings import match_or_create_listing
|
||
|
||
db = _make_db(
|
||
[
|
||
{"listing_id": 101}, # listing_sources hit (first and only execute call)
|
||
]
|
||
)
|
||
listing_id, _conf, method = match_or_create_listing(
|
||
db,
|
||
"avito",
|
||
"avito-888",
|
||
house_id=10,
|
||
)
|
||
assert listing_id == 101
|
||
assert method == "source_exact"
|
||
|
||
|
||
def test_match_listing_tier2_minhash():
|
||
from app.services.matching.listings import match_or_create_listing
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # listing_sources miss
|
||
{"id": 55}, # minhash hit
|
||
None, # _upsert_listing_source
|
||
]
|
||
)
|
||
listing_id, conf, method = match_or_create_listing(
|
||
db,
|
||
"cian",
|
||
"cian-999",
|
||
house_id=10,
|
||
description_minhash="abc123deadbeef",
|
||
)
|
||
assert listing_id == 55
|
||
assert conf == 0.85
|
||
assert method == "minhash"
|
||
|
||
|
||
def test_match_listing_tier3_composite():
|
||
"""Tier 3: no cadastral + no minhash provided → tiers 0/2 skipped.
|
||
Execute order: listing_sources miss → composite hit → _upsert.
|
||
"""
|
||
from app.services.matching.listings import match_or_create_listing
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # listing_sources miss (Tier 1)
|
||
{"id": 77, "diff": 0.1}, # composite hit (Tier 3; Tier 2 skipped — no hash)
|
||
None, # _upsert_listing_source
|
||
]
|
||
)
|
||
listing_id, conf, method = match_or_create_listing(
|
||
db,
|
||
"cian",
|
||
"cian-777",
|
||
house_id=10,
|
||
floor=3,
|
||
rooms_count=1,
|
||
area_m2=36.0,
|
||
)
|
||
assert listing_id == 77
|
||
assert conf == 0.75
|
||
assert method == "composite"
|
||
|
||
|
||
def test_match_listing_new():
|
||
from app.services.matching.listings import match_or_create_listing
|
||
|
||
db = _make_db(
|
||
[
|
||
None, # listing_sources miss
|
||
None, # composite miss (floor/area/rooms all None → tier skipped)
|
||
]
|
||
)
|
||
listing_id, _conf, method = match_or_create_listing(
|
||
db,
|
||
"avito",
|
||
"avito-new",
|
||
house_id=10,
|
||
)
|
||
assert listing_id == 0
|
||
assert method == "new"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# conflict_resolution
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_field_priority_dicts_not_empty():
|
||
assert HOUSE_FIELD_PRIORITY
|
||
assert LISTING_FIELD_PRIORITY
|
||
|
||
|
||
VALID_STRING_RULES = {"union", "cross_validate", "first_non_null"}
|
||
|
||
|
||
def test_field_priority_sources_are_lists_or_known_string_rules():
|
||
"""Each priority entry is either a source list or a known string rule.
|
||
|
||
PR #471 (main) introduced 'union' / 'cross_validate' / 'first_non_null' string
|
||
rules alongside list entries. Both are valid per LISTING_FIELD_PRIORITY type hint.
|
||
"""
|
||
for col, sources in HOUSE_FIELD_PRIORITY.items():
|
||
assert (
|
||
isinstance(sources, list) or sources in VALID_STRING_RULES
|
||
), f"HOUSE_FIELD_PRIORITY[{col!r}] must be list or known rule, got {sources!r}"
|
||
for col, sources in LISTING_FIELD_PRIORITY.items():
|
||
assert (
|
||
isinstance(sources, list) or sources in VALID_STRING_RULES
|
||
), f"LISTING_FIELD_PRIORITY[{col!r}] must be list or known rule, got {sources!r}"
|
||
|
||
|
||
def test_update_canonical_fields_is_callable():
|
||
"""Stage 8 v1 — legacy stub (no-op pass). Full arbitration deferred to Stage 8.x.
|
||
|
||
PR #471 (main) replaced NotImplementedError stub with silent pass to allow
|
||
backward-compat imports. Test updated accordingly.
|
||
"""
|
||
db = MagicMock()
|
||
# Should not raise — no-op stub returns None
|
||
result = update_canonical_fields(db, listing_id=1, ext_source="cian", lot_data=object())
|
||
assert result is None
|