gendesign/tradein-mvp/backend/tests/test_matching.py
bot-backend 384c4e7ac3
All checks were successful
CI / changes (pull_request) Successful in 8s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
fix(tradein/matching): не сваливать листинги в чужой дом (P1-P3)
Три гарда в match_or_create_house (+ P2 в match_house_readonly):
- P1: не матчить/алиасить голую улицу (normalized_address без номера дома)
- P2: отклонять Tier-3 гео-кандидата, если номера домов различаются
- P3: не цементировать слабый гео-матч (0.7) как alias-ключ здания

Хелперы has_house_number/house_number_token (по хвостовому токену —
«улица 8 марта» это улица, не дом). Корень mis-bucketing: house 131237
стянул 8 разных Мамина-номеров + голую улицу. Юнит + поведенческие тесты,
603 matching/normalize/house зелёных. Аффектит только НОВЫЕ матчи;
чистка cemented-данных — отдельной миграцией (P4).
2026-06-29 00:06:13 +03:00

660 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Unit tests for cross-source matching service.
Covers:
- normalize_address(): abbreviation expansion, punctuation stripping, edge cases
- address_fingerprint(): stability on case variation, sensitivity to coords, None handling
- match_or_create_house() / match_or_create_listing(): mock-DB tier routing
Reference: decisions/Cross_Source_Matching_Strategy.md
"""
from unittest.mock import MagicMock
from app.services.matching.conflict_resolution import (
HOUSE_FIELD_PRIORITY,
LISTING_FIELD_PRIORITY,
update_canonical_fields,
)
from app.services.matching.normalize import (
address_fingerprint,
has_house_number,
house_number_token,
normalize_address,
)
# ---------------------------------------------------------------------------
# normalize_address
# ---------------------------------------------------------------------------
def test_normalize_strips_punctuation_and_collapses_whitespace():
assert normalize_address(" Test, Address! ") == "test address"
def test_normalize_expands_ul_abbreviation():
result = normalize_address("ул Ленина, 5")
assert "улица" in result
assert "5" in result
def test_normalize_expands_pr_abbreviation():
result = normalize_address("пр Мира 100")
assert "проспект" in result
assert "100" in result
def test_normalize_expands_prkt_abbreviation():
result = normalize_address("пр-кт Ленина 10")
assert "проспект" in result
# 'кт' must NOT remain as a stray token after expansion
assert "кт" not in result.split()
def test_normalize_prkt_no_leftover_kt():
"""пр-кт must expand to 'проспект' cleanly — no leftover 'кт' token."""
result = normalize_address("пр-кт Ленина 10")
tokens = result.split()
assert "проспект" in tokens
assert "пр" not in tokens
assert "кт" not in tokens
def test_normalize_handles_none():
assert normalize_address(None) == ""
def test_normalize_handles_empty_string():
assert normalize_address("") == ""
def test_normalize_lowercases():
assert normalize_address("ТЕСТ") == "тест"
def test_normalize_unicode_nfc():
# Composed vs decomposed е (U+0435 vs U+0435)
s1 = normalize_address("улица")
s2 = normalize_address("улица")
assert s1 == s2
# ---------------------------------------------------------------------------
# has_house_number / house_number_token (mis-bucketing hardening)
# ---------------------------------------------------------------------------
def test_has_house_number_true_when_trailing_number():
assert has_house_number("улица мамина сибиряка 126а") is True
def test_has_house_number_false_for_street_name_digit_only():
# 'улица 8 марта' — the digit is part of the street NAME, not a house number
assert has_house_number("улица 8 марта") is False
def test_has_house_number_true_for_street_name_digit_plus_house():
# 'улица 8 марта 5' — '5' is the house number, '8' is in the name
assert has_house_number("улица 8 марта 5") is True
def test_has_house_number_false_for_bare_street():
assert has_house_number("улица мамина сибиряка") is False
def test_has_house_number_false_for_none_and_empty():
assert has_house_number(None) is False
assert has_house_number("") is False
def test_house_number_token_extracts_trailing_token():
assert house_number_token("улица мамина сибиряка 126а") == "126а"
def test_house_number_token_for_street_name_digit_plus_house():
assert house_number_token("улица 8 марта 5") == "5"
def test_house_number_token_none_for_bare_street():
assert house_number_token("мамина сибиряка") is None
def test_house_number_token_none_for_none():
assert house_number_token(None) is None
# ---------------------------------------------------------------------------
# address_fingerprint
# ---------------------------------------------------------------------------
def test_fingerprint_stable_on_case_variation():
f1 = address_fingerprint("Ленина 5", 56.8378, 60.5946)
f2 = address_fingerprint("ЛЕНИНА 5", 56.8378, 60.5946)
assert f1 == f2
def test_fingerprint_distinct_on_different_lat():
f1 = address_fingerprint("Same Address", 56.8, 60.5)
f2 = address_fingerprint("Same Address", 56.9, 60.5)
assert f1 != f2
def test_fingerprint_distinct_on_different_lon():
f1 = address_fingerprint("Same Address", 56.8, 60.5)
f2 = address_fingerprint("Same Address", 56.8, 60.6)
assert f1 != f2
def test_fingerprint_stable_within_11m(monkeypatch):
# 4 decimal places → ~11 m resolution; coords differing in 5th decimal should match
f1 = address_fingerprint("Addr", 56.83780, 60.59460)
f2 = address_fingerprint("Addr", 56.83781, 60.59461)
# 4dp rounding: 56.8378 == 56.8378, 60.5946 == 60.5946 → same fingerprint
assert f1 == f2
def test_fingerprint_handles_none_coords():
fp = address_fingerprint("Some Address", None, None)
assert len(fp) == 32 # truncated sha256 hex
def test_fingerprint_handles_none_address():
fp = address_fingerprint(None, 56.8, 60.5)
assert len(fp) == 32
def test_fingerprint_all_none():
fp = address_fingerprint(None, None, None)
assert len(fp) == 32
# ---------------------------------------------------------------------------
# match_or_create_house — mock DB tier routing
# ---------------------------------------------------------------------------
def _make_db(rows: list[dict | None]) -> MagicMock:
"""Build a Session mock that returns rows sequentially per execute() call."""
db = MagicMock()
side_effects = []
for row in rows:
result = MagicMock()
mappings = MagicMock()
if row is None:
mappings.first.return_value = None
mappings.one.return_value = None
else:
mappings.first.return_value = row
mappings.one.return_value = row
result.mappings.return_value = mappings
side_effects.append(result)
db.execute.side_effect = side_effects
return db
def test_match_house_tier0_cadastr():
"""Tier 0: cadastral_number match → returns cadastr_exact without geo query."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
{"id": 42}, # cadastral match
None, # _insert_alias (added by #1543: register fp/addr alias after cadastr_exact)
None, # _upsert_house_source (INSERT ... ON CONFLICT)
]
)
house_id, conf, method = match_or_create_house(
db,
"cian",
"ext-001",
address="ул Ленина 5",
lat=56.8,
lon=60.5,
cadastral_number="66:41:0000000:999",
)
assert house_id == 42
assert conf == 1.0
assert method == "cadastr_exact"
def test_match_house_tier1_source_exact():
"""Tier 1: ext_source+ext_id already in house_sources.
No cadastral_number supplied → Tier 0 skipped entirely.
First db.execute call goes to house_sources query.
source_exact returns immediately without calling _upsert_house_source.
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
{"house_id": 7}, # house_sources hit
]
)
house_id, _conf, method = match_or_create_house(
db,
"avito",
"ext-999",
address="пр Мира 10",
lat=56.8,
lon=60.5,
)
assert house_id == 7
assert method == "source_exact"
def test_match_house_tier2_fingerprint():
"""Tier 2a: fingerprint match in house_address_aliases (exact fp hit).
After Tier 2a hit, _insert_alias is called to ensure fp is kept in sync
(#849 Part 2 — alias sync).
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
{"house_id": 15}, # fingerprint hit (Tier 2a)
None, # _upsert_house_source
None, # _insert_alias (sync fingerprint — Part 2)
]
)
house_id, conf, method = match_or_create_house(
db,
"yandex",
"ext-123",
address="Тестовая 1",
lat=56.84,
lon=60.60,
)
assert house_id == 15
assert conf == 0.9
assert method == "fingerprint"
def test_match_house_tier3_geo():
"""Tier 3: geo-proximity match.
Tier 2b (normalized_address fallback) is tried after Tier 2a miss.
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a)
None, # normalized_address miss (Tier 2b — Part 2)
{"id": 22, "dist": 15.5}, # geo hit (Tier 3)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db,
"n1",
"ext-456",
address="Новая 3",
lat=56.83,
lon=60.59,
)
assert house_id == 22
assert conf == 0.7
assert method == "geo_proximity"
def test_match_house_new():
"""All tiers miss → new house created.
Tier 2b (normalized_address fallback) is tried after Tier 2a miss.
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a)
None, # normalized_address miss (Tier 2b — Part 2)
None, # geo miss
{"id": 99}, # INSERT RETURNING id
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db,
"avito",
"new-ext",
address="Новостройка 1",
lat=56.85,
lon=60.61,
)
assert house_id == 99
assert conf == 1.0
assert method == "new"
def test_match_house_advisory_lock_called_first():
"""Regression: pg_advisory_xact_lock must be the first execute() call
on match_or_create_house. Re-ordering would re-introduce the race
(finding #1 from 2026-05-24 audit). PR #501."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
{"house_id": 7}, # house_sources Tier 1 hit (short path)
]
)
match_or_create_house(
db,
"avito",
"regress-001",
address="Тестовая 1",
lat=56.8,
lon=60.5,
)
# First execute MUST be the lock acquisition, with bind {'fp': <hex>}.
first_call = db.execute.call_args_list[0]
sql_obj = first_call[0][0] # TextClause
bind = first_call[0][1] # dict
assert "pg_advisory_xact_lock" in str(
sql_obj
), f"first execute must be advisory lock, got: {sql_obj}"
assert "fp" in bind, f"lock bind must include fp, got: {bind}"
assert (
isinstance(bind["fp"], str) and len(bind["fp"]) == 32
), f"fp must be 32-char sha256 hex, got: {bind.get('fp')!r}"
# ---------------------------------------------------------------------------
# Mis-bucketing hardening — bare-street + geo address-consistency guards
# ---------------------------------------------------------------------------
def _executed_sqls(db: MagicMock) -> list[str]:
"""Collapsed SQL text of every db.execute() call (whitespace-normalized)."""
out = []
for call in db.execute.call_args_list:
if call.args:
out.append(" ".join(str(call.args[0]).split()))
return out
def test_geo_match_does_not_register_alias():
"""P3: a geo-proximity match must NOT INSERT a house_address_aliases row.
A 0.7-confidence proximity hit is too weak to cement as a building key.
Execute order: lock, source miss, fp miss, norm_addr miss, geo hit, upsert.
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a)
None, # normalized_address miss (Tier 2b)
{"id": 22, "dist": 12.0, "h_addr": "улица новая 3"}, # geo hit, same number
None, # _upsert_house_source
]
)
house_id, conf, method = match_or_create_house(
db, "n1", "ext-geo", address="улица Новая 3", lat=56.83, lon=60.59
)
assert (house_id, conf, method) == (22, 0.7, "geo_proximity")
assert not any(
"INSERT INTO house_address_aliases" in s for s in _executed_sqls(db)
), "geo match must not write an alias (P3)"
def test_geo_match_rejected_when_house_number_differs():
"""P2: geo candidate with a DIFFERENT house number is rejected → New house.
Listing 'Мамина Сибиряка 126' must not bucket into nearby house '... 52'.
On rejection the flow falls through to the New-house INSERT.
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a)
None, # normalized_address miss (Tier 2b)
{"id": 99, "dist": 5.0, "h_addr": "улица мамина сибиряка 52"}, # geo hit, WRONG number
{"id": 500}, # INSERT RETURNING id (New house)
None, # _upsert_house_source
None, # _insert_alias (address has number → executes)
]
)
house_id, _conf, method = match_or_create_house(
db, "yandex", "ext-rej", address="улица Мамина-Сибиряка 126", lat=56.84, lon=60.61
)
assert method == "new", "geo candidate with different house number must be rejected"
assert house_id == 500
def test_bare_street_listing_skips_tier2b_and_does_not_alias():
"""P1: a numberless ('bare street') address must not match or register an alias.
'екатеринбург улица мамина сибиряка' has no house number → Tier 2b is skipped
and _insert_alias is a no-op, so it cannot mass-collapse into a wrong house.
Execute order: lock, source miss, fp miss, INSERT, upsert (no Tier 2b, no alias).
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a)
{"id": 700}, # INSERT RETURNING id (New house — Tier 2b skipped, no lat/lon)
None, # _upsert_house_source
]
)
house_id, _conf, method = match_or_create_house(
db, "yandex", "ext-bare", address="екатеринбург улица мамина сибиряка"
)
assert method == "new"
assert house_id == 700
sqls = _executed_sqls(db)
assert not any(
"normalized_address = :na" in s for s in sqls
), "bare-street address must not run Tier 2b normalized_address lookup (P1)"
assert not any(
"INSERT INTO house_address_aliases" in s for s in sqls
), "bare-street address must not register an alias (P1)"
def test_insert_alias_noop_for_bare_street():
"""_insert_alias is a no-op when the normalized address has no house number."""
from app.services.matching.houses import _insert_alias
db = MagicMock()
_insert_alias(
db,
house_id=1,
address="екатеринбург улица мамина сибиряка",
fp="a" * 32,
source="yandex",
)
db.execute.assert_not_called()
def test_insert_alias_executes_when_house_number_present():
"""_insert_alias still writes when the normalized address carries a house number."""
from app.services.matching.houses import _insert_alias
db = MagicMock()
_insert_alias(
db,
house_id=1,
address="улица Мамина-Сибиряка 126",
fp="b" * 32,
source="yandex",
)
db.execute.assert_called_once()
# ---------------------------------------------------------------------------
# match_or_create_listing — mock DB tier routing
# ---------------------------------------------------------------------------
def test_match_listing_tier0_cadastr():
from app.services.matching.listings import match_or_create_listing
db = _make_db(
[
{"id": 200}, # cadastral hit
None, # _upsert_listing_source
]
)
listing_id, _conf, method = match_or_create_listing(
db,
"cian",
"cian-555",
house_id=10,
floor=5,
rooms_count=2,
area_m2=48.5,
price_rub=5_000_000.0,
cadastral_number="66:41:0204016:1234",
)
assert listing_id == 200
assert method == "cadastr_exact"
def test_match_listing_tier1_source_exact():
"""Tier 1: no cadastral_number → Tier 0 skipped; first execute is listing_sources hit."""
from app.services.matching.listings import match_or_create_listing
db = _make_db(
[
{"listing_id": 101}, # listing_sources hit (first and only execute call)
]
)
listing_id, _conf, method = match_or_create_listing(
db,
"avito",
"avito-888",
house_id=10,
)
assert listing_id == 101
assert method == "source_exact"
def test_match_listing_tier2_minhash():
from app.services.matching.listings import match_or_create_listing
db = _make_db(
[
None, # listing_sources miss
{"id": 55}, # minhash hit
None, # _upsert_listing_source
]
)
listing_id, conf, method = match_or_create_listing(
db,
"cian",
"cian-999",
house_id=10,
description_minhash="abc123deadbeef",
)
assert listing_id == 55
assert conf == 0.85
assert method == "minhash"
def test_match_listing_tier3_composite():
"""Tier 3: no cadastral + no minhash provided → tiers 0/2 skipped.
Execute order: listing_sources miss → composite hit → _upsert.
"""
from app.services.matching.listings import match_or_create_listing
db = _make_db(
[
None, # listing_sources miss (Tier 1)
{"id": 77, "diff": 0.1}, # composite hit (Tier 3; Tier 2 skipped — no hash)
None, # _upsert_listing_source
]
)
listing_id, conf, method = match_or_create_listing(
db,
"cian",
"cian-777",
house_id=10,
floor=3,
rooms_count=1,
area_m2=36.0,
)
assert listing_id == 77
assert conf == 0.75
assert method == "composite"
def test_match_listing_new():
from app.services.matching.listings import match_or_create_listing
db = _make_db(
[
None, # listing_sources miss
None, # composite miss (floor/area/rooms all None → tier skipped)
]
)
listing_id, _conf, method = match_or_create_listing(
db,
"avito",
"avito-new",
house_id=10,
)
assert listing_id == 0
assert method == "new"
# ---------------------------------------------------------------------------
# conflict_resolution
# ---------------------------------------------------------------------------
def test_field_priority_dicts_not_empty():
assert HOUSE_FIELD_PRIORITY
assert LISTING_FIELD_PRIORITY
VALID_STRING_RULES = {"union", "cross_validate", "first_non_null"}
def test_field_priority_sources_are_lists_or_known_string_rules():
"""Each priority entry is either a source list or a known string rule.
PR #471 (main) introduced 'union' / 'cross_validate' / 'first_non_null' string
rules alongside list entries. Both are valid per LISTING_FIELD_PRIORITY type hint.
"""
for col, sources in HOUSE_FIELD_PRIORITY.items():
assert (
isinstance(sources, list) or sources in VALID_STRING_RULES
), f"HOUSE_FIELD_PRIORITY[{col!r}] must be list or known rule, got {sources!r}"
for col, sources in LISTING_FIELD_PRIORITY.items():
assert (
isinstance(sources, list) or sources in VALID_STRING_RULES
), f"LISTING_FIELD_PRIORITY[{col!r}] must be list or known rule, got {sources!r}"
def test_update_canonical_fields_is_callable():
"""Stage 8 v1 — legacy stub (no-op pass). Full arbitration deferred to Stage 8.x.
PR #471 (main) replaced NotImplementedError stub with silent pass to allow
backward-compat imports. Test updated accordingly.
"""
db = MagicMock()
# Should not raise — no-op stub returns None
result = update_canonical_fields(db, listing_id=1, ext_source="cian", lot_data=object())
assert result is None