diff --git a/tradein-mvp/backend/app/services/matching/houses.py b/tradein-mvp/backend/app/services/matching/houses.py index 2322dbda..2b130d29 100644 --- a/tradein-mvp/backend/app/services/matching/houses.py +++ b/tradein-mvp/backend/app/services/matching/houses.py @@ -16,11 +16,13 @@ from sqlalchemy import text from sqlalchemy.orm import Session from app.services.matching.normalize import ( + EKB_CITY_TOKEN, address_fingerprint, has_city_token, has_house_number, house_number_token, normalize_address, + resolve_city_token, ) logger = logging.getLogger(__name__) @@ -189,15 +191,45 @@ def match_or_create_house( # Without 2b, two scrapers for the same house with slightly different lat/lon (beyond # the 4-decimal rounding tolerance) would produce distinct fingerprints, miss Tier 2a, # and each potentially create a duplicate house row. - row = ( - db.execute( - text("SELECT house_id FROM house_address_aliases " "WHERE fingerprint = :fp LIMIT 1"), - {"fp": fp}, + # + # Tier-2a/2b oblast city guard (#2500 follow-up, closes the Tier-2a mis-bucket that the + # #2500 Tier-2b geo-guard could not reach): a coord-less card whose address explicitly + # resolves a NON-ЕКБ обл.66 city must NOT match a globally-unique alias (fingerprint OR + # normalized_address) that almost certainly belongs to ЕКБ/another city — that would + # corrupt ЕКБ house data. When it fires we skip both alias lookups and fall through to a + # New house (Tier 3 is coord-gated, so coord-less cards skip it too). + # + # This PREVENTS ЕКБ CORRUPTION. Full oblast-internal dedup (deduping two listings of the + # SAME oblast building) still needs city-keyed aliases — a separate follow-up, out of + # scope, only relevant once the oblast sweep is enabled. + # + # EKB happy-path is byte-identical: the guard fires ONLY when the address names a non-ЕКБ + # city AND no coords disambiguate. ЕКБ cards (resolved city = екатеринбург) and the + # dominant bare/city-less Avito coord-less cards (resolved city None) run Tier-2a/2b + # exactly as before. NB: a BARE oblast card (no city token in the address — today's Avito + # SERP format) carries no signal here and is deliberately left on the unchanged path; that + # residual needs sweep-context and is out of this fix's scope. + _resolved_city = resolve_city_token(norm_addr) if (lat is None and lon is None) else None + _skip_oblast_alias = _resolved_city is not None and _resolved_city != EKB_CITY_TOKEN + if _skip_oblast_alias: + logger.info( + "house tier2a/2b skip: coord-less non-ЕКБ city %r na=%r src=%s", + _resolved_city, + norm_addr, + ext_source, ) - .mappings() - .first() - ) - if row is None and norm_addr and has_num: + + row = None + if not _skip_oblast_alias: + row = ( + db.execute( + text("SELECT house_id FROM house_address_aliases WHERE fingerprint = :fp LIMIT 1"), + {"fp": fp}, + ) + .mappings() + .first() + ) + if row is None and norm_addr and has_num and not _skip_oblast_alias: # Tier 2b: same normalized address, possibly different coords fingerprint. # Gated on has_num (P1): never match a bare-street normalized_address — any # numberless listing would otherwise collapse into whichever house first diff --git a/tradein-mvp/backend/app/services/matching/normalize.py b/tradein-mvp/backend/app/services/matching/normalize.py index f1ac4099..a7b9c9bd 100644 --- a/tradein-mvp/backend/app/services/matching/normalize.py +++ b/tradein-mvp/backend/app/services/matching/normalize.py @@ -112,12 +112,17 @@ def house_number_token(normalized: str | None) -> str | None: return m.group(1) if m else None +# Екатеринбург — the home city; every other _CITY_TOKENS entry is a non-ЕКБ обл.66 city. +# The Tier-2a oblast guard treats a resolved non-ЕКБ city specially (see resolve_city_token). +EKB_CITY_TOKEN = "екатеринбург" + # Cities covered by the обл.66 sweep rollout (ЕКБ + oblast per-city schedules). # Normalized form: lowercase, hyphens collapsed to spaces (mirrors normalize_address, # e.g. 'каменск-уральский' -> 'каменск уральский'). Mirrors CITY_ANCHORS in # scraper_kit.orchestration.pipeline — extend together as the sweep adds cities. +# EKB_CITY_TOKEN must be first (leftmost-alternation determinism / documentation). _CITY_TOKENS: tuple[str, ...] = ( - "екатеринбург", + EKB_CITY_TOKEN, "нижний тагил", "каменск уральский", "первоуральск", @@ -131,6 +136,25 @@ _CITY_TOKEN_RE = re.compile( ) +def resolve_city_token(normalized: str | None) -> str | None: + """Return the обл.66 city token present in the normalized address, else None. + + Token-bounded (see _CITY_TOKEN_RE) so a street named after a city never matches + ('улица серова 5' -> None, not 'серов'). When the address names ЕКБ, returns + EKB_CITY_TOKEN; a non-ЕКБ oblast city returns its own token. Used by the Tier-2a + oblast guard in match_or_create_house: a coord-less card that resolves a NON-ЕКБ city + must not fingerprint/normalized_address-match a globally-unique alias that (almost + certainly) belongs to ЕКБ/another city — that would corrupt ЕКБ house data. + """ + if not normalized: + return None + m = _CITY_TOKEN_RE.search(normalized) + if m is None: + return None + # group(0) carries the (?:^|\s)...(?:\s|$) boundary whitespace — strip it to the token. + return m.group(0).strip() + + def has_city_token(normalized: str | None) -> bool: """True if the normalized address carries a known обл.66 city token. @@ -141,9 +165,7 @@ def has_city_token(normalized: str | None) -> bool: globally-unique normalized_address alias (it would mis-bucket an oblast card into a same-named ЕКБ house — bug #2 oblast rollout). """ - if not normalized: - return False - return _CITY_TOKEN_RE.search(normalized) is not None + return resolve_city_token(normalized) is not None def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str: diff --git a/tradein-mvp/backend/tests/test_matching.py b/tradein-mvp/backend/tests/test_matching.py index 1b5d524b..622e2732 100644 --- a/tradein-mvp/backend/tests/test_matching.py +++ b/tradein-mvp/backend/tests/test_matching.py @@ -16,11 +16,13 @@ from app.services.matching.conflict_resolution import ( update_canonical_fields, ) from app.services.matching.normalize import ( + EKB_CITY_TOKEN, address_fingerprint, has_city_token, has_house_number, house_number_token, normalize_address, + resolve_city_token, ) # --------------------------------------------------------------------------- @@ -200,6 +202,37 @@ def test_has_city_token_handles_empty(): assert has_city_token(None) is False +# --------------------------------------------------------------------------- +# resolve_city_token — which обл.66 city (or None); drives the Tier-2a oblast guard +# --------------------------------------------------------------------------- + + +def test_resolve_city_token_returns_ekb(): + assert resolve_city_token("екатеринбург улица ленина 5") == EKB_CITY_TOKEN + + +def test_resolve_city_token_returns_non_ekb_city(): + assert resolve_city_token("нижний тагил улица ленина 100") == "нижний тагил" + assert resolve_city_token("свердловская область каменск уральский улица мира 7") == ( + "каменск уральский" + ) + + +def test_resolve_city_token_bare_street_is_none(): + # The dominant Avito SERP format — no city token → None (guard stays dormant). + assert resolve_city_token("улица ленина 100") is None + + +def test_resolve_city_token_no_false_positive_on_street_named_after_city(): + assert resolve_city_token("улица серова 5") is None + assert resolve_city_token("первоуральская улица 3") is None + + +def test_resolve_city_token_handles_empty(): + assert resolve_city_token("") is None + assert resolve_city_token(None) is None + + # --------------------------------------------------------------------------- # match_or_create_house — mock DB tier routing # --------------------------------------------------------------------------- @@ -773,6 +806,85 @@ def test_tier2b_no_coords_with_city_token_matches(): assert "ST_DWithin" not in tier2b_sql +# --------------------------------------------------------------------------- +# match_or_create_house — Tier-2a coord-less oblast city guard (#2500 follow-up) +# --------------------------------------------------------------------------- + + +def test_tier2a_coord_less_non_ekb_city_skips_both_lookups_and_creates_new(): + """A coord-less card whose address resolves a NON-ЕКБ city (Н.Тагил) must NOT match a + globally-unique alias by fingerprint OR normalized_address (either would corrupt ЕКБ + house data). The guard skips both alias lookups → New house.""" + from app.services.matching.houses import match_or_create_house + + db = _make_db( + [ + None, # pg_advisory_xact_lock + None, # house_sources miss (Tier 1) + # Tier 2a (fingerprint) SKIPPED, Tier 2b (normalized_address) SKIPPED, + # Tier 3 SKIPPED (no coords) — guard forces fall-through to New. + {"id": 701}, # INSERT RETURNING id (New house) + None, # _upsert_house_source + None, # _insert_alias (address carries a house number → executes) + ] + ) + house_id, conf, method = match_or_create_house( + db, "avito", "ext-2a-nt", address="Нижний Тагил, ул. Ленина, 100" + ) + assert (house_id, conf, method) == (701, 1.0, "new") + sqls = _executed_sqls(db) + assert not any( + "fingerprint = :fp" in s for s in sqls + ), "coord-less non-ЕКБ card must NOT run the Tier-2a fingerprint lookup" + assert not any( + "normalized_address = :na" in s for s in sqls + ), "coord-less non-ЕКБ card must NOT run the Tier-2b normalized_address lookup" + + +def test_tier2a_coord_less_ekb_city_still_matches(): + """EKB happy-path: a coord-less card that resolves ЕКБ runs Tier-2a EXACTLY as before — + the guard does not fire (екатеринбург == EKB_CITY_TOKEN).""" + from app.services.matching.houses import match_or_create_house + + db = _make_db( + [ + None, # pg_advisory_xact_lock + None, # house_sources miss + {"house_id": 88}, # Tier 2a fingerprint HIT (guard did NOT skip it) + None, # _upsert_house_source + None, # _insert_alias + ] + ) + house_id, conf, method = match_or_create_house( + db, "avito", "ext-2a-ekb", address="Екатеринбург, ул. Ленина, 5" + ) + assert (house_id, conf, method) == (88, 0.9, "fingerprint") + assert any( + "fingerprint = :fp" in s for s in _executed_sqls(db) + ), "ЕКБ coord-less card must still run the Tier-2a fingerprint lookup" + + +def test_tier2a_coord_less_bare_street_still_runs_tier2a(): + """EKB happy-path (dominant): a coord-less BARE card (no city token — today's Avito SERP + format) resolves to None → guard dormant → Tier-2a fingerprint dedup UNCHANGED.""" + from app.services.matching.houses import match_or_create_house + + db = _make_db( + [ + None, # pg_advisory_xact_lock + None, # house_sources miss + {"house_id": 99}, # Tier 2a fingerprint HIT (bare card → guard did not skip) + None, # _upsert_house_source + None, # _insert_alias + ] + ) + house_id, conf, method = match_or_create_house( + db, "avito", "ext-2a-bare", address="ул. Ленина, 5" + ) + assert (house_id, conf, method) == (99, 0.9, "fingerprint") + assert any("fingerprint = :fp" in s for s in _executed_sqls(db)) + + # --------------------------------------------------------------------------- # match_or_create_listing — mock DB tier routing # ---------------------------------------------------------------------------