Merge pull request 'fix(tradein/matching): Tier-2a city-guard для non-ЕКБ coord-less (п.2, defense-in-depth)' (#2502) from fix/tradein-oblast-tier2a-cityguard into main

tradein-mvp/backend/app/services/matching/houses.py

@@ -16,11 +16,13 @@ from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from app.services.matching.normalize import (
+    EKB_CITY_TOKEN,
     address_fingerprint,
     has_city_token,
     has_house_number,
     house_number_token,
     normalize_address,
+    resolve_city_token,
 )
 
 logger = logging.getLogger(__name__)
@@ -189,15 +191,45 @@ def match_or_create_house(
     # Without 2b, two scrapers for the same house with slightly different lat/lon (beyond
     # the 4-decimal rounding tolerance) would produce distinct fingerprints, miss Tier 2a,
     # and each potentially create a duplicate house row.
+    #
+    # Tier-2a/2b oblast city guard (#2500 follow-up, closes the Tier-2a mis-bucket that the
+    # #2500 Tier-2b geo-guard could not reach): a coord-less card whose address explicitly
+    # resolves a NON-ЕКБ обл.66 city must NOT match a globally-unique alias (fingerprint OR
+    # normalized_address) that almost certainly belongs to ЕКБ/another city — that would
+    # corrupt ЕКБ house data. When it fires we skip both alias lookups and fall through to a
+    # New house (Tier 3 is coord-gated, so coord-less cards skip it too).
+    #
+    # This PREVENTS ЕКБ CORRUPTION. Full oblast-internal dedup (deduping two listings of the
+    # SAME oblast building) still needs city-keyed aliases — a separate follow-up, out of
+    # scope, only relevant once the oblast sweep is enabled.
+    #
+    # EKB happy-path is byte-identical: the guard fires ONLY when the address names a non-ЕКБ
+    # city AND no coords disambiguate. ЕКБ cards (resolved city = екатеринбург) and the
+    # dominant bare/city-less Avito coord-less cards (resolved city None) run Tier-2a/2b
+    # exactly as before. NB: a BARE oblast card (no city token in the address — today's Avito
+    # SERP format) carries no signal here and is deliberately left on the unchanged path; that
+    # residual needs sweep-context and is out of this fix's scope.
+    _resolved_city = resolve_city_token(norm_addr) if (lat is None and lon is None) else None
+    _skip_oblast_alias = _resolved_city is not None and _resolved_city != EKB_CITY_TOKEN
+    if _skip_oblast_alias:
+        logger.info(
+            "house tier2a/2b skip: coord-less non-ЕКБ city %r na=%r src=%s",
+            _resolved_city,
+            norm_addr,
+            ext_source,
+        )
+
+    row = None
+    if not _skip_oblast_alias:
         row = (
             db.execute(
-            text("SELECT house_id FROM house_address_aliases " "WHERE fingerprint = :fp LIMIT 1"),
+                text("SELECT house_id FROM house_address_aliases WHERE fingerprint = :fp LIMIT 1"),
                 {"fp": fp},
             )
             .mappings()
             .first()
         )
-    if row is None and norm_addr and has_num:
+    if row is None and norm_addr and has_num and not _skip_oblast_alias:
         # Tier 2b: same normalized address, possibly different coords fingerprint.
         # Gated on has_num (P1): never match a bare-street normalized_address — any
         # numberless listing would otherwise collapse into whichever house first

tradein-mvp/backend/app/services/matching/normalize.py

@@ -112,12 +112,17 @@ def house_number_token(normalized: str | None) -> str | None:
     return m.group(1) if m else None
 
 
+# Екатеринбург — the home city; every other _CITY_TOKENS entry is a non-ЕКБ обл.66 city.
+# The Tier-2a oblast guard treats a resolved non-ЕКБ city specially (see resolve_city_token).
+EKB_CITY_TOKEN = "екатеринбург"
+
 # Cities covered by the обл.66 sweep rollout (ЕКБ + oblast per-city schedules).
 # Normalized form: lowercase, hyphens collapsed to spaces (mirrors normalize_address,
 # e.g. 'каменск-уральский' -> 'каменск уральский'). Mirrors CITY_ANCHORS in
 # scraper_kit.orchestration.pipeline — extend together as the sweep adds cities.
+# EKB_CITY_TOKEN must be first (leftmost-alternation determinism / documentation).
 _CITY_TOKENS: tuple[str, ...] = (
-    "екатеринбург",
+    EKB_CITY_TOKEN,
     "нижний тагил",
     "каменск уральский",
     "первоуральск",
@@ -131,6 +136,25 @@ _CITY_TOKEN_RE = re.compile(
 )
 
 
+def resolve_city_token(normalized: str | None) -> str | None:
+    """Return the обл.66 city token present in the normalized address, else None.
+
+    Token-bounded (see _CITY_TOKEN_RE) so a street named after a city never matches
+    ('улица серова 5' -> None, not 'серов'). When the address names ЕКБ, returns
+    EKB_CITY_TOKEN; a non-ЕКБ oblast city returns its own token. Used by the Tier-2a
+    oblast guard in match_or_create_house: a coord-less card that resolves a NON-ЕКБ city
+    must not fingerprint/normalized_address-match a globally-unique alias that (almost
+    certainly) belongs to ЕКБ/another city — that would corrupt ЕКБ house data.
+    """
+    if not normalized:
+        return None
+    m = _CITY_TOKEN_RE.search(normalized)
+    if m is None:
+        return None
+    # group(0) carries the (?:^|\s)...(?:\s|$) boundary whitespace — strip it to the token.
+    return m.group(0).strip()
+
+
 def has_city_token(normalized: str | None) -> bool:
     """True if the normalized address carries a known обл.66 city token.
 
@@ -141,9 +165,7 @@ def has_city_token(normalized: str | None) -> bool:
     globally-unique normalized_address alias (it would mis-bucket an oblast card into a
     same-named ЕКБ house — bug #2 oblast rollout).
     """
-    if not normalized:
+    return resolve_city_token(normalized) is not None
-        return False
-    return _CITY_TOKEN_RE.search(normalized) is not None
 
 
 def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str:

tradein-mvp/backend/tests/test_matching.py

@@ -16,11 +16,13 @@ from app.services.matching.conflict_resolution import (
     update_canonical_fields,
 )
 from app.services.matching.normalize import (
+    EKB_CITY_TOKEN,
     address_fingerprint,
     has_city_token,
     has_house_number,
     house_number_token,
     normalize_address,
+    resolve_city_token,
 )
 
 # ---------------------------------------------------------------------------
@@ -200,6 +202,37 @@ def test_has_city_token_handles_empty():
     assert has_city_token(None) is False
 
 
+# ---------------------------------------------------------------------------
+# resolve_city_token — which обл.66 city (or None); drives the Tier-2a oblast guard
+# ---------------------------------------------------------------------------
+
+
+def test_resolve_city_token_returns_ekb():
+    assert resolve_city_token("екатеринбург улица ленина 5") == EKB_CITY_TOKEN
+
+
+def test_resolve_city_token_returns_non_ekb_city():
+    assert resolve_city_token("нижний тагил улица ленина 100") == "нижний тагил"
+    assert resolve_city_token("свердловская область каменск уральский улица мира 7") == (
+        "каменск уральский"
+    )
+
+
+def test_resolve_city_token_bare_street_is_none():
+    # The dominant Avito SERP format — no city token → None (guard stays dormant).
+    assert resolve_city_token("улица ленина 100") is None
+
+
+def test_resolve_city_token_no_false_positive_on_street_named_after_city():
+    assert resolve_city_token("улица серова 5") is None
+    assert resolve_city_token("первоуральская улица 3") is None
+
+
+def test_resolve_city_token_handles_empty():
+    assert resolve_city_token("") is None
+    assert resolve_city_token(None) is None
+
+
 # ---------------------------------------------------------------------------
 # match_or_create_house — mock DB tier routing
 # ---------------------------------------------------------------------------
@@ -773,6 +806,85 @@ def test_tier2b_no_coords_with_city_token_matches():
     assert "ST_DWithin" not in tier2b_sql
 
 
+# ---------------------------------------------------------------------------
+# match_or_create_house — Tier-2a coord-less oblast city guard (#2500 follow-up)
+# ---------------------------------------------------------------------------
+
+
+def test_tier2a_coord_less_non_ekb_city_skips_both_lookups_and_creates_new():
+    """A coord-less card whose address resolves a NON-ЕКБ city (Н.Тагил) must NOT match a
+    globally-unique alias by fingerprint OR normalized_address (either would corrupt ЕКБ
+    house data). The guard skips both alias lookups → New house."""
+    from app.services.matching.houses import match_or_create_house
+
+    db = _make_db(
+        [
+            None,  # pg_advisory_xact_lock
+            None,  # house_sources miss (Tier 1)
+            # Tier 2a (fingerprint) SKIPPED, Tier 2b (normalized_address) SKIPPED,
+            # Tier 3 SKIPPED (no coords) — guard forces fall-through to New.
+            {"id": 701},  # INSERT RETURNING id (New house)
+            None,  # _upsert_house_source
+            None,  # _insert_alias (address carries a house number → executes)
+        ]
+    )
+    house_id, conf, method = match_or_create_house(
+        db, "avito", "ext-2a-nt", address="Нижний Тагил, ул. Ленина, 100"
+    )
+    assert (house_id, conf, method) == (701, 1.0, "new")
+    sqls = _executed_sqls(db)
+    assert not any(
+        "fingerprint = :fp" in s for s in sqls
+    ), "coord-less non-ЕКБ card must NOT run the Tier-2a fingerprint lookup"
+    assert not any(
+        "normalized_address = :na" in s for s in sqls
+    ), "coord-less non-ЕКБ card must NOT run the Tier-2b normalized_address lookup"
+
+
+def test_tier2a_coord_less_ekb_city_still_matches():
+    """EKB happy-path: a coord-less card that resolves ЕКБ runs Tier-2a EXACTLY as before —
+    the guard does not fire (екатеринбург == EKB_CITY_TOKEN)."""
+    from app.services.matching.houses import match_or_create_house
+
+    db = _make_db(
+        [
+            None,  # pg_advisory_xact_lock
+            None,  # house_sources miss
+            {"house_id": 88},  # Tier 2a fingerprint HIT (guard did NOT skip it)
+            None,  # _upsert_house_source
+            None,  # _insert_alias
+        ]
+    )
+    house_id, conf, method = match_or_create_house(
+        db, "avito", "ext-2a-ekb", address="Екатеринбург, ул. Ленина, 5"
+    )
+    assert (house_id, conf, method) == (88, 0.9, "fingerprint")
+    assert any(
+        "fingerprint = :fp" in s for s in _executed_sqls(db)
+    ), "ЕКБ coord-less card must still run the Tier-2a fingerprint lookup"
+
+
+def test_tier2a_coord_less_bare_street_still_runs_tier2a():
+    """EKB happy-path (dominant): a coord-less BARE card (no city token — today's Avito SERP
+    format) resolves to None → guard dormant → Tier-2a fingerprint dedup UNCHANGED."""
+    from app.services.matching.houses import match_or_create_house
+
+    db = _make_db(
+        [
+            None,  # pg_advisory_xact_lock
+            None,  # house_sources miss
+            {"house_id": 99},  # Tier 2a fingerprint HIT (bare card → guard did not skip)
+            None,  # _upsert_house_source
+            None,  # _insert_alias
+        ]
+    )
+    house_id, conf, method = match_or_create_house(
+        db, "avito", "ext-2a-bare", address="ул. Ленина, 5"
+    )
+    assert (house_id, conf, method) == (99, 0.9, "fingerprint")
+    assert any("fingerprint = :fp" in s for s in _executed_sqls(db))
+
+
 # ---------------------------------------------------------------------------
 # match_or_create_listing — mock DB tier routing
 # ---------------------------------------------------------------------------