fix(tradein/oblast): Tier-2a guard — coord-less non-ЕКБ карточка не матчит ЕКБ-alias
All checks were successful
CI Trade-In / changes (pull_request) Successful in 7s
CI / changes (pull_request) Successful in 7s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 52s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped

Follow-up к #2500: закрывает Tier-2a-дыру, которую geo-guard Tier-2b не доставал.
Coord-less карточка, чей адрес РАЗРЕШАЕТ non-ЕКБ город обл.66 (resolve_city_token),
больше не матчит глобально-уникальный alias (ни по fingerprint, ни по
normalized_address) — иначе non-ЕКБ карточка баккетилась бы в одноимённый ЕКБ-дом и
корраптила ЕКБ-данные. При срабатывании guard'а обе alias-выборки пропускаются →
fall-through в New house (Tier-3 coord-gated, тоже пропускается).

normalize.py: + resolve_city_token() (город обл.66 или None), + EKB_CITY_TOKEN;
has_city_token переиспользует resolve_city_token.

EKB happy-path байт-в-байт: guard срабатывает ТОЛЬКО когда адрес называет non-ЕКБ
город И нет координат. ЕКБ-карточки (resolved city = екатеринбург) и доминирующие
bare/city-less coord-less карточки Avito (resolved None) идут Tier-2a/2b как раньше.

ВАЖНО (документировано в коде и отчёте): реальные Avito SERP-адреса — bare (без
city-токена; 2% из 5037 avito-alias'ов несут 'екатеринбург', 0% — oblast). Значит для
bare oblast-карточки resolve_city_token=None и guard дремлет: полное закрытие
bare-Tier-2a-остатка требует sweep-context/city-keyed aliases — отдельный follow-up,
вне scope, актуален лишь при включённом oblast-sweep. Zero prod-impact сегодня.
This commit is contained in:
bot-backend 2026-07-12 23:26:01 +03:00
parent 51473fc841
commit d92ff5c3d3
3 changed files with 178 additions and 12 deletions

View file

@ -16,11 +16,13 @@ from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.services.matching.normalize import ( from app.services.matching.normalize import (
EKB_CITY_TOKEN,
address_fingerprint, address_fingerprint,
has_city_token, has_city_token,
has_house_number, has_house_number,
house_number_token, house_number_token,
normalize_address, normalize_address,
resolve_city_token,
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -189,15 +191,45 @@ def match_or_create_house(
# Without 2b, two scrapers for the same house with slightly different lat/lon (beyond # Without 2b, two scrapers for the same house with slightly different lat/lon (beyond
# the 4-decimal rounding tolerance) would produce distinct fingerprints, miss Tier 2a, # the 4-decimal rounding tolerance) would produce distinct fingerprints, miss Tier 2a,
# and each potentially create a duplicate house row. # and each potentially create a duplicate house row.
row = ( #
db.execute( # Tier-2a/2b oblast city guard (#2500 follow-up, closes the Tier-2a mis-bucket that the
text("SELECT house_id FROM house_address_aliases " "WHERE fingerprint = :fp LIMIT 1"), # #2500 Tier-2b geo-guard could not reach): a coord-less card whose address explicitly
{"fp": fp}, # resolves a NON-ЕКБ обл.66 city must NOT match a globally-unique alias (fingerprint OR
# normalized_address) that almost certainly belongs to ЕКБ/another city — that would
# corrupt ЕКБ house data. When it fires we skip both alias lookups and fall through to a
# New house (Tier 3 is coord-gated, so coord-less cards skip it too).
#
# This PREVENTS ЕКБ CORRUPTION. Full oblast-internal dedup (deduping two listings of the
# SAME oblast building) still needs city-keyed aliases — a separate follow-up, out of
# scope, only relevant once the oblast sweep is enabled.
#
# EKB happy-path is byte-identical: the guard fires ONLY when the address names a non-ЕКБ
# city AND no coords disambiguate. ЕКБ cards (resolved city = екатеринбург) and the
# dominant bare/city-less Avito coord-less cards (resolved city None) run Tier-2a/2b
# exactly as before. NB: a BARE oblast card (no city token in the address — today's Avito
# SERP format) carries no signal here and is deliberately left on the unchanged path; that
# residual needs sweep-context and is out of this fix's scope.
_resolved_city = resolve_city_token(norm_addr) if (lat is None and lon is None) else None
_skip_oblast_alias = _resolved_city is not None and _resolved_city != EKB_CITY_TOKEN
if _skip_oblast_alias:
logger.info(
"house tier2a/2b skip: coord-less non-ЕКБ city %r na=%r src=%s",
_resolved_city,
norm_addr,
ext_source,
) )
.mappings()
.first() row = None
) if not _skip_oblast_alias:
if row is None and norm_addr and has_num: row = (
db.execute(
text("SELECT house_id FROM house_address_aliases WHERE fingerprint = :fp LIMIT 1"),
{"fp": fp},
)
.mappings()
.first()
)
if row is None and norm_addr and has_num and not _skip_oblast_alias:
# Tier 2b: same normalized address, possibly different coords fingerprint. # Tier 2b: same normalized address, possibly different coords fingerprint.
# Gated on has_num (P1): never match a bare-street normalized_address — any # Gated on has_num (P1): never match a bare-street normalized_address — any
# numberless listing would otherwise collapse into whichever house first # numberless listing would otherwise collapse into whichever house first

View file

@ -112,12 +112,17 @@ def house_number_token(normalized: str | None) -> str | None:
return m.group(1) if m else None return m.group(1) if m else None
# Екатеринбург — the home city; every other _CITY_TOKENS entry is a non-ЕКБ обл.66 city.
# The Tier-2a oblast guard treats a resolved non-ЕКБ city specially (see resolve_city_token).
EKB_CITY_TOKEN = "екатеринбург"
# Cities covered by the обл.66 sweep rollout (ЕКБ + oblast per-city schedules). # Cities covered by the обл.66 sweep rollout (ЕКБ + oblast per-city schedules).
# Normalized form: lowercase, hyphens collapsed to spaces (mirrors normalize_address, # Normalized form: lowercase, hyphens collapsed to spaces (mirrors normalize_address,
# e.g. 'каменск-уральский' -> 'каменск уральский'). Mirrors CITY_ANCHORS in # e.g. 'каменск-уральский' -> 'каменск уральский'). Mirrors CITY_ANCHORS in
# scraper_kit.orchestration.pipeline — extend together as the sweep adds cities. # scraper_kit.orchestration.pipeline — extend together as the sweep adds cities.
# EKB_CITY_TOKEN must be first (leftmost-alternation determinism / documentation).
_CITY_TOKENS: tuple[str, ...] = ( _CITY_TOKENS: tuple[str, ...] = (
"екатеринбург", EKB_CITY_TOKEN,
"нижний тагил", "нижний тагил",
"каменск уральский", "каменск уральский",
"первоуральск", "первоуральск",
@ -131,6 +136,25 @@ _CITY_TOKEN_RE = re.compile(
) )
def resolve_city_token(normalized: str | None) -> str | None:
"""Return the обл.66 city token present in the normalized address, else None.
Token-bounded (see _CITY_TOKEN_RE) so a street named after a city never matches
('улица серова 5' -> None, not 'серов'). When the address names ЕКБ, returns
EKB_CITY_TOKEN; a non-ЕКБ oblast city returns its own token. Used by the Tier-2a
oblast guard in match_or_create_house: a coord-less card that resolves a NON-ЕКБ city
must not fingerprint/normalized_address-match a globally-unique alias that (almost
certainly) belongs to ЕКБ/another city that would corrupt ЕКБ house data.
"""
if not normalized:
return None
m = _CITY_TOKEN_RE.search(normalized)
if m is None:
return None
# group(0) carries the (?:^|\s)...(?:\s|$) boundary whitespace — strip it to the token.
return m.group(0).strip()
def has_city_token(normalized: str | None) -> bool: def has_city_token(normalized: str | None) -> bool:
"""True if the normalized address carries a known обл.66 city token. """True if the normalized address carries a known обл.66 city token.
@ -141,9 +165,7 @@ def has_city_token(normalized: str | None) -> bool:
globally-unique normalized_address alias (it would mis-bucket an oblast card into a globally-unique normalized_address alias (it would mis-bucket an oblast card into a
same-named ЕКБ house bug #2 oblast rollout). same-named ЕКБ house bug #2 oblast rollout).
""" """
if not normalized: return resolve_city_token(normalized) is not None
return False
return _CITY_TOKEN_RE.search(normalized) is not None
def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str: def address_fingerprint(address: str | None, lat: float | None, lon: float | None) -> str:

View file

@ -16,11 +16,13 @@ from app.services.matching.conflict_resolution import (
update_canonical_fields, update_canonical_fields,
) )
from app.services.matching.normalize import ( from app.services.matching.normalize import (
EKB_CITY_TOKEN,
address_fingerprint, address_fingerprint,
has_city_token, has_city_token,
has_house_number, has_house_number,
house_number_token, house_number_token,
normalize_address, normalize_address,
resolve_city_token,
) )
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -200,6 +202,37 @@ def test_has_city_token_handles_empty():
assert has_city_token(None) is False assert has_city_token(None) is False
# ---------------------------------------------------------------------------
# resolve_city_token — which обл.66 city (or None); drives the Tier-2a oblast guard
# ---------------------------------------------------------------------------
def test_resolve_city_token_returns_ekb():
assert resolve_city_token("екатеринбург улица ленина 5") == EKB_CITY_TOKEN
def test_resolve_city_token_returns_non_ekb_city():
assert resolve_city_token("нижний тагил улица ленина 100") == "нижний тагил"
assert resolve_city_token("свердловская область каменск уральский улица мира 7") == (
"каменск уральский"
)
def test_resolve_city_token_bare_street_is_none():
# The dominant Avito SERP format — no city token → None (guard stays dormant).
assert resolve_city_token("улица ленина 100") is None
def test_resolve_city_token_no_false_positive_on_street_named_after_city():
assert resolve_city_token("улица серова 5") is None
assert resolve_city_token("первоуральская улица 3") is None
def test_resolve_city_token_handles_empty():
assert resolve_city_token("") is None
assert resolve_city_token(None) is None
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# match_or_create_house — mock DB tier routing # match_or_create_house — mock DB tier routing
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -773,6 +806,85 @@ def test_tier2b_no_coords_with_city_token_matches():
assert "ST_DWithin" not in tier2b_sql assert "ST_DWithin" not in tier2b_sql
# ---------------------------------------------------------------------------
# match_or_create_house — Tier-2a coord-less oblast city guard (#2500 follow-up)
# ---------------------------------------------------------------------------
def test_tier2a_coord_less_non_ekb_city_skips_both_lookups_and_creates_new():
"""A coord-less card whose address resolves a NON-ЕКБ city (Н.Тагил) must NOT match a
globally-unique alias by fingerprint OR normalized_address (either would corrupt ЕКБ
house data). The guard skips both alias lookups New house."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss (Tier 1)
# Tier 2a (fingerprint) SKIPPED, Tier 2b (normalized_address) SKIPPED,
# Tier 3 SKIPPED (no coords) — guard forces fall-through to New.
{"id": 701}, # INSERT RETURNING id (New house)
None, # _upsert_house_source
None, # _insert_alias (address carries a house number → executes)
]
)
house_id, conf, method = match_or_create_house(
db, "avito", "ext-2a-nt", address="Нижний Тагил, ул. Ленина, 100"
)
assert (house_id, conf, method) == (701, 1.0, "new")
sqls = _executed_sqls(db)
assert not any(
"fingerprint = :fp" in s for s in sqls
), "coord-less non-ЕКБ card must NOT run the Tier-2a fingerprint lookup"
assert not any(
"normalized_address = :na" in s for s in sqls
), "coord-less non-ЕКБ card must NOT run the Tier-2b normalized_address lookup"
def test_tier2a_coord_less_ekb_city_still_matches():
"""EKB happy-path: a coord-less card that resolves ЕКБ runs Tier-2a EXACTLY as before —
the guard does not fire (екатеринбург == EKB_CITY_TOKEN)."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
{"house_id": 88}, # Tier 2a fingerprint HIT (guard did NOT skip it)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db, "avito", "ext-2a-ekb", address="Екатеринбург, ул. Ленина, 5"
)
assert (house_id, conf, method) == (88, 0.9, "fingerprint")
assert any(
"fingerprint = :fp" in s for s in _executed_sqls(db)
), "ЕКБ coord-less card must still run the Tier-2a fingerprint lookup"
def test_tier2a_coord_less_bare_street_still_runs_tier2a():
"""EKB happy-path (dominant): a coord-less BARE card (no city token — today's Avito SERP
format) resolves to None guard dormant Tier-2a fingerprint dedup UNCHANGED."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
{"house_id": 99}, # Tier 2a fingerprint HIT (bare card → guard did not skip)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db, "avito", "ext-2a-bare", address="ул. Ленина, 5"
)
assert (house_id, conf, method) == (99, 0.9, "fingerprint")
assert any("fingerprint = :fp" in s for s in _executed_sqls(db))
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# match_or_create_listing — mock DB tier routing # match_or_create_listing — mock DB tier routing
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------