fix(tradein/matching): P1 extended — never create a house from a numberless address (no cadastral) #2194

Merged
lekss361 merged 2 commits from fix/tradein-no-numberless-house-insert into main 2026-07-02 18:46:21 +00:00
6 changed files with 188 additions and 30 deletions

View file

@ -38,7 +38,7 @@ def match_or_create_house(
cadastral_number: str | None = None, cadastral_number: str | None = None,
house_fias_id: str | None = None, house_fias_id: str | None = None,
source_url: str | None = None, source_url: str | None = None,
) -> tuple[int, float, str]: ) -> tuple[int | None, float, str]:
"""Match existing house or create new canonical record. """Match existing house or create new canonical record.
Concurrency-safe: serializes concurrent calls for the same address fingerprint Concurrency-safe: serializes concurrent calls for the same address fingerprint
@ -55,16 +55,26 @@ def match_or_create_house(
Returns: Returns:
(house_id, confidence [0.0, 1.0], method { (house_id, confidence [0.0, 1.0], method {
'cadastr_exact', 'fias_exact', 'source_exact', 'fingerprint', 'cadastr_exact', 'fias_exact', 'source_exact', 'fingerprint',
'geo_proximity', 'new' 'geo_proximity', 'new', 'no_house_number'
}) })
house_id is None only for the 'no_house_number' terminal case below.
Method values: Method values:
'cadastr_exact' matched by cadastral number (confidence 1.0) 'cadastr_exact' matched by cadastral number (confidence 1.0)
'fias_exact' matched by house_fias_id (ГАР OBJECTGUID) (confidence 0.95) 'fias_exact' matched by house_fias_id (ГАР OBJECTGUID) (confidence 0.95)
'source_exact' already in house_sources for this source+ext_id (confidence 1.0) 'source_exact' already in house_sources for this source+ext_id (confidence 1.0)
'fingerprint' matched by address fingerprint (confidence 0.9) 'fingerprint' matched by address fingerprint (confidence 0.9)
'geo_proximity' matched by geo within 30 m (confidence 0.7) 'geo_proximity' matched by geo within 30 m (confidence 0.7)
'new' new house created (confidence 1.0) 'new' new house created (confidence 1.0)
'no_house_number' REFUSED: numberless address with no cadastral number
house_id=None, confidence 0.0. No house/house_source/alias is written.
P1 philosophy: a bare-street building-key (no house number) mass-mis-buckets
active listings into one "house". The alias/Tier-2b guards already refuse to
MATCH such a key (house 131237 collapsed 8 numbers); this closes the last hole
where the New-house INSERT would still MINT one. On prod the missing gate spawned
3 384 numberless mega-buckets ('Москва' 664 listings, 'Екатеринбург (Avito)' 587,
'р-н Чкаловский, мкр. Вторчермет' 480). A cadastral number is a precise building
identity, so cad-carrying rows stay exempt (Tier 0 owns them).
""" """
cad = building_cadastral_number or cadastral_number cad = building_cadastral_number or cadastral_number
@ -265,6 +275,16 @@ def match_or_create_house(
) )
return (house_id, 0.7, "geo_proximity") return (house_id, 0.7, "geo_proximity")
# P1 (extended to INSERT-new): never CREATE a house from a numberless address that
# carries no cadastral number. `has_num` is False for a bare-street normalized_address
# (e.g. 'екатеринбург улица мамина сибиряка') AND for address=None. Such a building-key
# is too ambiguous to mint a canonical row — it becomes a mega-bucket that every later
# numberless listing geo-collapses into. Tier 0/aliases already refuse to MATCH it; here
# we also refuse to CREATE it. A cadastral number (`cad`) is a precise identity → exempt.
if not has_num and not cad:
logger.info("house create skipped: numberless address %r src=%s", address, ext_source)
return (None, 0.0, "no_house_number")
# New house — INSERT canonical record. # New house — INSERT canonical record.
# geom column is auto-populated by houses_set_geom_trg BEFORE INSERT trigger from lat/lon. # geom column is auto-populated by houses_set_geom_trg BEFORE INSERT trigger from lat/lon.
# Do NOT include geom in the INSERT column list — trigger handles it. # Do NOT include geom in the INSERT column list — trigger handles it.

View file

@ -62,7 +62,9 @@ class RealMatcherAdapter:
building_cadastral_number: str | None = None, building_cadastral_number: str | None = None,
cadastral_number: str | None = None, cadastral_number: str | None = None,
source_url: str | None = None, source_url: str | None = None,
) -> tuple[int, float, str]: ) -> tuple[int | None, float, str]:
# house_id is None when the matcher refuses a numberless address without a
# cadastral number (method 'no_house_number', P1). Callers must tolerate None.
return _match_or_create_house( return _match_or_create_house(
db, db,
ext_source, ext_source,

View file

@ -919,10 +919,11 @@ async def fetch_house_catalog(
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def upsert_house(db: Session, h: HouseInfo) -> int: def upsert_house(db: Session, h: HouseInfo) -> int | None:
"""Match-or-create канонический дом через match_or_create_house, затем обновить enrichment. """Match-or-create канонический дом через match_or_create_house, затем обновить enrichment.
Source='avito'. Возвращает houses.id. Source='avito'. Возвращает houses.id, либо None если matcher отказал (P1:
безномерный адрес без кадастра, method='no_house_number').
Делегирует в _persist_house (без url url пуст как раньше для этого пути). Делегирует в _persist_house (без url url пуст как раньше для этого пути).
Маппинг полей HouseInfo DDL колонки (009 + 010): Маппинг полей HouseInfo DDL колонки (009 + 010):
@ -937,15 +938,16 @@ def upsert_house(db: Session, h: HouseInfo) -> int:
return _persist_house(db, h, "") return _persist_house(db, h, "")
def _upsert_house_with_url(db: Session, h: HouseInfo, house_url: str) -> int: def _upsert_house_with_url(db: Session, h: HouseInfo, house_url: str) -> int | None:
"""Внутренний вариант upsert_house с правильным URL. """Внутренний вариант upsert_house с правильным URL.
Делегирует в _persist_house используй его вместо прямого вызова. Делегирует в _persist_house используй его вместо прямого вызова.
Возвращает None если matcher отказал (P1 numberless без кадастра).
""" """
return _persist_house(db, h, house_url) return _persist_house(db, h, house_url)
def _persist_house(db: Session, h: HouseInfo, house_url: str) -> int: def _persist_house(db: Session, h: HouseInfo, house_url: str) -> int | None:
"""Канонический путь сохранения дома-avito через match_or_create_house. """Канонический путь сохранения дома-avito через match_or_create_house.
Шаги: Шаги:
@ -974,6 +976,17 @@ def _persist_house(db: Session, h: HouseInfo, house_url: str) -> int:
source_url=house_url or None, source_url=house_url or None,
) )
# P1: matcher refuses a numberless address without a cadastral number
# (method 'no_house_number' → house_id None). No canonical row exists to enrich,
# so skip the UPDATE and signal the graceful skip up to save_house_catalog_enrichment.
if house_id is None:
logger.info(
"_persist_house: skip numberless avito house ext_id=%s addr=%r (no cadastral)",
h.ext_id,
h.short_address or h.full_address,
)
return None
db.execute( db.execute(
text(""" text("""
UPDATE houses SET UPDATE houses SET
@ -1275,6 +1288,25 @@ def save_house_catalog_enrichment(
} }
house_id = _upsert_house_with_url(db, e.house, e.house_url) house_id = _upsert_house_with_url(db, e.house, e.house_url)
# P1: matcher refused to create a canonical house for a numberless address without a
# cadastral number (house_id None). Downstream saves (reviews / placement history) are
# house-scoped and meaningless with a NULL house_id, so graceful-skip the whole
# enrichment — same zero-counter contract as the id-less guard above.
if house_id is None:
logger.warning(
"save_house_catalog_enrichment: пропуск дома с безномерным адресом без кадастра"
" (ext_id=%s addr=%r) — matcher вернул no_house_number",
e.house.ext_id,
e.house.short_address or e.house.full_address,
)
return {
"house_id": 0,
"reviews": 0,
"sellers": 0,
"listings_linked": 0,
"placement_history": 0,
}
# 2. Сохранить отзывы # 2. Сохранить отзывы
reviews_count = save_house_reviews(db, house_id, e.reviews) reviews_count = save_house_reviews(db, house_id, e.reviews)

View file

@ -494,12 +494,14 @@ def test_geo_match_rejected_when_house_number_differs():
assert house_id == 500 assert house_id == 500
def test_bare_street_listing_skips_tier2b_and_does_not_alias(): def test_bare_street_numberless_no_house_created():
"""P1: a numberless ('bare street') address must not match or register an alias. """P1 (extended to INSERT-new): a numberless address with no cadastral → NO house.
'екатеринбург улица мамина сибиряка' has no house number Tier 2b is skipped 'екатеринбург улица мамина сибиряка' has no house number and no cadastral number,
and _insert_alias is a no-op, so it cannot mass-collapse into a wrong house. so match_or_create_house refuses to MINT a canonical row (which becomes a mega-bucket
Execute order: lock, source miss, fp miss, INSERT, upsert (no Tier 2b, no alias). that later numberless listings collapse into). Returns (None, 0.0, 'no_house_number')
and writes nothing: no Tier 2b lookup, no INSERT INTO houses, no alias, no house_sources.
Execute order: lock, source miss, fp miss then the gate returns.
""" """
from app.services.matching.houses import match_or_create_house from app.services.matching.houses import match_or_create_house
@ -508,22 +510,86 @@ def test_bare_street_listing_skips_tier2b_and_does_not_alias():
None, # pg_advisory_xact_lock None, # pg_advisory_xact_lock
None, # house_sources miss None, # house_sources miss
None, # fingerprint miss (Tier 2a) None, # fingerprint miss (Tier 2a)
{"id": 700}, # INSERT RETURNING id (New house — Tier 2b skipped, no lat/lon)
None, # _upsert_house_source
] ]
) )
house_id, _conf, method = match_or_create_house( house_id, conf, method = match_or_create_house(
db, "yandex", "ext-bare", address="екатеринбург улица мамина сибиряка" db, "yandex", "ext-bare", address="екатеринбург улица мамина сибиряка"
) )
assert method == "new" assert (house_id, conf, method) == (None, 0.0, "no_house_number")
assert house_id == 700
sqls = _executed_sqls(db) sqls = _executed_sqls(db)
assert not any( assert not any(
"normalized_address = :na" in s for s in sqls "normalized_address = :na" in s for s in sqls
), "bare-street address must not run Tier 2b normalized_address lookup (P1)" ), "bare-street address must not run Tier 2b normalized_address lookup (P1)"
assert not any(
"INSERT INTO houses" in s for s in sqls
), "numberless address must not create a house (P1 extended)"
assert not any( assert not any(
"INSERT INTO house_address_aliases" in s for s in sqls "INSERT INTO house_address_aliases" in s for s in sqls
), "bare-street address must not register an alias (P1)" ), "numberless address must not register an alias (P1)"
assert not any(
"INSERT INTO house_sources" in s for s in sqls
), "numberless refusal must not upsert house_sources"
def test_numberless_none_address_with_coords_no_house_created():
"""P1 edge: address=None but lat/lon present → still gated (no address-NULL house).
Previously the New-house INSERT would mint a house with address NULL keyed only by
coords exactly the mis-bucket the gate refuses. has_num is False for an empty
normalized address, so with no cadastral the flow returns 'no_house_number'.
Execute order: lock, source miss, fp miss, geo miss then the gate returns.
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a); Tier 2b skipped (empty norm_addr)
None, # geo miss (Tier 3 — lat/lon present)
]
)
house_id, conf, method = match_or_create_house(
db, "yandex", "ext-none-addr", address=None, lat=56.83, lon=60.59
)
assert (house_id, conf, method) == (None, 0.0, "no_house_number")
assert not any(
"INSERT INTO houses" in s for s in _executed_sqls(db)
), "address-NULL coords-only listing must not create a house (P1 extended)"
def test_numberless_address_with_cadastral_creates_house():
"""P1 exemption: a numberless address WITH a cadastral number still creates a house.
The cadastral number is a precise building identity, so the numberless gate does not
apply. Tier 0 cadastr lookup misses (new cad), remaining tiers miss, and the New-house
INSERT proceeds because `cad` is truthy.
Execute order: lock, cadastr miss, source miss, fp miss, INSERT, upsert
(_insert_alias is a no-op bare-street normalized_address has no number).
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # Tier 0 cadastr miss
None, # house_sources miss
None, # fingerprint miss (Tier 2a; no lat/lon → no Tier 2b/geo)
{"id": 321}, # INSERT RETURNING id (New house — cad exempts the gate)
None, # _upsert_house_source
]
)
house_id, conf, method = match_or_create_house(
db,
"cian",
"ext-cad",
address="екатеринбург улица мамина сибиряка",
cadastral_number="66:41:0000000:12345",
)
assert (house_id, conf, method) == (321, 1.0, "new")
assert any(
"INSERT INTO houses" in s for s in _executed_sqls(db)
), "numberless + cadastral must still create a house (cadastral = identity)"
def test_insert_alias_noop_for_bare_street(): def test_insert_alias_noop_for_bare_street():

View file

@ -50,13 +50,17 @@ class HouseMatcher(Protocol):
building_cadastral_number: str | None = ..., building_cadastral_number: str | None = ...,
cadastral_number: str | None = ..., cadastral_number: str | None = ...,
source_url: str | None = ..., source_url: str | None = ...,
) -> tuple[int, float, str]: ) -> tuple[int | None, float, str]:
"""Найти или создать канонический дом. """Найти или создать канонический дом.
Returns: Returns:
(house_id, confidence [0.0, 1.0], method), где method { (house_id, confidence [0.0, 1.0], method), где method {
'cadastr_exact', 'source_exact', 'fingerprint', 'geo_proximity', 'new' 'cadastr_exact', 'source_exact', 'fingerprint', 'geo_proximity', 'new',
'no_house_number'
}. }.
house_id=None только при method='no_house_number' matcher отказался
создавать дом из безномерного адреса без кадастра (P1). Вызывающий обязан
обработать None.
""" """
... ...

View file

@ -919,10 +919,11 @@ async def fetch_house_catalog(
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def upsert_house(db: Session, h: HouseInfo, *, matcher: HouseMatcher) -> int: def upsert_house(db: Session, h: HouseInfo, *, matcher: HouseMatcher) -> int | None:
"""Match-or-create канонический дом через matcher, затем обновить enrichment. """Match-or-create канонический дом через matcher, затем обновить enrichment.
Source='avito'. Возвращает houses.id. Source='avito'. Возвращает houses.id, либо None если matcher отказал (P1:
безномерный адрес без кадастра, method='no_house_number').
Делегирует в _persist_house (без url url пуст как раньше для этого пути). Делегирует в _persist_house (без url url пуст как раньше для этого пути).
`matcher` инжектируемый `HouseMatcher` (scraper_kit.contracts, #2133) вместо `matcher` инжектируемый `HouseMatcher` (scraper_kit.contracts, #2133) вместо
прямого импорта app.services.matching.houses.match_or_create_house. прямого импорта app.services.matching.houses.match_or_create_house.
@ -941,15 +942,18 @@ def upsert_house(db: Session, h: HouseInfo, *, matcher: HouseMatcher) -> int:
def _upsert_house_with_url( def _upsert_house_with_url(
db: Session, h: HouseInfo, house_url: str, *, matcher: HouseMatcher db: Session, h: HouseInfo, house_url: str, *, matcher: HouseMatcher
) -> int: ) -> int | None:
"""Внутренний вариант upsert_house с правильным URL. """Внутренний вариант upsert_house с правильным URL.
Делегирует в _persist_house используй его вместо прямого вызова. Делегирует в _persist_house используй его вместо прямого вызова.
Возвращает None если matcher отказал (P1 numberless без кадастра).
""" """
return _persist_house(db, h, house_url, matcher=matcher) return _persist_house(db, h, house_url, matcher=matcher)
def _persist_house(db: Session, h: HouseInfo, house_url: str, *, matcher: HouseMatcher) -> int: def _persist_house(
db: Session, h: HouseInfo, house_url: str, *, matcher: HouseMatcher
) -> int | None:
"""Канонический путь сохранения дома-avito через match_or_create_house. """Канонический путь сохранения дома-avito через match_or_create_house.
Шаги: Шаги:
@ -978,6 +982,17 @@ def _persist_house(db: Session, h: HouseInfo, house_url: str, *, matcher: HouseM
source_url=house_url or None, source_url=house_url or None,
) )
# P1: matcher refuses a numberless address without a cadastral number
# (method 'no_house_number' → house_id None). No canonical row exists to enrich,
# so skip the UPDATE and signal the graceful skip up to save_house_catalog_enrichment.
if house_id is None:
logger.info(
"_persist_house: skip numberless avito house ext_id=%s addr=%r (no cadastral)",
h.ext_id,
h.short_address or h.full_address,
)
return None
db.execute( db.execute(
text(""" text("""
UPDATE houses SET UPDATE houses SET
@ -1281,6 +1296,25 @@ def save_house_catalog_enrichment(
} }
house_id = _upsert_house_with_url(db, e.house, e.house_url, matcher=matcher) house_id = _upsert_house_with_url(db, e.house, e.house_url, matcher=matcher)
# P1: matcher refused to create a canonical house for a numberless address without a
# cadastral number (house_id None). Downstream saves (reviews / placement history) are
# house-scoped and meaningless with a NULL house_id, so graceful-skip the whole
# enrichment — same zero-counter contract as the id-less guard above.
if house_id is None:
logger.warning(
"save_house_catalog_enrichment: пропуск дома с безномерным адресом без кадастра"
" (ext_id=%s addr=%r) — matcher вернул no_house_number",
e.house.ext_id,
e.house.short_address or e.house.full_address,
)
return {
"house_id": 0,
"reviews": 0,
"sellers": 0,
"listings_linked": 0,
"placement_history": 0,
}
# 2. Сохранить отзывы # 2. Сохранить отзывы
reviews_count = save_house_reviews(db, house_id, e.reviews) reviews_count = save_house_reviews(db, house_id, e.reviews)