Compare commits

..

No commits in common. "a9d06c9c4479d935d1a00431ff224ce3950ab76b" and "bcdec5ebd4db8212c10515614942f2e549e4fb11" have entirely different histories.

6 changed files with 30 additions and 188 deletions

View file

@ -38,7 +38,7 @@ def match_or_create_house(
cadastral_number: str | None = None, cadastral_number: str | None = None,
house_fias_id: str | None = None, house_fias_id: str | None = None,
source_url: str | None = None, source_url: str | None = None,
) -> tuple[int | None, float, str]: ) -> tuple[int, float, str]:
"""Match existing house or create new canonical record. """Match existing house or create new canonical record.
Concurrency-safe: serializes concurrent calls for the same address fingerprint Concurrency-safe: serializes concurrent calls for the same address fingerprint
@ -55,26 +55,16 @@ def match_or_create_house(
Returns: Returns:
(house_id, confidence [0.0, 1.0], method { (house_id, confidence [0.0, 1.0], method {
'cadastr_exact', 'fias_exact', 'source_exact', 'fingerprint', 'cadastr_exact', 'fias_exact', 'source_exact', 'fingerprint',
'geo_proximity', 'new', 'no_house_number' 'geo_proximity', 'new'
}) })
house_id is None only for the 'no_house_number' terminal case below.
Method values: Method values:
'cadastr_exact' matched by cadastral number (confidence 1.0) 'cadastr_exact' matched by cadastral number (confidence 1.0)
'fias_exact' matched by house_fias_id (ГАР OBJECTGUID) (confidence 0.95) 'fias_exact' matched by house_fias_id (ГАР OBJECTGUID) (confidence 0.95)
'source_exact' already in house_sources for this source+ext_id (confidence 1.0) 'source_exact' already in house_sources for this source+ext_id (confidence 1.0)
'fingerprint' matched by address fingerprint (confidence 0.9) 'fingerprint' matched by address fingerprint (confidence 0.9)
'geo_proximity' matched by geo within 30 m (confidence 0.7) 'geo_proximity' matched by geo within 30 m (confidence 0.7)
'new' new house created (confidence 1.0) 'new' new house created (confidence 1.0)
'no_house_number' REFUSED: numberless address with no cadastral number
house_id=None, confidence 0.0. No house/house_source/alias is written.
P1 philosophy: a bare-street building-key (no house number) mass-mis-buckets
active listings into one "house". The alias/Tier-2b guards already refuse to
MATCH such a key (house 131237 collapsed 8 numbers); this closes the last hole
where the New-house INSERT would still MINT one. On prod the missing gate spawned
3 384 numberless mega-buckets ('Москва' 664 listings, 'Екатеринбург (Avito)' 587,
'р-н Чкаловский, мкр. Вторчермет' 480). A cadastral number is a precise building
identity, so cad-carrying rows stay exempt (Tier 0 owns them).
""" """
cad = building_cadastral_number or cadastral_number cad = building_cadastral_number or cadastral_number
@ -275,16 +265,6 @@ def match_or_create_house(
) )
return (house_id, 0.7, "geo_proximity") return (house_id, 0.7, "geo_proximity")
# P1 (extended to INSERT-new): never CREATE a house from a numberless address that
# carries no cadastral number. `has_num` is False for a bare-street normalized_address
# (e.g. 'екатеринбург улица мамина сибиряка') AND for address=None. Such a building-key
# is too ambiguous to mint a canonical row — it becomes a mega-bucket that every later
# numberless listing geo-collapses into. Tier 0/aliases already refuse to MATCH it; here
# we also refuse to CREATE it. A cadastral number (`cad`) is a precise identity → exempt.
if not has_num and not cad:
logger.info("house create skipped: numberless address %r src=%s", address, ext_source)
return (None, 0.0, "no_house_number")
# New house — INSERT canonical record. # New house — INSERT canonical record.
# geom column is auto-populated by houses_set_geom_trg BEFORE INSERT trigger from lat/lon. # geom column is auto-populated by houses_set_geom_trg BEFORE INSERT trigger from lat/lon.
# Do NOT include geom in the INSERT column list — trigger handles it. # Do NOT include geom in the INSERT column list — trigger handles it.

View file

@ -62,9 +62,7 @@ class RealMatcherAdapter:
building_cadastral_number: str | None = None, building_cadastral_number: str | None = None,
cadastral_number: str | None = None, cadastral_number: str | None = None,
source_url: str | None = None, source_url: str | None = None,
) -> tuple[int | None, float, str]: ) -> tuple[int, float, str]:
# house_id is None when the matcher refuses a numberless address without a
# cadastral number (method 'no_house_number', P1). Callers must tolerate None.
return _match_or_create_house( return _match_or_create_house(
db, db,
ext_source, ext_source,

View file

@ -919,11 +919,10 @@ async def fetch_house_catalog(
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def upsert_house(db: Session, h: HouseInfo) -> int | None: def upsert_house(db: Session, h: HouseInfo) -> int:
"""Match-or-create канонический дом через match_or_create_house, затем обновить enrichment. """Match-or-create канонический дом через match_or_create_house, затем обновить enrichment.
Source='avito'. Возвращает houses.id, либо None если matcher отказал (P1: Source='avito'. Возвращает houses.id.
безномерный адрес без кадастра, method='no_house_number').
Делегирует в _persist_house (без url url пуст как раньше для этого пути). Делегирует в _persist_house (без url url пуст как раньше для этого пути).
Маппинг полей HouseInfo DDL колонки (009 + 010): Маппинг полей HouseInfo DDL колонки (009 + 010):
@ -938,16 +937,15 @@ def upsert_house(db: Session, h: HouseInfo) -> int | None:
return _persist_house(db, h, "") return _persist_house(db, h, "")
def _upsert_house_with_url(db: Session, h: HouseInfo, house_url: str) -> int | None: def _upsert_house_with_url(db: Session, h: HouseInfo, house_url: str) -> int:
"""Внутренний вариант upsert_house с правильным URL. """Внутренний вариант upsert_house с правильным URL.
Делегирует в _persist_house используй его вместо прямого вызова. Делегирует в _persist_house используй его вместо прямого вызова.
Возвращает None если matcher отказал (P1 numberless без кадастра).
""" """
return _persist_house(db, h, house_url) return _persist_house(db, h, house_url)
def _persist_house(db: Session, h: HouseInfo, house_url: str) -> int | None: def _persist_house(db: Session, h: HouseInfo, house_url: str) -> int:
"""Канонический путь сохранения дома-avito через match_or_create_house. """Канонический путь сохранения дома-avito через match_or_create_house.
Шаги: Шаги:
@ -976,17 +974,6 @@ def _persist_house(db: Session, h: HouseInfo, house_url: str) -> int | None:
source_url=house_url or None, source_url=house_url or None,
) )
# P1: matcher refuses a numberless address without a cadastral number
# (method 'no_house_number' → house_id None). No canonical row exists to enrich,
# so skip the UPDATE and signal the graceful skip up to save_house_catalog_enrichment.
if house_id is None:
logger.info(
"_persist_house: skip numberless avito house ext_id=%s addr=%r (no cadastral)",
h.ext_id,
h.short_address or h.full_address,
)
return None
db.execute( db.execute(
text(""" text("""
UPDATE houses SET UPDATE houses SET
@ -1288,25 +1275,6 @@ def save_house_catalog_enrichment(
} }
house_id = _upsert_house_with_url(db, e.house, e.house_url) house_id = _upsert_house_with_url(db, e.house, e.house_url)
# P1: matcher refused to create a canonical house for a numberless address without a
# cadastral number (house_id None). Downstream saves (reviews / placement history) are
# house-scoped and meaningless with a NULL house_id, so graceful-skip the whole
# enrichment — same zero-counter contract as the id-less guard above.
if house_id is None:
logger.warning(
"save_house_catalog_enrichment: пропуск дома с безномерным адресом без кадастра"
" (ext_id=%s addr=%r) — matcher вернул no_house_number",
e.house.ext_id,
e.house.short_address or e.house.full_address,
)
return {
"house_id": 0,
"reviews": 0,
"sellers": 0,
"listings_linked": 0,
"placement_history": 0,
}
# 2. Сохранить отзывы # 2. Сохранить отзывы
reviews_count = save_house_reviews(db, house_id, e.reviews) reviews_count = save_house_reviews(db, house_id, e.reviews)

View file

@ -494,14 +494,12 @@ def test_geo_match_rejected_when_house_number_differs():
assert house_id == 500 assert house_id == 500
def test_bare_street_numberless_no_house_created(): def test_bare_street_listing_skips_tier2b_and_does_not_alias():
"""P1 (extended to INSERT-new): a numberless address with no cadastral → NO house. """P1: a numberless ('bare street') address must not match or register an alias.
'екатеринбург улица мамина сибиряка' has no house number and no cadastral number, 'екатеринбург улица мамина сибиряка' has no house number Tier 2b is skipped
so match_or_create_house refuses to MINT a canonical row (which becomes a mega-bucket and _insert_alias is a no-op, so it cannot mass-collapse into a wrong house.
that later numberless listings collapse into). Returns (None, 0.0, 'no_house_number') Execute order: lock, source miss, fp miss, INSERT, upsert (no Tier 2b, no alias).
and writes nothing: no Tier 2b lookup, no INSERT INTO houses, no alias, no house_sources.
Execute order: lock, source miss, fp miss then the gate returns.
""" """
from app.services.matching.houses import match_or_create_house from app.services.matching.houses import match_or_create_house
@ -510,86 +508,22 @@ def test_bare_street_numberless_no_house_created():
None, # pg_advisory_xact_lock None, # pg_advisory_xact_lock
None, # house_sources miss None, # house_sources miss
None, # fingerprint miss (Tier 2a) None, # fingerprint miss (Tier 2a)
{"id": 700}, # INSERT RETURNING id (New house — Tier 2b skipped, no lat/lon)
None, # _upsert_house_source
] ]
) )
house_id, conf, method = match_or_create_house( house_id, _conf, method = match_or_create_house(
db, "yandex", "ext-bare", address="екатеринбург улица мамина сибиряка" db, "yandex", "ext-bare", address="екатеринбург улица мамина сибиряка"
) )
assert (house_id, conf, method) == (None, 0.0, "no_house_number") assert method == "new"
assert house_id == 700
sqls = _executed_sqls(db) sqls = _executed_sqls(db)
assert not any( assert not any(
"normalized_address = :na" in s for s in sqls "normalized_address = :na" in s for s in sqls
), "bare-street address must not run Tier 2b normalized_address lookup (P1)" ), "bare-street address must not run Tier 2b normalized_address lookup (P1)"
assert not any(
"INSERT INTO houses" in s for s in sqls
), "numberless address must not create a house (P1 extended)"
assert not any( assert not any(
"INSERT INTO house_address_aliases" in s for s in sqls "INSERT INTO house_address_aliases" in s for s in sqls
), "numberless address must not register an alias (P1)" ), "bare-street address must not register an alias (P1)"
assert not any(
"INSERT INTO house_sources" in s for s in sqls
), "numberless refusal must not upsert house_sources"
def test_numberless_none_address_with_coords_no_house_created():
"""P1 edge: address=None but lat/lon present → still gated (no address-NULL house).
Previously the New-house INSERT would mint a house with address NULL keyed only by
coords exactly the mis-bucket the gate refuses. has_num is False for an empty
normalized address, so with no cadastral the flow returns 'no_house_number'.
Execute order: lock, source miss, fp miss, geo miss then the gate returns.
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # house_sources miss
None, # fingerprint miss (Tier 2a); Tier 2b skipped (empty norm_addr)
None, # geo miss (Tier 3 — lat/lon present)
]
)
house_id, conf, method = match_or_create_house(
db, "yandex", "ext-none-addr", address=None, lat=56.83, lon=60.59
)
assert (house_id, conf, method) == (None, 0.0, "no_house_number")
assert not any(
"INSERT INTO houses" in s for s in _executed_sqls(db)
), "address-NULL coords-only listing must not create a house (P1 extended)"
def test_numberless_address_with_cadastral_creates_house():
"""P1 exemption: a numberless address WITH a cadastral number still creates a house.
The cadastral number is a precise building identity, so the numberless gate does not
apply. Tier 0 cadastr lookup misses (new cad), remaining tiers miss, and the New-house
INSERT proceeds because `cad` is truthy.
Execute order: lock, cadastr miss, source miss, fp miss, INSERT, upsert
(_insert_alias is a no-op bare-street normalized_address has no number).
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
None, # Tier 0 cadastr miss
None, # house_sources miss
None, # fingerprint miss (Tier 2a; no lat/lon → no Tier 2b/geo)
{"id": 321}, # INSERT RETURNING id (New house — cad exempts the gate)
None, # _upsert_house_source
]
)
house_id, conf, method = match_or_create_house(
db,
"cian",
"ext-cad",
address="екатеринбург улица мамина сибиряка",
cadastral_number="66:41:0000000:12345",
)
assert (house_id, conf, method) == (321, 1.0, "new")
assert any(
"INSERT INTO houses" in s for s in _executed_sqls(db)
), "numberless + cadastral must still create a house (cadastral = identity)"
def test_insert_alias_noop_for_bare_street(): def test_insert_alias_noop_for_bare_street():

View file

@ -50,17 +50,13 @@ class HouseMatcher(Protocol):
building_cadastral_number: str | None = ..., building_cadastral_number: str | None = ...,
cadastral_number: str | None = ..., cadastral_number: str | None = ...,
source_url: str | None = ..., source_url: str | None = ...,
) -> tuple[int | None, float, str]: ) -> tuple[int, float, str]:
"""Найти или создать канонический дом. """Найти или создать канонический дом.
Returns: Returns:
(house_id, confidence [0.0, 1.0], method), где method { (house_id, confidence [0.0, 1.0], method), где method {
'cadastr_exact', 'source_exact', 'fingerprint', 'geo_proximity', 'new', 'cadastr_exact', 'source_exact', 'fingerprint', 'geo_proximity', 'new'
'no_house_number'
}. }.
house_id=None только при method='no_house_number' matcher отказался
создавать дом из безномерного адреса без кадастра (P1). Вызывающий обязан
обработать None.
""" """
... ...

View file

@ -919,11 +919,10 @@ async def fetch_house_catalog(
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def upsert_house(db: Session, h: HouseInfo, *, matcher: HouseMatcher) -> int | None: def upsert_house(db: Session, h: HouseInfo, *, matcher: HouseMatcher) -> int:
"""Match-or-create канонический дом через matcher, затем обновить enrichment. """Match-or-create канонический дом через matcher, затем обновить enrichment.
Source='avito'. Возвращает houses.id, либо None если matcher отказал (P1: Source='avito'. Возвращает houses.id.
безномерный адрес без кадастра, method='no_house_number').
Делегирует в _persist_house (без url url пуст как раньше для этого пути). Делегирует в _persist_house (без url url пуст как раньше для этого пути).
`matcher` инжектируемый `HouseMatcher` (scraper_kit.contracts, #2133) вместо `matcher` инжектируемый `HouseMatcher` (scraper_kit.contracts, #2133) вместо
прямого импорта app.services.matching.houses.match_or_create_house. прямого импорта app.services.matching.houses.match_or_create_house.
@ -942,18 +941,15 @@ def upsert_house(db: Session, h: HouseInfo, *, matcher: HouseMatcher) -> int | N
def _upsert_house_with_url( def _upsert_house_with_url(
db: Session, h: HouseInfo, house_url: str, *, matcher: HouseMatcher db: Session, h: HouseInfo, house_url: str, *, matcher: HouseMatcher
) -> int | None: ) -> int:
"""Внутренний вариант upsert_house с правильным URL. """Внутренний вариант upsert_house с правильным URL.
Делегирует в _persist_house используй его вместо прямого вызова. Делегирует в _persist_house используй его вместо прямого вызова.
Возвращает None если matcher отказал (P1 numberless без кадастра).
""" """
return _persist_house(db, h, house_url, matcher=matcher) return _persist_house(db, h, house_url, matcher=matcher)
def _persist_house( def _persist_house(db: Session, h: HouseInfo, house_url: str, *, matcher: HouseMatcher) -> int:
db: Session, h: HouseInfo, house_url: str, *, matcher: HouseMatcher
) -> int | None:
"""Канонический путь сохранения дома-avito через match_or_create_house. """Канонический путь сохранения дома-avito через match_or_create_house.
Шаги: Шаги:
@ -982,17 +978,6 @@ def _persist_house(
source_url=house_url or None, source_url=house_url or None,
) )
# P1: matcher refuses a numberless address without a cadastral number
# (method 'no_house_number' → house_id None). No canonical row exists to enrich,
# so skip the UPDATE and signal the graceful skip up to save_house_catalog_enrichment.
if house_id is None:
logger.info(
"_persist_house: skip numberless avito house ext_id=%s addr=%r (no cadastral)",
h.ext_id,
h.short_address or h.full_address,
)
return None
db.execute( db.execute(
text(""" text("""
UPDATE houses SET UPDATE houses SET
@ -1296,25 +1281,6 @@ def save_house_catalog_enrichment(
} }
house_id = _upsert_house_with_url(db, e.house, e.house_url, matcher=matcher) house_id = _upsert_house_with_url(db, e.house, e.house_url, matcher=matcher)
# P1: matcher refused to create a canonical house for a numberless address without a
# cadastral number (house_id None). Downstream saves (reviews / placement history) are
# house-scoped and meaningless with a NULL house_id, so graceful-skip the whole
# enrichment — same zero-counter contract as the id-less guard above.
if house_id is None:
logger.warning(
"save_house_catalog_enrichment: пропуск дома с безномерным адресом без кадастра"
" (ext_id=%s addr=%r) — matcher вернул no_house_number",
e.house.ext_id,
e.house.short_address or e.house.full_address,
)
return {
"house_id": 0,
"reviews": 0,
"sellers": 0,
"listings_linked": 0,
"placement_history": 0,
}
# 2. Сохранить отзывы # 2. Сохранить отзывы
reviews_count = save_house_reviews(db, house_id, e.reviews) reviews_count = save_house_reviews(db, house_id, e.reviews)