feat(tradein/matching): fias_exact tier + estimate fias persist + suggest fias_id contract #2184

Merged
lekss361 merged 1 commit from feat/tradein-fias-tier-matching into main 2026-07-02 17:56:13 +00:00
9 changed files with 362 additions and 18 deletions

View file

@ -40,6 +40,10 @@ class SuggestItem(BaseModel):
lat: float lat: float
lon: float lon: float
kind: str kind: str
# ГАР OBJECTGUID (ФИАС) дома — присутствует ТОЛЬКО у house-level кандидатов
# DaData-тира. У прочих тиров / street/locality-кандидатов null. Фронт
# прокидывает его в POST /estimate как target_fias_id (Tier 0.5 fias_exact).
fias_id: str | None = None
class SuggestResponse(BaseModel): class SuggestResponse(BaseModel):
@ -62,13 +66,19 @@ async def suggest_addresses(
/api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет /api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет
""" """
items = await suggest(q, db=db, limit=limit) items = await suggest(q, db=db, limit=limit)
return SuggestResponse(items=[ return SuggestResponse(
SuggestItem( items=[
label=s.label, SuggestItem(
full_address=s.full_address, label=s.label,
lat=s.lat, lon=s.lon, kind=s.kind, full_address=s.full_address,
) for s in items lat=s.lat,
]) lon=s.lon,
kind=s.kind,
fias_id=s.fias_id,
)
for s in items
]
)
class ReverseResponse(BaseModel): class ReverseResponse(BaseModel):

View file

@ -27,6 +27,11 @@ class TradeInEstimateInput(BaseModel):
# geocode() (который падает на DaData-формах при мёртвом Yandex-ключе). # geocode() (который падает на DaData-формах при мёртвом Yandex-ключе).
lat: float | None = Field(default=None, ge=-90, le=90) lat: float | None = Field(default=None, ge=-90, le=90)
lon: float | None = Field(default=None, ge=-180, le=180) lon: float | None = Field(default=None, ge=-180, le=180)
# ФИАС/ГАР OBJECTGUID целевого дома, если фронт разрешил его через suggest
# (SuggestItem.fias_id у house-level кандидата). Прокидывается в матчер
# (Tier 0.5 fias_exact) ПЕРВЫМ, до fias из DaData /clean. Additive/optional —
# отсутствие поля сохраняет прежнее поведение.
target_fias_id: str | None = Field(default=None, max_length=64)
# #2044: опциональный радиус анализа (контрол РАДИУС на /trade-in/v2), метры. # #2044: опциональный радиус анализа (контрол РАДИУС на /trade-in/v2), метры.
# None → текущее дефолтное поведение (1000 м поиск аналогов, 2000 м fallback). # None → текущее дефолтное поведение (1000 м поиск аналогов, 2000 м fallback).
# Задан → и первичный, и fallback-поиск аналогов/сделок используют ровно этот # Задан → и первичный, и fallback-поиск аналогов/сделок используют ровно этот

View file

@ -2663,6 +2663,11 @@ async def estimate_quality(
# cadastr от DaData → cadastr_exact tier заработает по мере backfill houses. # cadastr от DaData → cadastr_exact tier заработает по мере backfill houses.
# Best-effort: None при любой ошибке, estimator продолжает на гео-tier'ах. # Best-effort: None при любой ошибке, estimator продолжает на гео-tier'ах.
target_house_id: int | None = None target_house_id: int | None = None
# ФИАС как first-class ключ: приоритет клиентскому target_fias_id (разрешён
# автокомплитом suggest), иначе fias из DaData /clean. Прокидываем в Tier 0.5
# fias_exact матчера — самый стабильный ключ дома, точнее адрес-строки.
dadata_fias = dadata.house_fias_id if dadata else None
match_fias = payload.target_fias_id or dadata_fias
try: try:
match = match_house_readonly( match = match_house_readonly(
db, db,
@ -2670,12 +2675,39 @@ async def estimate_quality(
lat=geo.lat, lat=geo.lat,
lon=geo.lon, lon=geo.lon,
cadastral_number=(dadata.house_cadnum if dadata else None), cadastral_number=(dadata.house_cadnum if dadata else None),
house_fias_id=match_fias,
) )
if match is not None: if match is not None:
target_house_id = match[0] target_house_id = match[0]
logger.info( logger.info(
"estimate target → house_id=%s via %s (conf=%.2f)", match[0], match[2], match[1] "estimate target → house_id=%s via %s (conf=%.2f)", match[0], match[2], match[1]
) )
# Fill-null persist: если DaData вернул fias, а у канонического дома его
# ещё нет — дописываем (+ qc_geo / enriched_at, сохраняя существующие
# значения через COALESCE). Семантика scripts/backfill_houses_dadata.py.
# Guard `house_fias_id IS NULL` — никогда не перетираем curated id.
# SAVEPOINT изолирует сбой UPDATE от внешней estimate-транзакции.
if dadata_fias and dadata is not None:
try:
with db.begin_nested():
db.execute(
text(
"UPDATE houses "
" SET house_fias_id = CAST(:fias AS text), "
" dadata_qc_geo = COALESCE(dadata_qc_geo, "
" CAST(:qc_geo AS integer)), "
" dadata_enriched_at = COALESCE(dadata_enriched_at, NOW()) "
" WHERE id = CAST(:hid AS bigint) "
" AND house_fias_id IS NULL"
),
{
"fias": dadata_fias,
"qc_geo": dadata.qc_geo,
"hid": target_house_id,
},
)
except Exception as exc: # pragma: no cover — defensive
logger.warning("estimate fias back-fill failed (graceful): %s", exc)
except Exception as exc: # pragma: no cover — defensive except Exception as exc: # pragma: no cover — defensive
logger.warning("target house match failed (graceful): %s", exc) logger.warning("target house match failed (graceful): %s", exc)

View file

@ -324,6 +324,10 @@ class GeocodeSuggestion:
lat: float lat: float
lon: float lon: float
kind: str # 'house' / 'street' / 'locality' kind: str # 'house' / 'street' / 'locality'
# ГАР OBJECTGUID (ФИАС) дома — заполняется ТОЛЬКО для house-level кандидатов
# DaData-тира (fias_level 8/9). У Yandex/Nominatim/cadastral-тиров и у
# street/locality-кандидатов остаётся None (нет стабильного house-fias).
fias_id: str | None = None
def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]: def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]:
@ -378,13 +382,18 @@ async def _dadata_suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion]
for s in raw: for s in raw:
if s.lat is None or s.lon is None: if s.lat is None or s.lon is None:
continue continue
mapped_kind = _DADATA_KIND_MAP.get(s.kind, "locality")
# ФИАС отдаём наружу ТОЛЬКО для house-level кандидата (DaData kind='house'
# ⇔ fias_level 8/9). Для street/city/plot fias_id указывает не на дом —
# не годится как ключ матчинга целевого дома, оставляем None.
out.append( out.append(
GeocodeSuggestion( GeocodeSuggestion(
label=s.value, label=s.value,
full_address=s.unrestricted_value, full_address=s.unrestricted_value,
lat=s.lat, lat=s.lat,
lon=s.lon, lon=s.lon,
kind=_DADATA_KIND_MAP.get(s.kind, "locality"), kind=mapped_kind,
fias_id=s.fias_id if mapped_kind == "house" else None,
) )
) )
return out return out

View file

@ -1,10 +1,11 @@
"""House cross-source matching — 3-tier algorithm. """House cross-source matching — tiered algorithm.
Tier 0 (confidence 1.0): cadastral_number exact match on houses table. Tier 0 (confidence 1.0): cadastral_number exact match on houses table.
Tier 1 (confidence 1.0): ext_source + ext_id already in house_sources. Tier 0.5 (confidence 0.95): house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive.
Tier 2 (confidence 0.9): address_fingerprint match in house_address_aliases. Tier 1 (confidence 1.0): ext_source + ext_id already in house_sources.
Tier 3 (confidence 0.7): geo-proximity within 30 m (PostGIS ST_DWithin). Tier 2 (confidence 0.9): address_fingerprint match in house_address_aliases.
New (confidence 1.0): INSERT new canonical house. Tier 3 (confidence 0.7): geo-proximity within 30 m (PostGIS ST_DWithin).
New (confidence 1.0): INSERT new canonical house.
Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3 Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3
""" """
@ -35,6 +36,7 @@ def match_or_create_house(
year_built: int | None = None, year_built: int | None = None,
building_cadastral_number: str | None = None, building_cadastral_number: str | None = None,
cadastral_number: str | None = None, cadastral_number: str | None = None,
house_fias_id: str | None = None,
source_url: str | None = None, source_url: str | None = None,
) -> tuple[int, float, str]: ) -> tuple[int, float, str]:
"""Match existing house or create new canonical record. """Match existing house or create new canonical record.
@ -45,13 +47,20 @@ def match_or_create_house(
for an unknown address could both miss Tier 0-3 and each INSERT a duplicate for an unknown address could both miss Tier 0-3 and each INSERT a duplicate
house row. Closes finding #1 from 2026-05-24 audit. house row. Closes finding #1 from 2026-05-24 audit.
Args:
house_fias_id: ГАР OBJECTGUID (UUID) of the building, when known upstream
(e.g. DaData /clean/address). Enables Tier 0.5 fias_exact additive and
optional, existing callers are unaffected.
Returns: Returns:
(house_id, confidence [0.0, 1.0], method { (house_id, confidence [0.0, 1.0], method {
'cadastr_exact', 'source_exact', 'fingerprint', 'geo_proximity', 'new' 'cadastr_exact', 'fias_exact', 'source_exact', 'fingerprint',
'geo_proximity', 'new'
}) })
Method values: Method values:
'cadastr_exact' matched by cadastral number (confidence 1.0) 'cadastr_exact' matched by cadastral number (confidence 1.0)
'fias_exact' matched by house_fias_id (ГАР OBJECTGUID) (confidence 0.95)
'source_exact' already in house_sources for this source+ext_id (confidence 1.0) 'source_exact' already in house_sources for this source+ext_id (confidence 1.0)
'fingerprint' matched by address fingerprint (confidence 0.9) 'fingerprint' matched by address fingerprint (confidence 0.9)
'geo_proximity' matched by geo within 30 m (confidence 0.7) 'geo_proximity' matched by geo within 30 m (confidence 0.7)
@ -106,6 +115,35 @@ def match_or_create_house(
logger.info("house match cadastr_exact house_id=%s cad=%s", house_id, cad) logger.info("house match cadastr_exact house_id=%s cad=%s", house_id, cad)
return (house_id, 1.0, "cadastr_exact") return (house_id, 1.0, "cadastr_exact")
# Tier 0.5: house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive.
# Stable ORDER BY id so concurrent/duplicate rows resolve deterministically.
if house_fias_id:
row = (
db.execute(
text(
"SELECT id FROM houses "
"WHERE lower(house_fias_id) = lower(CAST(:fias AS text)) "
"ORDER BY id ASC LIMIT 1"
),
{"fias": house_fias_id},
)
.mappings()
.first()
)
if row:
house_id = int(row["id"])
_upsert_house_source(
db,
house_id=house_id,
ext_source=ext_source,
ext_id=ext_id,
method="fias_exact",
confidence=0.95,
)
_insert_alias(db, house_id=house_id, address=address, fp=fp, source=ext_source)
logger.info("house match fias_exact house_id=%s fias=%s", house_id, house_fias_id)
return (house_id, 0.95, "fias_exact")
# Tier 1: source+ext_id already registered in house_sources # Tier 1: source+ext_id already registered in house_sources
row = ( row = (
db.execute( db.execute(
@ -285,6 +323,7 @@ def match_house_readonly(
lat: float | None = None, lat: float | None = None,
lon: float | None = None, lon: float | None = None,
cadastral_number: str | None = None, cadastral_number: str | None = None,
house_fias_id: str | None = None,
) -> tuple[int, float, str] | None: ) -> tuple[int, float, str] | None:
"""Match a target address to an EXISTING canonical house WITHOUT creating one. """Match a target address to an EXISTING canonical house WITHOUT creating one.
@ -295,9 +334,15 @@ def match_house_readonly(
Tiers (same lookups as match_or_create_house, minus source_exact which needs an Tiers (same lookups as match_or_create_house, minus source_exact which needs an
ext_id): ext_id):
1. cadastr_exact houses.cadastral_number = :cad (confidence 1.0) 0. cadastr_exact houses.cadastral_number = :cad (confidence 1.0)
2. fingerprint house_address_aliases.fingerprint (confidence 0.9) 0.5 fias_exact houses.house_fias_id = :fias (ci) (confidence 0.95)
3. geo_proximity within 50 m of an existing house (confidence 0.7) 1. fingerprint house_address_aliases.fingerprint (confidence 0.9)
2. geo_proximity within 50 m of an existing house (confidence 0.7)
Args:
house_fias_id: ГАР OBJECTGUID (UUID) of the target building when known
(e.g. DaData /clean/address or a client-supplied target_fias_id).
Additive/optional omitting it preserves prior behaviour exactly.
Returns (house_id, confidence, method) or None if nothing confident matches. Returns (house_id, confidence, method) or None if nothing confident matches.
Uses 50 m for geo (vs 30 m in match_or_create_house) to absorb geocoder jitter Uses 50 m for geo (vs 30 m in match_or_create_house) to absorb geocoder jitter
@ -320,6 +365,27 @@ def match_house_readonly(
) )
return (house_id, 1.0, "cadastr_exact") return (house_id, 1.0, "cadastr_exact")
# Tier 0.5: house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive.
if house_fias_id:
row = (
db.execute(
text(
"SELECT id FROM houses "
"WHERE lower(house_fias_id) = lower(CAST(:fias AS text)) "
"ORDER BY id ASC LIMIT 1"
),
{"fias": house_fias_id},
)
.mappings()
.first()
)
if row:
house_id = int(row["id"])
logger.info(
"house readonly match fias_exact house_id=%s fias=%s", house_id, house_fias_id
)
return (house_id, 0.95, "fias_exact")
# Tier 1: address fingerprint lookup # Tier 1: address fingerprint lookup
fp = address_fingerprint(address, lat, lon) fp = address_fingerprint(address, lat, lon)
row = ( row = (

View file

@ -0,0 +1,26 @@
-- 159_houses_fias_idx.sql
-- Индекс под Tier 0.5 fias_exact матчинга домов (feat/tradein-fias-tier-matching).
--
-- WHY:
-- houses.house_fias_id (ГАР OBJECTGUID, migration 070) теперь заполнен у ~5.1k
-- из 11.8k домов (DaData backfill 2026-07-02). Матчинг-каскад получил новый
-- tier `fias_exact` (services/matching/houses.py) с предикатом
-- `lower(house_fias_id) = lower(:fias)`. Без функционального индекса каждый
-- estimate/scrape делал бы seq-scan по houses.
--
-- SCOPE:
-- Только создание partial functional index. Данные не трогаются.
--
-- IDEMPOTENCY / SAFETY:
-- - BEGIN/COMMIT одной транзакцией.
-- - CREATE INDEX IF NOT EXISTS → повторный прогон no-op (auto-apply strict).
-- - Partial (WHERE house_fias_id IS NOT NULL) — не индексируем ~6.7k NULL-строк.
-- - lower(...) выражение соответствует case-insensitive предикату матчера.
BEGIN;
CREATE INDEX IF NOT EXISTS houses_fias_lower_idx
ON houses (lower(house_fias_id))
WHERE house_fias_id IS NOT NULL;
COMMIT;

View file

@ -218,6 +218,59 @@ def test_match_house_tier0_cadastr():
assert method == "cadastr_exact" assert method == "cadastr_exact"
def test_match_house_tier05_fias_exact():
"""Tier 0.5: house_fias_id match → fias_exact (conf 0.95) before source/fp/geo.
No cadastral_number supplied Tier 0 skipped; the fias SELECT is the first
lookup after the advisory lock. On hit, _upsert_house_source + _insert_alias
fire (address carries a house number so the alias is registered).
"""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
{"id": 55}, # fias_exact hit (Tier 0.5)
None, # _upsert_house_source
None, # _insert_alias
]
)
house_id, conf, method = match_or_create_house(
db,
"avito",
"ext-fias-1",
address="ул Ленина 5",
lat=56.8,
lon=60.5,
house_fias_id="0a1b2c3d-0000-4000-8000-000000000001",
)
assert house_id == 55
assert conf == 0.95
assert method == "fias_exact"
def test_match_house_fias_skipped_when_absent():
"""No house_fias_id → Tier 0.5 does not run; first lookup is house_sources."""
from app.services.matching.houses import match_or_create_house
db = _make_db(
[
None, # pg_advisory_xact_lock
{"house_id": 8}, # house_sources hit (fias tier skipped, no fias SELECT)
]
)
house_id, _conf, method = match_or_create_house(
db,
"avito",
"ext-nofias",
address="пр Мира 10",
lat=56.8,
lon=60.5,
)
assert house_id == 8
assert method == "source_exact"
def test_match_house_tier1_source_exact(): def test_match_house_tier1_source_exact():
"""Tier 1: ext_source+ext_id already in house_sources. """Tier 1: ext_source+ext_id already in house_sources.

View file

@ -66,6 +66,26 @@ def test_readonly_cadastr_exact_hit():
assert result == (42, 1.0, "cadastr_exact") assert result == (42, 1.0, "cadastr_exact")
def test_readonly_fias_exact_hit():
"""Tier 0.5: house_fias_id match → fias_exact (conf 0.95), no cad supplied."""
db = MagicMock()
db.execute.return_value.mappings.return_value.first.side_effect = [{"id": 55}]
result = match_house_readonly(
db,
address="ул Малышева 125",
house_fias_id="0a1b2c3d-0000-4000-8000-000000000001",
)
assert result == (55, 0.95, "fias_exact")
def test_readonly_fias_skipped_when_absent():
"""No house_fias_id → fias tier skipped; first .first() is fingerprint tier."""
db = MagicMock()
db.execute.return_value.mappings.return_value.first.side_effect = [{"house_id": 7}]
result = match_house_readonly(db, address="Екатеринбург, ул Малышева, д 125")
assert result == (7, 0.9, "fingerprint")
def test_readonly_fingerprint_hit(): def test_readonly_fingerprint_hit():
db = MagicMock() db = MagicMock()
# cadastral_number=None → cadastr tier skipped; first .first() is fingerprint # cadastral_number=None → cadastr tier skipped; first .first() is fingerprint

View file

@ -0,0 +1,123 @@
"""Тесты контракта suggest→ФИАС (feat/tradein-fias-tier-matching).
Покрывают:
1. geocoder._dadata_suggest fias_id прокидывается ТОЛЬКО у house-level
кандидата (DaData kind='house' fias_level 8/9); у street/city/plot None.
2. SuggestItem сериализует fias_id (house-тир) и None у не-DaData тиров.
3. TradeInEstimateInput принимает target_fias_id и дефолтит None при отсутствии.
"""
import os
# Settings требует DATABASE_URL на импорте — ставим dummy DSN до app-импортов.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from unittest.mock import AsyncMock, patch
from app.api.v1.geocode import SuggestItem
from app.schemas.trade_in import TradeInEstimateInput
from app.services.dadata import DadataSuggestion
from app.services.geocoder import GeocodeSuggestion, _dadata_suggest
def _sugg(kind: str, fias_id: str | None) -> DadataSuggestion:
return DadataSuggestion(
value=f"тест {kind}",
unrestricted_value=f"г Екатеринбург, тест {kind}",
lat=56.84,
lon=60.60,
fias_id=fias_id,
house="1" if kind == "house" else None,
street="Тестовая",
city="Екатеринбург",
kind=kind,
)
# ── 1. _dadata_suggest: fias_id только у house-тира ──────────────────────────
async def test_dadata_suggest_fias_only_for_house_kind():
raw = [
_sugg("house", "0a1b2c3d-0000-4000-8000-00000000aaaa"),
_sugg("street", "0a1b2c3d-0000-4000-8000-00000000bbbb"),
_sugg("city", "0a1b2c3d-0000-4000-8000-00000000cccc"),
_sugg("plot", "0a1b2c3d-0000-4000-8000-00000000dddd"),
]
with patch(
"app.services.geocoder.dadata.suggest_addresses",
new=AsyncMock(return_value=raw),
):
out = await _dadata_suggest("тест", limit=8)
assert len(out) == 4
house = next(s for s in out if s.kind == "house")
assert house.fias_id == "0a1b2c3d-0000-4000-8000-00000000aaaa"
# У всех НЕ-house тиров fias_id обнулён, даже если DaData его прислал.
for s in out:
if s.kind != "house":
assert s.fias_id is None
async def test_dadata_suggest_house_without_fias_stays_none():
"""house-кандидат без fias у DaData → fias_id None (не падаем)."""
with patch(
"app.services.geocoder.dadata.suggest_addresses",
new=AsyncMock(return_value=[_sugg("house", None)]),
):
out = await _dadata_suggest("тест", limit=8)
assert len(out) == 1
assert out[0].fias_id is None
# ── 2. SuggestItem сериализация ──────────────────────────────────────────────
def test_suggest_item_serializes_fias_id():
item = SuggestItem(
label="ул Малышева, д 30",
full_address="г Екатеринбург, ул Малышева, д 30",
lat=56.84,
lon=60.60,
kind="house",
fias_id="0a1b2c3d-0000-4000-8000-00000000aaaa",
)
dumped = item.model_dump()
assert dumped["fias_id"] == "0a1b2c3d-0000-4000-8000-00000000aaaa"
def test_suggest_item_fias_id_defaults_none_for_non_dadata_tier():
"""Не-DaData тир (Yandex/Nominatim/cadastral) не передаёт fias_id → None."""
gs = GeocodeSuggestion(
label="Малышева 30",
full_address="г Екатеринбург, ул Малышева, д 30",
lat=56.84,
lon=60.60,
kind="house",
)
assert gs.fias_id is None
item = SuggestItem(
label=gs.label,
full_address=gs.full_address,
lat=gs.lat,
lon=gs.lon,
kind=gs.kind,
fias_id=gs.fias_id,
)
assert item.model_dump()["fias_id"] is None
# ── 3. TradeInEstimateInput.target_fias_id ───────────────────────────────────
def test_estimate_input_accepts_target_fias_id():
payload = TradeInEstimateInput(
address="ул Малышева 30, Екатеринбург",
area_m2=50.0,
rooms=2,
target_fias_id="0a1b2c3d-0000-4000-8000-00000000aaaa",
)
assert payload.target_fias_id == "0a1b2c3d-0000-4000-8000-00000000aaaa"
def test_estimate_input_target_fias_id_defaults_none():
payload = TradeInEstimateInput(
address="ул Малышева 30, Екатеринбург",
area_m2=50.0,
rooms=2,
)
assert payload.target_fias_id is None