From 67a852caf1f53bc76adc4420611281c6e86f9bcc Mon Sep 17 00:00:00 2001 From: bot-backend Date: Thu, 2 Jul 2026 20:49:04 +0300 Subject: [PATCH] feat(tradein/matching): fias_exact tier + estimate fias persist + suggest fias_id contract --- tradein-mvp/backend/app/api/v1/geocode.py | 24 +++- tradein-mvp/backend/app/schemas/trade_in.py | 5 + tradein-mvp/backend/app/services/estimator.py | 32 +++++ tradein-mvp/backend/app/services/geocoder.py | 11 +- .../backend/app/services/matching/houses.py | 86 ++++++++++-- .../backend/data/sql/159_houses_fias_idx.sql | 26 ++++ tradein-mvp/backend/tests/test_matching.py | 53 ++++++++ .../backend/tests/test_same_building_match.py | 20 +++ .../tests/test_suggest_fias_contract.py | 123 ++++++++++++++++++ 9 files changed, 362 insertions(+), 18 deletions(-) create mode 100644 tradein-mvp/backend/data/sql/159_houses_fias_idx.sql create mode 100644 tradein-mvp/backend/tests/test_suggest_fias_contract.py diff --git a/tradein-mvp/backend/app/api/v1/geocode.py b/tradein-mvp/backend/app/api/v1/geocode.py index 2c4da301..76fe0cfe 100644 --- a/tradein-mvp/backend/app/api/v1/geocode.py +++ b/tradein-mvp/backend/app/api/v1/geocode.py @@ -40,6 +40,10 @@ class SuggestItem(BaseModel): lat: float lon: float kind: str + # ГАР OBJECTGUID (ФИАС) дома — присутствует ТОЛЬКО у house-level кандидатов + # DaData-тира. У прочих тиров / street/locality-кандидатов null. Фронт + # прокидывает его в POST /estimate как target_fias_id (Tier 0.5 fias_exact). + fias_id: str | None = None class SuggestResponse(BaseModel): @@ -62,13 +66,19 @@ async def suggest_addresses( /api/v1/geocode/suggest?q=Цвиллинга # → пусто, такой улицы в ЕКБ нет """ items = await suggest(q, db=db, limit=limit) - return SuggestResponse(items=[ - SuggestItem( - label=s.label, - full_address=s.full_address, - lat=s.lat, lon=s.lon, kind=s.kind, - ) for s in items - ]) + return SuggestResponse( + items=[ + SuggestItem( + label=s.label, + full_address=s.full_address, + lat=s.lat, + lon=s.lon, + kind=s.kind, + fias_id=s.fias_id, + ) + for s in items + ] + ) class ReverseResponse(BaseModel): diff --git a/tradein-mvp/backend/app/schemas/trade_in.py b/tradein-mvp/backend/app/schemas/trade_in.py index 62f9ae54..86237492 100644 --- a/tradein-mvp/backend/app/schemas/trade_in.py +++ b/tradein-mvp/backend/app/schemas/trade_in.py @@ -27,6 +27,11 @@ class TradeInEstimateInput(BaseModel): # geocode() (который падает на DaData-формах при мёртвом Yandex-ключе). lat: float | None = Field(default=None, ge=-90, le=90) lon: float | None = Field(default=None, ge=-180, le=180) + # ФИАС/ГАР OBJECTGUID целевого дома, если фронт разрешил его через suggest + # (SuggestItem.fias_id у house-level кандидата). Прокидывается в матчер + # (Tier 0.5 fias_exact) ПЕРВЫМ, до fias из DaData /clean. Additive/optional — + # отсутствие поля сохраняет прежнее поведение. + target_fias_id: str | None = Field(default=None, max_length=64) # #2044: опциональный радиус анализа (контрол РАДИУС на /trade-in/v2), метры. # None → текущее дефолтное поведение (1000 м поиск аналогов, 2000 м fallback). # Задан → и первичный, и fallback-поиск аналогов/сделок используют ровно этот diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index 79b8c44e..096ab93f 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -2663,6 +2663,11 @@ async def estimate_quality( # cadastr от DaData → cadastr_exact tier заработает по мере backfill houses. # Best-effort: None при любой ошибке, estimator продолжает на гео-tier'ах. target_house_id: int | None = None + # ФИАС как first-class ключ: приоритет клиентскому target_fias_id (разрешён + # автокомплитом suggest), иначе fias из DaData /clean. Прокидываем в Tier 0.5 + # fias_exact матчера — самый стабильный ключ дома, точнее адрес-строки. + dadata_fias = dadata.house_fias_id if dadata else None + match_fias = payload.target_fias_id or dadata_fias try: match = match_house_readonly( db, @@ -2670,12 +2675,39 @@ async def estimate_quality( lat=geo.lat, lon=geo.lon, cadastral_number=(dadata.house_cadnum if dadata else None), + house_fias_id=match_fias, ) if match is not None: target_house_id = match[0] logger.info( "estimate target → house_id=%s via %s (conf=%.2f)", match[0], match[2], match[1] ) + # Fill-null persist: если DaData вернул fias, а у канонического дома его + # ещё нет — дописываем (+ qc_geo / enriched_at, сохраняя существующие + # значения через COALESCE). Семантика scripts/backfill_houses_dadata.py. + # Guard `house_fias_id IS NULL` — никогда не перетираем curated id. + # SAVEPOINT изолирует сбой UPDATE от внешней estimate-транзакции. + if dadata_fias and dadata is not None: + try: + with db.begin_nested(): + db.execute( + text( + "UPDATE houses " + " SET house_fias_id = CAST(:fias AS text), " + " dadata_qc_geo = COALESCE(dadata_qc_geo, " + " CAST(:qc_geo AS integer)), " + " dadata_enriched_at = COALESCE(dadata_enriched_at, NOW()) " + " WHERE id = CAST(:hid AS bigint) " + " AND house_fias_id IS NULL" + ), + { + "fias": dadata_fias, + "qc_geo": dadata.qc_geo, + "hid": target_house_id, + }, + ) + except Exception as exc: # pragma: no cover — defensive + logger.warning("estimate fias back-fill failed (graceful): %s", exc) except Exception as exc: # pragma: no cover — defensive logger.warning("target house match failed (graceful): %s", exc) diff --git a/tradein-mvp/backend/app/services/geocoder.py b/tradein-mvp/backend/app/services/geocoder.py index 984b9d2e..45835d75 100644 --- a/tradein-mvp/backend/app/services/geocoder.py +++ b/tradein-mvp/backend/app/services/geocoder.py @@ -324,6 +324,10 @@ class GeocodeSuggestion: lat: float lon: float kind: str # 'house' / 'street' / 'locality' + # ГАР OBJECTGUID (ФИАС) дома — заполняется ТОЛЬКО для house-level кандидатов + # DaData-тира (fias_level 8/9). У Yandex/Nominatim/cadastral-тиров и у + # street/locality-кандидатов остаётся None (нет стабильного house-fias). + fias_id: str | None = None def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]: @@ -378,13 +382,18 @@ async def _dadata_suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion] for s in raw: if s.lat is None or s.lon is None: continue + mapped_kind = _DADATA_KIND_MAP.get(s.kind, "locality") + # ФИАС отдаём наружу ТОЛЬКО для house-level кандидата (DaData kind='house' + # ⇔ fias_level 8/9). Для street/city/plot fias_id указывает не на дом — + # не годится как ключ матчинга целевого дома, оставляем None. out.append( GeocodeSuggestion( label=s.value, full_address=s.unrestricted_value, lat=s.lat, lon=s.lon, - kind=_DADATA_KIND_MAP.get(s.kind, "locality"), + kind=mapped_kind, + fias_id=s.fias_id if mapped_kind == "house" else None, ) ) return out diff --git a/tradein-mvp/backend/app/services/matching/houses.py b/tradein-mvp/backend/app/services/matching/houses.py index e3ff94b5..fc7639d8 100644 --- a/tradein-mvp/backend/app/services/matching/houses.py +++ b/tradein-mvp/backend/app/services/matching/houses.py @@ -1,10 +1,11 @@ -"""House cross-source matching — 3-tier algorithm. +"""House cross-source matching — tiered algorithm. -Tier 0 (confidence 1.0): cadastral_number exact match on houses table. -Tier 1 (confidence 1.0): ext_source + ext_id already in house_sources. -Tier 2 (confidence 0.9): address_fingerprint match in house_address_aliases. -Tier 3 (confidence 0.7): geo-proximity within 30 m (PostGIS ST_DWithin). -New (confidence 1.0): INSERT new canonical house. +Tier 0 (confidence 1.0): cadastral_number exact match on houses table. +Tier 0.5 (confidence 0.95): house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive. +Tier 1 (confidence 1.0): ext_source + ext_id already in house_sources. +Tier 2 (confidence 0.9): address_fingerprint match in house_address_aliases. +Tier 3 (confidence 0.7): geo-proximity within 30 m (PostGIS ST_DWithin). +New (confidence 1.0): INSERT new canonical house. Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3 """ @@ -35,6 +36,7 @@ def match_or_create_house( year_built: int | None = None, building_cadastral_number: str | None = None, cadastral_number: str | None = None, + house_fias_id: str | None = None, source_url: str | None = None, ) -> tuple[int, float, str]: """Match existing house or create new canonical record. @@ -45,13 +47,20 @@ def match_or_create_house( for an unknown address could both miss Tier 0-3 and each INSERT a duplicate house row. Closes finding #1 from 2026-05-24 audit. + Args: + house_fias_id: ГАР OBJECTGUID (UUID) of the building, when known upstream + (e.g. DaData /clean/address). Enables Tier 0.5 fias_exact — additive and + optional, existing callers are unaffected. + Returns: (house_id, confidence ∈ [0.0, 1.0], method ∈ { - 'cadastr_exact', 'source_exact', 'fingerprint', 'geo_proximity', 'new' + 'cadastr_exact', 'fias_exact', 'source_exact', 'fingerprint', + 'geo_proximity', 'new' }) Method values: 'cadastr_exact' — matched by cadastral number (confidence 1.0) + 'fias_exact' — matched by house_fias_id (ГАР OBJECTGUID) (confidence 0.95) 'source_exact' — already in house_sources for this source+ext_id (confidence 1.0) 'fingerprint' — matched by address fingerprint (confidence 0.9) 'geo_proximity' — matched by geo within 30 m (confidence 0.7) @@ -106,6 +115,35 @@ def match_or_create_house( logger.info("house match cadastr_exact house_id=%s cad=%s", house_id, cad) return (house_id, 1.0, "cadastr_exact") + # Tier 0.5: house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive. + # Stable ORDER BY id so concurrent/duplicate rows resolve deterministically. + if house_fias_id: + row = ( + db.execute( + text( + "SELECT id FROM houses " + "WHERE lower(house_fias_id) = lower(CAST(:fias AS text)) " + "ORDER BY id ASC LIMIT 1" + ), + {"fias": house_fias_id}, + ) + .mappings() + .first() + ) + if row: + house_id = int(row["id"]) + _upsert_house_source( + db, + house_id=house_id, + ext_source=ext_source, + ext_id=ext_id, + method="fias_exact", + confidence=0.95, + ) + _insert_alias(db, house_id=house_id, address=address, fp=fp, source=ext_source) + logger.info("house match fias_exact house_id=%s fias=%s", house_id, house_fias_id) + return (house_id, 0.95, "fias_exact") + # Tier 1: source+ext_id already registered in house_sources row = ( db.execute( @@ -285,6 +323,7 @@ def match_house_readonly( lat: float | None = None, lon: float | None = None, cadastral_number: str | None = None, + house_fias_id: str | None = None, ) -> tuple[int, float, str] | None: """Match a target address to an EXISTING canonical house WITHOUT creating one. @@ -295,9 +334,15 @@ def match_house_readonly( Tiers (same lookups as match_or_create_house, minus source_exact which needs an ext_id): - 1. cadastr_exact — houses.cadastral_number = :cad (confidence 1.0) - 2. fingerprint — house_address_aliases.fingerprint (confidence 0.9) - 3. geo_proximity — within 50 m of an existing house (confidence 0.7) + 0. cadastr_exact — houses.cadastral_number = :cad (confidence 1.0) + 0.5 fias_exact — houses.house_fias_id = :fias (ci) (confidence 0.95) + 1. fingerprint — house_address_aliases.fingerprint (confidence 0.9) + 2. geo_proximity — within 50 m of an existing house (confidence 0.7) + + Args: + house_fias_id: ГАР OBJECTGUID (UUID) of the target building when known + (e.g. DaData /clean/address or a client-supplied target_fias_id). + Additive/optional — omitting it preserves prior behaviour exactly. Returns (house_id, confidence, method) or None if nothing confident matches. Uses 50 m for geo (vs 30 m in match_or_create_house) to absorb geocoder jitter @@ -320,6 +365,27 @@ def match_house_readonly( ) return (house_id, 1.0, "cadastr_exact") + # Tier 0.5: house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive. + if house_fias_id: + row = ( + db.execute( + text( + "SELECT id FROM houses " + "WHERE lower(house_fias_id) = lower(CAST(:fias AS text)) " + "ORDER BY id ASC LIMIT 1" + ), + {"fias": house_fias_id}, + ) + .mappings() + .first() + ) + if row: + house_id = int(row["id"]) + logger.info( + "house readonly match fias_exact house_id=%s fias=%s", house_id, house_fias_id + ) + return (house_id, 0.95, "fias_exact") + # Tier 1: address fingerprint lookup fp = address_fingerprint(address, lat, lon) row = ( diff --git a/tradein-mvp/backend/data/sql/159_houses_fias_idx.sql b/tradein-mvp/backend/data/sql/159_houses_fias_idx.sql new file mode 100644 index 00000000..883810a9 --- /dev/null +++ b/tradein-mvp/backend/data/sql/159_houses_fias_idx.sql @@ -0,0 +1,26 @@ +-- 159_houses_fias_idx.sql +-- Индекс под Tier 0.5 fias_exact матчинга домов (feat/tradein-fias-tier-matching). +-- +-- WHY: +-- houses.house_fias_id (ГАР OBJECTGUID, migration 070) теперь заполнен у ~5.1k +-- из 11.8k домов (DaData backfill 2026-07-02). Матчинг-каскад получил новый +-- tier `fias_exact` (services/matching/houses.py) с предикатом +-- `lower(house_fias_id) = lower(:fias)`. Без функционального индекса каждый +-- estimate/scrape делал бы seq-scan по houses. +-- +-- SCOPE: +-- Только создание partial functional index. Данные не трогаются. +-- +-- IDEMPOTENCY / SAFETY: +-- - BEGIN/COMMIT одной транзакцией. +-- - CREATE INDEX IF NOT EXISTS → повторный прогон no-op (auto-apply strict). +-- - Partial (WHERE house_fias_id IS NOT NULL) — не индексируем ~6.7k NULL-строк. +-- - lower(...) выражение соответствует case-insensitive предикату матчера. + +BEGIN; + +CREATE INDEX IF NOT EXISTS houses_fias_lower_idx + ON houses (lower(house_fias_id)) + WHERE house_fias_id IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_matching.py b/tradein-mvp/backend/tests/test_matching.py index 1fccf83d..ea2bbb26 100644 --- a/tradein-mvp/backend/tests/test_matching.py +++ b/tradein-mvp/backend/tests/test_matching.py @@ -218,6 +218,59 @@ def test_match_house_tier0_cadastr(): assert method == "cadastr_exact" +def test_match_house_tier05_fias_exact(): + """Tier 0.5: house_fias_id match → fias_exact (conf 0.95) before source/fp/geo. + + No cadastral_number supplied → Tier 0 skipped; the fias SELECT is the first + lookup after the advisory lock. On hit, _upsert_house_source + _insert_alias + fire (address carries a house number so the alias is registered). + """ + from app.services.matching.houses import match_or_create_house + + db = _make_db( + [ + None, # pg_advisory_xact_lock + {"id": 55}, # fias_exact hit (Tier 0.5) + None, # _upsert_house_source + None, # _insert_alias + ] + ) + house_id, conf, method = match_or_create_house( + db, + "avito", + "ext-fias-1", + address="ул Ленина 5", + lat=56.8, + lon=60.5, + house_fias_id="0a1b2c3d-0000-4000-8000-000000000001", + ) + assert house_id == 55 + assert conf == 0.95 + assert method == "fias_exact" + + +def test_match_house_fias_skipped_when_absent(): + """No house_fias_id → Tier 0.5 does not run; first lookup is house_sources.""" + from app.services.matching.houses import match_or_create_house + + db = _make_db( + [ + None, # pg_advisory_xact_lock + {"house_id": 8}, # house_sources hit (fias tier skipped, no fias SELECT) + ] + ) + house_id, _conf, method = match_or_create_house( + db, + "avito", + "ext-nofias", + address="пр Мира 10", + lat=56.8, + lon=60.5, + ) + assert house_id == 8 + assert method == "source_exact" + + def test_match_house_tier1_source_exact(): """Tier 1: ext_source+ext_id already in house_sources. diff --git a/tradein-mvp/backend/tests/test_same_building_match.py b/tradein-mvp/backend/tests/test_same_building_match.py index fcc0b3f3..53020455 100644 --- a/tradein-mvp/backend/tests/test_same_building_match.py +++ b/tradein-mvp/backend/tests/test_same_building_match.py @@ -66,6 +66,26 @@ def test_readonly_cadastr_exact_hit(): assert result == (42, 1.0, "cadastr_exact") +def test_readonly_fias_exact_hit(): + """Tier 0.5: house_fias_id match → fias_exact (conf 0.95), no cad supplied.""" + db = MagicMock() + db.execute.return_value.mappings.return_value.first.side_effect = [{"id": 55}] + result = match_house_readonly( + db, + address="ул Малышева 125", + house_fias_id="0a1b2c3d-0000-4000-8000-000000000001", + ) + assert result == (55, 0.95, "fias_exact") + + +def test_readonly_fias_skipped_when_absent(): + """No house_fias_id → fias tier skipped; first .first() is fingerprint tier.""" + db = MagicMock() + db.execute.return_value.mappings.return_value.first.side_effect = [{"house_id": 7}] + result = match_house_readonly(db, address="Екатеринбург, ул Малышева, д 125") + assert result == (7, 0.9, "fingerprint") + + def test_readonly_fingerprint_hit(): db = MagicMock() # cadastral_number=None → cadastr tier skipped; first .first() is fingerprint diff --git a/tradein-mvp/backend/tests/test_suggest_fias_contract.py b/tradein-mvp/backend/tests/test_suggest_fias_contract.py new file mode 100644 index 00000000..dd5478d3 --- /dev/null +++ b/tradein-mvp/backend/tests/test_suggest_fias_contract.py @@ -0,0 +1,123 @@ +"""Тесты контракта suggest→ФИАС (feat/tradein-fias-tier-matching). + +Покрывают: + 1. geocoder._dadata_suggest — fias_id прокидывается ТОЛЬКО у house-level + кандидата (DaData kind='house' ⇔ fias_level 8/9); у street/city/plot None. + 2. SuggestItem — сериализует fias_id (house-тир) и None у не-DaData тиров. + 3. TradeInEstimateInput — принимает target_fias_id и дефолтит None при отсутствии. +""" + +import os + +# Settings требует DATABASE_URL на импорте — ставим dummy DSN до app-импортов. +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from unittest.mock import AsyncMock, patch + +from app.api.v1.geocode import SuggestItem +from app.schemas.trade_in import TradeInEstimateInput +from app.services.dadata import DadataSuggestion +from app.services.geocoder import GeocodeSuggestion, _dadata_suggest + + +def _sugg(kind: str, fias_id: str | None) -> DadataSuggestion: + return DadataSuggestion( + value=f"тест {kind}", + unrestricted_value=f"г Екатеринбург, тест {kind}", + lat=56.84, + lon=60.60, + fias_id=fias_id, + house="1" if kind == "house" else None, + street="Тестовая", + city="Екатеринбург", + kind=kind, + ) + + +# ── 1. _dadata_suggest: fias_id только у house-тира ────────────────────────── +async def test_dadata_suggest_fias_only_for_house_kind(): + raw = [ + _sugg("house", "0a1b2c3d-0000-4000-8000-00000000aaaa"), + _sugg("street", "0a1b2c3d-0000-4000-8000-00000000bbbb"), + _sugg("city", "0a1b2c3d-0000-4000-8000-00000000cccc"), + _sugg("plot", "0a1b2c3d-0000-4000-8000-00000000dddd"), + ] + with patch( + "app.services.geocoder.dadata.suggest_addresses", + new=AsyncMock(return_value=raw), + ): + out = await _dadata_suggest("тест", limit=8) + + assert len(out) == 4 + house = next(s for s in out if s.kind == "house") + assert house.fias_id == "0a1b2c3d-0000-4000-8000-00000000aaaa" + # У всех НЕ-house тиров fias_id обнулён, даже если DaData его прислал. + for s in out: + if s.kind != "house": + assert s.fias_id is None + + +async def test_dadata_suggest_house_without_fias_stays_none(): + """house-кандидат без fias у DaData → fias_id None (не падаем).""" + with patch( + "app.services.geocoder.dadata.suggest_addresses", + new=AsyncMock(return_value=[_sugg("house", None)]), + ): + out = await _dadata_suggest("тест", limit=8) + assert len(out) == 1 + assert out[0].fias_id is None + + +# ── 2. SuggestItem сериализация ────────────────────────────────────────────── +def test_suggest_item_serializes_fias_id(): + item = SuggestItem( + label="ул Малышева, д 30", + full_address="г Екатеринбург, ул Малышева, д 30", + lat=56.84, + lon=60.60, + kind="house", + fias_id="0a1b2c3d-0000-4000-8000-00000000aaaa", + ) + dumped = item.model_dump() + assert dumped["fias_id"] == "0a1b2c3d-0000-4000-8000-00000000aaaa" + + +def test_suggest_item_fias_id_defaults_none_for_non_dadata_tier(): + """Не-DaData тир (Yandex/Nominatim/cadastral) не передаёт fias_id → None.""" + gs = GeocodeSuggestion( + label="Малышева 30", + full_address="г Екатеринбург, ул Малышева, д 30", + lat=56.84, + lon=60.60, + kind="house", + ) + assert gs.fias_id is None + item = SuggestItem( + label=gs.label, + full_address=gs.full_address, + lat=gs.lat, + lon=gs.lon, + kind=gs.kind, + fias_id=gs.fias_id, + ) + assert item.model_dump()["fias_id"] is None + + +# ── 3. TradeInEstimateInput.target_fias_id ─────────────────────────────────── +def test_estimate_input_accepts_target_fias_id(): + payload = TradeInEstimateInput( + address="ул Малышева 30, Екатеринбург", + area_m2=50.0, + rooms=2, + target_fias_id="0a1b2c3d-0000-4000-8000-00000000aaaa", + ) + assert payload.target_fias_id == "0a1b2c3d-0000-4000-8000-00000000aaaa" + + +def test_estimate_input_target_fias_id_defaults_none(): + payload = TradeInEstimateInput( + address="ул Малышева 30, Екатеринбург", + area_m2=50.0, + rooms=2, + ) + assert payload.target_fias_id is None