fix(tradein/cian): персист bti_data/valuation house_info в houses + дедуп УК (#2435) #2437

Merged
lekss361 merged 2 commits from feat/tradein-cian-houses-bti-valuation into main 2026-07-04 20:21:33 +00:00
7 changed files with 810 additions and 10 deletions

View file

@ -1865,7 +1865,7 @@ async def scrape_cian_detail(
saved = False saved = False
if listing_id is not None: if listing_id is not None:
save_detail_enrichment(db, listing_id, enrichment) save_detail_enrichment(db, listing_id, enrichment, matcher=RealMatcherAdapter())
saved = True saved = True
return CianDetailTriggerResp( return CianDetailTriggerResp(

View file

@ -26,7 +26,7 @@ from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichme
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.services.scraper_adapters import RealScraperConfig from app.services.scraper_adapters import RealMatcherAdapter, RealScraperConfig
from app.services.scraper_settings import get_scraper_delay from app.services.scraper_settings import get_scraper_delay
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -147,7 +147,9 @@ async def backfill_cian_price_history(
), ),
{"lid": lid}, {"lid": lid},
).scalar_one() ).scalar_one()
save_detail_enrichment(db, lid, enrichment) # matcher (#2435): резолвит канонический дом из bti_data (address/geo
# листинга) — иначе BTI-снапшот распарсен, но выбрасывается.
save_detail_enrichment(db, lid, enrichment, matcher=RealMatcherAdapter())
after = db.execute( after = db.execute(
text( text(
"SELECT COUNT(*) FROM offer_price_history " "SELECT COUNT(*) FROM offer_price_history "

View file

@ -34,7 +34,7 @@ from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.config import settings from app.core.config import settings
from app.services.scraper_adapters import RealScraperConfig from app.services.scraper_adapters import RealMatcherAdapter, RealScraperConfig
from app.services.scraper_settings import get_scraper_delay from app.services.scraper_settings import get_scraper_delay
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -146,7 +146,11 @@ async def backfill_cian_history(
continue continue
try: try:
save_detail_enrichment(db, listing_id, enrichment) # matcher (#2435): резолвит канонический дом из bti_data (address/geo
# листинга) — иначе BTI-снапшот распарсен, но выбрасывается.
save_detail_enrichment(
db, listing_id, enrichment, matcher=RealMatcherAdapter()
)
result.listings_succeeded += 1 result.listings_succeeded += 1
result.price_changes_attempted += len(enrichment.price_changes or []) result.price_changes_attempted += len(enrichment.price_changes or [])
except Exception as exc: except Exception as exc:

View file

@ -0,0 +1,412 @@
"""Тесты для Cian BTI / Valuation house-persist (#2435).
`bti_data` (cian/detail.py) и valuation `house_info`/`managementCompany`
(cian/valuation.py) парсились, но раньше выбрасывались («это задача Stage 6
(houses), где есть house_id») теперь пишутся в канонический houses-ряд:
- detail.py: резолвит дом через инжектируемый `matcher: HouseMatcher`
(`match_or_create_house`, address/geo листинга) mirror avito's `_persist_house`.
- valuation.py: `house_id` уже резолвлен ВЫЗЫВАЮЩИМ (estimator.py, read-only
`match_house_readonly`) здесь только COALESCE-UPDATE, без создания дома.
MagicMock db без реальной БД, тот же паттерн, что test_snapshot_writer.py /
test_extval_house_id_write_path.py.
"""
from __future__ import annotations
import os
# Settings требует DATABASE_URL на import (см. test_extval_house_id_write_path.py).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from contextlib import contextmanager
from unittest.mock import MagicMock
from scraper_kit.providers.cian.detail import DetailEnrichment, save_detail_enrichment
from scraper_kit.providers.cian.valuation import CianValuationResult, _save_to_cache
# ── Part 1: bti_data → houses (detail.py) ────────────────────────────────────
_BTI_SAMPLE = {
"yearRelease": 1961,
"houseMaterialType": "brick",
"floorMax": 4,
"entrances": 4,
"flatCount": 58,
"isEmergency": False,
"houseHeatSupplyType": "central",
"houseGasSupplyType": "central",
"houseOverlapType": "concrete",
"lifts": 0,
"seriesName": "1-447",
}
def _mock_matcher(house_id: int | None = 501, method: str = "new") -> MagicMock:
matcher = MagicMock()
matcher.match_or_create_house.return_value = (house_id, 1.0, method)
return matcher
def _mock_db_bti(
address: str | None = "Екатеринбург, улица Малышева, 51",
lat: float = 56.83,
lon: float = 60.6,
) -> MagicMock:
"""Mock db: SELECT address/lat/lon FROM listings → mappings().first() dict."""
db = MagicMock()
@contextmanager
def _nested():
yield MagicMock()
db.begin_nested.side_effect = _nested
def _exec(sql, params=None):
sql_str = str(sql)
mock_result = MagicMock()
mock_result.fetchone.return_value = None
mock_result.scalar_one_or_none.return_value = None
if "SELECT address, lat, lon FROM listings" in sql_str:
mock_result.mappings.return_value.first.return_value = {
"address": address,
"lat": lat,
"lon": lon,
}
else:
mock_result.mappings.return_value.first.return_value = None
return mock_result
db.execute.side_effect = _exec
return db
def _bti_update_call(db: MagicMock) -> tuple[str, dict] | None:
for call_args in db.execute.call_args_list:
if not call_args.args:
continue
sql = str(call_args.args[0])
if "UPDATE houses" in sql and "series_name" in sql:
params = call_args.args[1] if len(call_args.args) >= 2 else {}
return sql, params
return None
def test_bti_data_present_updates_house_columns_via_coalesce():
"""bti_data + matcher → UPDATE houses SET series_name=COALESCE(...) с BTI-полями."""
db = _mock_db_bti()
matcher = _mock_matcher(house_id=501)
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
save_detail_enrichment(db, 123, enrichment, matcher=matcher)
matcher.match_or_create_house.assert_called_once()
kwargs = matcher.match_or_create_house.call_args.kwargs
assert kwargs["ext_source"] == "cian_bti"
assert kwargs["ext_id"] == "123"
assert kwargs["address"] == "Екатеринбург, улица Малышева, 51"
call = _bti_update_call(db)
assert call is not None, "UPDATE houses с BTI-полями не найден"
sql, params = call
assert "COALESCE" in sql
assert params["house_id"] == 501
assert params["series_name"] == "1-447"
assert params["entrances"] == 4
assert params["flat_count"] == 58
assert params["is_emergency"] is False
assert params["heat_supply_type"] == "central"
assert params["gas_supply_type"] == "central"
assert params["overlap_type"] == "concrete"
db.commit.assert_called_once()
def test_bti_data_absent_is_noop_no_crash():
"""bti_data=None (default) → matcher не вызывается, UPDATE houses не выполняется."""
db = _mock_db_bti()
matcher = _mock_matcher()
enrichment = DetailEnrichment() # bti_data=None
save_detail_enrichment(db, 124, enrichment, matcher=matcher)
matcher.match_or_create_house.assert_not_called()
assert _bti_update_call(db) is None
db.commit.assert_called_once()
def test_bti_data_present_but_no_matcher_is_noop():
"""matcher=None (default, backward-compat для существующих вызовов) → bti_data игнорируется."""
db = _mock_db_bti()
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
save_detail_enrichment(db, 125, enrichment) # no matcher kwarg
assert _bti_update_call(db) is None
db.commit.assert_called_once()
def test_bti_house_resolution_refused_no_crash():
"""matcher отказал (безномерный адрес, method='no_house_number') → graceful skip."""
db = _mock_db_bti()
matcher = _mock_matcher(house_id=None, method="no_house_number")
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
save_detail_enrichment(db, 126, enrichment, matcher=matcher)
assert _bti_update_call(db) is None
db.commit.assert_called_once()
def test_bti_persist_exception_does_not_abort_main_save():
"""Исключение в BTI house-persist (напр. UPDATE houses упал) не прерывает основной save."""
db = _mock_db_bti()
matcher = _mock_matcher(house_id=501)
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
original_side_effect = db.execute.side_effect
def _boom(sql, params=None):
if "UPDATE houses" in str(sql):
raise RuntimeError("boom")
return original_side_effect(sql, params)
db.execute.side_effect = _boom
# Не должно бросить исключение наружу.
save_detail_enrichment(db, 127, enrichment, matcher=matcher)
db.commit.assert_called_once()
def test_bti_missing_listing_address_skips_resolve():
"""Листинг без адреса → matcher не вызывается, no crash (best-effort)."""
db = _mock_db_bti(address="")
matcher = _mock_matcher()
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
save_detail_enrichment(db, 128, enrichment, matcher=matcher)
matcher.match_or_create_house.assert_not_called()
db.commit.assert_called_once()
# ── Part 2: valuation house_info/managementCompany → houses (valuation.py) ──
_HOUSE_INFO_SAMPLE = [
{"title": "Год постройки", "value": 2005},
{"title": "Тип дома", "value": "Кирпичный"},
{"title": "Этажность", "value": 16},
{"title": "Газоснабжение", "value": "Отсутствует"},
{"title": "Отопление", "value": "Центральное"},
{"title": "Тип перекрытий", "value": "Железобетонные"},
{"title": "Подъездов", "value": 1},
{"title": "Количество лифтов", "value": "1 пассажирский, 1 грузовой"},
{"title": "Мусоропровод", "value": "Нет"},
{"title": "Квартир", "value": 105},
{"title": "Реновация", "value": "Нет"},
{"title": "Аварийность", "value": "Нет"},
{"title": "Детская площадка", "value": "Нет"},
{"title": "Спортивная площадка", "value": "Нет"},
]
def _cian_result(
house_info: list[dict] | None = None,
management_company: dict | None = None,
external_house_id: int | None = 54016,
) -> CianValuationResult:
return CianValuationResult(
sale_price_rub=10_000_000,
sale_accuracy=85.0,
chart=[],
external_house_id=external_house_id,
house_info=house_info or [],
management_company=management_company,
)
def _mock_db_valuation(
mc_insert_id: int | None = 900, mc_existing_id: int | None = None
) -> MagicMock:
"""Mock db для _save_to_cache: SELECT (dedup) → UPDATE-or-INSERT management_companies,
затем UPDATE houses.
`mc_existing_id` simulates the dedup SELECT finding a prior row (existing.fetchone()
returns that id) the UPDATE branch is taken instead of INSERT. Default None means
"no existing row", exercising the INSERT branch (the original test-suite behavior).
"""
db = MagicMock()
@contextmanager
def _nested():
yield MagicMock()
db.begin_nested.side_effect = _nested
def _exec(sql, params=None):
sql_str = str(sql)
mock_result = MagicMock()
if "SELECT id FROM management_companies" in sql_str:
mock_result.fetchone.return_value = (
(mc_existing_id,) if mc_existing_id is not None else None
)
elif "INSERT INTO management_companies" in sql_str:
mock_result.fetchone.return_value = (
(mc_insert_id,) if mc_insert_id is not None else None
)
else:
mock_result.fetchone.return_value = None
return mock_result
db.execute.side_effect = _exec
return db
def _mc_calls(db: MagicMock) -> list[str]:
"""SQL statement kinds touching management_companies, in call order."""
kinds = []
for call_args in db.execute.call_args_list:
if not call_args.args:
continue
sql = str(call_args.args[0])
if "SELECT id FROM management_companies" in sql:
kinds.append("select")
elif "INSERT INTO management_companies" in sql:
kinds.append("insert")
elif "UPDATE management_companies" in sql:
kinds.append("update")
return kinds
def _houses_update_call(db: MagicMock) -> tuple[str, dict] | None:
for call_args in db.execute.call_args_list:
if not call_args.args:
continue
sql = str(call_args.args[0])
if "UPDATE houses" in sql:
params = call_args.args[1] if len(call_args.args) >= 2 else {}
return sql, params
return None
def _call_save_to_cache(db: MagicMock, result: CianValuationResult, house_id: int | None) -> None:
_save_to_cache(
db,
cache_key="ck",
address="Екатеринбург, улица Учителей, 18",
total_area=50.0,
rooms_count=2,
floor=3,
total_floors=16,
repair_type="cosmetic",
deal_type="sale",
result=result,
house_id=house_id,
)
def test_valuation_house_info_and_management_company_land_in_houses_update():
"""house_id задан, УК ранее не встречалась → SELECT (не найдено) → INSERT management_companies,
UPDATE houses с management_company_id + house_info-полями."""
db = _mock_db_valuation(mc_insert_id=900)
result = _cian_result(
house_info=_HOUSE_INFO_SAMPLE,
management_company={
"name": 'ТСЖ "Учителей, 18"',
"phones": ["+7 (343) 216-59-12"],
"email": "tsg.u.18@mail.ru",
"openingHours": [{"Пн-Пт": ["10:00-20:00"]}],
"chiefName": "Сабиров Наиль Вагизович",
},
)
_call_save_to_cache(db, result, house_id=42)
assert _mc_calls(db) == ["select", "insert"]
call = _houses_update_call(db)
assert call is not None
sql, params = call
assert "COALESCE" in sql
assert params["hid"] == 42
assert params["mcid"] == 900
assert params["cihi"] == 54016
assert params["yb"] == 2005
assert params["ht"] == "Кирпичный"
assert params["tf"] == 16
assert params["gst"] == "Отсутствует"
assert params["hst"] == "Центральное"
assert params["ot"] == "Железобетонные"
assert params["ent"] == 1
assert params["fc"] == 105
assert params["ie"] is False
assert params["hp"] is False
assert params["pe"] == 1
assert params["ce"] == 1
db.commit.assert_called_once()
def test_valuation_management_company_dedups_on_repeat_call_same_name():
"""Regression for the NULLS-DISTINCT dedup bug (#2435 review): a repeat valuation
call for a management company already persisted (same ext_source/name, ext_id=NULL)
must find it via the explicit SELECT and UPDATE it not INSERT a duplicate row.
`ON CONFLICT (ext_source, name, ext_id)` alone can't dedup ext_id=NULL rows under
Postgres's default NULLS DISTINCT (two NULLs never compare equal), which is why
dedup is done via SELECT-before-INSERT rather than relying on the constraint."""
db = _mock_db_valuation(mc_existing_id=900)
result = _cian_result(
house_info=[],
management_company={"name": 'ТСЖ "Учителей, 18"', "phones": ["+7 (343) 216-59-12"]},
)
_call_save_to_cache(db, result, house_id=42)
assert _mc_calls(db) == ["select", "update"]
call = _houses_update_call(db)
assert call is not None
_, params = call
assert params["mcid"] == 900
db.commit.assert_called_once()
def test_valuation_house_id_none_skips_house_update():
"""house_id=None → нет UPDATE houses (best-effort no-op, тот же no-crash контракт,
что test_extval_house_id_write_path.py::test_cian_save_null_house_id_no_exception)."""
db = _mock_db_valuation()
result = _cian_result(house_info=_HOUSE_INFO_SAMPLE, management_company={"name": "УК Тест"})
_call_save_to_cache(db, result, house_id=None)
assert _houses_update_call(db) is None
db.commit.assert_called_once()
def test_valuation_house_persist_exception_does_not_abort_main_save():
"""UPDATE houses упал → перехвачено, external_valuations INSERT + commit всё равно проходят."""
db = _mock_db_valuation()
result = _cian_result(house_info=[], management_company=None)
def _boom(sql, params=None):
if "UPDATE houses" in str(sql):
raise RuntimeError("boom")
mock_result = MagicMock()
mock_result.fetchone.return_value = None
return mock_result
db.execute.side_effect = _boom
_call_save_to_cache(db, result, house_id=42)
db.commit.assert_called_once()
def test_valuation_no_management_company_leaves_mcid_none():
"""management_company=None → mc_id=None → management_company_id COALESCE(None, existing)."""
db = _mock_db_valuation()
result = _cian_result(house_info=[], management_company=None)
_call_save_to_cache(db, result, house_id=42)
call = _houses_update_call(db)
assert call is not None
_sql, params = call
assert params["mcid"] is None

View file

@ -2344,7 +2344,9 @@ async def run_cian_city_sweep(
source_url, config=config, proxy_provider=proxy_provider source_url, config=config, proxy_provider=proxy_provider
) )
if cian_enrichment is not None: if cian_enrichment is not None:
cian_save_detail_enrichment(db, listing_id, cian_enrichment) cian_save_detail_enrichment(
db, listing_id, cian_enrichment, matcher=matcher
)
counters.detail_enriched += 1 counters.detail_enriched += 1
else: else:
counters.detail_failed += 1 counters.detail_failed += 1
@ -2818,7 +2820,9 @@ async def run_cian_full_load(
source_url, config=config, proxy_provider=proxy_provider source_url, config=config, proxy_provider=proxy_provider
) )
if cian_enrichment is not None: if cian_enrichment is not None:
cian_save_detail_enrichment(db, listing_id, cian_enrichment) cian_save_detail_enrichment(
db, listing_id, cian_enrichment, matcher=matcher
)
counters.detail_enriched += 1 counters.detail_enriched += 1
else: else:
counters.detail_failed += 1 counters.detail_failed += 1

View file

@ -34,7 +34,7 @@ if TYPE_CHECKING:
from curl_cffi.requests import AsyncSession from curl_cffi.requests import AsyncSession
from scraper_kit.browser_fetcher import BrowserFetcher from scraper_kit.browser_fetcher import BrowserFetcher
from scraper_kit.contracts import ProxyProvider, ScraperConfig from scraper_kit.contracts import HouseMatcher, ProxyProvider, ScraperConfig
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -307,13 +307,22 @@ def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
# ── Save helpers ────────────────────────────────────────────────────────────── # ── Save helpers ──────────────────────────────────────────────────────────────
def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None: def save_detail_enrichment(
db: Session,
listing_id: int,
enrichment: DetailEnrichment,
*,
matcher: HouseMatcher | None = None,
) -> None:
"""Persist DetailEnrichment to DB: """Persist DetailEnrichment to DB:
- UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id - UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
- INSERT INTO offer_price_history rows (if price_changes non-empty) - INSERT INTO offer_price_history rows (if price_changes non-empty)
- INSERT INTO agents (if agent_profile) and link via agent_id_fk - INSERT INTO agents (if agent_profile) and link via agent_id_fk
- bti_data не сохраняется здесь это задача Stage 6 (houses), где есть house_id - bti_data канонический houses-ряд (#2435), best-effort, только если `matcher`
передан вызывающим (инжектируемый `HouseMatcher`, тот же паттерн, что
avito/houses.py::_persist_house). `matcher=None` (default) backward-compat
no-op для существующих вызовов, которые ещё не прокинули matcher.
Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics). Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
""" """
@ -447,6 +456,23 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
{"aid": agent_db_id, "lid": listing_id}, {"aid": agent_db_id, "lid": listing_id},
) )
# BTI house-persist (#2435): bti_data распарсен (fetch_detail), но раньше выбрасывался —
# ранее здесь стоял комментарий "это задача Stage 6 (houses)". matcher резолвит
# канонический дом по address/geo листинга (см. _persist_cian_bti_house) — best-effort,
# SAVEPOINT изолирует сбой от уже выполненных выше UPDATE/INSERT (тот же паттерн, что
# estimator.py::_backfill_house_fias).
if enrichment.bti_data and matcher is not None:
try:
with db.begin_nested():
_persist_cian_bti_house(db, listing_id, enrichment.bti_data, matcher=matcher)
except Exception as exc:
logger.warning(
"Cian BTI house-persist failed for listing_id=%s (graceful): %s",
listing_id,
exc,
exc_info=True,
)
db.commit() db.commit()
logger.info( logger.info(
"Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)", "Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)",
@ -455,3 +481,110 @@ def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnric
bool(enrichment.agent_profile), bool(enrichment.agent_profile),
_snap_written, _snap_written,
) )
# ── BTI house-persist (#2435) ─────────────────────────────────────────────────
def _persist_cian_bti_house(
db: Session,
listing_id: int,
bti_data: dict[str, Any],
*,
matcher: HouseMatcher,
) -> int | None:
"""Резолвит канонический дом и пишет BTI-поля дома (#2435).
`bti_data` raw ``offerData.bti.houseData`` снапшот (см. `DetailEnrichment.bti_data`),
shape (Schema_Cian_SERP_Inventory sec 20.2):
{yearRelease, houseMaterialType, floorMax, entrances, flatCount, isEmergency,
demolishedInMoscowProgramm, heatIndex, houseHeatSupplyType, houseOverhaulFundType,
houseGasSupplyType, houseOverlapType, lifts, seriesName}
Персистятся только BTI-эксклюзивные колонки из `020_houses_alter_cian.sql`
(cian_bti единственный источник для них, ни один другой скраппер их не пишет):
seriesName series_name
entrances entrances
flatCount flat_count
isEmergency is_emergency
houseHeatSupplyType heat_supply_type
houseGasSupplyType gas_supply_type
houseOverlapType overlap_type
COALESCE(new, existing) свежий BTI-снапшот побеждает, но не затирает существующее
значение NULL'ом (напр. если повторный fetch вернул неполный houseData).
NOT persisted (gap нет чистого маппинга на существующую колонку, см. summary):
yearRelease/houseMaterialType/floorMax (генерик-колонки year_built/house_type/
total_floors существуют, но вне explicit scope БTI-колонок #2435 — future work),
demolishedInMoscowProgramm, heatIndex, houseOverhaulFundType, lifts (нет колонки
под недифференцированное «лифтов всего» passenger/cargo split недоступен в BTI).
address/lat/lon листинга bti_data адреса не несёт, читаем из `listings` по listing_id.
Returns:
houses.id, либо None если у листинга нет адреса, либо matcher отказал
(безномерный адрес без кадастра метод 'no_house_number', #1772/P1).
"""
row = (
db.execute(
text("SELECT address, lat, lon FROM listings WHERE id = CAST(:lid AS bigint)"),
{"lid": listing_id},
)
.mappings()
.first()
)
if row is None or not row["address"]:
logger.info(
"_persist_cian_bti_house: listing_id=%s missing address — skip house resolve",
listing_id,
)
return None
house_id, _conf, method = matcher.match_or_create_house(
db,
ext_source="cian_bti",
ext_id=str(listing_id),
address=row["address"],
lat=row["lat"],
lon=row["lon"],
)
if house_id is None:
logger.info(
"_persist_cian_bti_house: skip listing_id=%s addr=%r (method=%s)",
listing_id,
row["address"],
method,
)
return None
db.execute(
text("""
UPDATE houses SET
series_name = COALESCE(CAST(:series_name AS text), series_name),
entrances = COALESCE(CAST(:entrances AS int), entrances),
flat_count = COALESCE(CAST(:flat_count AS int), flat_count),
is_emergency = COALESCE(CAST(:is_emergency AS boolean), is_emergency),
heat_supply_type = COALESCE(CAST(:heat_supply_type AS text), heat_supply_type),
gas_supply_type = COALESCE(CAST(:gas_supply_type AS text), gas_supply_type),
overlap_type = COALESCE(CAST(:overlap_type AS text), overlap_type)
WHERE id = CAST(:house_id AS bigint)
"""),
{
"house_id": house_id,
"series_name": bti_data.get("seriesName"),
"entrances": bti_data.get("entrances"),
"flat_count": bti_data.get("flatCount"),
"is_emergency": bti_data.get("isEmergency"),
"heat_supply_type": bti_data.get("houseHeatSupplyType"),
"gas_supply_type": bti_data.get("houseGasSupplyType"),
"overlap_type": bti_data.get("houseOverlapType"),
},
)
logger.info(
"_persist_cian_bti_house: listing_id=%s → house_id=%s (method=%s)",
listing_id,
house_id,
method,
)
return int(house_id)

View file

@ -23,6 +23,7 @@ from __future__ import annotations
import hashlib import hashlib
import json import json
import logging import logging
import re
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode from urllib.parse import urlencode
@ -557,7 +558,251 @@ def _save_to_cache(
"raw": json.dumps(result.raw_state) if result.raw_state else None, "raw": json.dumps(result.raw_state) if result.raw_state else None,
}, },
) )
# Best-effort house enrichment (#2435): management_company/house_info/
# cian_internal_house_id → канонический houses-ряд. house_id уже резолвлен
# ВЫЗЫВАЮЩИМ (estimator.py::match_house_readonly — read-only, не создаёт дом из
# ephemeral valuation-запроса) — здесь только COALESCE-UPDATE, никакого
# match_or_create_house. SAVEPOINT изолирует сбой от уже выполненного INSERT
# выше (тот же паттерн, что estimator.py::_backfill_house_fias).
try:
with db.begin_nested():
_persist_cian_valuation_house(db, house_id=house_id, result=result)
except Exception as exc:
logger.warning(
"Cian valuation house-persist failed for house_id=%s (graceful): %s",
house_id,
exc,
exc_info=True,
)
db.commit() db.commit()
except Exception as exc: except Exception as exc:
logger.error("Cian valuation cache save failed: %s", exc, exc_info=True) logger.error("Cian valuation cache save failed: %s", exc, exc_info=True)
raise raise
# ── House enrichment (#2435) ──────────────────────────────────────────────────
# houseInfo.data.items title (RU, lowercased) → houses column. Только поля с чистым
# 1:1 маппингом на существующую колонку (Schema_Cian_SERP_Inventory sec 24.6).
# Пропущенные (мусоропровод/реновация/спортплощадка/фонд капремонта) — нет колонки
# в houses, см. gap-заметку в docstring _persist_cian_valuation_house.
_HOUSE_INFO_TITLE_MAP: dict[str, str] = {
"год постройки": "year_built",
"тип дома": "house_type",
"этажность": "total_floors",
"газоснабжение": "gas_supply_type",
"отопление": "heat_supply_type",
"тип перекрытий": "overlap_type",
"подъездов": "entrances",
"квартир": "flat_count",
"аварийность": "is_emergency",
"детская площадка": "has_playground",
}
_HOUSE_INFO_BOOL_FIELDS = {"is_emergency", "has_playground"}
_BOOL_YES_RU = {"да"}
_BOOL_NO_RU = {"нет"}
# "Количество лифтов": free-text "1 пассажирский, 1 грузовой" (не структурировано Cian'ом).
_ELEVATORS_PASSENGER_RE = re.compile(r"(\d+)\s*пассажир", re.IGNORECASE)
_ELEVATORS_CARGO_RE = re.compile(r"(\d+)\s*груз", re.IGNORECASE)
def _map_house_info_items(items: list[dict[str, Any]]) -> dict[str, Any]:
"""houseInfo.data.items (RU title/value пары) → known houses columns (#2435).
Best-effort парсинг: неизвестные/непонятные title просто пропускаются (нет
исключений). "Количество лифтов" единственное поле с free-text значением
("N пассажирский, M грузовой") парсится regex'ом, при несовпадении паттерна
соответствующий счётчик остаётся не заполненным (не 0 0 означало бы "нет лифтов").
"""
out: dict[str, Any] = {}
for item in items:
if not isinstance(item, dict):
continue
title = str(item.get("title") or "").strip().lower()
value = item.get("value")
if title == "количество лифтов" and isinstance(value, str):
pm = _ELEVATORS_PASSENGER_RE.search(value)
if pm:
out["passenger_elevators"] = int(pm.group(1))
cm = _ELEVATORS_CARGO_RE.search(value)
if cm:
out["cargo_elevators"] = int(cm.group(1))
continue
field_name = _HOUSE_INFO_TITLE_MAP.get(title)
if field_name is None:
continue
if field_name in _HOUSE_INFO_BOOL_FIELDS:
if isinstance(value, bool):
out[field_name] = value
elif isinstance(value, str):
v_norm = value.strip().lower()
if v_norm in _BOOL_YES_RU:
out[field_name] = True
elif v_norm in _BOOL_NO_RU:
out[field_name] = False
else:
out[field_name] = value
return out
def _upsert_cian_valuation_management_company(
db: Session, mc: dict[str, Any] | None
) -> int | None:
"""UPSERT management_companies из Cian Valuation Calculator's managementCompany.data.
Тот же намерение, что providers/cian/newbuilding.py::save_newbuilding_enrichment
(независимая копия newbuilding.py вне scope #2435, не рефакторим/не переиспользуем),
НО другой механизм dedup. ext_source='cian_valuation' (не 'cian') валюация не даёт
числового id УК (в отличие от newbuilding.managementCompany.id), поэтому пишем
ext_id=NULL. `UNIQUE (ext_source, name, ext_id)` под Postgres-дефолтом NULLS DISTINCT
не считает два NULL равными `ON CONFLICT` с ext_id=NULL никогда бы не сработал (каждый
вызов создавал бы новую дублирующую строку). Поэтому dedup сделан явно: SELECT по
(ext_source, name) WHERE ext_id IS NULL перед INSERT, а не через constraint.
opening_hours Cian отдаёт список словарей ([{"Пн–Пт": ["10:00-20:00"]}, ...]), не
строку; json.dumps в text-колонку (в отличие от newbuilding.py, которая пишет объект
напрямую тот путь не в scope этой задачи).
"""
if not mc or not isinstance(mc, dict):
return None
name = mc.get("name")
if not name:
return None
opening_hours = mc.get("openingHours")
phones = mc.get("phones") or []
email = mc.get("email")
oh_json = json.dumps(opening_hours, ensure_ascii=False) if opening_hours else None
chief = mc.get("chiefName")
existing = db.execute(
text("""
SELECT id FROM management_companies
WHERE ext_source = 'cian_valuation' AND name = :name AND ext_id IS NULL
LIMIT 1
"""),
{"name": name},
).fetchone()
if existing:
mc_id = int(existing[0])
db.execute(
text("""
UPDATE management_companies SET
phones = COALESCE(CAST(:phones AS text[]), phones),
email = COALESCE(:email, email)
WHERE id = CAST(:mc_id AS bigint)
"""),
{"phones": phones, "email": email, "mc_id": mc_id},
)
return mc_id
row = db.execute(
text("""
INSERT INTO management_companies (
name, phones, email, opening_hours, chief_name, ext_source, ext_id
) VALUES (
:name, CAST(:phones AS text[]), :email, :oh, :chief, 'cian_valuation', NULL
)
RETURNING id
"""),
{
"name": name,
"phones": phones,
"email": email,
"oh": oh_json,
"chief": chief,
},
).fetchone()
return int(row[0]) if row else None
def _persist_cian_valuation_house(
db: Session,
*,
house_id: int | None,
result: CianValuationResult,
) -> None:
"""Пишет managementCompany/houseInfo/houseId в канонический houses-ряд (#2435).
`house_id` уже резолвлен ВЫЗЫВАЮЩИМ (estimator.py, read-only match_house_readonly)
valuation-запрос по эфемерному целевому адресу не должен САМ создавать дом (в отличие
от cian_bti #2435 Part 1, где match_or_create_house уместен — там источник привязан
к конкретному листингу/дому, а не к произвольному введённому адресу). house_id=None
no-op, best-effort.
COALESCE-направление по app.services.matching.conflict_resolution.HOUSE_FIELD_PRIORITY:
- management_company_id: HOUSE_FIELD_PRIORITY["management_company_id"] == ["cian_valuation"]
единственный источник COALESCE(new, existing) (свежее значение побеждает).
- cian_internal_house_id (filters.houseId): не в реестре, но тот же принцип, что
newbuilding.py применяет к этой же колонке COALESCE(new, existing).
- houseInfo-производные (year_built/house_type/total_floors/entrances/flat_count/
is_emergency/heat_supply_type/gas_supply_type/overlap_type/has_playground/
passenger_elevators/cargo_elevators): cian_valuation НЕ входит в приоритет для этих
полей вовсе (или cian_bti/serp/avito ранжированы выше) COALESCE(existing, new)
валюация только заполняет пробелы, не перетирает более авторитетные источники.
"""
if house_id is None:
return
mc_id = _upsert_cian_valuation_management_company(db, result.management_company)
info = _map_house_info_items(result.house_info)
db.execute(
text("""
UPDATE houses SET
management_company_id = COALESCE(
CAST(:mcid AS bigint), houses.management_company_id
),
cian_internal_house_id = COALESCE(
CAST(:cihi AS bigint), houses.cian_internal_house_id
),
year_built = COALESCE(houses.year_built, CAST(:yb AS int)),
house_type = COALESCE(houses.house_type, CAST(:ht AS text)),
total_floors = COALESCE(houses.total_floors, CAST(:tf AS int)),
entrances = COALESCE(houses.entrances, CAST(:ent AS int)),
flat_count = COALESCE(houses.flat_count, CAST(:fc AS int)),
is_emergency = COALESCE(houses.is_emergency, CAST(:ie AS boolean)),
heat_supply_type = COALESCE(
houses.heat_supply_type, CAST(:hst AS text)
),
gas_supply_type = COALESCE(houses.gas_supply_type, CAST(:gst AS text)),
overlap_type = COALESCE(houses.overlap_type, CAST(:ot AS text)),
has_playground = COALESCE(
houses.has_playground, CAST(:hp AS boolean)
),
passenger_elevators = COALESCE(
houses.passenger_elevators, CAST(:pe AS int)
),
cargo_elevators = COALESCE(houses.cargo_elevators, CAST(:ce AS int))
WHERE id = CAST(:hid AS bigint)
"""),
{
"hid": house_id,
"mcid": mc_id,
"cihi": result.external_house_id,
"yb": info.get("year_built"),
"ht": info.get("house_type"),
"tf": info.get("total_floors"),
"ent": info.get("entrances"),
"fc": info.get("flat_count"),
"ie": info.get("is_emergency"),
"hst": info.get("heat_supply_type"),
"gst": info.get("gas_supply_type"),
"ot": info.get("overlap_type"),
"hp": info.get("has_playground"),
"pe": info.get("passenger_elevators"),
"ce": info.get("cargo_elevators"),
},
)
logger.info(
"_persist_cian_valuation_house: house_id=%s mc_id=%s cian_internal_house_id=%s",
house_id,
mc_id,
result.external_house_id,
)