bot-backend
272abac4d2
All checks were successful
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 3m6s
Deploy Trade-In / build-backend (push) Successful in 1m37s
Deploy Trade-In / deploy (push) Successful in 2m3s
473 lines
18 KiB
Python
473 lines
18 KiB
Python
"""Тесты для Cian BTI / Valuation house-persist (#2435).
|
||
|
||
`bti_data` (cian/detail.py) и valuation `house_info`/`managementCompany`
|
||
(cian/valuation.py) парсились, но раньше выбрасывались («это задача Stage 6
|
||
(houses), где есть house_id») — теперь пишутся в канонический houses-ряд:
|
||
|
||
- detail.py: резолвит дом через инжектируемый `matcher: HouseMatcher`
|
||
(`match_or_create_house`, address/geo листинга) — mirror avito's `_persist_house`.
|
||
- valuation.py: `house_id` уже резолвлен ВЫЗЫВАЮЩИМ (estimator.py, read-only
|
||
`match_house_readonly`) — здесь только COALESCE-UPDATE, без создания дома.
|
||
|
||
MagicMock db — без реальной БД, тот же паттерн, что test_snapshot_writer.py /
|
||
test_extval_house_id_write_path.py.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
# Settings требует DATABASE_URL на import (см. test_extval_house_id_write_path.py).
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from contextlib import contextmanager
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
from scraper_kit.providers.cian.detail import (
|
||
DetailEnrichment,
|
||
fetch_detail,
|
||
save_detail_enrichment,
|
||
)
|
||
from scraper_kit.providers.cian.valuation import CianValuationResult, _save_to_cache
|
||
|
||
# ── Part 1: bti_data → houses (detail.py) ────────────────────────────────────
|
||
|
||
_BTI_SAMPLE = {
|
||
"yearRelease": 1961,
|
||
"houseMaterialType": "brick",
|
||
"floorMax": 4,
|
||
"entrances": 4,
|
||
"flatCount": 58,
|
||
"isEmergency": False,
|
||
"houseHeatSupplyType": "central",
|
||
"houseGasSupplyType": "central",
|
||
"houseOverlapType": "concrete",
|
||
"lifts": 0,
|
||
"seriesName": "1-447",
|
||
}
|
||
|
||
|
||
def _mock_matcher(house_id: int | None = 501, method: str = "new") -> MagicMock:
|
||
matcher = MagicMock()
|
||
matcher.match_or_create_house.return_value = (house_id, 1.0, method)
|
||
return matcher
|
||
|
||
|
||
def _mock_db_bti(
|
||
address: str | None = "Екатеринбург, улица Малышева, 51",
|
||
lat: float = 56.83,
|
||
lon: float = 60.6,
|
||
city: str | None = "Екатеринбург",
|
||
) -> MagicMock:
|
||
"""Mock db: SELECT address/city/lat/lon FROM listings → mappings().first() dict."""
|
||
db = MagicMock()
|
||
|
||
@contextmanager
|
||
def _nested():
|
||
yield MagicMock()
|
||
|
||
db.begin_nested.side_effect = _nested
|
||
|
||
def _exec(sql, params=None):
|
||
sql_str = str(sql)
|
||
mock_result = MagicMock()
|
||
mock_result.fetchone.return_value = None
|
||
mock_result.scalar_one_or_none.return_value = None
|
||
if "SELECT address, city, lat, lon FROM listings" in sql_str:
|
||
mock_result.mappings.return_value.first.return_value = {
|
||
"address": address,
|
||
"city": city,
|
||
"lat": lat,
|
||
"lon": lon,
|
||
}
|
||
else:
|
||
mock_result.mappings.return_value.first.return_value = None
|
||
return mock_result
|
||
|
||
db.execute.side_effect = _exec
|
||
return db
|
||
|
||
|
||
def _bti_update_call(db: MagicMock) -> tuple[str, dict] | None:
|
||
for call_args in db.execute.call_args_list:
|
||
if not call_args.args:
|
||
continue
|
||
sql = str(call_args.args[0])
|
||
if "UPDATE houses" in sql and "series_name" in sql:
|
||
params = call_args.args[1] if len(call_args.args) >= 2 else {}
|
||
return sql, params
|
||
return None
|
||
|
||
|
||
def test_bti_data_present_updates_house_columns_via_coalesce():
|
||
"""bti_data + matcher → UPDATE houses SET series_name=COALESCE(...) с BTI-полями."""
|
||
db = _mock_db_bti()
|
||
matcher = _mock_matcher(house_id=501)
|
||
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
|
||
|
||
save_detail_enrichment(db, 123, enrichment, matcher=matcher)
|
||
|
||
matcher.match_or_create_house.assert_called_once()
|
||
kwargs = matcher.match_or_create_house.call_args.kwargs
|
||
assert kwargs["ext_source"] == "cian_bti"
|
||
assert kwargs["ext_id"] == "123"
|
||
assert kwargs["address"] == "Екатеринбург, улица Малышева, 51"
|
||
# #2777: город-цель развёртки (listings.city) — независимое от адреса наблюдение;
|
||
# без него бескоординатная BTI-карточка матчится в одноимённый дом другого города.
|
||
assert kwargs["city"] == "Екатеринбург"
|
||
|
||
call = _bti_update_call(db)
|
||
assert call is not None, "UPDATE houses с BTI-полями не найден"
|
||
sql, params = call
|
||
assert "COALESCE" in sql
|
||
assert params["house_id"] == 501
|
||
assert params["series_name"] == "1-447"
|
||
assert params["entrances"] == 4
|
||
assert params["flat_count"] == 58
|
||
assert params["is_emergency"] is False
|
||
assert params["heat_supply_type"] == "central"
|
||
assert params["gas_supply_type"] == "central"
|
||
assert params["overlap_type"] == "concrete"
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_bti_data_absent_is_noop_no_crash():
|
||
"""bti_data=None (default) → matcher не вызывается, UPDATE houses не выполняется."""
|
||
db = _mock_db_bti()
|
||
matcher = _mock_matcher()
|
||
enrichment = DetailEnrichment() # bti_data=None
|
||
|
||
save_detail_enrichment(db, 124, enrichment, matcher=matcher)
|
||
|
||
matcher.match_or_create_house.assert_not_called()
|
||
assert _bti_update_call(db) is None
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_bti_data_present_but_no_matcher_is_noop():
|
||
"""matcher=None (default, backward-compat для существующих вызовов) → bti_data игнорируется."""
|
||
db = _mock_db_bti()
|
||
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
|
||
|
||
save_detail_enrichment(db, 125, enrichment) # no matcher kwarg
|
||
|
||
assert _bti_update_call(db) is None
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_bti_house_resolution_refused_no_crash():
|
||
"""matcher отказал (безномерный адрес, method='no_house_number') → graceful skip."""
|
||
db = _mock_db_bti()
|
||
matcher = _mock_matcher(house_id=None, method="no_house_number")
|
||
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
|
||
|
||
save_detail_enrichment(db, 126, enrichment, matcher=matcher)
|
||
|
||
assert _bti_update_call(db) is None
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_bti_persist_exception_does_not_abort_main_save():
|
||
"""Исключение в BTI house-persist (напр. UPDATE houses упал) не прерывает основной save."""
|
||
db = _mock_db_bti()
|
||
matcher = _mock_matcher(house_id=501)
|
||
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
|
||
|
||
original_side_effect = db.execute.side_effect
|
||
|
||
def _boom(sql, params=None):
|
||
if "UPDATE houses" in str(sql):
|
||
raise RuntimeError("boom")
|
||
return original_side_effect(sql, params)
|
||
|
||
db.execute.side_effect = _boom
|
||
|
||
# Не должно бросить исключение наружу.
|
||
save_detail_enrichment(db, 127, enrichment, matcher=matcher)
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_bti_missing_listing_address_skips_resolve():
|
||
"""Листинг без адреса → matcher не вызывается, no crash (best-effort)."""
|
||
db = _mock_db_bti(address="")
|
||
matcher = _mock_matcher()
|
||
enrichment = DetailEnrichment(bti_data=dict(_BTI_SAMPLE))
|
||
|
||
save_detail_enrichment(db, 128, enrichment, matcher=matcher)
|
||
|
||
matcher.match_or_create_house.assert_not_called()
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
# ── Part 1b: bti_data реально извлекается с живой страницы (регрессия) ───────
|
||
#
|
||
# Тесты выше кормят `bti_data` напрямую в `save_detail_enrichment`, поэтому они
|
||
# оставались зелёными, пока `fetch_detail` на живых страницах НИКОГДА его не
|
||
# заполнял: bti читался только как соседний с `defaultState` ключ контейнера
|
||
# frontend-offer-card, а Cian отдаёт его ВНУТРИ defaultState — `offerData.bti`.
|
||
# Итог на проде: 628 detail-обогащённых Cian-листингов, 0 домов с BTI-полями.
|
||
# Здесь прогоняем настоящий сохранённый HTML через настоящий `fetch_detail`.
|
||
|
||
|
||
def _detail_fixture_html() -> str:
|
||
path = os.path.join(os.path.dirname(__file__), "fixtures", "cian_flat_330982715.html")
|
||
with open(path, encoding="utf-8") as fh:
|
||
return fh.read()
|
||
|
||
|
||
async def test_fetch_detail_extracts_bti_from_offer_data_on_real_page():
|
||
"""Реальная страница Cian: bti лежит в offerData → bti_data должен заполниться."""
|
||
fetcher = MagicMock()
|
||
fetcher.fetch = AsyncMock(return_value=_detail_fixture_html())
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/330982715/", browser_fetcher=fetcher)
|
||
|
||
assert result is not None
|
||
assert result.bti_data, "bti_data пуст — BTI-персист (#2435) в проде остаётся no-op"
|
||
# Поля, которые пишет _persist_cian_bti_house (см. Part 1).
|
||
assert result.bti_data["entrances"] == 3
|
||
assert result.bti_data["flatCount"] == 24
|
||
assert result.bti_data["isEmergency"] is False
|
||
assert result.bti_data["houseHeatSupplyType"] == "central"
|
||
assert result.bti_data["houseGasSupplyType"] == "central"
|
||
assert result.bti_data["houseOverlapType"] == "concrete"
|
||
|
||
|
||
async def test_fetch_detail_bti_legacy_sibling_location_still_read():
|
||
"""Фоллбэк: если Cian вернёт bti соседним ключом контейнера — читаем и оттуда."""
|
||
html = (
|
||
"<script>window._cianConfig['frontend-offer-card'] = "
|
||
"(window._cianConfig['frontend-offer-card'] || []).concat(["
|
||
'{"key":"defaultState","value":{"offerData":{"offer":{"cianId":777}}}},'
|
||
'{"key":"bti","value":{"houseData":{"seriesName":"1-464","flatCount":58}}}'
|
||
"]);</script>"
|
||
)
|
||
fetcher = MagicMock()
|
||
fetcher.fetch = AsyncMock(return_value=html)
|
||
|
||
result = await fetch_detail("https://ekb.cian.ru/sale/flat/777/", browser_fetcher=fetcher)
|
||
|
||
assert result is not None
|
||
assert result.bti_data == {"seriesName": "1-464", "flatCount": 58}
|
||
|
||
|
||
# ── Part 2: valuation house_info/managementCompany → houses (valuation.py) ──
|
||
|
||
_HOUSE_INFO_SAMPLE = [
|
||
{"title": "Год постройки", "value": 2005},
|
||
{"title": "Тип дома", "value": "Кирпичный"},
|
||
{"title": "Этажность", "value": 16},
|
||
{"title": "Газоснабжение", "value": "Отсутствует"},
|
||
{"title": "Отопление", "value": "Центральное"},
|
||
{"title": "Тип перекрытий", "value": "Железобетонные"},
|
||
{"title": "Подъездов", "value": 1},
|
||
{"title": "Количество лифтов", "value": "1 пассажирский, 1 грузовой"},
|
||
{"title": "Мусоропровод", "value": "Нет"},
|
||
{"title": "Квартир", "value": 105},
|
||
{"title": "Реновация", "value": "Нет"},
|
||
{"title": "Аварийность", "value": "Нет"},
|
||
{"title": "Детская площадка", "value": "Нет"},
|
||
{"title": "Спортивная площадка", "value": "Нет"},
|
||
]
|
||
|
||
|
||
def _cian_result(
|
||
house_info: list[dict] | None = None,
|
||
management_company: dict | None = None,
|
||
external_house_id: int | None = 54016,
|
||
) -> CianValuationResult:
|
||
return CianValuationResult(
|
||
sale_price_rub=10_000_000,
|
||
sale_accuracy=85.0,
|
||
chart=[],
|
||
external_house_id=external_house_id,
|
||
house_info=house_info or [],
|
||
management_company=management_company,
|
||
)
|
||
|
||
|
||
def _mock_db_valuation(
|
||
mc_insert_id: int | None = 900, mc_existing_id: int | None = None
|
||
) -> MagicMock:
|
||
"""Mock db для _save_to_cache: SELECT (dedup) → UPDATE-or-INSERT management_companies,
|
||
затем UPDATE houses.
|
||
|
||
`mc_existing_id` simulates the dedup SELECT finding a prior row (existing.fetchone()
|
||
returns that id) — the UPDATE branch is taken instead of INSERT. Default None means
|
||
"no existing row", exercising the INSERT branch (the original test-suite behavior).
|
||
"""
|
||
db = MagicMock()
|
||
|
||
@contextmanager
|
||
def _nested():
|
||
yield MagicMock()
|
||
|
||
db.begin_nested.side_effect = _nested
|
||
|
||
def _exec(sql, params=None):
|
||
sql_str = str(sql)
|
||
mock_result = MagicMock()
|
||
if "SELECT id FROM management_companies" in sql_str:
|
||
mock_result.fetchone.return_value = (
|
||
(mc_existing_id,) if mc_existing_id is not None else None
|
||
)
|
||
elif "INSERT INTO management_companies" in sql_str:
|
||
mock_result.fetchone.return_value = (
|
||
(mc_insert_id,) if mc_insert_id is not None else None
|
||
)
|
||
else:
|
||
mock_result.fetchone.return_value = None
|
||
return mock_result
|
||
|
||
db.execute.side_effect = _exec
|
||
return db
|
||
|
||
|
||
def _mc_calls(db: MagicMock) -> list[str]:
|
||
"""SQL statement kinds touching management_companies, in call order."""
|
||
kinds = []
|
||
for call_args in db.execute.call_args_list:
|
||
if not call_args.args:
|
||
continue
|
||
sql = str(call_args.args[0])
|
||
if "SELECT id FROM management_companies" in sql:
|
||
kinds.append("select")
|
||
elif "INSERT INTO management_companies" in sql:
|
||
kinds.append("insert")
|
||
elif "UPDATE management_companies" in sql:
|
||
kinds.append("update")
|
||
return kinds
|
||
|
||
|
||
def _houses_update_call(db: MagicMock) -> tuple[str, dict] | None:
|
||
for call_args in db.execute.call_args_list:
|
||
if not call_args.args:
|
||
continue
|
||
sql = str(call_args.args[0])
|
||
if "UPDATE houses" in sql:
|
||
params = call_args.args[1] if len(call_args.args) >= 2 else {}
|
||
return sql, params
|
||
return None
|
||
|
||
|
||
def _call_save_to_cache(db: MagicMock, result: CianValuationResult, house_id: int | None) -> None:
|
||
_save_to_cache(
|
||
db,
|
||
cache_key="ck",
|
||
address="Екатеринбург, улица Учителей, 18",
|
||
total_area=50.0,
|
||
rooms_count=2,
|
||
floor=3,
|
||
total_floors=16,
|
||
repair_type="cosmetic",
|
||
deal_type="sale",
|
||
result=result,
|
||
house_id=house_id,
|
||
)
|
||
|
||
|
||
def test_valuation_house_info_and_management_company_land_in_houses_update():
|
||
"""house_id задан, УК ранее не встречалась → SELECT (не найдено) → INSERT management_companies,
|
||
UPDATE houses с management_company_id + house_info-полями."""
|
||
db = _mock_db_valuation(mc_insert_id=900)
|
||
result = _cian_result(
|
||
house_info=_HOUSE_INFO_SAMPLE,
|
||
management_company={
|
||
"name": 'ТСЖ "Учителей, 18"',
|
||
"phones": ["+7 (343) 216-59-12"],
|
||
"email": "tsg.u.18@mail.ru",
|
||
"openingHours": [{"Пн-Пт": ["10:00-20:00"]}],
|
||
"chiefName": "Сабиров Наиль Вагизович",
|
||
},
|
||
)
|
||
|
||
_call_save_to_cache(db, result, house_id=42)
|
||
|
||
assert _mc_calls(db) == ["select", "insert"]
|
||
call = _houses_update_call(db)
|
||
assert call is not None
|
||
sql, params = call
|
||
assert "COALESCE" in sql
|
||
assert params["hid"] == 42
|
||
assert params["mcid"] == 900
|
||
assert params["cihi"] == 54016
|
||
assert params["yb"] == 2005
|
||
assert params["ht"] == "Кирпичный"
|
||
assert params["tf"] == 16
|
||
assert params["gst"] == "Отсутствует"
|
||
assert params["hst"] == "Центральное"
|
||
assert params["ot"] == "Железобетонные"
|
||
assert params["ent"] == 1
|
||
assert params["fc"] == 105
|
||
assert params["ie"] is False
|
||
assert params["hp"] is False
|
||
assert params["pe"] == 1
|
||
assert params["ce"] == 1
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_valuation_management_company_dedups_on_repeat_call_same_name():
|
||
"""Regression for the NULLS-DISTINCT dedup bug (#2435 review): a repeat valuation
|
||
call for a management company already persisted (same ext_source/name, ext_id=NULL)
|
||
must find it via the explicit SELECT and UPDATE it — not INSERT a duplicate row.
|
||
`ON CONFLICT (ext_source, name, ext_id)` alone can't dedup ext_id=NULL rows under
|
||
Postgres's default NULLS DISTINCT (two NULLs never compare equal), which is why
|
||
dedup is done via SELECT-before-INSERT rather than relying on the constraint."""
|
||
db = _mock_db_valuation(mc_existing_id=900)
|
||
result = _cian_result(
|
||
house_info=[],
|
||
management_company={"name": 'ТСЖ "Учителей, 18"', "phones": ["+7 (343) 216-59-12"]},
|
||
)
|
||
|
||
_call_save_to_cache(db, result, house_id=42)
|
||
|
||
assert _mc_calls(db) == ["select", "update"]
|
||
call = _houses_update_call(db)
|
||
assert call is not None
|
||
_, params = call
|
||
assert params["mcid"] == 900
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_valuation_house_id_none_skips_house_update():
|
||
"""house_id=None → нет UPDATE houses (best-effort no-op, тот же no-crash контракт,
|
||
что test_extval_house_id_write_path.py::test_cian_save_null_house_id_no_exception)."""
|
||
db = _mock_db_valuation()
|
||
result = _cian_result(house_info=_HOUSE_INFO_SAMPLE, management_company={"name": "УК Тест"})
|
||
|
||
_call_save_to_cache(db, result, house_id=None)
|
||
|
||
assert _houses_update_call(db) is None
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_valuation_house_persist_exception_does_not_abort_main_save():
|
||
"""UPDATE houses упал → перехвачено, external_valuations INSERT + commit всё равно проходят."""
|
||
db = _mock_db_valuation()
|
||
result = _cian_result(house_info=[], management_company=None)
|
||
|
||
def _boom(sql, params=None):
|
||
if "UPDATE houses" in str(sql):
|
||
raise RuntimeError("boom")
|
||
mock_result = MagicMock()
|
||
mock_result.fetchone.return_value = None
|
||
return mock_result
|
||
|
||
db.execute.side_effect = _boom
|
||
|
||
_call_save_to_cache(db, result, house_id=42)
|
||
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_valuation_no_management_company_leaves_mcid_none():
|
||
"""management_company=None → mc_id=None → management_company_id COALESCE(None, existing)."""
|
||
db = _mock_db_valuation()
|
||
result = _cian_result(house_info=[], management_company=None)
|
||
|
||
_call_save_to_cache(db, result, house_id=42)
|
||
|
||
call = _houses_update_call(db)
|
||
assert call is not None
|
||
_sql, params = call
|
||
assert params["mcid"] is None
|