bot-backend
480fbba3ca
All checks were successful
CI / changes (push) Successful in 7s
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / openapi-codegen-check (push) Has been skipped
CI / changes (pull_request) Successful in 6s
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
curl_cffi получает zhk-*.cian.ru без initialState; tradein-browser (camoufox) тянет страницу с 1.17MB initialState. Заменяем транспорт только для page-fetch; session= param сохранён для backward-compat вызывающих. resolve_cian_zhk_url_via_search остаётся на curl_cffi SERP. Тесты: 21 passed.
380 lines
14 KiB
Python
380 lines
14 KiB
Python
"""Tests for cian_newbuilding scraper (Stage 6)."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.cian_newbuilding import (
|
||
NewbuildingEnrichment,
|
||
_extract_address,
|
||
_extract_advantages,
|
||
_extract_chart,
|
||
_extract_deadline_year,
|
||
_extract_nested_offers,
|
||
_extract_reliability_checks,
|
||
fetch_newbuilding,
|
||
)
|
||
|
||
# ---- unit tests for extractors ----
|
||
|
||
|
||
def test_extract_chart_primary_shape():
|
||
"""realtyValuation primary shape: data.priceDynamics.chart.data.{labels, values}."""
|
||
rv_state = {
|
||
"data": {
|
||
"priceDynamics": {
|
||
"chart": {
|
||
"data": {
|
||
"labels": ["2025-11-01", "2025-12-01", "2026-01-01"],
|
||
"values": [13800000, 14100000, 14200000],
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
points = _extract_chart(rv_state)
|
||
assert len(points) == 3
|
||
assert points[0]["month_date"] == "2025-11-01"
|
||
assert points[0]["price_per_sqm"] == 13800000.0
|
||
assert points[0]["room_count"] == "all"
|
||
assert points[0]["prices_type"] == "price"
|
||
assert points[0]["period"] == "halfYear"
|
||
assert points[2]["month_date"] == "2026-01-01"
|
||
assert points[2]["price_per_sqm"] == 14200000.0
|
||
|
||
|
||
def test_extract_chart_normalizes_points():
|
||
"""Fallback shape: rv_state.chart[] with {month, price, roomType, period}."""
|
||
rv_state = {
|
||
"chart": [
|
||
{"month": "2025-12-01", "price": 150000, "roomType": "1", "period": "monthly"},
|
||
{"month": "2026-01-01", "price": 152000, "roomType": "1", "period": "monthly"},
|
||
]
|
||
}
|
||
points = _extract_chart(rv_state)
|
||
assert len(points) == 2
|
||
assert points[0]["month_date"] == "2025-12-01"
|
||
assert points[0]["price_per_sqm"] == 150000.0
|
||
assert points[0]["room_count"] == "1"
|
||
assert points[1]["month_date"] == "2026-01-01"
|
||
|
||
|
||
def test_extract_chart_handles_empty():
|
||
assert _extract_chart({}) == []
|
||
assert _extract_chart({"chart": None}) == []
|
||
assert _extract_chart({"data": None}) == []
|
||
assert _extract_chart({"data": {"priceDynamics": None}}) == []
|
||
|
||
|
||
def test_extract_chart_handles_none_values():
|
||
"""Entries with None month or value are skipped."""
|
||
rv_state = {
|
||
"chart": [
|
||
{"month": "2025-12-01", "price": 150000},
|
||
{"month": None, "price": 152000},
|
||
{"month": "2026-01-01", "price": None},
|
||
]
|
||
}
|
||
points = _extract_chart(rv_state)
|
||
assert len(points) == 1
|
||
assert points[0]["month_date"] == "2025-12-01"
|
||
|
||
|
||
def test_extract_chart_truncates_date():
|
||
"""Only first 10 chars of date string used (YYYY-MM-DD prefix)."""
|
||
rv_state = {
|
||
"chart": [
|
||
{"month": "2025-12-01T00:00:00Z", "price": 100000},
|
||
]
|
||
}
|
||
points = _extract_chart(rv_state)
|
||
assert points[0]["month_date"] == "2025-12-01"
|
||
|
||
|
||
def test_extract_reliability_primary_shape():
|
||
"""Primary shape: {checkStatus: {status, title}, details: [{type, title, iconType}]}."""
|
||
rel = {
|
||
"checkStatus": {
|
||
"status": "reliable",
|
||
"title": "Проверено",
|
||
"mobileTitle": "Проверено нашдом.рф",
|
||
"date": "Обновлено 5 февраля 2026",
|
||
},
|
||
"details": [
|
||
{
|
||
"title": "Нет в реестре проблемных объектов",
|
||
"iconType": "success",
|
||
"type": "problemHousesRegistry",
|
||
},
|
||
{
|
||
"title": "Застройщик не банкрот",
|
||
"iconType": "success",
|
||
"type": "developerBankruptStatus",
|
||
},
|
||
],
|
||
"banner": {"title": "ЖК проверен нашдом.рф"},
|
||
}
|
||
result = _extract_reliability_checks(rel)
|
||
assert len(result) == 1
|
||
assert result[0]["check_status"] == "reliable"
|
||
assert result[0]["check_name"] == "Проверено"
|
||
assert isinstance(result[0]["details"], list)
|
||
assert len(result[0]["details"]) == 2
|
||
assert result[0]["details"][0]["type"] == "problemHousesRegistry"
|
||
|
||
|
||
def test_extract_reliability_handles_empty():
|
||
assert _extract_reliability_checks({}) == []
|
||
assert _extract_reliability_checks({"checkStatus": None}) == []
|
||
|
||
|
||
def test_extract_reliability_fallback_shape():
|
||
"""Fallback: {checks: [{name, status, details}]}."""
|
||
rel = {
|
||
"checks": [
|
||
{"name": "License", "status": "ok", "details": {"doc": "lic-123"}},
|
||
{"name": "Permission", "status": "expired", "details": {}},
|
||
]
|
||
}
|
||
result = _extract_reliability_checks(rel)
|
||
assert len(result) == 2
|
||
assert result[0]["check_name"] == "License"
|
||
assert result[0]["check_status"] == "ok"
|
||
assert result[1]["check_name"] == "Permission"
|
||
assert result[1]["check_status"] == "expired"
|
||
|
||
|
||
def test_extract_address_handles_string_and_dict():
|
||
assert _extract_address({"fullAddress": "Екатеринбург"}) == "Екатеринбург"
|
||
assert _extract_address({"address": "ул. Ленина"}) == "ул. Ленина"
|
||
assert _extract_address({"address": {"full": "ЕКБ, ул. Ленина 1"}}) == "ЕКБ, ул. Ленина 1"
|
||
assert _extract_address({"address": {"display": "ЕКБ"}}) == "ЕКБ"
|
||
assert _extract_address({}) is None
|
||
|
||
|
||
def test_extract_address_prefers_full_address():
|
||
"""fullAddress wins over address."""
|
||
nb = {"fullAddress": "Полный адрес", "address": "Краткий"}
|
||
assert _extract_address(nb) == "Полный адрес"
|
||
|
||
|
||
def test_extract_deadline_year_dict_or_int():
|
||
assert _extract_deadline_year({"deadline": {"year": 2027}}) == 2027
|
||
assert _extract_deadline_year({"buildingDeadline": {"year": 2028}}) == 2028
|
||
assert _extract_deadline_year({"deadline": 2029}) == 2029
|
||
assert _extract_deadline_year({}) is None
|
||
|
||
|
||
def test_extract_advantages_returns_dicts_only():
|
||
nb = {
|
||
"advantages": [
|
||
{"title": "Отделка", "advantageType": "Finish"},
|
||
"not_a_dict",
|
||
None,
|
||
{"title": "Парковка", "advantageType": "Parking"},
|
||
]
|
||
}
|
||
result = _extract_advantages(nb)
|
||
assert len(result) == 2
|
||
assert result[0]["advantageType"] == "Finish"
|
||
assert result[1]["advantageType"] == "Parking"
|
||
|
||
|
||
def test_extract_advantages_handles_empty():
|
||
assert _extract_advantages({}) == []
|
||
assert _extract_advantages({"advantages": None}) == []
|
||
assert _extract_advantages({"advantages": "invalid"}) == []
|
||
|
||
|
||
def test_extract_nested_offers_developer_rooms_shape():
|
||
"""data.total.fromDeveloperRooms[].layouts[] → flat list with room_count."""
|
||
offers_state = {
|
||
"data": {
|
||
"total": {
|
||
"fromDeveloperRooms": [
|
||
{
|
||
"roomType": "oneRoom",
|
||
"layouts": [
|
||
{"offerId": 100, "price": 10000000},
|
||
{"offerId": 101, "price": 10500000},
|
||
],
|
||
},
|
||
{
|
||
"roomType": "twoRoom",
|
||
"layouts": [
|
||
{"offerId": 200, "price": 15000000},
|
||
],
|
||
},
|
||
]
|
||
}
|
||
}
|
||
}
|
||
result = _extract_nested_offers(offers_state)
|
||
assert len(result) == 3
|
||
assert result[0]["room_count"] == "oneRoom"
|
||
assert result[0]["offerId"] == 100
|
||
assert result[2]["room_count"] == "twoRoom"
|
||
assert result[2]["offerId"] == 200
|
||
|
||
|
||
def test_extract_nested_offers_items_shape():
|
||
"""items[] fast path."""
|
||
offers_state = {"items": [{"offerId": 1}, {"offerId": 2}]}
|
||
result = _extract_nested_offers(offers_state)
|
||
assert len(result) == 2
|
||
assert result[0]["offerId"] == 1
|
||
|
||
|
||
def test_extract_nested_offers_handles_empty():
|
||
assert _extract_nested_offers({}) == []
|
||
assert _extract_nested_offers({"data": {}}) == []
|
||
assert _extract_nested_offers({"data": {"total": {}}}) == []
|
||
|
||
|
||
def test_dataclass_defaults():
|
||
"""NewbuildingEnrichment initialises with safe defaults."""
|
||
nb = NewbuildingEnrichment()
|
||
assert nb.cian_internal_house_id is None
|
||
assert nb.realty_valuation_chart == []
|
||
assert nb.reliability_checks == []
|
||
assert nb.advantages == []
|
||
assert nb.builders == []
|
||
assert nb.banks == []
|
||
assert nb.houses_by_turn == []
|
||
assert nb.reviews == []
|
||
assert nb.nested_offers == []
|
||
assert nb.raw_state is None
|
||
assert nb.raw_sister_states == {}
|
||
|
||
|
||
# ---- async integration-style tests ----
|
||
# Transport: fetch_newbuilding теперь использует BrowserFetcher (camoufox) вместо
|
||
# curl_cffi AsyncSession (#972). Мокируем BrowserFetcher.fetch через monkeypatch.
|
||
|
||
|
||
def _make_browser_fetcher_mock(html: str, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Мокирует BrowserFetcher так, чтобы fetch() возвращал заданный html."""
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
mock_fetcher = MagicMock()
|
||
mock_fetcher.fetch = AsyncMock(return_value=html)
|
||
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
||
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_newbuilding.BrowserFetcher",
|
||
lambda: mock_fetcher,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch):
|
||
"""Если state extraction fails → return None gracefully."""
|
||
_make_browser_fetcher_mock("<html></html>", monkeypatch)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_newbuilding.extract_state",
|
||
lambda html, mfe, key: None,
|
||
)
|
||
|
||
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
|
||
assert result is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_newbuilding_browser_error_propagates(monkeypatch):
|
||
"""BrowserFetcher.fetch raises → fetch_newbuilding propagates the exception."""
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
mock_fetcher = MagicMock()
|
||
mock_fetcher.fetch = AsyncMock(side_effect=RuntimeError("browser timeout"))
|
||
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
||
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_newbuilding.BrowserFetcher",
|
||
lambda: mock_fetcher,
|
||
)
|
||
|
||
with pytest.raises(RuntimeError, match="browser timeout"):
|
||
await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_newbuilding_session_param_ignored(monkeypatch):
|
||
"""session= param принимается для backward-compat, но не используется для page-fetch."""
|
||
_make_browser_fetcher_mock("<html></html>", monkeypatch)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_newbuilding.extract_state",
|
||
lambda html, mfe, key: None,
|
||
)
|
||
|
||
from unittest.mock import MagicMock
|
||
|
||
sentinel_session = MagicMock(name="should_not_be_called")
|
||
# Передаём session= — должно работать без ошибок, session.get НЕ вызывается
|
||
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=sentinel_session)
|
||
assert result is None
|
||
sentinel_session.get.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_newbuilding_parses_state(monkeypatch):
|
||
"""Happy-path: valid state → NewbuildingEnrichment populated."""
|
||
fake_state = {
|
||
"newbuilding": {
|
||
"id": 54016,
|
||
"name": "ЖК Тест",
|
||
"fullAddress": "Екатеринбург, ул. Теста",
|
||
"newbuildingClassName": "comfort",
|
||
"transportAccessibilityRate": 7,
|
||
"advantages": [{"title": "Парковка", "advantageType": "Parking"}],
|
||
"builders": [{"name": "Застройщик Тест"}],
|
||
"banks": [{"name": "Банк Тест"}],
|
||
"housesByTurn": [{"turn": 1, "houses": []}],
|
||
},
|
||
"realtyValuation": {
|
||
"data": {
|
||
"priceDynamics": {
|
||
"chart": {
|
||
"data": {
|
||
"labels": ["2025-11-01", "2025-12-01"],
|
||
"values": [14000000, 14500000],
|
||
}
|
||
}
|
||
}
|
||
}
|
||
},
|
||
"reliability": {
|
||
"checkStatus": {"status": "reliable", "title": "Проверено"},
|
||
"details": [{"type": "problemHousesRegistry", "iconType": "success"}],
|
||
},
|
||
}
|
||
|
||
_make_browser_fetcher_mock("<html>fake</html>", monkeypatch)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_newbuilding.extract_state",
|
||
lambda html, mfe, key: fake_state,
|
||
)
|
||
monkeypatch.setattr(
|
||
"app.services.scrapers.cian_newbuilding.extract_all_states",
|
||
lambda html: {},
|
||
)
|
||
|
||
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
|
||
|
||
assert result is not None
|
||
assert result.cian_internal_house_id == 54016
|
||
assert result.name == "ЖК Тест"
|
||
assert result.address == "Екатеринбург, ул. Теста"
|
||
assert result.building_class == "comfort"
|
||
assert result.transport_accessibility_rate == 7
|
||
assert len(result.advantages) == 1
|
||
assert len(result.builders) == 1
|
||
assert len(result.banks) == 1
|
||
assert len(result.houses_by_turn) == 1
|
||
# realtyValuation chart
|
||
assert len(result.realty_valuation_chart) == 2
|
||
assert result.realty_valuation_chart[0]["month_date"] == "2025-11-01"
|
||
assert result.realty_valuation_chart[0]["price_per_sqm"] == 14000000.0
|
||
# reliability
|
||
assert len(result.reliability_checks) == 1
|
||
assert result.reliability_checks[0]["check_status"] == "reliable"
|