lekss361
d28bf4e66a
Stage 6 / Wave 4 Worker C of CianScraper v1. Cian.ru newbuilding (ЖК) catalog page scraper — extracts newbuilding metadata + 6 sister state containers (realtyValuation chart / reliability / reviews / offers / builders / housesByTurn). Main Stage 6 deliverable: 7-month price dynamics chart → houses_price_dynamics table (cross-source benchmark для Stage 9 estimator). Files: - app/services/scrapers/cian_newbuilding.py (NEW, 548 LOC) - tests/test_cian_newbuilding.py (NEW, 357 LOC, 20 tests) Reviewed: schema match (migrations 020/021/025/029), ON CONFLICT constraint name verified (houses_price_dynamics_dim_key), graceful chart extraction (empty/None/fallback shapes), atomic transaction. Known minor follow-ups (non-blocking): - management_companies UPSERT: ext_id=NULL → potential duplicates (Postgres NULL!=NULL); bounded, document in vault - offers items[] fast path: no dict-only filter / no room_count injection - houses_price_dynamics primary shape stores total price into price_per_sqm column with prices_type='price' discriminator — Stage 9 estimator must respect prices_type filter - house_reliability_checks no UNIQUE → unbounded growth on rescrape (deferred to future migration 030+) Verdict: APPROVE (deep-code-reviewer).
357 lines
12 KiB
Python
357 lines
12 KiB
Python
"""Tests for cian_newbuilding scraper (Stage 6)."""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from app.services.scrapers.cian_newbuilding import (
|
|
NewbuildingEnrichment,
|
|
_extract_address,
|
|
_extract_advantages,
|
|
_extract_chart,
|
|
_extract_deadline_year,
|
|
_extract_nested_offers,
|
|
_extract_reliability_checks,
|
|
fetch_newbuilding,
|
|
)
|
|
|
|
# ---- unit tests for extractors ----
|
|
|
|
|
|
def test_extract_chart_primary_shape():
|
|
"""realtyValuation primary shape: data.priceDynamics.chart.data.{labels, values}."""
|
|
rv_state = {
|
|
"data": {
|
|
"priceDynamics": {
|
|
"chart": {
|
|
"data": {
|
|
"labels": ["2025-11-01", "2025-12-01", "2026-01-01"],
|
|
"values": [13800000, 14100000, 14200000],
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
points = _extract_chart(rv_state)
|
|
assert len(points) == 3
|
|
assert points[0]["month_date"] == "2025-11-01"
|
|
assert points[0]["price_per_sqm"] == 13800000.0
|
|
assert points[0]["room_count"] == "all"
|
|
assert points[0]["prices_type"] == "price"
|
|
assert points[0]["period"] == "halfYear"
|
|
assert points[2]["month_date"] == "2026-01-01"
|
|
assert points[2]["price_per_sqm"] == 14200000.0
|
|
|
|
|
|
def test_extract_chart_normalizes_points():
|
|
"""Fallback shape: rv_state.chart[] with {month, price, roomType, period}."""
|
|
rv_state = {
|
|
"chart": [
|
|
{"month": "2025-12-01", "price": 150000, "roomType": "1", "period": "monthly"},
|
|
{"month": "2026-01-01", "price": 152000, "roomType": "1", "period": "monthly"},
|
|
]
|
|
}
|
|
points = _extract_chart(rv_state)
|
|
assert len(points) == 2
|
|
assert points[0]["month_date"] == "2025-12-01"
|
|
assert points[0]["price_per_sqm"] == 150000.0
|
|
assert points[0]["room_count"] == "1"
|
|
assert points[1]["month_date"] == "2026-01-01"
|
|
|
|
|
|
def test_extract_chart_handles_empty():
|
|
assert _extract_chart({}) == []
|
|
assert _extract_chart({"chart": None}) == []
|
|
assert _extract_chart({"data": None}) == []
|
|
assert _extract_chart({"data": {"priceDynamics": None}}) == []
|
|
|
|
|
|
def test_extract_chart_handles_none_values():
|
|
"""Entries with None month or value are skipped."""
|
|
rv_state = {
|
|
"chart": [
|
|
{"month": "2025-12-01", "price": 150000},
|
|
{"month": None, "price": 152000},
|
|
{"month": "2026-01-01", "price": None},
|
|
]
|
|
}
|
|
points = _extract_chart(rv_state)
|
|
assert len(points) == 1
|
|
assert points[0]["month_date"] == "2025-12-01"
|
|
|
|
|
|
def test_extract_chart_truncates_date():
|
|
"""Only first 10 chars of date string used (YYYY-MM-DD prefix)."""
|
|
rv_state = {
|
|
"chart": [
|
|
{"month": "2025-12-01T00:00:00Z", "price": 100000},
|
|
]
|
|
}
|
|
points = _extract_chart(rv_state)
|
|
assert points[0]["month_date"] == "2025-12-01"
|
|
|
|
|
|
def test_extract_reliability_primary_shape():
|
|
"""Primary shape: {checkStatus: {status, title}, details: [{type, title, iconType}]}."""
|
|
rel = {
|
|
"checkStatus": {
|
|
"status": "reliable",
|
|
"title": "Проверено",
|
|
"mobileTitle": "Проверено нашдом.рф",
|
|
"date": "Обновлено 5 февраля 2026",
|
|
},
|
|
"details": [
|
|
{
|
|
"title": "Нет в реестре проблемных объектов",
|
|
"iconType": "success",
|
|
"type": "problemHousesRegistry",
|
|
},
|
|
{
|
|
"title": "Застройщик не банкрот",
|
|
"iconType": "success",
|
|
"type": "developerBankruptStatus",
|
|
},
|
|
],
|
|
"banner": {"title": "ЖК проверен нашдом.рф"},
|
|
}
|
|
result = _extract_reliability_checks(rel)
|
|
assert len(result) == 1
|
|
assert result[0]["check_status"] == "reliable"
|
|
assert result[0]["check_name"] == "Проверено"
|
|
assert isinstance(result[0]["details"], list)
|
|
assert len(result[0]["details"]) == 2
|
|
assert result[0]["details"][0]["type"] == "problemHousesRegistry"
|
|
|
|
|
|
def test_extract_reliability_handles_empty():
|
|
assert _extract_reliability_checks({}) == []
|
|
assert _extract_reliability_checks({"checkStatus": None}) == []
|
|
|
|
|
|
def test_extract_reliability_fallback_shape():
|
|
"""Fallback: {checks: [{name, status, details}]}."""
|
|
rel = {
|
|
"checks": [
|
|
{"name": "License", "status": "ok", "details": {"doc": "lic-123"}},
|
|
{"name": "Permission", "status": "expired", "details": {}},
|
|
]
|
|
}
|
|
result = _extract_reliability_checks(rel)
|
|
assert len(result) == 2
|
|
assert result[0]["check_name"] == "License"
|
|
assert result[0]["check_status"] == "ok"
|
|
assert result[1]["check_name"] == "Permission"
|
|
assert result[1]["check_status"] == "expired"
|
|
|
|
|
|
def test_extract_address_handles_string_and_dict():
|
|
assert _extract_address({"fullAddress": "Екатеринбург"}) == "Екатеринбург"
|
|
assert _extract_address({"address": "ул. Ленина"}) == "ул. Ленина"
|
|
assert _extract_address({"address": {"full": "ЕКБ, ул. Ленина 1"}}) == "ЕКБ, ул. Ленина 1"
|
|
assert _extract_address({"address": {"display": "ЕКБ"}}) == "ЕКБ"
|
|
assert _extract_address({}) is None
|
|
|
|
|
|
def test_extract_address_prefers_full_address():
|
|
"""fullAddress wins over address."""
|
|
nb = {"fullAddress": "Полный адрес", "address": "Краткий"}
|
|
assert _extract_address(nb) == "Полный адрес"
|
|
|
|
|
|
def test_extract_deadline_year_dict_or_int():
|
|
assert _extract_deadline_year({"deadline": {"year": 2027}}) == 2027
|
|
assert _extract_deadline_year({"buildingDeadline": {"year": 2028}}) == 2028
|
|
assert _extract_deadline_year({"deadline": 2029}) == 2029
|
|
assert _extract_deadline_year({}) is None
|
|
|
|
|
|
def test_extract_advantages_returns_dicts_only():
|
|
nb = {
|
|
"advantages": [
|
|
{"title": "Отделка", "advantageType": "Finish"},
|
|
"not_a_dict",
|
|
None,
|
|
{"title": "Парковка", "advantageType": "Parking"},
|
|
]
|
|
}
|
|
result = _extract_advantages(nb)
|
|
assert len(result) == 2
|
|
assert result[0]["advantageType"] == "Finish"
|
|
assert result[1]["advantageType"] == "Parking"
|
|
|
|
|
|
def test_extract_advantages_handles_empty():
|
|
assert _extract_advantages({}) == []
|
|
assert _extract_advantages({"advantages": None}) == []
|
|
assert _extract_advantages({"advantages": "invalid"}) == []
|
|
|
|
|
|
def test_extract_nested_offers_developer_rooms_shape():
|
|
"""data.total.fromDeveloperRooms[].layouts[] → flat list with room_count."""
|
|
offers_state = {
|
|
"data": {
|
|
"total": {
|
|
"fromDeveloperRooms": [
|
|
{
|
|
"roomType": "oneRoom",
|
|
"layouts": [
|
|
{"offerId": 100, "price": 10000000},
|
|
{"offerId": 101, "price": 10500000},
|
|
],
|
|
},
|
|
{
|
|
"roomType": "twoRoom",
|
|
"layouts": [
|
|
{"offerId": 200, "price": 15000000},
|
|
],
|
|
},
|
|
]
|
|
}
|
|
}
|
|
}
|
|
result = _extract_nested_offers(offers_state)
|
|
assert len(result) == 3
|
|
assert result[0]["room_count"] == "oneRoom"
|
|
assert result[0]["offerId"] == 100
|
|
assert result[2]["room_count"] == "twoRoom"
|
|
assert result[2]["offerId"] == 200
|
|
|
|
|
|
def test_extract_nested_offers_items_shape():
|
|
"""items[] fast path."""
|
|
offers_state = {"items": [{"offerId": 1}, {"offerId": 2}]}
|
|
result = _extract_nested_offers(offers_state)
|
|
assert len(result) == 2
|
|
assert result[0]["offerId"] == 1
|
|
|
|
|
|
def test_extract_nested_offers_handles_empty():
|
|
assert _extract_nested_offers({}) == []
|
|
assert _extract_nested_offers({"data": {}}) == []
|
|
assert _extract_nested_offers({"data": {"total": {}}}) == []
|
|
|
|
|
|
def test_dataclass_defaults():
|
|
"""NewbuildingEnrichment initialises with safe defaults."""
|
|
nb = NewbuildingEnrichment()
|
|
assert nb.cian_internal_house_id is None
|
|
assert nb.realty_valuation_chart == []
|
|
assert nb.reliability_checks == []
|
|
assert nb.advantages == []
|
|
assert nb.builders == []
|
|
assert nb.banks == []
|
|
assert nb.houses_by_turn == []
|
|
assert nb.reviews == []
|
|
assert nb.nested_offers == []
|
|
assert nb.raw_state is None
|
|
assert nb.raw_sister_states == {}
|
|
|
|
|
|
# ---- async integration-style tests ----
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch):
|
|
"""Если state extraction fails → return None gracefully."""
|
|
monkeypatch.setattr(
|
|
"app.services.scrapers.cian_newbuilding.extract_state",
|
|
lambda html, mfe, key: None,
|
|
)
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
session = MagicMock()
|
|
response = MagicMock()
|
|
response.status_code = 200
|
|
response.text = "<html></html>"
|
|
session.get = AsyncMock(return_value=response)
|
|
session.close = AsyncMock()
|
|
|
|
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
|
|
assert result is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_newbuilding_returns_none_on_http_error(monkeypatch):
|
|
"""HTTP non-200 → return None gracefully."""
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
session = MagicMock()
|
|
response = MagicMock()
|
|
response.status_code = 403
|
|
response.text = "Forbidden"
|
|
session.get = AsyncMock(return_value=response)
|
|
session.close = AsyncMock()
|
|
|
|
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
|
|
assert result is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_newbuilding_parses_state(monkeypatch):
|
|
"""Happy-path: valid state → NewbuildingEnrichment populated."""
|
|
fake_state = {
|
|
"newbuilding": {
|
|
"id": 54016,
|
|
"name": "ЖК Тест",
|
|
"fullAddress": "Екатеринбург, ул. Теста",
|
|
"newbuildingClassName": "comfort",
|
|
"transportAccessibilityRate": 7,
|
|
"advantages": [{"title": "Парковка", "advantageType": "Parking"}],
|
|
"builders": [{"name": "Застройщик Тест"}],
|
|
"banks": [{"name": "Банк Тест"}],
|
|
"housesByTurn": [{"turn": 1, "houses": []}],
|
|
},
|
|
"realtyValuation": {
|
|
"data": {
|
|
"priceDynamics": {
|
|
"chart": {
|
|
"data": {
|
|
"labels": ["2025-11-01", "2025-12-01"],
|
|
"values": [14000000, 14500000],
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"reliability": {
|
|
"checkStatus": {"status": "reliable", "title": "Проверено"},
|
|
"details": [{"type": "problemHousesRegistry", "iconType": "success"}],
|
|
},
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
"app.services.scrapers.cian_newbuilding.extract_state",
|
|
lambda html, mfe, key: fake_state,
|
|
)
|
|
monkeypatch.setattr(
|
|
"app.services.scrapers.cian_newbuilding.extract_all_states",
|
|
lambda html: {},
|
|
)
|
|
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
session = MagicMock()
|
|
response = MagicMock()
|
|
response.status_code = 200
|
|
response.text = "<html>fake</html>"
|
|
session.get = AsyncMock(return_value=response)
|
|
session.close = AsyncMock()
|
|
|
|
result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
|
|
|
|
assert result is not None
|
|
assert result.cian_internal_house_id == 54016
|
|
assert result.name == "ЖК Тест"
|
|
assert result.address == "Екатеринбург, ул. Теста"
|
|
assert result.building_class == "comfort"
|
|
assert result.transport_accessibility_rate == 7
|
|
assert len(result.advantages) == 1
|
|
assert len(result.builders) == 1
|
|
assert len(result.banks) == 1
|
|
assert len(result.houses_by_turn) == 1
|
|
# realtyValuation chart
|
|
assert len(result.realty_valuation_chart) == 2
|
|
assert result.realty_valuation_chart[0]["month_date"] == "2025-11-01"
|
|
assert result.realty_valuation_chart[0]["price_per_sqm"] == 14000000.0
|
|
# reliability
|
|
assert len(result.reliability_checks) == 1
|
|
assert result.reliability_checks[0]["check_status"] == "reliable"
|