"""Tests for cian_newbuilding scraper (Stage 6).""" from __future__ import annotations import pytest from app.services.scrapers.cian_newbuilding import ( NewbuildingEnrichment, _extract_address, _extract_advantages, _extract_chart, _extract_deadline_year, _extract_nested_offers, _extract_reliability_checks, fetch_newbuilding, ) # ---- unit tests for extractors ---- def test_extract_chart_primary_shape(): """realtyValuation primary shape: data.priceDynamics.chart.data.{labels, values}.""" rv_state = { "data": { "priceDynamics": { "chart": { "data": { "labels": ["2025-11-01", "2025-12-01", "2026-01-01"], "values": [13800000, 14100000, 14200000], } } } } } points = _extract_chart(rv_state) assert len(points) == 3 assert points[0]["month_date"] == "2025-11-01" assert points[0]["price_per_sqm"] == 13800000.0 assert points[0]["room_count"] == "all" assert points[0]["prices_type"] == "price" assert points[0]["period"] == "halfYear" assert points[2]["month_date"] == "2026-01-01" assert points[2]["price_per_sqm"] == 14200000.0 def test_extract_chart_normalizes_points(): """Fallback shape: rv_state.chart[] with {month, price, roomType, period}.""" rv_state = { "chart": [ {"month": "2025-12-01", "price": 150000, "roomType": "1", "period": "monthly"}, {"month": "2026-01-01", "price": 152000, "roomType": "1", "period": "monthly"}, ] } points = _extract_chart(rv_state) assert len(points) == 2 assert points[0]["month_date"] == "2025-12-01" assert points[0]["price_per_sqm"] == 150000.0 assert points[0]["room_count"] == "1" assert points[1]["month_date"] == "2026-01-01" def test_extract_chart_handles_empty(): assert _extract_chart({}) == [] assert _extract_chart({"chart": None}) == [] assert _extract_chart({"data": None}) == [] assert _extract_chart({"data": {"priceDynamics": None}}) == [] def test_extract_chart_handles_none_values(): """Entries with None month or value are skipped.""" rv_state = { "chart": [ {"month": "2025-12-01", "price": 150000}, {"month": None, "price": 152000}, {"month": "2026-01-01", "price": None}, ] } points = _extract_chart(rv_state) assert len(points) == 1 assert points[0]["month_date"] == "2025-12-01" def test_extract_chart_truncates_date(): """Only first 10 chars of date string used (YYYY-MM-DD prefix).""" rv_state = { "chart": [ {"month": "2025-12-01T00:00:00Z", "price": 100000}, ] } points = _extract_chart(rv_state) assert points[0]["month_date"] == "2025-12-01" def test_extract_reliability_primary_shape(): """Primary shape: {checkStatus: {status, title}, details: [{type, title, iconType}]}.""" rel = { "checkStatus": { "status": "reliable", "title": "Проверено", "mobileTitle": "Проверено нашдом.рф", "date": "Обновлено 5 февраля 2026", }, "details": [ { "title": "Нет в реестре проблемных объектов", "iconType": "success", "type": "problemHousesRegistry", }, { "title": "Застройщик не банкрот", "iconType": "success", "type": "developerBankruptStatus", }, ], "banner": {"title": "ЖК проверен нашдом.рф"}, } result = _extract_reliability_checks(rel) assert len(result) == 1 assert result[0]["check_status"] == "reliable" assert result[0]["check_name"] == "Проверено" assert isinstance(result[0]["details"], list) assert len(result[0]["details"]) == 2 assert result[0]["details"][0]["type"] == "problemHousesRegistry" def test_extract_reliability_handles_empty(): assert _extract_reliability_checks({}) == [] assert _extract_reliability_checks({"checkStatus": None}) == [] def test_extract_reliability_fallback_shape(): """Fallback: {checks: [{name, status, details}]}.""" rel = { "checks": [ {"name": "License", "status": "ok", "details": {"doc": "lic-123"}}, {"name": "Permission", "status": "expired", "details": {}}, ] } result = _extract_reliability_checks(rel) assert len(result) == 2 assert result[0]["check_name"] == "License" assert result[0]["check_status"] == "ok" assert result[1]["check_name"] == "Permission" assert result[1]["check_status"] == "expired" def test_extract_address_handles_string_and_dict(): assert _extract_address({"fullAddress": "Екатеринбург"}) == "Екатеринбург" assert _extract_address({"address": "ул. Ленина"}) == "ул. Ленина" assert _extract_address({"address": {"full": "ЕКБ, ул. Ленина 1"}}) == "ЕКБ, ул. Ленина 1" assert _extract_address({"address": {"display": "ЕКБ"}}) == "ЕКБ" assert _extract_address({}) is None def test_extract_address_prefers_full_address(): """fullAddress wins over address.""" nb = {"fullAddress": "Полный адрес", "address": "Краткий"} assert _extract_address(nb) == "Полный адрес" def test_extract_deadline_year_dict_or_int(): assert _extract_deadline_year({"deadline": {"year": 2027}}) == 2027 assert _extract_deadline_year({"buildingDeadline": {"year": 2028}}) == 2028 assert _extract_deadline_year({"deadline": 2029}) == 2029 assert _extract_deadline_year({}) is None def test_extract_advantages_returns_dicts_only(): nb = { "advantages": [ {"title": "Отделка", "advantageType": "Finish"}, "not_a_dict", None, {"title": "Парковка", "advantageType": "Parking"}, ] } result = _extract_advantages(nb) assert len(result) == 2 assert result[0]["advantageType"] == "Finish" assert result[1]["advantageType"] == "Parking" def test_extract_advantages_handles_empty(): assert _extract_advantages({}) == [] assert _extract_advantages({"advantages": None}) == [] assert _extract_advantages({"advantages": "invalid"}) == [] def test_extract_nested_offers_developer_rooms_shape(): """data.total.fromDeveloperRooms[].layouts[] → flat list with room_count.""" offers_state = { "data": { "total": { "fromDeveloperRooms": [ { "roomType": "oneRoom", "layouts": [ {"offerId": 100, "price": 10000000}, {"offerId": 101, "price": 10500000}, ], }, { "roomType": "twoRoom", "layouts": [ {"offerId": 200, "price": 15000000}, ], }, ] } } } result = _extract_nested_offers(offers_state) assert len(result) == 3 assert result[0]["room_count"] == "oneRoom" assert result[0]["offerId"] == 100 assert result[2]["room_count"] == "twoRoom" assert result[2]["offerId"] == 200 def test_extract_nested_offers_items_shape(): """items[] fast path.""" offers_state = {"items": [{"offerId": 1}, {"offerId": 2}]} result = _extract_nested_offers(offers_state) assert len(result) == 2 assert result[0]["offerId"] == 1 def test_extract_nested_offers_handles_empty(): assert _extract_nested_offers({}) == [] assert _extract_nested_offers({"data": {}}) == [] assert _extract_nested_offers({"data": {"total": {}}}) == [] def test_dataclass_defaults(): """NewbuildingEnrichment initialises with safe defaults.""" nb = NewbuildingEnrichment() assert nb.cian_internal_house_id is None assert nb.realty_valuation_chart == [] assert nb.reliability_checks == [] assert nb.advantages == [] assert nb.builders == [] assert nb.banks == [] assert nb.houses_by_turn == [] assert nb.reviews == [] assert nb.nested_offers == [] assert nb.raw_state is None assert nb.raw_sister_states == {} # ---- async integration-style tests ---- @pytest.mark.asyncio async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch): """Если state extraction fails → return None gracefully.""" monkeypatch.setattr( "app.services.scrapers.cian_newbuilding.extract_state", lambda html, mfe, key: None, ) from unittest.mock import AsyncMock, MagicMock session = MagicMock() response = MagicMock() response.status_code = 200 response.text = "" session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) assert result is None @pytest.mark.asyncio async def test_fetch_newbuilding_returns_none_on_http_error(monkeypatch): """HTTP non-200 → return None gracefully.""" from unittest.mock import AsyncMock, MagicMock session = MagicMock() response = MagicMock() response.status_code = 403 response.text = "Forbidden" session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) assert result is None @pytest.mark.asyncio async def test_fetch_newbuilding_parses_state(monkeypatch): """Happy-path: valid state → NewbuildingEnrichment populated.""" fake_state = { "newbuilding": { "id": 54016, "name": "ЖК Тест", "fullAddress": "Екатеринбург, ул. Теста", "newbuildingClassName": "comfort", "transportAccessibilityRate": 7, "advantages": [{"title": "Парковка", "advantageType": "Parking"}], "builders": [{"name": "Застройщик Тест"}], "banks": [{"name": "Банк Тест"}], "housesByTurn": [{"turn": 1, "houses": []}], }, "realtyValuation": { "data": { "priceDynamics": { "chart": { "data": { "labels": ["2025-11-01", "2025-12-01"], "values": [14000000, 14500000], } } } } }, "reliability": { "checkStatus": {"status": "reliable", "title": "Проверено"}, "details": [{"type": "problemHousesRegistry", "iconType": "success"}], }, } monkeypatch.setattr( "app.services.scrapers.cian_newbuilding.extract_state", lambda html, mfe, key: fake_state, ) monkeypatch.setattr( "app.services.scrapers.cian_newbuilding.extract_all_states", lambda html: {}, ) from unittest.mock import AsyncMock, MagicMock session = MagicMock() response = MagicMock() response.status_code = 200 response.text = "fake" session.get = AsyncMock(return_value=response) session.close = AsyncMock() result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) assert result is not None assert result.cian_internal_house_id == 54016 assert result.name == "ЖК Тест" assert result.address == "Екатеринбург, ул. Теста" assert result.building_class == "comfort" assert result.transport_accessibility_rate == 7 assert len(result.advantages) == 1 assert len(result.builders) == 1 assert len(result.banks) == 1 assert len(result.houses_by_turn) == 1 # realtyValuation chart assert len(result.realty_valuation_chart) == 2 assert result.realty_valuation_chart[0]["month_date"] == "2025-11-01" assert result.realty_valuation_chart[0]["price_per_sqm"] == 14000000.0 # reliability assert len(result.reliability_checks) == 1 assert result.reliability_checks[0]["check_status"] == "reliable"