From ec196339a6971f1a441bad798f3d4637b7df0060 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 16:16:19 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20cian=5Fnewbuilding.py=20?= =?UTF-8?q?=E2=80=94=20=D0=96=D0=9A=20catalog=20scraper=20(chart=20+=20rel?= =?UTF-8?q?iability=20+=20offers,=20Wave=204=20Worker=20C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/services/scrapers/cian_newbuilding.py | 548 ++++++++++++++++++ .../backend/tests/test_cian_newbuilding.py | 357 ++++++++++++ 2 files changed, 905 insertions(+) create mode 100644 tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py create mode 100644 tradein-mvp/backend/tests/test_cian_newbuilding.py diff --git a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py new file mode 100644 index 00000000..10e458e0 --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py @@ -0,0 +1,548 @@ +"""Cian.ru newbuilding (ЖК) catalog page scraper. + +URL: https://zhk---i.cian.ru/ +MFE: 'newbuilding-card-desktop-fichering-frontend', key: 'initialState' + +Sister state containers extracted from same MFE initialState top-level keys: +- realtyValuation: 7-month price chart (data.priceDynamics.chart.data.{labels,values}) + → houses_price_dynamics +- reliability: наш.дом.рф checkStatus + details[] → house_reliability_checks +- offers: grouped by roomType (fromDeveloperRooms[].layouts[]) +- reviews: house reviews list +- housesByTurn: корпуса по очередям (stored as jsonb on houses) + +Stage 6 of CianScraper v1. +""" +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Any + +from curl_cffi.requests import AsyncSession # type: ignore[import-untyped] + +from app.services.scrapers.cian_state_parser import extract_all_states, extract_state + +logger = logging.getLogger(__name__) + + +@dataclass +class NewbuildingEnrichment: + """Result of fetch_newbuilding() — houses row + sister containers.""" + + # Core house metadata (from state.newbuilding) + cian_internal_house_id: int | None = None + name: str | None = None + address: str | None = None + year_built: int | None = None + deadline_year: int | None = None + floors_count_min: int | None = None + floors_count_max: int | None = None + building_class: str | None = None # 'comfort' / 'business' / 'premium' + + # Management + management_company: dict[str, Any] | None = None # name, phones[], email, ... + + # Aggregated metrics + transport_accessibility_rate: int | None = None + advantages: list[dict[str, Any]] = field(default_factory=list) + + # Time-series price chart (Stage 6 main deliverable) + # Each entry: {'month_date': str, 'room_count': str, 'prices_type': str, 'period': str, + # 'price_per_sqm': float | None} + realty_valuation_chart: list[dict[str, Any]] = field(default_factory=list) + + # Reliability (наш.дом.рф) — overall status + checks detail array + # {check_status: str, check_name: str, details: list[dict]} + reliability_checks: list[dict[str, Any]] = field(default_factory=list) + + # Builders / banks (related orgs) + builders: list[dict[str, Any]] = field(default_factory=list) + banks: list[dict[str, Any]] = field(default_factory=list) + + # Corpuses (houses_by_turn — корпуса по очередям) + houses_by_turn: list[dict[str, Any]] = field(default_factory=list) + + # Reviews + reviews: list[dict[str, Any]] = field(default_factory=list) + + # Mini-SERP offers (grouped by roomType, layouts) + nested_offers: list[dict[str, Any]] = field(default_factory=list) + + # Raw payloads для debugging + raw_state: dict[str, Any] | None = None + raw_sister_states: dict[str, Any] = field(default_factory=dict) + + +async def fetch_newbuilding( + zhk_url: str, + *, + session: AsyncSession | None = None, +) -> NewbuildingEnrichment | None: + """Fetch ЖК catalog page и extract все containers. + + Args: + zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/' + session: optional shared curl_cffi session (caller owns lifecycle) + + Returns: NewbuildingEnrichment, or None если fetch / parse failed. + """ + close_session = False + if session is None: + session = AsyncSession( + impersonate="chrome120", + timeout=30.0, + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", + }, + ) + close_session = True + + try: + resp = await session.get(zhk_url, allow_redirects=True) + if resp.status_code != 200: + logger.warning("Cian newbuilding fetch %s → HTTP %d", zhk_url, resp.status_code) + return None + html = resp.text + + mfe = "newbuilding-card-desktop-fichering-frontend" + + nb_state = extract_state(html, mfe=mfe, key="initialState") + if nb_state is None: + logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url) + return None + + # Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4) + nb = nb_state.get("newbuilding") or {} + if not isinstance(nb, dict): + nb = {} + + result = NewbuildingEnrichment( + cian_internal_house_id=nb.get("id"), + name=nb.get("name") or nb.get("displayName"), + address=_extract_address(nb), + year_built=nb.get("yearBuilt") or nb.get("buildYear"), + deadline_year=_extract_deadline_year(nb), + floors_count_min=nb.get("floorsCountMin"), + floors_count_max=nb.get("floorsCountMax"), + building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"), + management_company=nb.get("managementCompany"), + transport_accessibility_rate=nb.get("transportAccessibilityRate"), + advantages=_extract_advantages(nb), + builders=_safe_list(nb.get("builders")), + banks=_safe_list(nb.get("banks")), + houses_by_turn=_safe_list(nb.get("housesByTurn")), + raw_state=nb_state, + ) + + # Sister containers — all live under the same MFE initialState top-level keys + # (not separate push() entries) per schema sec 15.3 + all_states = extract_all_states(html) + mfe_states = all_states.get(mfe, {}) + # initialState top-level keys contain sister data: realtyValuation, reliability, offers + top = nb_state + + # realtyValuation 7-month chart (Stage 6 main deliverable) + # Located at initialState.realtyValuation (sec 15.5) + rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation") + if rv_state and isinstance(rv_state, dict): + result.realty_valuation_chart = _extract_chart(rv_state) + result.raw_sister_states["realtyValuation"] = rv_state + + # reliability (наш.дом.рф) — located at initialState.reliability (sec 15.6) + rel_state = top.get("reliability") or mfe_states.get("reliability") + if rel_state and isinstance(rel_state, dict): + result.reliability_checks = _extract_reliability_checks(rel_state) + result.raw_sister_states["reliability"] = rel_state + + # reviews — located at initialState or separate state + reviews_state = top.get("reviews") or mfe_states.get("reviews") + if reviews_state and isinstance(reviews_state, dict): + items = reviews_state.get("items") or reviews_state.get("data") or [] + result.reviews = items if isinstance(items, list) else [] + result.raw_sister_states["reviews"] = reviews_state + + # offers (grouped by roomType) — located at initialState.offers (sec 15.7) + offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp") + if offers_state and isinstance(offers_state, dict): + result.nested_offers = _extract_nested_offers(offers_state) + result.raw_sister_states["offers"] = offers_state + + logger.info( + "Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d", + zhk_url, + result.cian_internal_house_id, + len(result.realty_valuation_chart), + len(result.reliability_checks), + len(result.nested_offers), + ) + return result + finally: + if close_session: + await session.close() + + +# ---- private extractors ---- + + +def _safe_list(v: Any) -> list[dict[str, Any]]: + """Return v as list[dict] or empty list.""" + if not isinstance(v, list): + return [] + return [item for item in v if isinstance(item, dict)] + + +def _extract_address(nb: dict[str, Any]) -> str | None: + """Extract display address string from newbuilding object.""" + addr = nb.get("fullAddress") or nb.get("address") + if isinstance(addr, str): + return addr + if isinstance(addr, dict): + return addr.get("full") or addr.get("display") or addr.get("name") + return None + + +def _extract_deadline_year(nb: dict[str, Any]) -> int | None: + """Extract planned completion year from newbuilding.deadline or similar field.""" + dl = nb.get("deadline") or nb.get("buildingDeadline") + if isinstance(dl, dict): + return dl.get("year") + if isinstance(dl, int): + return dl + return None + + +def _extract_advantages(nb: dict[str, Any]) -> list[dict[str, Any]]: + """Extract typed advantages list from newbuilding.advantages.""" + raw = nb.get("advantages") or nb.get("advantagesTyped") or [] + if not isinstance(raw, list): + return [] + return [a for a in raw if isinstance(a, dict)] + + +def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]: + """Extract realtyValuation time-series chart points. + + Actual Cian shape (sec 15.5): + rv_state.data.priceDynamics.chart.data = {labels: [date...], values: [int...]} + + Falls back to alternative shapes if structure varies. + Each output point: + {month_date: str, room_count: str, prices_type: str, period: str, price_per_sqm: float|None} + + Note: The initial page state provides aggregate (all room types) chart only. + Per-room-type data requires XHR requests (not done here). + """ + points: list[dict[str, Any]] = [] + + # Primary shape: data.priceDynamics.chart.data.{labels, values} + data_block = rv_state.get("data") + if isinstance(data_block, dict): + price_dynamics = data_block.get("priceDynamics") or {} + if isinstance(price_dynamics, dict): + chart = price_dynamics.get("chart") or {} + if isinstance(chart, dict): + chart_data = chart.get("data") or {} + if isinstance(chart_data, dict): + labels = chart_data.get("labels") or [] + values = chart_data.get("values") or [] + if isinstance(labels, list) and isinstance(values, list): + for label, value in zip(labels, values, strict=False): + if label is None or value is None: + continue + points.append({ + "month_date": str(label)[:10], # YYYY-MM-DD + "room_count": "all", # aggregate (initial state) + "prices_type": "price", # total price (not priceSqm) + "period": "halfYear", # default period shown + "price_per_sqm": None, # chart shows total price, not sqm + }) + # Store raw value as price_per_sqm placeholder for DB + # (actual sqm price would require area context) + # Use the value as-is; caller / downstream can enrich + points[-1]["price_per_sqm"] = float(value) if isinstance( + value, (int, float) + ) else None + if points: + return points + + # Fallback shape A: rv_state.chart (flat list with {month/date, price, roomType}) + chart_flat = rv_state.get("chart") or rv_state.get("prices") or rv_state.get("dynamics") + if isinstance(chart_flat, list): + for entry in chart_flat: + if not isinstance(entry, dict): + continue + month = entry.get("month") or entry.get("date") or entry.get("monthDate") + price = entry.get("price") or entry.get("value") or entry.get("pricePerSqm") + if month is None or price is None: + continue + price_val: float | None = None + try: + price_val = float(price) + except (TypeError, ValueError): + pass + points.append({ + "month_date": str(month)[:10], + "room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"), + "prices_type": str(entry.get("pricesType") or entry.get("priceType") or "price"), + "period": str(entry.get("period") or "halfYear"), + "price_per_sqm": price_val, + }) + return points + + return [] + + +def _extract_reliability_checks(rel_state: dict[str, Any]) -> list[dict[str, Any]]: + """Extract reliability check summary from наш.дом.рф state. + + Actual Cian shape (sec 15.6): + rel_state = { + checkStatus: {status: 'reliable', title: '...', date: '...'}, + details: [{title, iconType, type}, ...], + ... + } + + Returns a list with ONE entry representing the overall check (per DB schema): + [{check_status, check_name, details}] + + The 'details' array is stored as jsonb (full check list) in house_reliability_checks. + """ + check_status_block = rel_state.get("checkStatus") + if not isinstance(check_status_block, dict): + # Alternative: flat list shape + raw = rel_state.get("checks") or rel_state.get("items") or [] + if isinstance(raw, list) and raw: + return [ + { + "check_name": entry.get("name") or entry.get("title") or entry.get("checkName"), + "check_status": entry.get("status") or entry.get("checkStatus"), + "details": entry.get("details") or entry, + } + for entry in raw + if isinstance(entry, dict) + ] + return [] + + status = check_status_block.get("status") + name = check_status_block.get("title") or check_status_block.get("mobileTitle") + details = rel_state.get("details") or [] + + return [ + { + "check_name": name, + "check_status": status, + "details": details if isinstance(details, list) else [details], + } + ] + + +def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]]: + """Extract mini-SERP offers from offers container. + + Actual Cian shape (sec 15.7): + offers_state.data.total.fromDeveloperRooms[].{roomType, layouts[]} + + Returns flat list of layout offers with room_count injected. + """ + result: list[dict[str, Any]] = [] + + # Shape A: miniSerp / items list + if "items" in offers_state: + items = offers_state["items"] + return items if isinstance(items, list) else [] + + # Shape B: data.total.fromDeveloperRooms[].layouts[] + data = offers_state.get("data") + if isinstance(data, dict): + total = data.get("total") or {} + if isinstance(total, dict): + rooms = total.get("fromDeveloperRooms") or [] + else: + rooms = [] + else: + rooms = [] + + if not isinstance(rooms, list): + return result + + for room in rooms: + if not isinstance(room, dict): + continue + room_type = room.get("roomType") or room.get("roomTypeDisplay") + layouts = room.get("layouts") or [] + if not isinstance(layouts, list): + continue + for layout in layouts: + if isinstance(layout, dict): + result.append({ + "room_count": room_type, + **layout, + }) + + return result + + +# ---- save helpers ---- + + +async def save_newbuilding_enrichment( + db: Any, + house_id: int, + enrichment: NewbuildingEnrichment, +) -> None: + """Persist NewbuildingEnrichment to DB. + + Steps: + 1. UPSERT management_companies (if present) → get mc_id + 2. UPDATE houses with Cian metadata + 3. INSERT INTO houses_price_dynamics (chart points, ON CONFLICT DO UPDATE) + 4. INSERT INTO house_reliability_checks (overall + details) + """ + from sqlalchemy import text + + # 1. UPSERT management_company + mc_id: int | None = None + if enrichment.management_company and isinstance(enrichment.management_company, dict): + mc = enrichment.management_company + mc_name = mc.get("name") or "Unknown" + mc_ext_id = mc.get("id") + mc_result = db.execute( + text(""" + INSERT INTO management_companies ( + name, phones, email, opening_hours, chief_name, ext_source, ext_id + ) VALUES ( + :name, + CAST(:phones AS text[]), + :email, + :oh, + :chief, + 'cian', + CAST(:ext_id AS bigint) + ) + ON CONFLICT (ext_source, name, ext_id) DO UPDATE SET + phones = EXCLUDED.phones, + email = COALESCE(EXCLUDED.email, management_companies.email) + RETURNING id + """), + { + "name": mc_name, + "phones": mc.get("phones") or [], + "email": mc.get("email"), + "oh": mc.get("openingHours"), + "chief": mc.get("chiefName"), + "ext_id": mc_ext_id, + }, + ) + row = mc_result.fetchone() + mc_id = row[0] if row else None + + # 2. UPDATE houses + db.execute( + text(""" + UPDATE houses SET + cian_internal_house_id = COALESCE( + CAST(:cihi AS bigint), cian_internal_house_id + ), + year_built = COALESCE(:yb, year_built), + management_company_id = COALESCE(CAST(:mcid AS bigint), management_company_id), + transport_accessibility_rate = COALESCE( + :tar, transport_accessibility_rate + ), + advantages = CAST(:adv AS jsonb), + banks = CAST(:bk AS jsonb), + builders = CAST(:bld AS jsonb), + houses_by_turn = CAST(:hbt AS jsonb) + WHERE id = CAST(:hid AS bigint) + """), + { + "hid": house_id, + "cihi": enrichment.cian_internal_house_id, + "yb": enrichment.year_built, + "mcid": mc_id, + "tar": enrichment.transport_accessibility_rate, + "adv": json.dumps(enrichment.advantages, ensure_ascii=False), + "bk": json.dumps(enrichment.banks, ensure_ascii=False), + "bld": json.dumps(enrichment.builders, ensure_ascii=False), + "hbt": json.dumps(enrichment.houses_by_turn, ensure_ascii=False), + }, + ) + + # 3. INSERT houses_price_dynamics + # UNIQUE constraint: houses_price_dynamics_dim_key + # (house_id, source, room_count, prices_type, period, month_date) — per migration 029 + chart_saved = 0 + for point in enrichment.realty_valuation_chart: + if point.get("price_per_sqm") is None: + continue + room_count = point.get("room_count") or "all" + prices_type = point.get("prices_type") or "price" + period = point.get("period") or "halfYear" + db.execute( + text(""" + INSERT INTO houses_price_dynamics ( + house_id, month_date, source, + room_count, prices_type, period, + price_per_sqm, recorded_at + ) VALUES ( + CAST(:hid AS bigint), + CAST(:md AS date), + 'cian_realty_valuation', + :rc, :pt, :pd, + CAST(:pps AS numeric), + NOW() + ) + ON CONFLICT ON CONSTRAINT houses_price_dynamics_dim_key DO UPDATE SET + price_per_sqm = EXCLUDED.price_per_sqm, + recorded_at = NOW() + """), + { + "hid": house_id, + "md": point["month_date"], + "rc": room_count, + "pt": prices_type, + "pd": period, + "pps": point["price_per_sqm"], + }, + ) + chart_saved += 1 + + # 4. INSERT house_reliability_checks (stores overall check + details array) + # Schema (025): (house_id, check_status, check_name, details jsonb, source, recorded_at) + # No UNIQUE constraint — caller should manage duplicates if needed + for check in enrichment.reliability_checks: + if not check.get("check_name") and not check.get("check_status"): + continue + db.execute( + text(""" + INSERT INTO house_reliability_checks ( + house_id, check_status, check_name, details, source, recorded_at + ) VALUES ( + CAST(:hid AS bigint), + :cs, + :cn, + CAST(:det AS jsonb), + 'cian_nashdom', + NOW() + ) + """), + { + "hid": house_id, + "cs": check.get("check_status"), + "cn": check.get("check_name"), + "det": json.dumps( + check.get("details") or [], ensure_ascii=False + ), + }, + ) + + db.commit() + logger.info( + "Cian newbuilding saved house_id=%s (chart=%d points, reliability=%d checks, mc_id=%s)", + house_id, + chart_saved, + len(enrichment.reliability_checks), + mc_id, + ) diff --git a/tradein-mvp/backend/tests/test_cian_newbuilding.py b/tradein-mvp/backend/tests/test_cian_newbuilding.py new file mode 100644 index 00000000..e5d02b65 --- /dev/null +++ b/tradein-mvp/backend/tests/test_cian_newbuilding.py @@ -0,0 +1,357 @@ +"""Tests for cian_newbuilding scraper (Stage 6).""" +from __future__ import annotations + +import pytest + +from app.services.scrapers.cian_newbuilding import ( + NewbuildingEnrichment, + _extract_address, + _extract_advantages, + _extract_chart, + _extract_deadline_year, + _extract_nested_offers, + _extract_reliability_checks, + fetch_newbuilding, +) + +# ---- unit tests for extractors ---- + + +def test_extract_chart_primary_shape(): + """realtyValuation primary shape: data.priceDynamics.chart.data.{labels, values}.""" + rv_state = { + "data": { + "priceDynamics": { + "chart": { + "data": { + "labels": ["2025-11-01", "2025-12-01", "2026-01-01"], + "values": [13800000, 14100000, 14200000], + } + } + } + } + } + points = _extract_chart(rv_state) + assert len(points) == 3 + assert points[0]["month_date"] == "2025-11-01" + assert points[0]["price_per_sqm"] == 13800000.0 + assert points[0]["room_count"] == "all" + assert points[0]["prices_type"] == "price" + assert points[0]["period"] == "halfYear" + assert points[2]["month_date"] == "2026-01-01" + assert points[2]["price_per_sqm"] == 14200000.0 + + +def test_extract_chart_normalizes_points(): + """Fallback shape: rv_state.chart[] with {month, price, roomType, period}.""" + rv_state = { + "chart": [ + {"month": "2025-12-01", "price": 150000, "roomType": "1", "period": "monthly"}, + {"month": "2026-01-01", "price": 152000, "roomType": "1", "period": "monthly"}, + ] + } + points = _extract_chart(rv_state) + assert len(points) == 2 + assert points[0]["month_date"] == "2025-12-01" + assert points[0]["price_per_sqm"] == 150000.0 + assert points[0]["room_count"] == "1" + assert points[1]["month_date"] == "2026-01-01" + + +def test_extract_chart_handles_empty(): + assert _extract_chart({}) == [] + assert _extract_chart({"chart": None}) == [] + assert _extract_chart({"data": None}) == [] + assert _extract_chart({"data": {"priceDynamics": None}}) == [] + + +def test_extract_chart_handles_none_values(): + """Entries with None month or value are skipped.""" + rv_state = { + "chart": [ + {"month": "2025-12-01", "price": 150000}, + {"month": None, "price": 152000}, + {"month": "2026-01-01", "price": None}, + ] + } + points = _extract_chart(rv_state) + assert len(points) == 1 + assert points[0]["month_date"] == "2025-12-01" + + +def test_extract_chart_truncates_date(): + """Only first 10 chars of date string used (YYYY-MM-DD prefix).""" + rv_state = { + "chart": [ + {"month": "2025-12-01T00:00:00Z", "price": 100000}, + ] + } + points = _extract_chart(rv_state) + assert points[0]["month_date"] == "2025-12-01" + + +def test_extract_reliability_primary_shape(): + """Primary shape: {checkStatus: {status, title}, details: [{type, title, iconType}]}.""" + rel = { + "checkStatus": { + "status": "reliable", + "title": "Проверено", + "mobileTitle": "Проверено нашдом.рф", + "date": "Обновлено 5 февраля 2026", + }, + "details": [ + { + "title": "Нет в реестре проблемных объектов", + "iconType": "success", + "type": "problemHousesRegistry", + }, + { + "title": "Застройщик не банкрот", + "iconType": "success", + "type": "developerBankruptStatus", + }, + ], + "banner": {"title": "ЖК проверен нашдом.рф"}, + } + result = _extract_reliability_checks(rel) + assert len(result) == 1 + assert result[0]["check_status"] == "reliable" + assert result[0]["check_name"] == "Проверено" + assert isinstance(result[0]["details"], list) + assert len(result[0]["details"]) == 2 + assert result[0]["details"][0]["type"] == "problemHousesRegistry" + + +def test_extract_reliability_handles_empty(): + assert _extract_reliability_checks({}) == [] + assert _extract_reliability_checks({"checkStatus": None}) == [] + + +def test_extract_reliability_fallback_shape(): + """Fallback: {checks: [{name, status, details}]}.""" + rel = { + "checks": [ + {"name": "License", "status": "ok", "details": {"doc": "lic-123"}}, + {"name": "Permission", "status": "expired", "details": {}}, + ] + } + result = _extract_reliability_checks(rel) + assert len(result) == 2 + assert result[0]["check_name"] == "License" + assert result[0]["check_status"] == "ok" + assert result[1]["check_name"] == "Permission" + assert result[1]["check_status"] == "expired" + + +def test_extract_address_handles_string_and_dict(): + assert _extract_address({"fullAddress": "Екатеринбург"}) == "Екатеринбург" + assert _extract_address({"address": "ул. Ленина"}) == "ул. Ленина" + assert _extract_address({"address": {"full": "ЕКБ, ул. Ленина 1"}}) == "ЕКБ, ул. Ленина 1" + assert _extract_address({"address": {"display": "ЕКБ"}}) == "ЕКБ" + assert _extract_address({}) is None + + +def test_extract_address_prefers_full_address(): + """fullAddress wins over address.""" + nb = {"fullAddress": "Полный адрес", "address": "Краткий"} + assert _extract_address(nb) == "Полный адрес" + + +def test_extract_deadline_year_dict_or_int(): + assert _extract_deadline_year({"deadline": {"year": 2027}}) == 2027 + assert _extract_deadline_year({"buildingDeadline": {"year": 2028}}) == 2028 + assert _extract_deadline_year({"deadline": 2029}) == 2029 + assert _extract_deadline_year({}) is None + + +def test_extract_advantages_returns_dicts_only(): + nb = { + "advantages": [ + {"title": "Отделка", "advantageType": "Finish"}, + "not_a_dict", + None, + {"title": "Парковка", "advantageType": "Parking"}, + ] + } + result = _extract_advantages(nb) + assert len(result) == 2 + assert result[0]["advantageType"] == "Finish" + assert result[1]["advantageType"] == "Parking" + + +def test_extract_advantages_handles_empty(): + assert _extract_advantages({}) == [] + assert _extract_advantages({"advantages": None}) == [] + assert _extract_advantages({"advantages": "invalid"}) == [] + + +def test_extract_nested_offers_developer_rooms_shape(): + """data.total.fromDeveloperRooms[].layouts[] → flat list with room_count.""" + offers_state = { + "data": { + "total": { + "fromDeveloperRooms": [ + { + "roomType": "oneRoom", + "layouts": [ + {"offerId": 100, "price": 10000000}, + {"offerId": 101, "price": 10500000}, + ], + }, + { + "roomType": "twoRoom", + "layouts": [ + {"offerId": 200, "price": 15000000}, + ], + }, + ] + } + } + } + result = _extract_nested_offers(offers_state) + assert len(result) == 3 + assert result[0]["room_count"] == "oneRoom" + assert result[0]["offerId"] == 100 + assert result[2]["room_count"] == "twoRoom" + assert result[2]["offerId"] == 200 + + +def test_extract_nested_offers_items_shape(): + """items[] fast path.""" + offers_state = {"items": [{"offerId": 1}, {"offerId": 2}]} + result = _extract_nested_offers(offers_state) + assert len(result) == 2 + assert result[0]["offerId"] == 1 + + +def test_extract_nested_offers_handles_empty(): + assert _extract_nested_offers({}) == [] + assert _extract_nested_offers({"data": {}}) == [] + assert _extract_nested_offers({"data": {"total": {}}}) == [] + + +def test_dataclass_defaults(): + """NewbuildingEnrichment initialises with safe defaults.""" + nb = NewbuildingEnrichment() + assert nb.cian_internal_house_id is None + assert nb.realty_valuation_chart == [] + assert nb.reliability_checks == [] + assert nb.advantages == [] + assert nb.builders == [] + assert nb.banks == [] + assert nb.houses_by_turn == [] + assert nb.reviews == [] + assert nb.nested_offers == [] + assert nb.raw_state is None + assert nb.raw_sister_states == {} + + +# ---- async integration-style tests ---- + + +@pytest.mark.asyncio +async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch): + """Если state extraction fails → return None gracefully.""" + monkeypatch.setattr( + "app.services.scrapers.cian_newbuilding.extract_state", + lambda html, mfe, key: None, + ) + from unittest.mock import AsyncMock, MagicMock + + session = MagicMock() + response = MagicMock() + response.status_code = 200 + response.text = "" + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) + assert result is None + + +@pytest.mark.asyncio +async def test_fetch_newbuilding_returns_none_on_http_error(monkeypatch): + """HTTP non-200 → return None gracefully.""" + from unittest.mock import AsyncMock, MagicMock + + session = MagicMock() + response = MagicMock() + response.status_code = 403 + response.text = "Forbidden" + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) + assert result is None + + +@pytest.mark.asyncio +async def test_fetch_newbuilding_parses_state(monkeypatch): + """Happy-path: valid state → NewbuildingEnrichment populated.""" + fake_state = { + "newbuilding": { + "id": 54016, + "name": "ЖК Тест", + "fullAddress": "Екатеринбург, ул. Теста", + "newbuildingClassName": "comfort", + "transportAccessibilityRate": 7, + "advantages": [{"title": "Парковка", "advantageType": "Parking"}], + "builders": [{"name": "Застройщик Тест"}], + "banks": [{"name": "Банк Тест"}], + "housesByTurn": [{"turn": 1, "houses": []}], + }, + "realtyValuation": { + "data": { + "priceDynamics": { + "chart": { + "data": { + "labels": ["2025-11-01", "2025-12-01"], + "values": [14000000, 14500000], + } + } + } + } + }, + "reliability": { + "checkStatus": {"status": "reliable", "title": "Проверено"}, + "details": [{"type": "problemHousesRegistry", "iconType": "success"}], + }, + } + + monkeypatch.setattr( + "app.services.scrapers.cian_newbuilding.extract_state", + lambda html, mfe, key: fake_state, + ) + monkeypatch.setattr( + "app.services.scrapers.cian_newbuilding.extract_all_states", + lambda html: {}, + ) + + from unittest.mock import AsyncMock, MagicMock + + session = MagicMock() + response = MagicMock() + response.status_code = 200 + response.text = "fake" + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) + + assert result is not None + assert result.cian_internal_house_id == 54016 + assert result.name == "ЖК Тест" + assert result.address == "Екатеринбург, ул. Теста" + assert result.building_class == "comfort" + assert result.transport_accessibility_rate == 7 + assert len(result.advantages) == 1 + assert len(result.builders) == 1 + assert len(result.banks) == 1 + assert len(result.houses_by_turn) == 1 + # realtyValuation chart + assert len(result.realty_valuation_chart) == 2 + assert result.realty_valuation_chart[0]["month_date"] == "2025-11-01" + assert result.realty_valuation_chart[0]["price_per_sqm"] == 14000000.0 + # reliability + assert len(result.reliability_checks) == 1 + assert result.reliability_checks[0]["check_status"] == "reliable"