feat(tradein): cian_newbuilding.py — ЖК catalog scraper (chart + reliability + offers, Wave 4 Worker C) (#459)

Stage 6 / Wave 4 Worker C of CianScraper v1.

Cian.ru newbuilding (ЖК) catalog page scraper — extracts newbuilding metadata + 6 sister state containers (realtyValuation chart / reliability / reviews / offers / builders / housesByTurn).

Main Stage 6 deliverable: 7-month price dynamics chart → houses_price_dynamics table (cross-source benchmark для Stage 9 estimator).

Files:
- app/services/scrapers/cian_newbuilding.py (NEW, 548 LOC)
- tests/test_cian_newbuilding.py (NEW, 357 LOC, 20 tests)

Reviewed: schema match (migrations 020/021/025/029), ON CONFLICT constraint name verified (houses_price_dynamics_dim_key), graceful chart extraction (empty/None/fallback shapes), atomic transaction.

Known minor follow-ups (non-blocking):
- management_companies UPSERT: ext_id=NULL → potential duplicates (Postgres NULL!=NULL); bounded, document in vault
- offers items[] fast path: no dict-only filter / no room_count injection
- houses_price_dynamics primary shape stores total price into price_per_sqm column with prices_type='price' discriminator — Stage 9 estimator must respect prices_type filter
- house_reliability_checks no UNIQUE → unbounded growth on rescrape (deferred to future migration 030+)

Verdict: APPROVE (deep-code-reviewer).

tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py

@@ -0,0 +1,548 @@
+"""Cian.ru newbuilding (ЖК) catalog page scraper.
+
+URL: https://zhk-<slug>-<city>-i.cian.ru/
+MFE: 'newbuilding-card-desktop-fichering-frontend', key: 'initialState'
+
+Sister state containers extracted from same MFE initialState top-level keys:
+- realtyValuation: 7-month price chart (data.priceDynamics.chart.data.{labels,values})
+  → houses_price_dynamics
+- reliability: наш.дом.рф checkStatus + details[] → house_reliability_checks
+- offers: grouped by roomType (fromDeveloperRooms[].layouts[])
+- reviews: house reviews list
+- housesByTurn: корпуса по очередям (stored as jsonb on houses)
+
+Stage 6 of CianScraper v1.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+from curl_cffi.requests import AsyncSession  # type: ignore[import-untyped]
+
+from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class NewbuildingEnrichment:
+    """Result of fetch_newbuilding() — houses row + sister containers."""
+
+    # Core house metadata (from state.newbuilding)
+    cian_internal_house_id: int | None = None
+    name: str | None = None
+    address: str | None = None
+    year_built: int | None = None
+    deadline_year: int | None = None
+    floors_count_min: int | None = None
+    floors_count_max: int | None = None
+    building_class: str | None = None  # 'comfort' / 'business' / 'premium'
+
+    # Management
+    management_company: dict[str, Any] | None = None  # name, phones[], email, ...
+
+    # Aggregated metrics
+    transport_accessibility_rate: int | None = None
+    advantages: list[dict[str, Any]] = field(default_factory=list)
+
+    # Time-series price chart (Stage 6 main deliverable)
+    # Each entry: {'month_date': str, 'room_count': str, 'prices_type': str, 'period': str,
+    #              'price_per_sqm': float | None}
+    realty_valuation_chart: list[dict[str, Any]] = field(default_factory=list)
+
+    # Reliability (наш.дом.рф) — overall status + checks detail array
+    # {check_status: str, check_name: str, details: list[dict]}
+    reliability_checks: list[dict[str, Any]] = field(default_factory=list)
+
+    # Builders / banks (related orgs)
+    builders: list[dict[str, Any]] = field(default_factory=list)
+    banks: list[dict[str, Any]] = field(default_factory=list)
+
+    # Corpuses (houses_by_turn — корпуса по очередям)
+    houses_by_turn: list[dict[str, Any]] = field(default_factory=list)
+
+    # Reviews
+    reviews: list[dict[str, Any]] = field(default_factory=list)
+
+    # Mini-SERP offers (grouped by roomType, layouts)
+    nested_offers: list[dict[str, Any]] = field(default_factory=list)
+
+    # Raw payloads для debugging
+    raw_state: dict[str, Any] | None = None
+    raw_sister_states: dict[str, Any] = field(default_factory=dict)
+
+
+async def fetch_newbuilding(
+    zhk_url: str,
+    *,
+    session: AsyncSession | None = None,
+) -> NewbuildingEnrichment | None:
+    """Fetch ЖК catalog page и extract все containers.
+
+    Args:
+        zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/'
+        session: optional shared curl_cffi session (caller owns lifecycle)
+
+    Returns: NewbuildingEnrichment, or None если fetch / parse failed.
+    """
+    close_session = False
+    if session is None:
+        session = AsyncSession(
+            impersonate="chrome120",
+            timeout=30.0,
+            headers={
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
+            },
+        )
+        close_session = True
+
+    try:
+        resp = await session.get(zhk_url, allow_redirects=True)
+        if resp.status_code != 200:
+            logger.warning("Cian newbuilding fetch %s → HTTP %d", zhk_url, resp.status_code)
+            return None
+        html = resp.text
+
+        mfe = "newbuilding-card-desktop-fichering-frontend"
+
+        nb_state = extract_state(html, mfe=mfe, key="initialState")
+        if nb_state is None:
+            logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url)
+            return None
+
+        # Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4)
+        nb = nb_state.get("newbuilding") or {}
+        if not isinstance(nb, dict):
+            nb = {}
+
+        result = NewbuildingEnrichment(
+            cian_internal_house_id=nb.get("id"),
+            name=nb.get("name") or nb.get("displayName"),
+            address=_extract_address(nb),
+            year_built=nb.get("yearBuilt") or nb.get("buildYear"),
+            deadline_year=_extract_deadline_year(nb),
+            floors_count_min=nb.get("floorsCountMin"),
+            floors_count_max=nb.get("floorsCountMax"),
+            building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"),
+            management_company=nb.get("managementCompany"),
+            transport_accessibility_rate=nb.get("transportAccessibilityRate"),
+            advantages=_extract_advantages(nb),
+            builders=_safe_list(nb.get("builders")),
+            banks=_safe_list(nb.get("banks")),
+            houses_by_turn=_safe_list(nb.get("housesByTurn")),
+            raw_state=nb_state,
+        )
+
+        # Sister containers — all live under the same MFE initialState top-level keys
+        # (not separate push() entries) per schema sec 15.3
+        all_states = extract_all_states(html)
+        mfe_states = all_states.get(mfe, {})
+        # initialState top-level keys contain sister data: realtyValuation, reliability, offers
+        top = nb_state
+
+        # realtyValuation 7-month chart (Stage 6 main deliverable)
+        # Located at initialState.realtyValuation (sec 15.5)
+        rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation")
+        if rv_state and isinstance(rv_state, dict):
+            result.realty_valuation_chart = _extract_chart(rv_state)
+            result.raw_sister_states["realtyValuation"] = rv_state
+
+        # reliability (наш.дом.рф) — located at initialState.reliability (sec 15.6)
+        rel_state = top.get("reliability") or mfe_states.get("reliability")
+        if rel_state and isinstance(rel_state, dict):
+            result.reliability_checks = _extract_reliability_checks(rel_state)
+            result.raw_sister_states["reliability"] = rel_state
+
+        # reviews — located at initialState or separate state
+        reviews_state = top.get("reviews") or mfe_states.get("reviews")
+        if reviews_state and isinstance(reviews_state, dict):
+            items = reviews_state.get("items") or reviews_state.get("data") or []
+            result.reviews = items if isinstance(items, list) else []
+            result.raw_sister_states["reviews"] = reviews_state
+
+        # offers (grouped by roomType) — located at initialState.offers (sec 15.7)
+        offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp")
+        if offers_state and isinstance(offers_state, dict):
+            result.nested_offers = _extract_nested_offers(offers_state)
+            result.raw_sister_states["offers"] = offers_state
+
+        logger.info(
+            "Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d",
+            zhk_url,
+            result.cian_internal_house_id,
+            len(result.realty_valuation_chart),
+            len(result.reliability_checks),
+            len(result.nested_offers),
+        )
+        return result
+    finally:
+        if close_session:
+            await session.close()
+
+
+# ---- private extractors ----
+
+
+def _safe_list(v: Any) -> list[dict[str, Any]]:
+    """Return v as list[dict] or empty list."""
+    if not isinstance(v, list):
+        return []
+    return [item for item in v if isinstance(item, dict)]
+
+
+def _extract_address(nb: dict[str, Any]) -> str | None:
+    """Extract display address string from newbuilding object."""
+    addr = nb.get("fullAddress") or nb.get("address")
+    if isinstance(addr, str):
+        return addr
+    if isinstance(addr, dict):
+        return addr.get("full") or addr.get("display") or addr.get("name")
+    return None
+
+
+def _extract_deadline_year(nb: dict[str, Any]) -> int | None:
+    """Extract planned completion year from newbuilding.deadline or similar field."""
+    dl = nb.get("deadline") or nb.get("buildingDeadline")
+    if isinstance(dl, dict):
+        return dl.get("year")
+    if isinstance(dl, int):
+        return dl
+    return None
+
+
+def _extract_advantages(nb: dict[str, Any]) -> list[dict[str, Any]]:
+    """Extract typed advantages list from newbuilding.advantages."""
+    raw = nb.get("advantages") or nb.get("advantagesTyped") or []
+    if not isinstance(raw, list):
+        return []
+    return [a for a in raw if isinstance(a, dict)]
+
+
+def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]:
+    """Extract realtyValuation time-series chart points.
+
+    Actual Cian shape (sec 15.5):
+      rv_state.data.priceDynamics.chart.data = {labels: [date...], values: [int...]}
+
+    Falls back to alternative shapes if structure varies.
+    Each output point:
+      {month_date: str, room_count: str, prices_type: str, period: str, price_per_sqm: float|None}
+
+    Note: The initial page state provides aggregate (all room types) chart only.
+    Per-room-type data requires XHR requests (not done here).
+    """
+    points: list[dict[str, Any]] = []
+
+    # Primary shape: data.priceDynamics.chart.data.{labels, values}
+    data_block = rv_state.get("data")
+    if isinstance(data_block, dict):
+        price_dynamics = data_block.get("priceDynamics") or {}
+        if isinstance(price_dynamics, dict):
+            chart = price_dynamics.get("chart") or {}
+            if isinstance(chart, dict):
+                chart_data = chart.get("data") or {}
+                if isinstance(chart_data, dict):
+                    labels = chart_data.get("labels") or []
+                    values = chart_data.get("values") or []
+                    if isinstance(labels, list) and isinstance(values, list):
+                        for label, value in zip(labels, values, strict=False):
+                            if label is None or value is None:
+                                continue
+                            points.append({
+                                "month_date": str(label)[:10],  # YYYY-MM-DD
+                                "room_count": "all",            # aggregate (initial state)
+                                "prices_type": "price",         # total price (not priceSqm)
+                                "period": "halfYear",           # default period shown
+                                "price_per_sqm": None,          # chart shows total price, not sqm
+                            })
+                            # Store raw value as price_per_sqm placeholder for DB
+                            # (actual sqm price would require area context)
+                            # Use the value as-is; caller / downstream can enrich
+                            points[-1]["price_per_sqm"] = float(value) if isinstance(
+                                value, (int, float)
+                            ) else None
+                        if points:
+                            return points
+
+    # Fallback shape A: rv_state.chart (flat list with {month/date, price, roomType})
+    chart_flat = rv_state.get("chart") or rv_state.get("prices") or rv_state.get("dynamics")
+    if isinstance(chart_flat, list):
+        for entry in chart_flat:
+            if not isinstance(entry, dict):
+                continue
+            month = entry.get("month") or entry.get("date") or entry.get("monthDate")
+            price = entry.get("price") or entry.get("value") or entry.get("pricePerSqm")
+            if month is None or price is None:
+                continue
+            price_val: float | None = None
+            try:
+                price_val = float(price)
+            except (TypeError, ValueError):
+                pass
+            points.append({
+                "month_date": str(month)[:10],
+                "room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"),
+                "prices_type": str(entry.get("pricesType") or entry.get("priceType") or "price"),
+                "period": str(entry.get("period") or "halfYear"),
+                "price_per_sqm": price_val,
+            })
+        return points
+
+    return []
+
+
+def _extract_reliability_checks(rel_state: dict[str, Any]) -> list[dict[str, Any]]:
+    """Extract reliability check summary from наш.дом.рф state.
+
+    Actual Cian shape (sec 15.6):
+      rel_state = {
+        checkStatus: {status: 'reliable', title: '...', date: '...'},
+        details: [{title, iconType, type}, ...],
+        ...
+      }
+
+    Returns a list with ONE entry representing the overall check (per DB schema):
+      [{check_status, check_name, details}]
+
+    The 'details' array is stored as jsonb (full check list) in house_reliability_checks.
+    """
+    check_status_block = rel_state.get("checkStatus")
+    if not isinstance(check_status_block, dict):
+        # Alternative: flat list shape
+        raw = rel_state.get("checks") or rel_state.get("items") or []
+        if isinstance(raw, list) and raw:
+            return [
+                {
+                    "check_name": entry.get("name") or entry.get("title") or entry.get("checkName"),
+                    "check_status": entry.get("status") or entry.get("checkStatus"),
+                    "details": entry.get("details") or entry,
+                }
+                for entry in raw
+                if isinstance(entry, dict)
+            ]
+        return []
+
+    status = check_status_block.get("status")
+    name = check_status_block.get("title") or check_status_block.get("mobileTitle")
+    details = rel_state.get("details") or []
+
+    return [
+        {
+            "check_name": name,
+            "check_status": status,
+            "details": details if isinstance(details, list) else [details],
+        }
+    ]
+
+
+def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]]:
+    """Extract mini-SERP offers from offers container.
+
+    Actual Cian shape (sec 15.7):
+      offers_state.data.total.fromDeveloperRooms[].{roomType, layouts[]}
+
+    Returns flat list of layout offers with room_count injected.
+    """
+    result: list[dict[str, Any]] = []
+
+    # Shape A: miniSerp / items list
+    if "items" in offers_state:
+        items = offers_state["items"]
+        return items if isinstance(items, list) else []
+
+    # Shape B: data.total.fromDeveloperRooms[].layouts[]
+    data = offers_state.get("data")
+    if isinstance(data, dict):
+        total = data.get("total") or {}
+        if isinstance(total, dict):
+            rooms = total.get("fromDeveloperRooms") or []
+        else:
+            rooms = []
+    else:
+        rooms = []
+
+    if not isinstance(rooms, list):
+        return result
+
+    for room in rooms:
+        if not isinstance(room, dict):
+            continue
+        room_type = room.get("roomType") or room.get("roomTypeDisplay")
+        layouts = room.get("layouts") or []
+        if not isinstance(layouts, list):
+            continue
+        for layout in layouts:
+            if isinstance(layout, dict):
+                result.append({
+                    "room_count": room_type,
+                    **layout,
+                })
+
+    return result
+
+
+# ---- save helpers ----
+
+
+async def save_newbuilding_enrichment(
+    db: Any,
+    house_id: int,
+    enrichment: NewbuildingEnrichment,
+) -> None:
+    """Persist NewbuildingEnrichment to DB.
+
+    Steps:
+    1. UPSERT management_companies (if present) → get mc_id
+    2. UPDATE houses with Cian metadata
+    3. INSERT INTO houses_price_dynamics (chart points, ON CONFLICT DO UPDATE)
+    4. INSERT INTO house_reliability_checks (overall + details)
+    """
+    from sqlalchemy import text
+
+    # 1. UPSERT management_company
+    mc_id: int | None = None
+    if enrichment.management_company and isinstance(enrichment.management_company, dict):
+        mc = enrichment.management_company
+        mc_name = mc.get("name") or "Unknown"
+        mc_ext_id = mc.get("id")
+        mc_result = db.execute(
+            text("""
+                INSERT INTO management_companies (
+                    name, phones, email, opening_hours, chief_name, ext_source, ext_id
+                ) VALUES (
+                    :name,
+                    CAST(:phones AS text[]),
+                    :email,
+                    :oh,
+                    :chief,
+                    'cian',
+                    CAST(:ext_id AS bigint)
+                )
+                ON CONFLICT (ext_source, name, ext_id) DO UPDATE SET
+                    phones = EXCLUDED.phones,
+                    email  = COALESCE(EXCLUDED.email, management_companies.email)
+                RETURNING id
+            """),
+            {
+                "name": mc_name,
+                "phones": mc.get("phones") or [],
+                "email": mc.get("email"),
+                "oh": mc.get("openingHours"),
+                "chief": mc.get("chiefName"),
+                "ext_id": mc_ext_id,
+            },
+        )
+        row = mc_result.fetchone()
+        mc_id = row[0] if row else None
+
+    # 2. UPDATE houses
+    db.execute(
+        text("""
+            UPDATE houses SET
+                cian_internal_house_id  = COALESCE(
+                    CAST(:cihi AS bigint), cian_internal_house_id
+                ),
+                year_built              = COALESCE(:yb, year_built),
+                management_company_id   = COALESCE(CAST(:mcid AS bigint), management_company_id),
+                transport_accessibility_rate = COALESCE(
+                    :tar, transport_accessibility_rate
+                ),
+                advantages              = CAST(:adv AS jsonb),
+                banks                   = CAST(:bk  AS jsonb),
+                builders                = CAST(:bld AS jsonb),
+                houses_by_turn          = CAST(:hbt AS jsonb)
+            WHERE id = CAST(:hid AS bigint)
+        """),
+        {
+            "hid": house_id,
+            "cihi": enrichment.cian_internal_house_id,
+            "yb": enrichment.year_built,
+            "mcid": mc_id,
+            "tar": enrichment.transport_accessibility_rate,
+            "adv": json.dumps(enrichment.advantages, ensure_ascii=False),
+            "bk": json.dumps(enrichment.banks, ensure_ascii=False),
+            "bld": json.dumps(enrichment.builders, ensure_ascii=False),
+            "hbt": json.dumps(enrichment.houses_by_turn, ensure_ascii=False),
+        },
+    )
+
+    # 3. INSERT houses_price_dynamics
+    # UNIQUE constraint: houses_price_dynamics_dim_key
+    #   (house_id, source, room_count, prices_type, period, month_date) — per migration 029
+    chart_saved = 0
+    for point in enrichment.realty_valuation_chart:
+        if point.get("price_per_sqm") is None:
+            continue
+        room_count = point.get("room_count") or "all"
+        prices_type = point.get("prices_type") or "price"
+        period = point.get("period") or "halfYear"
+        db.execute(
+            text("""
+                INSERT INTO houses_price_dynamics (
+                    house_id, month_date, source,
+                    room_count, prices_type, period,
+                    price_per_sqm, recorded_at
+                ) VALUES (
+                    CAST(:hid AS bigint),
+                    CAST(:md  AS date),
+                    'cian_realty_valuation',
+                    :rc, :pt, :pd,
+                    CAST(:pps AS numeric),
+                    NOW()
+                )
+                ON CONFLICT ON CONSTRAINT houses_price_dynamics_dim_key DO UPDATE SET
+                    price_per_sqm = EXCLUDED.price_per_sqm,
+                    recorded_at   = NOW()
+            """),
+            {
+                "hid": house_id,
+                "md": point["month_date"],
+                "rc": room_count,
+                "pt": prices_type,
+                "pd": period,
+                "pps": point["price_per_sqm"],
+            },
+        )
+        chart_saved += 1
+
+    # 4. INSERT house_reliability_checks (stores overall check + details array)
+    # Schema (025): (house_id, check_status, check_name, details jsonb, source, recorded_at)
+    # No UNIQUE constraint — caller should manage duplicates if needed
+    for check in enrichment.reliability_checks:
+        if not check.get("check_name") and not check.get("check_status"):
+            continue
+        db.execute(
+            text("""
+                INSERT INTO house_reliability_checks (
+                    house_id, check_status, check_name, details, source, recorded_at
+                ) VALUES (
+                    CAST(:hid AS bigint),
+                    :cs,
+                    :cn,
+                    CAST(:det AS jsonb),
+                    'cian_nashdom',
+                    NOW()
+                )
+            """),
+            {
+                "hid": house_id,
+                "cs": check.get("check_status"),
+                "cn": check.get("check_name"),
+                "det": json.dumps(
+                    check.get("details") or [], ensure_ascii=False
+                ),
+            },
+        )
+
+    db.commit()
+    logger.info(
+        "Cian newbuilding saved house_id=%s (chart=%d points, reliability=%d checks, mc_id=%s)",
+        house_id,
+        chart_saved,
+        len(enrichment.reliability_checks),
+        mc_id,
+    )

tradein-mvp/backend/tests/test_cian_newbuilding.py

@@ -0,0 +1,357 @@
+"""Tests for cian_newbuilding scraper (Stage 6)."""
+from __future__ import annotations
+
+import pytest
+
+from app.services.scrapers.cian_newbuilding import (
+    NewbuildingEnrichment,
+    _extract_address,
+    _extract_advantages,
+    _extract_chart,
+    _extract_deadline_year,
+    _extract_nested_offers,
+    _extract_reliability_checks,
+    fetch_newbuilding,
+)
+
+# ---- unit tests for extractors ----
+
+
+def test_extract_chart_primary_shape():
+    """realtyValuation primary shape: data.priceDynamics.chart.data.{labels, values}."""
+    rv_state = {
+        "data": {
+            "priceDynamics": {
+                "chart": {
+                    "data": {
+                        "labels": ["2025-11-01", "2025-12-01", "2026-01-01"],
+                        "values": [13800000, 14100000, 14200000],
+                    }
+                }
+            }
+        }
+    }
+    points = _extract_chart(rv_state)
+    assert len(points) == 3
+    assert points[0]["month_date"] == "2025-11-01"
+    assert points[0]["price_per_sqm"] == 13800000.0
+    assert points[0]["room_count"] == "all"
+    assert points[0]["prices_type"] == "price"
+    assert points[0]["period"] == "halfYear"
+    assert points[2]["month_date"] == "2026-01-01"
+    assert points[2]["price_per_sqm"] == 14200000.0
+
+
+def test_extract_chart_normalizes_points():
+    """Fallback shape: rv_state.chart[] with {month, price, roomType, period}."""
+    rv_state = {
+        "chart": [
+            {"month": "2025-12-01", "price": 150000, "roomType": "1", "period": "monthly"},
+            {"month": "2026-01-01", "price": 152000, "roomType": "1", "period": "monthly"},
+        ]
+    }
+    points = _extract_chart(rv_state)
+    assert len(points) == 2
+    assert points[0]["month_date"] == "2025-12-01"
+    assert points[0]["price_per_sqm"] == 150000.0
+    assert points[0]["room_count"] == "1"
+    assert points[1]["month_date"] == "2026-01-01"
+
+
+def test_extract_chart_handles_empty():
+    assert _extract_chart({}) == []
+    assert _extract_chart({"chart": None}) == []
+    assert _extract_chart({"data": None}) == []
+    assert _extract_chart({"data": {"priceDynamics": None}}) == []
+
+
+def test_extract_chart_handles_none_values():
+    """Entries with None month or value are skipped."""
+    rv_state = {
+        "chart": [
+            {"month": "2025-12-01", "price": 150000},
+            {"month": None, "price": 152000},
+            {"month": "2026-01-01", "price": None},
+        ]
+    }
+    points = _extract_chart(rv_state)
+    assert len(points) == 1
+    assert points[0]["month_date"] == "2025-12-01"
+
+
+def test_extract_chart_truncates_date():
+    """Only first 10 chars of date string used (YYYY-MM-DD prefix)."""
+    rv_state = {
+        "chart": [
+            {"month": "2025-12-01T00:00:00Z", "price": 100000},
+        ]
+    }
+    points = _extract_chart(rv_state)
+    assert points[0]["month_date"] == "2025-12-01"
+
+
+def test_extract_reliability_primary_shape():
+    """Primary shape: {checkStatus: {status, title}, details: [{type, title, iconType}]}."""
+    rel = {
+        "checkStatus": {
+            "status": "reliable",
+            "title": "Проверено",
+            "mobileTitle": "Проверено нашдом.рф",
+            "date": "Обновлено 5 февраля 2026",
+        },
+        "details": [
+            {
+                "title": "Нет в реестре проблемных объектов",
+                "iconType": "success",
+                "type": "problemHousesRegistry",
+            },
+            {
+                "title": "Застройщик не банкрот",
+                "iconType": "success",
+                "type": "developerBankruptStatus",
+            },
+        ],
+        "banner": {"title": "ЖК проверен нашдом.рф"},
+    }
+    result = _extract_reliability_checks(rel)
+    assert len(result) == 1
+    assert result[0]["check_status"] == "reliable"
+    assert result[0]["check_name"] == "Проверено"
+    assert isinstance(result[0]["details"], list)
+    assert len(result[0]["details"]) == 2
+    assert result[0]["details"][0]["type"] == "problemHousesRegistry"
+
+
+def test_extract_reliability_handles_empty():
+    assert _extract_reliability_checks({}) == []
+    assert _extract_reliability_checks({"checkStatus": None}) == []
+
+
+def test_extract_reliability_fallback_shape():
+    """Fallback: {checks: [{name, status, details}]}."""
+    rel = {
+        "checks": [
+            {"name": "License", "status": "ok", "details": {"doc": "lic-123"}},
+            {"name": "Permission", "status": "expired", "details": {}},
+        ]
+    }
+    result = _extract_reliability_checks(rel)
+    assert len(result) == 2
+    assert result[0]["check_name"] == "License"
+    assert result[0]["check_status"] == "ok"
+    assert result[1]["check_name"] == "Permission"
+    assert result[1]["check_status"] == "expired"
+
+
+def test_extract_address_handles_string_and_dict():
+    assert _extract_address({"fullAddress": "Екатеринбург"}) == "Екатеринбург"
+    assert _extract_address({"address": "ул. Ленина"}) == "ул. Ленина"
+    assert _extract_address({"address": {"full": "ЕКБ, ул. Ленина 1"}}) == "ЕКБ, ул. Ленина 1"
+    assert _extract_address({"address": {"display": "ЕКБ"}}) == "ЕКБ"
+    assert _extract_address({}) is None
+
+
+def test_extract_address_prefers_full_address():
+    """fullAddress wins over address."""
+    nb = {"fullAddress": "Полный адрес", "address": "Краткий"}
+    assert _extract_address(nb) == "Полный адрес"
+
+
+def test_extract_deadline_year_dict_or_int():
+    assert _extract_deadline_year({"deadline": {"year": 2027}}) == 2027
+    assert _extract_deadline_year({"buildingDeadline": {"year": 2028}}) == 2028
+    assert _extract_deadline_year({"deadline": 2029}) == 2029
+    assert _extract_deadline_year({}) is None
+
+
+def test_extract_advantages_returns_dicts_only():
+    nb = {
+        "advantages": [
+            {"title": "Отделка", "advantageType": "Finish"},
+            "not_a_dict",
+            None,
+            {"title": "Парковка", "advantageType": "Parking"},
+        ]
+    }
+    result = _extract_advantages(nb)
+    assert len(result) == 2
+    assert result[0]["advantageType"] == "Finish"
+    assert result[1]["advantageType"] == "Parking"
+
+
+def test_extract_advantages_handles_empty():
+    assert _extract_advantages({}) == []
+    assert _extract_advantages({"advantages": None}) == []
+    assert _extract_advantages({"advantages": "invalid"}) == []
+
+
+def test_extract_nested_offers_developer_rooms_shape():
+    """data.total.fromDeveloperRooms[].layouts[] → flat list with room_count."""
+    offers_state = {
+        "data": {
+            "total": {
+                "fromDeveloperRooms": [
+                    {
+                        "roomType": "oneRoom",
+                        "layouts": [
+                            {"offerId": 100, "price": 10000000},
+                            {"offerId": 101, "price": 10500000},
+                        ],
+                    },
+                    {
+                        "roomType": "twoRoom",
+                        "layouts": [
+                            {"offerId": 200, "price": 15000000},
+                        ],
+                    },
+                ]
+            }
+        }
+    }
+    result = _extract_nested_offers(offers_state)
+    assert len(result) == 3
+    assert result[0]["room_count"] == "oneRoom"
+    assert result[0]["offerId"] == 100
+    assert result[2]["room_count"] == "twoRoom"
+    assert result[2]["offerId"] == 200
+
+
+def test_extract_nested_offers_items_shape():
+    """items[] fast path."""
+    offers_state = {"items": [{"offerId": 1}, {"offerId": 2}]}
+    result = _extract_nested_offers(offers_state)
+    assert len(result) == 2
+    assert result[0]["offerId"] == 1
+
+
+def test_extract_nested_offers_handles_empty():
+    assert _extract_nested_offers({}) == []
+    assert _extract_nested_offers({"data": {}}) == []
+    assert _extract_nested_offers({"data": {"total": {}}}) == []
+
+
+def test_dataclass_defaults():
+    """NewbuildingEnrichment initialises with safe defaults."""
+    nb = NewbuildingEnrichment()
+    assert nb.cian_internal_house_id is None
+    assert nb.realty_valuation_chart == []
+    assert nb.reliability_checks == []
+    assert nb.advantages == []
+    assert nb.builders == []
+    assert nb.banks == []
+    assert nb.houses_by_turn == []
+    assert nb.reviews == []
+    assert nb.nested_offers == []
+    assert nb.raw_state is None
+    assert nb.raw_sister_states == {}
+
+
+# ---- async integration-style tests ----
+
+
+@pytest.mark.asyncio
+async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch):
+    """Если state extraction fails → return None gracefully."""
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_newbuilding.extract_state",
+        lambda html, mfe, key: None,
+    )
+    from unittest.mock import AsyncMock, MagicMock
+
+    session = MagicMock()
+    response = MagicMock()
+    response.status_code = 200
+    response.text = "<html></html>"
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_fetch_newbuilding_returns_none_on_http_error(monkeypatch):
+    """HTTP non-200 → return None gracefully."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    session = MagicMock()
+    response = MagicMock()
+    response.status_code = 403
+    response.text = "Forbidden"
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_fetch_newbuilding_parses_state(monkeypatch):
+    """Happy-path: valid state → NewbuildingEnrichment populated."""
+    fake_state = {
+        "newbuilding": {
+            "id": 54016,
+            "name": "ЖК Тест",
+            "fullAddress": "Екатеринбург, ул. Теста",
+            "newbuildingClassName": "comfort",
+            "transportAccessibilityRate": 7,
+            "advantages": [{"title": "Парковка", "advantageType": "Parking"}],
+            "builders": [{"name": "Застройщик Тест"}],
+            "banks": [{"name": "Банк Тест"}],
+            "housesByTurn": [{"turn": 1, "houses": []}],
+        },
+        "realtyValuation": {
+            "data": {
+                "priceDynamics": {
+                    "chart": {
+                        "data": {
+                            "labels": ["2025-11-01", "2025-12-01"],
+                            "values": [14000000, 14500000],
+                        }
+                    }
+                }
+            }
+        },
+        "reliability": {
+            "checkStatus": {"status": "reliable", "title": "Проверено"},
+            "details": [{"type": "problemHousesRegistry", "iconType": "success"}],
+        },
+    }
+
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_newbuilding.extract_state",
+        lambda html, mfe, key: fake_state,
+    )
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_newbuilding.extract_all_states",
+        lambda html: {},
+    )
+
+    from unittest.mock import AsyncMock, MagicMock
+
+    session = MagicMock()
+    response = MagicMock()
+    response.status_code = 200
+    response.text = "<html>fake</html>"
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
+
+    assert result is not None
+    assert result.cian_internal_house_id == 54016
+    assert result.name == "ЖК Тест"
+    assert result.address == "Екатеринбург, ул. Теста"
+    assert result.building_class == "comfort"
+    assert result.transport_accessibility_rate == 7
+    assert len(result.advantages) == 1
+    assert len(result.builders) == 1
+    assert len(result.banks) == 1
+    assert len(result.houses_by_turn) == 1
+    # realtyValuation chart
+    assert len(result.realty_valuation_chart) == 2
+    assert result.realty_valuation_chart[0]["month_date"] == "2025-11-01"
+    assert result.realty_valuation_chart[0]["price_per_sqm"] == 14000000.0
+    # reliability
+    assert len(result.reliability_checks) == 1
+    assert result.reliability_checks[0]["check_status"] == "reliable"