"""Cian.ru newbuilding (ЖК) catalog page scraper. URL: https://zhk---i.cian.ru/ MFE: 'newbuilding-card-desktop-fichering-frontend', key: 'initialState' Sister state containers extracted from same MFE initialState top-level keys: - realtyValuation: 7-month price chart (data.priceDynamics.chart.data.{labels,values}) → houses_price_dynamics - reliability: наш.дом.рф checkStatus + details[] → house_reliability_checks - offers: grouped by roomType (fromDeveloperRooms[].layouts[]) - reviews: house reviews list - housesByTurn: корпуса по очередям (stored as jsonb on houses) Stage 6 of CianScraper v1. """ from __future__ import annotations import json import logging from dataclasses import dataclass, field from typing import Any from curl_cffi.requests import AsyncSession # type: ignore[import-untyped] from app.services.scrapers.cian_state_parser import extract_all_states, extract_state logger = logging.getLogger(__name__) @dataclass class NewbuildingEnrichment: """Result of fetch_newbuilding() — houses row + sister containers.""" # Core house metadata (from state.newbuilding) cian_internal_house_id: int | None = None # ZHK page URL — canonical slug form stored after fetch / redirect resolution cian_zhk_url: str | None = None name: str | None = None address: str | None = None year_built: int | None = None deadline_year: int | None = None floors_count_min: int | None = None floors_count_max: int | None = None building_class: str | None = None # 'comfort' / 'business' / 'premium' # Management management_company: dict[str, Any] | None = None # name, phones[], email, ... # Aggregated metrics transport_accessibility_rate: int | None = None advantages: list[dict[str, Any]] = field(default_factory=list) # Time-series price chart (Stage 6 main deliverable) # Each entry: {'month_date': str, 'room_count': str, 'prices_type': str, 'period': str, # 'price_per_sqm': float | None} realty_valuation_chart: list[dict[str, Any]] = field(default_factory=list) # Reliability (наш.дом.рф) — overall status + checks detail array # {check_status: str, check_name: str, details: list[dict]} reliability_checks: list[dict[str, Any]] = field(default_factory=list) # Builders / banks (related orgs) builders: list[dict[str, Any]] = field(default_factory=list) banks: list[dict[str, Any]] = field(default_factory=list) # Corpuses (houses_by_turn — корпуса по очередям) houses_by_turn: list[dict[str, Any]] = field(default_factory=list) # Reviews reviews: list[dict[str, Any]] = field(default_factory=list) # Mini-SERP offers (grouped by roomType, layouts) nested_offers: list[dict[str, Any]] = field(default_factory=list) # Raw payloads для debugging raw_state: dict[str, Any] | None = None raw_sister_states: dict[str, Any] = field(default_factory=dict) async def fetch_newbuilding( zhk_url: str, *, session: AsyncSession | None = None, ) -> NewbuildingEnrichment | None: """Fetch ЖК catalog page и extract все containers. Args: zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/' session: optional shared curl_cffi session (caller owns lifecycle) Returns: NewbuildingEnrichment, or None если fetch / parse failed. """ close_session = False if session is None: session = AsyncSession( impersonate="chrome120", timeout=30.0, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", }, ) close_session = True try: resp = await session.get(zhk_url, allow_redirects=True) if resp.status_code != 200: logger.warning("Cian newbuilding fetch %s → HTTP %d", zhk_url, resp.status_code) return None html = resp.text mfe = "newbuilding-card-desktop-fichering-frontend" nb_state = extract_state(html, mfe=mfe, key="initialState") if nb_state is None: logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url) return None # Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4) nb = nb_state.get("newbuilding") or {} if not isinstance(nb, dict): nb = {} result = NewbuildingEnrichment( cian_internal_house_id=nb.get("id"), cian_zhk_url=zhk_url, name=nb.get("name") or nb.get("displayName"), address=_extract_address(nb), year_built=nb.get("yearBuilt") or nb.get("buildYear"), deadline_year=_extract_deadline_year(nb), floors_count_min=nb.get("floorsCountMin"), floors_count_max=nb.get("floorsCountMax"), building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"), management_company=nb.get("managementCompany"), transport_accessibility_rate=nb.get("transportAccessibilityRate"), advantages=_extract_advantages(nb), builders=_safe_list(nb.get("builders")), banks=_safe_list(nb.get("banks")), houses_by_turn=_safe_list(nb.get("housesByTurn")), raw_state=nb_state, ) # Sister containers — all live under the same MFE initialState top-level keys # (not separate push() entries) per schema sec 15.3 all_states = extract_all_states(html) mfe_states = all_states.get(mfe, {}) # initialState top-level keys contain sister data: realtyValuation, reliability, offers top = nb_state # realtyValuation 7-month chart (Stage 6 main deliverable) # Located at initialState.realtyValuation (sec 15.5) rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation") if rv_state and isinstance(rv_state, dict): result.realty_valuation_chart = _extract_chart(rv_state) result.raw_sister_states["realtyValuation"] = rv_state # reliability (наш.дом.рф) — located at initialState.reliability (sec 15.6) rel_state = top.get("reliability") or mfe_states.get("reliability") if rel_state and isinstance(rel_state, dict): result.reliability_checks = _extract_reliability_checks(rel_state) result.raw_sister_states["reliability"] = rel_state # reviews — located at initialState or separate state reviews_state = top.get("reviews") or mfe_states.get("reviews") if reviews_state and isinstance(reviews_state, dict): items = reviews_state.get("items") or reviews_state.get("data") or [] result.reviews = items if isinstance(items, list) else [] result.raw_sister_states["reviews"] = reviews_state # offers (grouped by roomType) — located at initialState.offers (sec 15.7) offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp") if offers_state and isinstance(offers_state, dict): result.nested_offers = _extract_nested_offers(offers_state) result.raw_sister_states["offers"] = offers_state logger.info( "Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d", zhk_url, result.cian_internal_house_id, len(result.realty_valuation_chart), len(result.reliability_checks), len(result.nested_offers), ) return result finally: if close_session: await session.close() # ---- private extractors ---- def _safe_list(v: Any) -> list[dict[str, Any]]: """Return v as list[dict] or empty list.""" if not isinstance(v, list): return [] return [item for item in v if isinstance(item, dict)] def _extract_address(nb: dict[str, Any]) -> str | None: """Extract display address string from newbuilding object.""" addr = nb.get("fullAddress") or nb.get("address") if isinstance(addr, str): return addr if isinstance(addr, dict): return addr.get("full") or addr.get("display") or addr.get("name") return None def _extract_deadline_year(nb: dict[str, Any]) -> int | None: """Extract planned completion year from newbuilding.deadline or similar field.""" dl = nb.get("deadline") or nb.get("buildingDeadline") if isinstance(dl, dict): return dl.get("year") if isinstance(dl, int): return dl return None def _extract_advantages(nb: dict[str, Any]) -> list[dict[str, Any]]: """Extract typed advantages list from newbuilding.advantages.""" raw = nb.get("advantages") or nb.get("advantagesTyped") or [] if not isinstance(raw, list): return [] return [a for a in raw if isinstance(a, dict)] def _extract_chart(rv_state: dict[str, Any]) -> list[dict[str, Any]]: """Extract realtyValuation time-series chart points. Actual Cian shape (sec 15.5): rv_state.data.priceDynamics.chart.data = {labels: [date...], values: [int...]} Falls back to alternative shapes if structure varies. Each output point: {month_date: str, room_count: str, prices_type: str, period: str, price_per_sqm: float|None} Note: The initial page state provides aggregate (all room types) chart only. Per-room-type data requires XHR requests (not done here). """ points: list[dict[str, Any]] = [] # Primary shape: data.priceDynamics.chart.data.{labels, values} data_block = rv_state.get("data") if isinstance(data_block, dict): price_dynamics = data_block.get("priceDynamics") or {} if isinstance(price_dynamics, dict): chart = price_dynamics.get("chart") or {} if isinstance(chart, dict): chart_data = chart.get("data") or {} if isinstance(chart_data, dict): labels = chart_data.get("labels") or [] values = chart_data.get("values") or [] if isinstance(labels, list) and isinstance(values, list): for label, value in zip(labels, values, strict=False): if label is None or value is None: continue points.append({ "month_date": str(label)[:10], # YYYY-MM-DD "room_count": "all", # aggregate (initial state) "prices_type": "price", # total price (not priceSqm) "period": "halfYear", # default period shown "price_per_sqm": None, # chart shows total price, not sqm }) # Store raw value as price_per_sqm placeholder for DB # (actual sqm price would require area context) # Use the value as-is; caller / downstream can enrich points[-1]["price_per_sqm"] = float(value) if isinstance( value, (int, float) ) else None if points: return points # Fallback shape A: rv_state.chart (flat list with {month/date, price, roomType}) chart_flat = rv_state.get("chart") or rv_state.get("prices") or rv_state.get("dynamics") if isinstance(chart_flat, list): for entry in chart_flat: if not isinstance(entry, dict): continue month = entry.get("month") or entry.get("date") or entry.get("monthDate") price = entry.get("price") or entry.get("value") or entry.get("pricePerSqm") if month is None or price is None: continue price_val: float | None = None try: price_val = float(price) except (TypeError, ValueError): pass points.append({ "month_date": str(month)[:10], "room_count": str(entry.get("roomType") or entry.get("roomCount") or "all"), "prices_type": str(entry.get("pricesType") or entry.get("priceType") or "price"), "period": str(entry.get("period") or "halfYear"), "price_per_sqm": price_val, }) return points return [] def _extract_reliability_checks(rel_state: dict[str, Any]) -> list[dict[str, Any]]: """Extract reliability check summary from наш.дом.рф state. Actual Cian shape (sec 15.6): rel_state = { checkStatus: {status: 'reliable', title: '...', date: '...'}, details: [{title, iconType, type}, ...], ... } Returns a list with ONE entry representing the overall check (per DB schema): [{check_status, check_name, details}] The 'details' array is stored as jsonb (full check list) in house_reliability_checks. """ check_status_block = rel_state.get("checkStatus") if not isinstance(check_status_block, dict): # Alternative: flat list shape raw = rel_state.get("checks") or rel_state.get("items") or [] if isinstance(raw, list) and raw: return [ { "check_name": entry.get("name") or entry.get("title") or entry.get("checkName"), "check_status": entry.get("status") or entry.get("checkStatus"), "details": entry.get("details") or entry, } for entry in raw if isinstance(entry, dict) ] return [] status = check_status_block.get("status") name = check_status_block.get("title") or check_status_block.get("mobileTitle") details = rel_state.get("details") or [] return [ { "check_name": name, "check_status": status, "details": details if isinstance(details, list) else [details], } ] def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]]: """Extract mini-SERP offers from offers container. Actual Cian shape (sec 15.7): offers_state.data.total.fromDeveloperRooms[].{roomType, layouts[]} Returns flat list of layout offers with room_count injected. """ result: list[dict[str, Any]] = [] # Shape A: miniSerp / items list if "items" in offers_state: items = offers_state["items"] return items if isinstance(items, list) else [] # Shape B: data.total.fromDeveloperRooms[].layouts[] data = offers_state.get("data") if isinstance(data, dict): total = data.get("total") or {} if isinstance(total, dict): rooms = total.get("fromDeveloperRooms") or [] else: rooms = [] else: rooms = [] if not isinstance(rooms, list): return result for room in rooms: if not isinstance(room, dict): continue room_type = room.get("roomType") or room.get("roomTypeDisplay") layouts = room.get("layouts") or [] if not isinstance(layouts, list): continue for layout in layouts: if isinstance(layout, dict): result.append({ "room_count": room_type, **layout, }) return result # ---- save helpers ---- def save_newbuilding_enrichment( db: Any, house_id: int, enrichment: NewbuildingEnrichment, ) -> None: """Persist NewbuildingEnrichment to DB. Steps: 1. UPSERT management_companies (if present) → get mc_id 2. UPDATE houses with Cian metadata (incl. cian_zhk_url if present) 3. INSERT INTO houses_price_dynamics (chart points, ON CONFLICT DO UPDATE) 4. INSERT INTO house_reliability_checks (overall + details) """ from sqlalchemy import text # 1. UPSERT management_company mc_id: int | None = None if enrichment.management_company and isinstance(enrichment.management_company, dict): mc = enrichment.management_company mc_name = mc.get("name") or "Unknown" mc_ext_id = mc.get("id") mc_result = db.execute( text(""" INSERT INTO management_companies ( name, phones, email, opening_hours, chief_name, ext_source, ext_id ) VALUES ( :name, CAST(:phones AS text[]), :email, :oh, :chief, 'cian', CAST(:ext_id AS bigint) ) ON CONFLICT (ext_source, name, ext_id) DO UPDATE SET phones = EXCLUDED.phones, email = COALESCE(EXCLUDED.email, management_companies.email) RETURNING id """), { "name": mc_name, "phones": mc.get("phones") or [], "email": mc.get("email"), "oh": mc.get("openingHours"), "chief": mc.get("chiefName"), "ext_id": mc_ext_id, }, ) row = mc_result.fetchone() mc_id = row[0] if row else None # 2. UPDATE houses db.execute( text(""" UPDATE houses SET cian_internal_house_id = COALESCE( CAST(:cihi AS bigint), cian_internal_house_id ), cian_zhk_url = COALESCE(:zhk_url, cian_zhk_url), year_built = COALESCE(:yb, year_built), management_company_id = COALESCE(CAST(:mcid AS bigint), management_company_id), transport_accessibility_rate = COALESCE( :tar, transport_accessibility_rate ), advantages = CAST(:adv AS jsonb), banks = CAST(:bk AS jsonb), builders = CAST(:bld AS jsonb), houses_by_turn = CAST(:hbt AS jsonb) WHERE id = CAST(:hid AS bigint) """), { "hid": house_id, "cihi": enrichment.cian_internal_house_id, "zhk_url": enrichment.cian_zhk_url, "yb": enrichment.year_built, "mcid": mc_id, "tar": enrichment.transport_accessibility_rate, "adv": json.dumps(enrichment.advantages, ensure_ascii=False), "bk": json.dumps(enrichment.banks, ensure_ascii=False), "bld": json.dumps(enrichment.builders, ensure_ascii=False), "hbt": json.dumps(enrichment.houses_by_turn, ensure_ascii=False), }, ) # 3. INSERT houses_price_dynamics # UNIQUE constraint: houses_price_dynamics_dim_key # (house_id, source, room_count, prices_type, period, month_date) — per migration 029 chart_saved = 0 for point in enrichment.realty_valuation_chart: if point.get("price_per_sqm") is None: continue room_count = point.get("room_count") or "all" prices_type = point.get("prices_type") or "price" period = point.get("period") or "halfYear" db.execute( text(""" INSERT INTO houses_price_dynamics ( house_id, month_date, source, room_count, prices_type, period, price_per_sqm, recorded_at ) VALUES ( CAST(:hid AS bigint), CAST(:md AS date), 'cian_realty_valuation', :rc, :pt, :pd, CAST(:pps AS numeric), NOW() ) ON CONFLICT ON CONSTRAINT houses_price_dynamics_dim_key DO UPDATE SET price_per_sqm = EXCLUDED.price_per_sqm, recorded_at = NOW() """), { "hid": house_id, "md": point["month_date"], "rc": room_count, "pt": prices_type, "pd": period, "pps": point["price_per_sqm"], }, ) chart_saved += 1 # 4. INSERT house_reliability_checks (stores overall check + details array) # Schema (025): (house_id, check_status, check_name, details jsonb, source, recorded_at) # No UNIQUE constraint — caller should manage duplicates if needed for check in enrichment.reliability_checks: if not check.get("check_name") and not check.get("check_status"): continue db.execute( text(""" INSERT INTO house_reliability_checks ( house_id, check_status, check_name, details, source, recorded_at ) VALUES ( CAST(:hid AS bigint), :cs, :cn, CAST(:det AS jsonb), 'cian_nashdom', NOW() ) """), { "hid": house_id, "cs": check.get("check_status"), "cn": check.get("check_name"), "det": json.dumps( check.get("details") or [], ensure_ascii=False ), }, ) db.commit() logger.info( "Cian newbuilding saved house_id=%s (chart=%d points, reliability=%d checks, mc_id=%s)", house_id, chart_saved, len(enrichment.reliability_checks), mc_id, ) async def resolve_cian_zhk_url( cian_internal_house_id: int, *, session: AsyncSession | None = None, ) -> str | None: """Resolve canonical ZHK slug URL from a Cian internal house ID. Cian redirects https://cian.ru/zhk// → canonical slug URL (e.g. https://zhk-park-ekb-i.cian.ru/). We follow the redirect and return the final URL. Used by backfill scripts (scripts/backfill_cian_zhk_url.py) for houses that have cian_internal_house_id set but cian_zhk_url IS NULL. Args: cian_internal_house_id: Cian's numeric ЖК identifier. session: optional shared curl_cffi AsyncSession (caller owns lifecycle). Returns: Canonical ZHK URL string, or None if request failed / redirect not followed. Note: This function makes a real HTTP request — do NOT call it without rate limiting. Caller must enforce per-request sleep matching scraper_settings 'cian' delay. """ close_session = False if session is None: session = AsyncSession( impersonate="chrome120", timeout=15.0, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", }, ) close_session = True fallback_url = f"https://cian.ru/zhk/{cian_internal_house_id}/" try: resp = await session.get(fallback_url, allow_redirects=True) final_url = str(resp.url) if final_url and final_url != fallback_url: logger.debug( "resolve_cian_zhk_url id=%s → %s", cian_internal_house_id, final_url ) return final_url # Redirect not followed or same URL — return fallback as canonical if resp.status_code == 200: return fallback_url logger.warning( "resolve_cian_zhk_url id=%s: HTTP %d, no usable URL", cian_internal_house_id, resp.status_code, ) return None except Exception as exc: logger.warning( "resolve_cian_zhk_url id=%s failed: %s", cian_internal_house_id, exc ) return None finally: if close_session: await session.close()