From f6ece38619b35fe9ed2c4645ae6f905081d2498c Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 16:12:15 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20cian=5Fdetail.py=20=E2=80=94?= =?UTF-8?q?=20detail=20page=20scraper=20(88=20fields=20+=20BTI=20+=20price?= =?UTF-8?q?Changes,=20Wave=204=20Worker=20B)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fetch_detail(): curl_cffi AsyncSession → HTML → extract_state(mfe='frontend-offer-card', key='defaultState') - DetailEnrichment dataclass: cian_id, windows_view_type, wcs counts, ceiling_height, repair_type, views_total/today, bti_data, price_changes, agent_profile, newbuilding_data, raw_offer/sister_states - extract_all_states() sister container stash: bti (houseData), newObject, plus raw dump for future extraction - save_detail_enrichment(): UPDATE listings (all 019 cols present), INSERT offer_price_history (ON CONFLICT DO NOTHING), UPSERT agents + link agent_id_fk, psycopg v3 CAST(:x AS type) everywhere - Tests: 19 tests (unit helpers + async mocked fetch_detail: no-state, 403, core fields, BTI sister state, newbuilding link) - Ruff clean, asyncio_mode=auto --- .../app/services/scrapers/cian_detail.py | 324 ++++++++++++++++++ tradein-mvp/backend/tests/test_cian_detail.py | 324 ++++++++++++++++++ 2 files changed, 648 insertions(+) create mode 100644 tradein-mvp/backend/app/services/scrapers/cian_detail.py create mode 100644 tradein-mvp/backend/tests/test_cian_detail.py diff --git a/tradein-mvp/backend/app/services/scrapers/cian_detail.py b/tradein-mvp/backend/app/services/scrapers/cian_detail.py new file mode 100644 index 00000000..59d281cf --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/cian_detail.py @@ -0,0 +1,324 @@ +"""Cian.ru detail page scraper — single offer enrichment. + +Different from SERP (cian.py): +- URL: https://ekb.cian.ru/sale/flat// or https://www.cian.ru/sale/flat// +- MFE: 'frontend-offer-card' +- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState) +- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject + +Parses ~88 offer fields + sister data → DetailEnrichment dataclass. +Stage 5 of CianScraper v1. +""" +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Any + +from curl_cffi.requests import AsyncSession +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.scrapers.cian_state_parser import extract_all_states, extract_state + +logger = logging.getLogger(__name__) + + +@dataclass +class DetailEnrichment: + """Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows.""" + + # Core listing enrichment (extend existing listings row) + cian_id: int | None = None + windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet' + separate_wcs_count: int | None = None + combined_wcs_count: int | None = None + ceiling_height: float | None = None # meters + repair_type: str | None = None # 'cosmetic' / 'design' / 'no' + views_total: int | None = None # Cian stats.totalViewsFormattedString → int + views_today: int | None = None + + # BTI (вторичка only — primary buildings don't have БТИ) + bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot + + # Price history (offer.priceChanges) + price_changes: list[dict[str, Any]] = field(default_factory=list) + + # Agent profile (offer.agent.* expanded) + agent_profile: dict[str, Any] | None = None + + # Newbuilding link (if offer is from a newbuilding) + newbuilding_data: dict[str, Any] | None = None + + # Raw payload для debugging / future-extraction + raw_offer: dict[str, Any] | None = None + raw_sister_states: dict[str, Any] = field(default_factory=dict) + + +async def fetch_detail( + offer_url: str, + *, + session: AsyncSession | None = None, +) -> DetailEnrichment | None: + """Fetch detail page and extract all containers. + + Args: + offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/' + session: optional shared curl_cffi session (для batched scrapes) + + Returns: DetailEnrichment, or None если fetch / parse failed. + """ + close_session = False + if session is None: + session = AsyncSession( + impersonate="chrome120", + timeout=25.0, + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", + }, + ) + close_session = True + + try: + resp = await session.get(offer_url, allow_redirects=True) + if resp.status_code != 200: + logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code) + return None + html = resp.text + + # Primary state container — defaultState in frontend-offer-card MFE + # NOTE: detail pages use 'defaultState', SERP uses 'initialState' + offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState") + if offer_state is None: + logger.warning("Cian detail %s: defaultState extraction failed", offer_url) + return None + + offer_data = offer_state.get("offerData", {}) + offer = offer_data.get("offer", {}) + + result = DetailEnrichment( + cian_id=offer.get("cianId") or offer.get("id"), + windows_view_type=_extract_windows_view(offer), + separate_wcs_count=offer.get("separateWcsCount"), + combined_wcs_count=offer.get("combinedWcsCount"), + ceiling_height=_parse_float(offer.get("ceilingHeight")), + repair_type=_extract_repair_type(offer), + raw_offer=offer, + ) + + # Stats (views) — from stats key in offerData or top-level state + stats = offer_data.get("stats") or offer_state.get("stats") or {} + result.views_total = _parse_views(stats.get("totalViewsFormattedString")) + result.views_today = _parse_views(stats.get("todayViewsFormattedString")) + + # Price changes — from offer.priceChanges or state-level priceChanges + result.price_changes = _extract_price_changes(offer, offer_state) + + # Agent profile expansion + result.agent_profile = _extract_agent_profile(offer) + + # Sister state containers: bti, newObject, plus raw stash of everything else + all_states = extract_all_states(html) + offer_card_states = all_states.get("frontend-offer-card", {}) + + bti_state = offer_card_states.get("bti") + if bti_state: + result.bti_data = bti_state.get("houseData") + result.raw_sister_states["bti"] = bti_state + + # Newbuilding link (if applicable) + newbuilding = offer.get("newbuilding") + if newbuilding and newbuilding.get("id"): + result.newbuilding_data = newbuilding + + # Stash all non-defaultState sister containers for debugging / future-extraction + for k, v in offer_card_states.items(): + if k != "defaultState": + result.raw_sister_states[k] = v + + return result + + finally: + if close_session: + await session.close() + + +# ── Field extractors ────────────────────────────────────────────────────────── + + +def _extract_windows_view(offer: dict[str, Any]) -> str | None: + return offer.get("windowsViewType") + + +def _extract_repair_type(offer: dict[str, Any]) -> str | None: + # Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ... + return offer.get("repairType") + + +def _parse_float(value: Any) -> float | None: + if value is None: + return None + try: + return float(value) + except (ValueError, TypeError): + return None + + +def _parse_views(formatted_str: str | None) -> int | None: + """Cian's stats.totalViewsFormattedString — '1 234' → 1234.""" + if not formatted_str: + return None + try: + return int(str(formatted_str).replace(" ", "").replace(",", "")) + except (ValueError, TypeError): + return None + + +def _extract_price_changes(offer: dict[str, Any], state: dict[str, Any]) -> list[dict[str, Any]]: + """offer.priceChanges or state.priceChanges — normalize to common form.""" + raw = offer.get("priceChanges") or state.get("priceChanges") or [] + if not isinstance(raw, list): + return [] + result = [] + for entry in raw: + if not isinstance(entry, dict): + continue + result.append( + { + "change_time": entry.get("changeTime") or entry.get("date"), + "price_rub": entry.get("price") or entry.get("priceRub"), + "diff_percent": entry.get("diffPercent") or entry.get("diff_percent"), + } + ) + return result + + +def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None: + """offer.agent or offer.agency or offer.user — extract canonical profile shape.""" + agent = offer.get("agent") or {} + user = offer.get("user") or {} + if not agent and not user: + return None + return { + "ext_agent_id": agent.get("id") or user.get("cianUserId"), + "company_name": agent.get("companyName") or user.get("companyName"), + "is_pro": agent.get("isPro") or user.get("isPro"), + "skills": agent.get("skills") or [], + "account_type": agent.get("accountType") or user.get("accountType"), + } + + +# ── Save helpers ────────────────────────────────────────────────────────────── + + +def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None: + """Persist DetailEnrichment to DB: + + - UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id + - INSERT INTO offer_price_history rows (if price_changes non-empty) + - INSERT INTO agents (if agent_profile) and link via agent_id_fk + - bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id + + Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics). + """ + # Extend listings row with detail-enriched fields + db.execute( + text(""" + UPDATE listings SET + windows_view_type = COALESCE(:wvt, windows_view_type), + separate_wcs_count = COALESCE(:swc, separate_wcs_count), + combined_wcs_count = COALESCE(:cwc, combined_wcs_count), + ceiling_height = COALESCE(:ch, ceiling_height), + repair_type = COALESCE(:rt, repair_type), + views_total = COALESCE(:vt, views_total), + views_today = COALESCE(:vd, views_today) + WHERE id = CAST(:lid AS bigint) + """), + { + "lid": listing_id, + "wvt": enrichment.windows_view_type, + "swc": enrichment.separate_wcs_count, + "cwc": enrichment.combined_wcs_count, + "ch": enrichment.ceiling_height, + "rt": enrichment.repair_type, + "vt": enrichment.views_total, + "vd": enrichment.views_today, + }, + ) + + # Price changes — INSERT new rows, de-dup by listing_id + change_time + for change in enrichment.price_changes: + if not change.get("change_time") or not change.get("price_rub"): + continue + db.execute( + text(""" + INSERT INTO offer_price_history + (listing_id, change_time, price_rub, diff_percent, source) + VALUES ( + CAST(:lid AS bigint), + CAST(:ct AS timestamptz), + CAST(:price AS numeric), + CAST(:diff AS numeric), + 'cian' + ) + ON CONFLICT DO NOTHING + """), + { + "lid": listing_id, + "ct": change["change_time"], + "price": change["price_rub"], + "diff": change.get("diff_percent"), + }, + ) + + # Agent profile UPSERT — store agent row and link to listing + if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"): + ap = enrichment.agent_profile + row = db.execute( + text(""" + INSERT INTO agents + (ext_source, ext_agent_id, company_name, is_pro, skills, account_type) + VALUES ( + 'cian', + :eai, + :cn, + :ip, + CAST(:sk AS jsonb), + :at + ) + ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET + company_name = COALESCE(EXCLUDED.company_name, agents.company_name), + is_pro = COALESCE(EXCLUDED.is_pro, agents.is_pro), + skills = COALESCE(EXCLUDED.skills, agents.skills), + account_type = COALESCE(EXCLUDED.account_type, agents.account_type) + RETURNING id + """), + { + "eai": str(ap["ext_agent_id"]), + "cn": ap.get("company_name"), + "ip": ap.get("is_pro"), + "sk": json.dumps(ap.get("skills") or []), + "at": ap.get("account_type"), + }, + ) + agent_db_id = row.scalar_one_or_none() + if agent_db_id is not None: + db.execute( + text(""" + UPDATE listings + SET agent_id_fk = CAST(:aid AS bigint) + WHERE id = CAST(:lid AS bigint) + AND agent_id_fk IS NULL + """), + {"aid": agent_db_id, "lid": listing_id}, + ) + + db.commit() + logger.info( + "Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s)", + listing_id, + len(enrichment.price_changes), + bool(enrichment.agent_profile), + ) diff --git a/tradein-mvp/backend/tests/test_cian_detail.py b/tradein-mvp/backend/tests/test_cian_detail.py new file mode 100644 index 00000000..22fc8dc2 --- /dev/null +++ b/tradein-mvp/backend/tests/test_cian_detail.py @@ -0,0 +1,324 @@ +"""Tests for cian_detail.py — Stage 5 of CianScraper v1.""" +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from app.services.scrapers.cian_detail import ( + DetailEnrichment, + _extract_agent_profile, + _extract_price_changes, + _parse_views, + fetch_detail, +) + + +# ── _parse_views ────────────────────────────────────────────────────────────── + + +def test_parse_views_space_formatted(): + assert _parse_views("1 234") == 1234 + + +def test_parse_views_plain(): + assert _parse_views("567") == 567 + + +def test_parse_views_none(): + assert _parse_views(None) is None + + +def test_parse_views_invalid(): + assert _parse_views("invalid") is None + + +def test_parse_views_comma_formatted(): + assert _parse_views("1,234") == 1234 + + +# ── _extract_price_changes ──────────────────────────────────────────────────── + + +def test_extract_price_changes_normalizes(): + offer = { + "priceChanges": [ + {"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5}, + {"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0}, + ] + } + result = _extract_price_changes(offer, {}) + assert len(result) == 2 + assert result[0]["change_time"] == "2026-04-01T00:00:00Z" + assert result[0]["price_rub"] == 5000000 + assert result[0]["diff_percent"] == -2.5 + + +def test_extract_price_changes_handles_empty_offer(): + assert _extract_price_changes({}, {}) == [] + + +def test_extract_price_changes_handles_none_value(): + assert _extract_price_changes({"priceChanges": None}, {}) == [] + + +def test_extract_price_changes_falls_back_to_state(): + state = { + "priceChanges": [ + {"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0}, + ] + } + result = _extract_price_changes({}, state) + assert len(result) == 1 + assert result[0]["price_rub"] == 6000000 + + +def test_extract_price_changes_skips_non_dict_entries(): + offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]} + result = _extract_price_changes(offer, {}) + assert len(result) == 1 + + +# ── _extract_agent_profile ──────────────────────────────────────────────────── + + +def test_extract_agent_profile_from_agent_key(): + offer = { + "agent": { + "id": 12345, + "companyName": "Новосёл", + "isPro": True, + "skills": [{"id": 1, "name": "Продажа"}], + "accountType": "specialist", + } + } + profile = _extract_agent_profile(offer) + assert profile is not None + assert profile["ext_agent_id"] == 12345 + assert profile["company_name"] == "Новосёл" + assert profile["is_pro"] is True + assert profile["account_type"] == "specialist" + + +def test_extract_agent_profile_from_user_key(): + offer = { + "user": { + "cianUserId": 99999, + "companyName": "Агентство Икс", + "isPro": False, + "accountType": "agency", + } + } + profile = _extract_agent_profile(offer) + assert profile is not None + assert profile["ext_agent_id"] == 99999 + assert profile["company_name"] == "Агентство Икс" + + +def test_extract_agent_profile_returns_none_when_no_agent(): + assert _extract_agent_profile({}) is None + assert _extract_agent_profile({"agent": {}, "user": {}}) is None + + +# ── DetailEnrichment defaults ───────────────────────────────────────────────── + + +def test_dataclass_defaults(): + enrich = DetailEnrichment() + assert enrich.cian_id is None + assert enrich.price_changes == [] + assert enrich.raw_sister_states == {} + assert enrich.bti_data is None + assert enrich.agent_profile is None + assert enrich.newbuilding_data is None + + +# ── fetch_detail integration (mocked) ──────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_fetch_detail_returns_none_on_no_state(monkeypatch): + """If state extraction returns None → fetch_detail returns None gracefully.""" + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: None, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session) + assert result is None + + +@pytest.mark.asyncio +async def test_fetch_detail_returns_none_on_non_200(monkeypatch): + """Non-200 HTTP response → returns None, no exception raised.""" + response = MagicMock() + response.status_code = 403 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session) + assert result is None + + +@pytest.mark.asyncio +async def test_fetch_detail_extracts_core_fields(monkeypatch): + """Verify core fields are extracted correctly from mocked state.""" + fake_state = { + "offerData": { + "offer": { + "cianId": 327830237, + "windowsViewType": "yardAndStreet", + "separateWcsCount": 1, + "combinedWcsCount": 0, + "ceilingHeight": 2.65, + "repairType": "cosmetic", + "priceChanges": [ + { + "changeTime": "2026-04-01T00:00:00Z", + "price": 5500000, + "diffPercent": -1.8, + } + ], + "agent": { + "id": 7654321, + "companyName": "Первичка Плюс", + "isPro": True, + "skills": [], + "accountType": "specialist", + }, + }, + "stats": { + "totalViewsFormattedString": "1 042", + "todayViewsFormattedString": "7", + }, + } + } + + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: fake_state if key == "defaultState" else None, + ) + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_all_states", + lambda html: {}, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session) + assert result is not None + assert result.cian_id == 327830237 + assert result.windows_view_type == "yardAndStreet" + assert result.separate_wcs_count == 1 + assert result.combined_wcs_count == 0 + assert result.ceiling_height == 2.65 + assert result.repair_type == "cosmetic" + assert result.views_total == 1042 + assert result.views_today == 7 + assert len(result.price_changes) == 1 + assert result.price_changes[0]["price_rub"] == 5500000 + assert result.agent_profile is not None + assert result.agent_profile["ext_agent_id"] == 7654321 + assert result.agent_profile["company_name"] == "Первичка Плюс" + + +@pytest.mark.asyncio +async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch): + """BTI sister state is captured into bti_data.""" + fake_state = { + "offerData": { + "offer": { + "cianId": 111111, + } + } + } + fake_all_states = { + "frontend-offer-card": { + "bti": { + "houseData": { + "seriesName": "П-44Т", + "floorsCount": 17, + "flatCount": 120, + } + } + } + } + + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: fake_state if key == "defaultState" else None, + ) + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_all_states", + lambda html: fake_all_states, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session) + assert result is not None + assert result.bti_data is not None + assert result.bti_data["seriesName"] == "П-44Т" + assert "bti" in result.raw_sister_states + + +@pytest.mark.asyncio +async def test_fetch_detail_extracts_newbuilding_link(monkeypatch): + """Newbuilding reference is captured from offer.newbuilding.""" + fake_state = { + "offerData": { + "offer": { + "cianId": 222222, + "newbuilding": { + "id": 54016, + "name": "Екатерининский парк", + "url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/", + }, + } + } + } + + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: fake_state if key == "defaultState" else None, + ) + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_all_states", + lambda html: {}, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session) + assert result is not None + assert result.newbuilding_data is not None + assert result.newbuilding_data["id"] == 54016