433 lines
17 KiB
Python
433 lines
17 KiB
Python
"""Cian.ru detail page scraper — single offer enrichment.
|
|
|
|
Different from SERP (cian.py):
|
|
- URL: https://ekb.cian.ru/sale/flat/<offer_id>/ or https://www.cian.ru/sale/flat/<offer_id>/
|
|
- MFE: 'frontend-offer-card'
|
|
- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState)
|
|
- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject
|
|
|
|
Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
|
|
Stage 5 of CianScraper v1.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from curl_cffi.requests import AsyncSession
|
|
from sqlalchemy import text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.config import settings
|
|
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
|
|
from app.services.scrapers.repair_state_normalizer import (
|
|
infer_repair_state_from_text,
|
|
normalize_repair_state,
|
|
)
|
|
from app.services.scrapers.snapshot_writer import upsert_listing_snapshot
|
|
|
|
if TYPE_CHECKING:
|
|
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class DetailEnrichment:
|
|
"""Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows."""
|
|
|
|
# Core listing enrichment (extend existing listings row)
|
|
cian_id: int | None = None
|
|
windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet'
|
|
separate_wcs_count: int | None = None
|
|
combined_wcs_count: int | None = None
|
|
ceiling_height: float | None = None # meters
|
|
repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value
|
|
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
|
kitchen_area_m2: float | None = None # offer.kitchenArea (м²)
|
|
description: str | None = None # offer.description (текст объявления)
|
|
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
|
views_today: int | None = None
|
|
|
|
# BTI (вторичка only — primary buildings don't have БТИ)
|
|
bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot
|
|
|
|
# Price history (offer.priceChanges)
|
|
price_changes: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
# Agent profile (offer.agent.* expanded)
|
|
agent_profile: dict[str, Any] | None = None
|
|
|
|
# Newbuilding link (if offer is from a newbuilding)
|
|
newbuilding_data: dict[str, Any] | None = None
|
|
|
|
# Raw payload для debugging / future-extraction
|
|
raw_offer: dict[str, Any] | None = None
|
|
raw_sister_states: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
async def fetch_detail(
|
|
offer_url: str,
|
|
*,
|
|
session: AsyncSession | None = None,
|
|
browser_fetcher: BrowserFetcher | None = None,
|
|
) -> DetailEnrichment | None:
|
|
"""Fetch detail page and extract all containers.
|
|
|
|
Args:
|
|
offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/'
|
|
session: optional shared curl_cffi session (для batched scrapes).
|
|
Ignored when browser_fetcher is provided.
|
|
browser_fetcher: optional BrowserFetcher instance (camoufox HTTP service).
|
|
When provided, fetches JS-rendered HTML via the browser service instead of
|
|
curl_cffi — enables priceChanges extraction which requires JS rendering.
|
|
Caller is responsible for the context-manager lifecycle of the fetcher.
|
|
|
|
Returns: DetailEnrichment, or None если fetch / parse failed.
|
|
"""
|
|
if browser_fetcher is not None:
|
|
# Browser path: get fully JS-rendered HTML; same parse path follows.
|
|
try:
|
|
html = await browser_fetcher.fetch(offer_url)
|
|
except Exception as exc:
|
|
logger.warning("Cian detail browser fetch failed %s: %s", offer_url, exc)
|
|
return None
|
|
else:
|
|
# curl_cffi path (legacy, back-compat).
|
|
close_session = False
|
|
if session is None:
|
|
# proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian.
|
|
# Пусто (env не задан) → прямое подключение (dev/no-op).
|
|
_proxy_url = settings.cian_proxy_url
|
|
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
|
session = AsyncSession(
|
|
impersonate="chrome120",
|
|
timeout=25.0,
|
|
proxies=_proxies,
|
|
headers={
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
|
},
|
|
)
|
|
close_session = True
|
|
|
|
try:
|
|
resp = await session.get(offer_url, allow_redirects=True)
|
|
if resp.status_code != 200:
|
|
logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code)
|
|
return None
|
|
html = resp.text
|
|
finally:
|
|
if close_session:
|
|
await session.close()
|
|
|
|
# ── Shared parse path (identical regardless of HTML source) ──────────────
|
|
|
|
# Primary state container — defaultState in frontend-offer-card MFE
|
|
# NOTE: detail pages use 'defaultState', SERP uses 'initialState'
|
|
offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
|
|
if offer_state is None:
|
|
logger.warning("Cian detail %s: defaultState extraction failed", offer_url)
|
|
return None
|
|
|
|
offer_data = offer_state.get("offerData", {})
|
|
offer = offer_data.get("offer", {})
|
|
|
|
result = DetailEnrichment(
|
|
cian_id=offer.get("cianId") or offer.get("id"),
|
|
windows_view_type=_extract_windows_view(offer),
|
|
separate_wcs_count=offer.get("separateWcsCount"),
|
|
combined_wcs_count=offer.get("combinedWcsCount"),
|
|
ceiling_height=_parse_float(offer.get("ceilingHeight")),
|
|
repair_type=_extract_repair_type(offer),
|
|
repair_state=_extract_repair_state(offer),
|
|
kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
|
|
description=offer.get("description") or None,
|
|
raw_offer=offer,
|
|
)
|
|
|
|
# Stats (views) — from stats key in offerData or top-level state
|
|
stats = offer_data.get("stats") or offer_state.get("stats") or {}
|
|
result.views_total = _parse_views(stats.get("totalViewsFormattedString"))
|
|
result.views_today = _parse_views(stats.get("todayViewsFormattedString"))
|
|
|
|
# Price changes — browser HTML: offerData.priceChanges (priceData.price format)
|
|
# curl_cffi HTML: offer.priceChanges or state.priceChanges (flat price field)
|
|
result.price_changes = _extract_price_changes(offer, offer_data, offer_state)
|
|
|
|
# Agent profile expansion
|
|
result.agent_profile = _extract_agent_profile(offer)
|
|
|
|
# Sister state containers: bti, newObject, plus raw stash of everything else
|
|
all_states = extract_all_states(html)
|
|
offer_card_states = all_states.get("frontend-offer-card", {})
|
|
|
|
bti_state = offer_card_states.get("bti")
|
|
if bti_state:
|
|
result.bti_data = bti_state.get("houseData")
|
|
result.raw_sister_states["bti"] = bti_state
|
|
|
|
# Newbuilding link (if applicable)
|
|
newbuilding = offer.get("newbuilding")
|
|
if newbuilding and newbuilding.get("id"):
|
|
result.newbuilding_data = newbuilding
|
|
|
|
# Stash all non-defaultState sister containers for debugging / future-extraction
|
|
for k, v in offer_card_states.items():
|
|
if k != "defaultState":
|
|
result.raw_sister_states[k] = v
|
|
|
|
return result
|
|
|
|
|
|
# ── Field extractors ──────────────────────────────────────────────────────────
|
|
|
|
|
|
def _extract_windows_view(offer: dict[str, Any]) -> str | None:
|
|
return offer.get("windowsViewType")
|
|
|
|
|
|
def _extract_repair_type(offer: dict[str, Any]) -> str | None:
|
|
# Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ...
|
|
return offer.get("repairType")
|
|
|
|
|
|
def _extract_repair_state(offer: dict[str, Any]) -> str | None:
|
|
"""Нормализованный enum из offer.repairType → needs_repair/standard/good/excellent.
|
|
|
|
Если структурное поле repairType пустое — fallback на инференс из описания (#622).
|
|
Структурное значение всегда в приоритете.
|
|
"""
|
|
state = normalize_repair_state(offer.get("repairType"))
|
|
if state is None:
|
|
state = infer_repair_state_from_text(offer.get("description"))
|
|
return state
|
|
|
|
|
|
def _parse_float(value: Any) -> float | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return float(value)
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def _parse_views(formatted_str: str | None) -> int | None:
|
|
"""Cian's stats.totalViewsFormattedString — '1 234' → 1234."""
|
|
if not formatted_str:
|
|
return None
|
|
try:
|
|
return int(str(formatted_str).replace(" ", "").replace(",", ""))
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def _extract_price_changes(
|
|
offer: dict[str, Any],
|
|
offer_data: dict[str, Any],
|
|
state: dict[str, Any],
|
|
) -> list[dict[str, Any]]:
|
|
"""Normalize priceChanges from any of the known Cian locations/formats.
|
|
|
|
Lookup priority:
|
|
1. offer.priceChanges — curl_cffi flat format: {changeTime, price, diffPercent}
|
|
2. offerData.priceChanges — browser-rendered format: {changeTime, priceData:{price}}
|
|
3. state.priceChanges — top-level fallback (rare)
|
|
|
|
Normalizes to: {change_time, price_rub, diff_percent}.
|
|
"""
|
|
raw = (
|
|
offer.get("priceChanges")
|
|
or offer_data.get("priceChanges")
|
|
or state.get("priceChanges")
|
|
or []
|
|
)
|
|
if not isinstance(raw, list):
|
|
return []
|
|
result = []
|
|
for entry in raw:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
# price_rub: flat field (curl_cffi) OR nested priceData.price (browser)
|
|
price_data = entry.get("priceData") or {}
|
|
price_rub = entry.get("price") or entry.get("priceRub") or price_data.get("price")
|
|
result.append(
|
|
{
|
|
"change_time": entry.get("changeTime") or entry.get("date"),
|
|
"price_rub": price_rub,
|
|
"diff_percent": entry.get("diffPercent") or entry.get("diff_percent"),
|
|
}
|
|
)
|
|
return result
|
|
|
|
|
|
def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
|
|
"""offer.agent or offer.agency or offer.user — extract canonical profile shape."""
|
|
agent = offer.get("agent") or {}
|
|
user = offer.get("user") or {}
|
|
if not agent and not user:
|
|
return None
|
|
return {
|
|
"ext_agent_id": agent.get("id") or user.get("cianUserId"),
|
|
"company_name": agent.get("companyName") or user.get("companyName"),
|
|
"is_pro": agent.get("isPro") or user.get("isPro"),
|
|
"skills": agent.get("skills") or [],
|
|
"account_type": agent.get("accountType") or user.get("accountType"),
|
|
}
|
|
|
|
|
|
# ── Save helpers ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None:
|
|
"""Persist DetailEnrichment to DB:
|
|
|
|
- UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
|
|
- INSERT INTO offer_price_history rows (if price_changes non-empty)
|
|
- INSERT INTO agents (if agent_profile) and link via agent_id_fk
|
|
- bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id
|
|
|
|
Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
|
|
"""
|
|
# Extend listings row with detail-enriched fields
|
|
db.execute(
|
|
text("""
|
|
UPDATE listings SET
|
|
windows_view_type = COALESCE(:wvt, windows_view_type),
|
|
separate_wcs_count = COALESCE(:swc, separate_wcs_count),
|
|
combined_wcs_count = COALESCE(:cwc, combined_wcs_count),
|
|
ceiling_height = COALESCE(:ch, ceiling_height),
|
|
repair_type = COALESCE(:rt, repair_type),
|
|
repair_state = COALESCE(:rs, repair_state),
|
|
kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
|
|
description = COALESCE(:descr, description),
|
|
views_total = COALESCE(:vt, views_total),
|
|
views_today = COALESCE(:vd, views_today),
|
|
detail_enriched_at = NOW()
|
|
WHERE id = CAST(:lid AS bigint)
|
|
"""),
|
|
{
|
|
"lid": listing_id,
|
|
"wvt": enrichment.windows_view_type,
|
|
"swc": enrichment.separate_wcs_count,
|
|
"cwc": enrichment.combined_wcs_count,
|
|
"ch": enrichment.ceiling_height,
|
|
"rt": enrichment.repair_type,
|
|
"rs": enrichment.repair_state,
|
|
"ka": enrichment.kitchen_area_m2,
|
|
"descr": enrichment.description,
|
|
"vt": enrichment.views_total,
|
|
"vd": enrichment.views_today,
|
|
},
|
|
)
|
|
|
|
# Snapshot: point-in-time observation в listings_snapshots.
|
|
# Берём price_rub / price_per_m2 из listings строки (только что обновлённой).
|
|
# Используем COALESCE — на случай если listings.price_rub NULL (не должно быть, но safe).
|
|
_snap_row = db.execute(
|
|
text("""
|
|
SELECT price_rub, price_per_m2
|
|
FROM listings
|
|
WHERE id = CAST(:lid AS bigint)
|
|
"""),
|
|
{"lid": listing_id},
|
|
).fetchone()
|
|
_snap_written = False
|
|
if _snap_row is not None and _snap_row.price_rub is not None:
|
|
try:
|
|
with db.begin_nested():
|
|
upsert_listing_snapshot(
|
|
db,
|
|
listing_id=listing_id,
|
|
price_rub=int(_snap_row.price_rub),
|
|
price_per_m2=int(_snap_row.price_per_m2) if _snap_row.price_per_m2 else None,
|
|
run_id=None,
|
|
status="active",
|
|
)
|
|
_snap_written = True
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"save_detail_enrichment: snapshot write failed listing_id=%s: %s",
|
|
listing_id,
|
|
exc,
|
|
)
|
|
|
|
# Price changes — INSERT new rows, de-dup by listing_id + change_time
|
|
for change in enrichment.price_changes:
|
|
if not change.get("change_time") or not change.get("price_rub"):
|
|
continue
|
|
db.execute(
|
|
text("""
|
|
INSERT INTO offer_price_history
|
|
(listing_id, change_time, price_rub, diff_percent, source)
|
|
VALUES (
|
|
CAST(:lid AS bigint),
|
|
CAST(:ct AS timestamptz),
|
|
CAST(:price AS numeric),
|
|
CAST(:diff AS numeric),
|
|
'cian'
|
|
)
|
|
ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq DO NOTHING
|
|
"""),
|
|
{
|
|
"lid": listing_id,
|
|
"ct": change["change_time"],
|
|
"price": change["price_rub"],
|
|
"diff": change.get("diff_percent"),
|
|
},
|
|
)
|
|
|
|
# Agent profile UPSERT — store agent row and link to listing
|
|
if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"):
|
|
ap = enrichment.agent_profile
|
|
row = db.execute(
|
|
text("""
|
|
INSERT INTO agents
|
|
(ext_source, ext_agent_id, company_name, is_pro, skills, account_type)
|
|
VALUES (
|
|
'cian',
|
|
:eai,
|
|
:cn,
|
|
:ip,
|
|
CAST(:sk AS jsonb),
|
|
:at
|
|
)
|
|
ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET
|
|
company_name = COALESCE(EXCLUDED.company_name, agents.company_name),
|
|
is_pro = COALESCE(EXCLUDED.is_pro, agents.is_pro),
|
|
skills = COALESCE(EXCLUDED.skills, agents.skills),
|
|
account_type = COALESCE(EXCLUDED.account_type, agents.account_type)
|
|
RETURNING id
|
|
"""),
|
|
{
|
|
"eai": str(ap["ext_agent_id"]),
|
|
"cn": ap.get("company_name"),
|
|
"ip": ap.get("is_pro"),
|
|
"sk": json.dumps(ap.get("skills") or []),
|
|
"at": ap.get("account_type"),
|
|
},
|
|
)
|
|
agent_db_id = row.scalar_one_or_none()
|
|
if agent_db_id is not None:
|
|
db.execute(
|
|
text("""
|
|
UPDATE listings
|
|
SET agent_id_fk = CAST(:aid AS bigint)
|
|
WHERE id = CAST(:lid AS bigint)
|
|
AND agent_id_fk IS NULL
|
|
"""),
|
|
{"aid": agent_db_id, "lid": listing_id},
|
|
)
|
|
|
|
db.commit()
|
|
logger.info(
|
|
"Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)",
|
|
listing_id,
|
|
len(enrichment.price_changes),
|
|
bool(enrichment.agent_profile),
|
|
_snap_written,
|
|
)
|