lekss361
2e345acb82
All checks were successful
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m38s
Deploy Trade-In / build-backend (push) Successful in 1m28s
Deploy Trade-In / deploy (push) Successful in 1m45s
464 lines
19 KiB
Python
464 lines
19 KiB
Python
"""Cian.ru detail page scraper — single offer enrichment.
|
||
|
||
Different from SERP (cian.py):
|
||
- URL: https://ekb.cian.ru/sale/flat/<offer_id>/ or https://www.cian.ru/sale/flat/<offer_id>/
|
||
- MFE: 'frontend-offer-card'
|
||
- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState)
|
||
- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject
|
||
|
||
Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
|
||
Stage 5 of CianScraper v1.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
from dataclasses import dataclass, field
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
from curl_cffi.requests import AsyncSession
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.config import settings
|
||
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
|
||
from app.services.scrapers.repair_state_normalizer import (
|
||
infer_repair_state_from_text,
|
||
normalize_repair_state,
|
||
)
|
||
from app.services.scrapers.snapshot_writer import upsert_listing_snapshot
|
||
|
||
if TYPE_CHECKING:
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# offer_price_history.diff_percent — NUMERIC(8,2) (миграция
|
||
# 131_fix_diff_percent_overflow.sql). Прямой INSERT ниже читает diff_percent из
|
||
# скрапленного Cian JSON и не проходит через БД-триггер record_listing_price_change()
|
||
# (тот срабатывает только на UPDATE listings) — гарбадж/экстремальное значение
|
||
# с редизайна source-страницы иначе переполняет колонку.
|
||
_DIFF_PERCENT_BOUND = 999999.99
|
||
|
||
|
||
def _clamp_diff_percent(value: float | int | None) -> float | None:
|
||
"""Клампит diff_percent в границы NUMERIC(8,2) (зеркало migration 131 trigger-clamp)."""
|
||
if value is None or isinstance(value, bool):
|
||
return None
|
||
try:
|
||
numeric = float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
return min(max(numeric, -_DIFF_PERCENT_BOUND), _DIFF_PERCENT_BOUND)
|
||
|
||
|
||
@dataclass
|
||
class DetailEnrichment:
|
||
"""Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows."""
|
||
|
||
# Core listing enrichment (extend existing listings row)
|
||
cian_id: int | None = None
|
||
windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet'
|
||
separate_wcs_count: int | None = None
|
||
combined_wcs_count: int | None = None
|
||
ceiling_height: float | None = None # meters — offer.building.ceilingHeight (#1791)
|
||
repair_type: str | None = None # raw Cian: offer.repairType OR offer.decoration (#1791)
|
||
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
||
kitchen_area_m2: float | None = None # offer.kitchenArea (м²)
|
||
description: str | None = None # offer.description (текст объявления)
|
||
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
||
views_today: int | None = None
|
||
|
||
# BTI (вторичка only — primary buildings don't have БТИ)
|
||
bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot
|
||
|
||
# Price history (offer.priceChanges)
|
||
price_changes: list[dict[str, Any]] = field(default_factory=list)
|
||
|
||
# Agent profile (offer.agent.* expanded)
|
||
agent_profile: dict[str, Any] | None = None
|
||
|
||
# Newbuilding link (if offer is from a newbuilding)
|
||
newbuilding_data: dict[str, Any] | None = None
|
||
|
||
# Raw payload для debugging / future-extraction
|
||
raw_offer: dict[str, Any] | None = None
|
||
raw_sister_states: dict[str, Any] = field(default_factory=dict)
|
||
|
||
|
||
async def fetch_detail(
|
||
offer_url: str,
|
||
*,
|
||
session: AsyncSession | None = None,
|
||
browser_fetcher: BrowserFetcher | None = None,
|
||
) -> DetailEnrichment | None:
|
||
"""Fetch detail page and extract all containers.
|
||
|
||
Args:
|
||
offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/'
|
||
session: optional shared curl_cffi session (для batched scrapes).
|
||
Ignored when browser_fetcher is provided.
|
||
browser_fetcher: optional BrowserFetcher instance (camoufox HTTP service).
|
||
When provided, fetches JS-rendered HTML via the browser service instead of
|
||
curl_cffi — enables priceChanges extraction which requires JS rendering.
|
||
Caller is responsible for the context-manager lifecycle of the fetcher.
|
||
|
||
Returns: DetailEnrichment, or None если fetch / parse failed.
|
||
"""
|
||
if browser_fetcher is not None:
|
||
# Browser path: get fully JS-rendered HTML; same parse path follows.
|
||
try:
|
||
html = await browser_fetcher.fetch(offer_url)
|
||
except Exception as exc:
|
||
logger.warning("Cian detail browser fetch failed %s: %s", offer_url, exc)
|
||
return None
|
||
else:
|
||
# curl_cffi path (legacy, back-compat).
|
||
close_session = False
|
||
if session is None:
|
||
# proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian.
|
||
# Пусто (env не задан) → прямое подключение (dev/no-op).
|
||
_proxy_url = settings.cian_proxy_url
|
||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||
session = AsyncSession(
|
||
impersonate="chrome120",
|
||
timeout=25.0,
|
||
proxies=_proxies,
|
||
headers={
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||
},
|
||
)
|
||
close_session = True
|
||
|
||
try:
|
||
resp = await session.get(offer_url, allow_redirects=True)
|
||
if resp.status_code != 200:
|
||
logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code)
|
||
return None
|
||
html = resp.text
|
||
finally:
|
||
if close_session:
|
||
await session.close()
|
||
|
||
# ── Shared parse path (identical regardless of HTML source) ──────────────
|
||
|
||
# Primary state container — defaultState in frontend-offer-card MFE
|
||
# NOTE: detail pages use 'defaultState', SERP uses 'initialState'
|
||
offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
|
||
if offer_state is None:
|
||
logger.warning("Cian detail %s: defaultState extraction failed", offer_url)
|
||
return None
|
||
|
||
offer_data = offer_state.get("offerData", {})
|
||
offer = offer_data.get("offer", {})
|
||
|
||
result = DetailEnrichment(
|
||
cian_id=offer.get("cianId") or offer.get("id"),
|
||
windows_view_type=_extract_windows_view(offer),
|
||
separate_wcs_count=offer.get("separateWcsCount"),
|
||
combined_wcs_count=offer.get("combinedWcsCount"),
|
||
ceiling_height=_extract_ceiling_height(offer),
|
||
repair_type=_extract_repair_type(offer),
|
||
repair_state=_extract_repair_state(offer),
|
||
kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
|
||
description=offer.get("description") or None,
|
||
raw_offer=offer,
|
||
)
|
||
|
||
# Stats (views) — from stats key in offerData or top-level state
|
||
stats = offer_data.get("stats") or offer_state.get("stats") or {}
|
||
result.views_total = _parse_views(stats.get("totalViewsFormattedString"))
|
||
result.views_today = _parse_views(stats.get("todayViewsFormattedString"))
|
||
|
||
# Price changes — browser HTML: offerData.priceChanges (priceData.price format)
|
||
# curl_cffi HTML: offer.priceChanges or state.priceChanges (flat price field)
|
||
result.price_changes = _extract_price_changes(offer, offer_data, offer_state)
|
||
|
||
# Agent profile expansion
|
||
result.agent_profile = _extract_agent_profile(offer)
|
||
|
||
# Sister state containers: bti, newObject, plus raw stash of everything else
|
||
all_states = extract_all_states(html)
|
||
offer_card_states = all_states.get("frontend-offer-card", {})
|
||
|
||
bti_state = offer_card_states.get("bti")
|
||
if bti_state:
|
||
result.bti_data = bti_state.get("houseData")
|
||
result.raw_sister_states["bti"] = bti_state
|
||
|
||
# Newbuilding link (if applicable)
|
||
newbuilding = offer.get("newbuilding")
|
||
if newbuilding and newbuilding.get("id"):
|
||
result.newbuilding_data = newbuilding
|
||
|
||
# Stash all non-defaultState sister containers for debugging / future-extraction
|
||
for k, v in offer_card_states.items():
|
||
if k != "defaultState":
|
||
result.raw_sister_states[k] = v
|
||
|
||
return result
|
||
|
||
|
||
# ── Field extractors ──────────────────────────────────────────────────────────
|
||
|
||
|
||
def _extract_windows_view(offer: dict[str, Any]) -> str | None:
|
||
return offer.get("windowsViewType")
|
||
|
||
|
||
def _extract_ceiling_height(offer: dict[str, Any]) -> float | None:
|
||
"""Высота потолков (м).
|
||
|
||
На detail-странице (defaultState) значение лежит в ``offer.building.ceilingHeight``,
|
||
а НЕ в ``offer.ceilingHeight`` (последнее всегда null) — #1791. Top-level оставлен
|
||
как fallback на случай иной структуры карточки.
|
||
"""
|
||
building = offer.get("building") or {}
|
||
return _parse_float(building.get("ceilingHeight") or offer.get("ceilingHeight"))
|
||
|
||
|
||
def _extract_repair_type(offer: dict[str, Any]) -> str | None:
|
||
# Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ...
|
||
# На detail-странице структурное поле чаще приходит как `decoration`
|
||
# ('preFine'/'fine'/...), а `repairType` пустое (#1791) — читаем оба.
|
||
return offer.get("repairType") or offer.get("decoration")
|
||
|
||
|
||
def _extract_repair_state(offer: dict[str, Any]) -> str | None:
|
||
"""Нормализованный enum из repairType/decoration → needs_repair/standard/good/excellent.
|
||
|
||
Если структурное поле пустое — fallback на инференс из описания (#622).
|
||
Структурное значение всегда в приоритете.
|
||
"""
|
||
state = normalize_repair_state(offer.get("repairType") or offer.get("decoration"))
|
||
if state is None:
|
||
state = infer_repair_state_from_text(offer.get("description"))
|
||
return state
|
||
|
||
|
||
def _parse_float(value: Any) -> float | None:
|
||
if value is None:
|
||
return None
|
||
try:
|
||
return float(value)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def _parse_views(formatted_str: str | None) -> int | None:
|
||
"""Cian's stats.totalViewsFormattedString — '1 234' → 1234."""
|
||
if not formatted_str:
|
||
return None
|
||
try:
|
||
return int(str(formatted_str).replace(" ", "").replace(",", ""))
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def _extract_price_changes(
|
||
offer: dict[str, Any],
|
||
offer_data: dict[str, Any],
|
||
state: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
"""Normalize priceChanges from any of the known Cian locations/formats.
|
||
|
||
Lookup priority:
|
||
1. offer.priceChanges — curl_cffi flat format: {changeTime, price, diffPercent}
|
||
2. offerData.priceChanges — browser-rendered format: {changeTime, priceData:{price}}
|
||
3. state.priceChanges — top-level fallback (rare)
|
||
|
||
Normalizes to: {change_time, price_rub, diff_percent}.
|
||
"""
|
||
raw = (
|
||
offer.get("priceChanges")
|
||
or offer_data.get("priceChanges")
|
||
or state.get("priceChanges")
|
||
or []
|
||
)
|
||
if not isinstance(raw, list):
|
||
return []
|
||
result = []
|
||
for entry in raw:
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
# price_rub: flat field (curl_cffi) OR nested priceData.price (browser)
|
||
price_data = entry.get("priceData") or {}
|
||
price_rub = entry.get("price") or entry.get("priceRub") or price_data.get("price")
|
||
result.append(
|
||
{
|
||
"change_time": entry.get("changeTime") or entry.get("date"),
|
||
"price_rub": price_rub,
|
||
"diff_percent": entry.get("diffPercent") or entry.get("diff_percent"),
|
||
}
|
||
)
|
||
return result
|
||
|
||
|
||
def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
|
||
"""offer.agent or offer.agency or offer.user — extract canonical profile shape."""
|
||
agent = offer.get("agent") or {}
|
||
user = offer.get("user") or {}
|
||
if not agent and not user:
|
||
return None
|
||
return {
|
||
"ext_agent_id": agent.get("id") or user.get("cianUserId"),
|
||
"company_name": agent.get("companyName") or user.get("companyName"),
|
||
"is_pro": agent.get("isPro") or user.get("isPro"),
|
||
"skills": agent.get("skills") or [],
|
||
"account_type": agent.get("accountType") or user.get("accountType"),
|
||
}
|
||
|
||
|
||
# ── Save helpers ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None:
|
||
"""Persist DetailEnrichment to DB:
|
||
|
||
- UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
|
||
- INSERT INTO offer_price_history rows (if price_changes non-empty)
|
||
- INSERT INTO agents (if agent_profile) and link via agent_id_fk
|
||
- bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id
|
||
|
||
Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
|
||
"""
|
||
# Extend listings row with detail-enriched fields
|
||
db.execute(
|
||
text("""
|
||
UPDATE listings SET
|
||
windows_view_type = COALESCE(:wvt, windows_view_type),
|
||
separate_wcs_count = COALESCE(:swc, separate_wcs_count),
|
||
combined_wcs_count = COALESCE(:cwc, combined_wcs_count),
|
||
ceiling_height = COALESCE(:ch, ceiling_height),
|
||
repair_type = COALESCE(:rt, repair_type),
|
||
repair_state = COALESCE(:rs, repair_state),
|
||
kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
|
||
description = COALESCE(:descr, description),
|
||
views_total = COALESCE(:vt, views_total),
|
||
views_today = COALESCE(:vd, views_today),
|
||
detail_enriched_at = NOW()
|
||
WHERE id = CAST(:lid AS bigint)
|
||
"""),
|
||
{
|
||
"lid": listing_id,
|
||
"wvt": enrichment.windows_view_type,
|
||
"swc": enrichment.separate_wcs_count,
|
||
"cwc": enrichment.combined_wcs_count,
|
||
"ch": enrichment.ceiling_height,
|
||
"rt": enrichment.repair_type,
|
||
"rs": enrichment.repair_state,
|
||
"ka": enrichment.kitchen_area_m2,
|
||
"descr": enrichment.description,
|
||
"vt": enrichment.views_total,
|
||
"vd": enrichment.views_today,
|
||
},
|
||
)
|
||
|
||
# Snapshot: point-in-time observation в listings_snapshots.
|
||
# Берём price_rub / price_per_m2 из listings строки (только что обновлённой).
|
||
# Используем COALESCE — на случай если listings.price_rub NULL (не должно быть, но safe).
|
||
_snap_row = db.execute(
|
||
text("""
|
||
SELECT price_rub, price_per_m2
|
||
FROM listings
|
||
WHERE id = CAST(:lid AS bigint)
|
||
"""),
|
||
{"lid": listing_id},
|
||
).fetchone()
|
||
_snap_written = False
|
||
if _snap_row is not None and _snap_row.price_rub is not None:
|
||
try:
|
||
with db.begin_nested():
|
||
upsert_listing_snapshot(
|
||
db,
|
||
listing_id=listing_id,
|
||
price_rub=int(_snap_row.price_rub),
|
||
price_per_m2=int(_snap_row.price_per_m2) if _snap_row.price_per_m2 else None,
|
||
run_id=None,
|
||
status="active",
|
||
)
|
||
_snap_written = True
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"save_detail_enrichment: snapshot write failed listing_id=%s: %s",
|
||
listing_id,
|
||
exc,
|
||
)
|
||
|
||
# Price changes — INSERT new rows, de-dup by listing_id + change_time
|
||
for change in enrichment.price_changes:
|
||
if not change.get("change_time") or not change.get("price_rub"):
|
||
continue
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO offer_price_history
|
||
(listing_id, change_time, price_rub, diff_percent, source)
|
||
VALUES (
|
||
CAST(:lid AS bigint),
|
||
CAST(:ct AS timestamptz),
|
||
CAST(:price AS numeric),
|
||
CAST(:diff AS numeric),
|
||
'cian'
|
||
)
|
||
ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq DO NOTHING
|
||
"""),
|
||
{
|
||
"lid": listing_id,
|
||
"ct": change["change_time"],
|
||
"price": change["price_rub"],
|
||
"diff": _clamp_diff_percent(change.get("diff_percent")),
|
||
},
|
||
)
|
||
|
||
# Agent profile UPSERT — store agent row and link to listing
|
||
if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"):
|
||
ap = enrichment.agent_profile
|
||
row = db.execute(
|
||
text("""
|
||
INSERT INTO agents
|
||
(ext_source, ext_agent_id, company_name, is_pro, skills, account_type)
|
||
VALUES (
|
||
'cian',
|
||
:eai,
|
||
:cn,
|
||
:ip,
|
||
CAST(:sk AS jsonb),
|
||
:at
|
||
)
|
||
ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET
|
||
company_name = COALESCE(EXCLUDED.company_name, agents.company_name),
|
||
is_pro = COALESCE(EXCLUDED.is_pro, agents.is_pro),
|
||
skills = COALESCE(EXCLUDED.skills, agents.skills),
|
||
account_type = COALESCE(EXCLUDED.account_type, agents.account_type)
|
||
RETURNING id
|
||
"""),
|
||
{
|
||
"eai": str(ap["ext_agent_id"]),
|
||
"cn": ap.get("company_name"),
|
||
"ip": ap.get("is_pro"),
|
||
"sk": json.dumps(ap.get("skills") or []),
|
||
"at": ap.get("account_type"),
|
||
},
|
||
)
|
||
agent_db_id = row.scalar_one_or_none()
|
||
if agent_db_id is not None:
|
||
db.execute(
|
||
text("""
|
||
UPDATE listings
|
||
SET agent_id_fk = CAST(:aid AS bigint)
|
||
WHERE id = CAST(:lid AS bigint)
|
||
AND agent_id_fk IS NULL
|
||
"""),
|
||
{"aid": agent_db_id, "lid": listing_id},
|
||
)
|
||
|
||
db.commit()
|
||
logger.info(
|
||
"Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)",
|
||
listing_id,
|
||
len(enrichment.price_changes),
|
||
bool(enrichment.agent_profile),
|
||
_snap_written,
|
||
)
|