lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

feat(tradein): cian_detail.py — detail page scraper (88 fields + BTI + priceChanges, Wave 4 Worker B)

Browse source 
- fetch_detail(): curl_cffi AsyncSession → HTML → extract_state(mfe='frontend-offer-card', key='defaultState')
- DetailEnrichment dataclass: cian_id, windows_view_type, wcs counts, ceiling_height, repair_type, views_total/today, bti_data, price_changes, agent_profile, newbuilding_data, raw_offer/sister_states
- extract_all_states() sister container stash: bti (houseData), newObject, plus raw dump for future extraction
- save_detail_enrichment(): UPDATE listings (all 019 cols present), INSERT offer_price_history (ON CONFLICT DO NOTHING), UPSERT agents + link agent_id_fk, psycopg v3 CAST(:x AS type) everywhere
- Tests: 19 tests (unit helpers + async mocked fetch_detail: no-state, 403, core fields, BTI sister state, newbuilding link)
- Ruff clean, asyncio_mode=auto
This commit is contained in:
lekss361 2026-05-23 16:12:15 +03:00
parent 5051ebde2e
commit f6ece38619
2 changed files with 648 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

324
tradein-mvp/backend/app/services/scrapers/cian_detail.py Normal file
View file

@ -0,0 +1,324 @@
"""Cian.ru detail page scraper — single offer enrichment.
Different from SERP (cian.py):
- URL: https://ekb.cian.ru/sale/flat/<offer_id>/ or https://www.cian.ru/sale/flat/<offer_id>/
- MFE: 'frontend-offer-card'
- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState)
- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject
Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
Stage 5 of CianScraper v1.
"""
from __future__ import annotations
import json
import logging
from dataclasses import dataclass, field
from typing import Any
from curl_cffi.requests import AsyncSession
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
logger = logging.getLogger(__name__)
@dataclass
class DetailEnrichment:
"""Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows."""
# Core listing enrichment (extend existing listings row)
cian_id: int | None = None
windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet'
separate_wcs_count: int | None = None
combined_wcs_count: int | None = None
ceiling_height: float | None = None # meters
repair_type: str | None = None # 'cosmetic' / 'design' / 'no'
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
views_today: int | None = None
# BTI (вторичка only — primary buildings don't have БТИ)
bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot
# Price history (offer.priceChanges)
price_changes: list[dict[str, Any]] = field(default_factory=list)
# Agent profile (offer.agent.* expanded)
agent_profile: dict[str, Any] | None = None
# Newbuilding link (if offer is from a newbuilding)
newbuilding_data: dict[str, Any] | None = None
# Raw payload для debugging / future-extraction
raw_offer: dict[str, Any] | None = None
raw_sister_states: dict[str, Any] = field(default_factory=dict)
async def fetch_detail(
offer_url: str,
*,
session: AsyncSession | None = None,
) -> DetailEnrichment | None:
"""Fetch detail page and extract all containers.
Args:
offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/'
session: optional shared curl_cffi session (для batched scrapes)
Returns: DetailEnrichment, or None если fetch / parse failed.
"""
close_session = False
if session is None:
session = AsyncSession(
impersonate="chrome120",
timeout=25.0,
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
},
)
close_session = True
try:
resp = await session.get(offer_url, allow_redirects=True)
if resp.status_code != 200:
logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code)
return None
html = resp.text
# Primary state container — defaultState in frontend-offer-card MFE
# NOTE: detail pages use 'defaultState', SERP uses 'initialState'
offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
if offer_state is None:
logger.warning("Cian detail %s: defaultState extraction failed", offer_url)
return None
offer_data = offer_state.get("offerData", {})
offer = offer_data.get("offer", {})
result = DetailEnrichment(
cian_id=offer.get("cianId") or offer.get("id"),
windows_view_type=_extract_windows_view(offer),
separate_wcs_count=offer.get("separateWcsCount"),
combined_wcs_count=offer.get("combinedWcsCount"),
ceiling_height=_parse_float(offer.get("ceilingHeight")),
repair_type=_extract_repair_type(offer),
raw_offer=offer,
)
# Stats (views) — from stats key in offerData or top-level state
stats = offer_data.get("stats") or offer_state.get("stats") or {}
result.views_total = _parse_views(stats.get("totalViewsFormattedString"))
result.views_today = _parse_views(stats.get("todayViewsFormattedString"))
# Price changes — from offer.priceChanges or state-level priceChanges
result.price_changes = _extract_price_changes(offer, offer_state)
# Agent profile expansion
result.agent_profile = _extract_agent_profile(offer)
# Sister state containers: bti, newObject, plus raw stash of everything else
all_states = extract_all_states(html)
offer_card_states = all_states.get("frontend-offer-card", {})
bti_state = offer_card_states.get("bti")
if bti_state:
result.bti_data = bti_state.get("houseData")
result.raw_sister_states["bti"] = bti_state
# Newbuilding link (if applicable)
newbuilding = offer.get("newbuilding")
if newbuilding and newbuilding.get("id"):
result.newbuilding_data = newbuilding
# Stash all non-defaultState sister containers for debugging / future-extraction
for k, v in offer_card_states.items():
if k != "defaultState":
result.raw_sister_states[k] = v
return result
finally:
if close_session:
await session.close()
# ── Field extractors ──────────────────────────────────────────────────────────
def _extract_windows_view(offer: dict[str, Any]) -> str | None:
return offer.get("windowsViewType")
def _extract_repair_type(offer: dict[str, Any]) -> str | None:
# Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ...
return offer.get("repairType")
def _parse_float(value: Any) -> float | None:
if value is None:
return None
try:
return float(value)
except (ValueError, TypeError):
return None
def _parse_views(formatted_str: str | None) -> int | None:
"""Cian's stats.totalViewsFormattedString — '1 234' → 1234."""
if not formatted_str:
return None
try:
return int(str(formatted_str).replace(" ", "").replace(",", ""))
except (ValueError, TypeError):
return None
def _extract_price_changes(offer: dict[str, Any], state: dict[str, Any]) -> list[dict[str, Any]]:
"""offer.priceChanges or state.priceChanges — normalize to common form."""
raw = offer.get("priceChanges") or state.get("priceChanges") or []
if not isinstance(raw, list):
return []
result = []
for entry in raw:
if not isinstance(entry, dict):
continue
result.append(
{
"change_time": entry.get("changeTime") or entry.get("date"),
"price_rub": entry.get("price") or entry.get("priceRub"),
"diff_percent": entry.get("diffPercent") or entry.get("diff_percent"),
}
)
return result
def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
"""offer.agent or offer.agency or offer.user — extract canonical profile shape."""
agent = offer.get("agent") or {}
user = offer.get("user") or {}
if not agent and not user:
return None
return {
"ext_agent_id": agent.get("id") or user.get("cianUserId"),
"company_name": agent.get("companyName") or user.get("companyName"),
"is_pro": agent.get("isPro") or user.get("isPro"),
"skills": agent.get("skills") or [],
"account_type": agent.get("accountType") or user.get("accountType"),
}
# ── Save helpers ──────────────────────────────────────────────────────────────
def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None:
"""Persist DetailEnrichment to DB:
- UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
- INSERT INTO offer_price_history rows (if price_changes non-empty)
- INSERT INTO agents (if agent_profile) and link via agent_id_fk
- bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id
Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
"""
# Extend listings row with detail-enriched fields
db.execute(
text("""
UPDATE listings SET
windows_view_type = COALESCE(:wvt, windows_view_type),
separate_wcs_count = COALESCE(:swc, separate_wcs_count),
combined_wcs_count = COALESCE(:cwc, combined_wcs_count),
ceiling_height = COALESCE(:ch, ceiling_height),
repair_type = COALESCE(:rt, repair_type),
views_total = COALESCE(:vt, views_total),
views_today = COALESCE(:vd, views_today)
WHERE id = CAST(:lid AS bigint)
"""),
{
"lid": listing_id,
"wvt": enrichment.windows_view_type,
"swc": enrichment.separate_wcs_count,
"cwc": enrichment.combined_wcs_count,
"ch": enrichment.ceiling_height,
"rt": enrichment.repair_type,
"vt": enrichment.views_total,
"vd": enrichment.views_today,
},
)
# Price changes — INSERT new rows, de-dup by listing_id + change_time
for change in enrichment.price_changes:
if not change.get("change_time") or not change.get("price_rub"):
continue
db.execute(
text("""
INSERT INTO offer_price_history
(listing_id, change_time, price_rub, diff_percent, source)
VALUES (
CAST(:lid AS bigint),
CAST(:ct AS timestamptz),
CAST(:price AS numeric),
CAST(:diff AS numeric),
'cian'
)
ON CONFLICT DO NOTHING
"""),
{
"lid": listing_id,
"ct": change["change_time"],
"price": change["price_rub"],
"diff": change.get("diff_percent"),
},
)
# Agent profile UPSERT — store agent row and link to listing
if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"):
ap = enrichment.agent_profile
row = db.execute(
text("""
INSERT INTO agents
(ext_source, ext_agent_id, company_name, is_pro, skills, account_type)
VALUES (
'cian',
:eai,
:cn,
:ip,
CAST(:sk AS jsonb),
:at
)
ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET
company_name = COALESCE(EXCLUDED.company_name, agents.company_name),
is_pro = COALESCE(EXCLUDED.is_pro, agents.is_pro),
skills = COALESCE(EXCLUDED.skills, agents.skills),
account_type = COALESCE(EXCLUDED.account_type, agents.account_type)
RETURNING id
"""),
{
"eai": str(ap["ext_agent_id"]),
"cn": ap.get("company_name"),
"ip": ap.get("is_pro"),
"sk": json.dumps(ap.get("skills") or []),
"at": ap.get("account_type"),
},
)
agent_db_id = row.scalar_one_or_none()
if agent_db_id is not None:
db.execute(
text("""
UPDATE listings
SET agent_id_fk = CAST(:aid AS bigint)
WHERE id = CAST(:lid AS bigint)
AND agent_id_fk IS NULL
"""),
{"aid": agent_db_id, "lid": listing_id},
)
db.commit()
logger.info(
"Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s)",
listing_id,
len(enrichment.price_changes),
bool(enrichment.agent_profile),
)

324
tradein-mvp/backend/tests/test_cian_detail.py Normal file
View file

@ -0,0 +1,324 @@
"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.services.scrapers.cian_detail import (
DetailEnrichment,
_extract_agent_profile,
_extract_price_changes,
_parse_views,
fetch_detail,
)
# ── _parse_views ──────────────────────────────────────────────────────────────
def test_parse_views_space_formatted():
assert _parse_views("1 234") == 1234
def test_parse_views_plain():
assert _parse_views("567") == 567
def test_parse_views_none():
assert _parse_views(None) is None
def test_parse_views_invalid():
assert _parse_views("invalid") is None
def test_parse_views_comma_formatted():
assert _parse_views("1,234") == 1234
# ── _extract_price_changes ────────────────────────────────────────────────────
def test_extract_price_changes_normalizes():
offer = {
"priceChanges": [
{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5},
{"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0},
]
}
result = _extract_price_changes(offer, {})
assert len(result) == 2
assert result[0]["change_time"] == "2026-04-01T00:00:00Z"
assert result[0]["price_rub"] == 5000000
assert result[0]["diff_percent"] == -2.5
def test_extract_price_changes_handles_empty_offer():
assert _extract_price_changes({}, {}) == []
def test_extract_price_changes_handles_none_value():
assert _extract_price_changes({"priceChanges": None}, {}) == []
def test_extract_price_changes_falls_back_to_state():
state = {
"priceChanges": [
{"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0},
]
}
result = _extract_price_changes({}, state)
assert len(result) == 1
assert result[0]["price_rub"] == 6000000
def test_extract_price_changes_skips_non_dict_entries():
offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]}
result = _extract_price_changes(offer, {})
assert len(result) == 1
# ── _extract_agent_profile ────────────────────────────────────────────────────
def test_extract_agent_profile_from_agent_key():
offer = {
"agent": {
"id": 12345,
"companyName": "Новосёл",
"isPro": True,
"skills": [{"id": 1, "name": "Продажа"}],
"accountType": "specialist",
}
}
profile = _extract_agent_profile(offer)
assert profile is not None
assert profile["ext_agent_id"] == 12345
assert profile["company_name"] == "Новосёл"
assert profile["is_pro"] is True
assert profile["account_type"] == "specialist"
def test_extract_agent_profile_from_user_key():
offer = {
"user": {
"cianUserId": 99999,
"companyName": "Агентство Икс",
"isPro": False,
"accountType": "agency",
}
}
profile = _extract_agent_profile(offer)
assert profile is not None
assert profile["ext_agent_id"] == 99999
assert profile["company_name"] == "Агентство Икс"
def test_extract_agent_profile_returns_none_when_no_agent():
assert _extract_agent_profile({}) is None
assert _extract_agent_profile({"agent": {}, "user": {}}) is None
# ── DetailEnrichment defaults ─────────────────────────────────────────────────
def test_dataclass_defaults():
enrich = DetailEnrichment()
assert enrich.cian_id is None
assert enrich.price_changes == []
assert enrich.raw_sister_states == {}
assert enrich.bti_data is None
assert enrich.agent_profile is None
assert enrich.newbuilding_data is None
# ── fetch_detail integration (mocked) ────────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_returns_none_on_no_state(monkeypatch):
"""If state extraction returns None → fetch_detail returns None gracefully."""
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: None,
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session)
assert result is None
@pytest.mark.asyncio
async def test_fetch_detail_returns_none_on_non_200(monkeypatch):
"""Non-200 HTTP response → returns None, no exception raised."""
response = MagicMock()
response.status_code = 403
response.text = ""
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session)
assert result is None
@pytest.mark.asyncio
async def test_fetch_detail_extracts_core_fields(monkeypatch):
"""Verify core fields are extracted correctly from mocked state."""
fake_state = {
"offerData": {
"offer": {
"cianId": 327830237,
"windowsViewType": "yardAndStreet",
"separateWcsCount": 1,
"combinedWcsCount": 0,
"ceilingHeight": 2.65,
"repairType": "cosmetic",
"priceChanges": [
{
"changeTime": "2026-04-01T00:00:00Z",
"price": 5500000,
"diffPercent": -1.8,
}
],
"agent": {
"id": 7654321,
"companyName": "Первичка Плюс",
"isPro": True,
"skills": [],
"accountType": "specialist",
},
},
"stats": {
"totalViewsFormattedString": "1 042",
"todayViewsFormattedString": "7",
},
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session)
assert result is not None
assert result.cian_id == 327830237
assert result.windows_view_type == "yardAndStreet"
assert result.separate_wcs_count == 1
assert result.combined_wcs_count == 0
assert result.ceiling_height == 2.65
assert result.repair_type == "cosmetic"
assert result.views_total == 1042
assert result.views_today == 7
assert len(result.price_changes) == 1
assert result.price_changes[0]["price_rub"] == 5500000
assert result.agent_profile is not None
assert result.agent_profile["ext_agent_id"] == 7654321
assert result.agent_profile["company_name"] == "Первичка Плюс"
@pytest.mark.asyncio
async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch):
"""BTI sister state is captured into bti_data."""
fake_state = {
"offerData": {
"offer": {
"cianId": 111111,
}
}
}
fake_all_states = {
"frontend-offer-card": {
"bti": {
"houseData": {
"seriesName": "П-44Т",
"floorsCount": 17,
"flatCount": 120,
}
}
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: fake_all_states,
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session)
assert result is not None
assert result.bti_data is not None
assert result.bti_data["seriesName"] == "П-44Т"
assert "bti" in result.raw_sister_states
@pytest.mark.asyncio
async def test_fetch_detail_extracts_newbuilding_link(monkeypatch):
"""Newbuilding reference is captured from offer.newbuilding."""
fake_state = {
"offerData": {
"offer": {
"cianId": 222222,
"newbuilding": {
"id": 54016,
"name": "Екатерининский парк",
"url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/",
},
}
}
}
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_state",
lambda html, mfe, key: fake_state if key == "defaultState" else None,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.extract_all_states",
lambda html: {},
)
response = MagicMock()
response.status_code = 200
response.text = "<html></html>"
session = MagicMock()
session.get = AsyncMock(return_value=response)
session.close = AsyncMock()
result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session)
assert result is not None
assert result.newbuilding_data is not None
assert result.newbuilding_data["id"] == 54016