From 2dd76ea5e8136c79f9fc23bdb34a8debd988b29f Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 13:22:00 +0000 Subject: [PATCH] feat(tradein): cian_session cookies management + admin endpoints (Wave 4 Worker A) (#457) --- tradein-mvp/backend/app/api/v1/admin.py | 70 ++++ tradein-mvp/backend/app/core/config.py | 4 + .../backend/app/services/cian_session.py | 196 +++++++++++ .../app/services/scrapers/cian_detail.py | 324 ++++++++++++++++++ tradein-mvp/backend/tests/test_cian_detail.py | 324 ++++++++++++++++++ .../backend/tests/test_cian_session.py | 238 +++++++++++++ tradein-mvp/scripts/upload-cian-cookies.sh | 50 +++ 7 files changed, 1206 insertions(+) create mode 100644 tradein-mvp/backend/app/services/cian_session.py create mode 100644 tradein-mvp/backend/app/services/scrapers/cian_detail.py create mode 100644 tradein-mvp/backend/tests/test_cian_detail.py create mode 100644 tradein-mvp/backend/tests/test_cian_session.py create mode 100644 tradein-mvp/scripts/upload-cian-cookies.sh diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index a38c47bb..f2c0aa1c 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -16,6 +16,7 @@ from sqlalchemy.orm import Session from app.core.config import settings from app.core.db import get_db +from app.services import cian_session as cian_session_svc from app.services.geocoder import geocode from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.base import save_listings @@ -233,3 +234,72 @@ async def geocode_missing( "skipped": skipped, "remaining": int(remaining or 0), } + + +@router.post("/scrape/cian/upload-cookies", status_code=200, dependencies=[Depends(require_admin)]) +async def upload_cian_cookies( + cookies: dict[str, str], + db: Annotated[Session, Depends(get_db)], +) -> dict: + """Upload Cian session cookies для аутентификации Valuation Calculator. + + Body: JSON object вида { "_ym_uid": "...", "_cian_visitor_session_id": "...", ... } + Cookies экспортируются из Chrome DevTools → Application → Cookies → www.cian.ru + или через расширение Cookie-Editor → Export → JSON (object format). + + Шаги: + 1. Фильтрует payload по CIAN_REQUIRED_COOKIES. + 2. Верифицирует через GET /kalkulator-nedvizhimosti/ — проверяет isAuthenticated. + 3. Сохраняет зашифрованно (pgp_sym_encrypt) в cian_session_cookies. + + Returns: {"ok": true, "userId": , "cookieCount": } + """ + if not settings.cookie_encryption_key: + raise HTTPException(status_code=503, detail="COOKIE_ENCRYPTION_KEY not configured") + + cleaned = {k: v for k, v in cookies.items() if k in cian_session_svc.CIAN_REQUIRED_COOKIES} + if not cleaned: + raise HTTPException( + status_code=400, + detail=( + f"No recognized Cian cookies in payload. " + f"Expected any of: {sorted(cian_session_svc.CIAN_REQUIRED_COOKIES)}" + ), + ) + + state = await cian_session_svc.verify_session(cleaned) + if state is None: + raise HTTPException( + status_code=401, + detail="Cookies invalid or session not authenticated on cian.ru", + ) + + user_id = state.get("user", {}).get("userId") + if not user_id: + raise HTTPException(status_code=400, detail="Authenticated state missing userId") + + cian_session_svc.save_session(db, account_user_id=int(user_id), cookies=cleaned) + return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)} + + +@router.get("/scrape/cian/test-auth", status_code=200, dependencies=[Depends(require_admin)]) +async def test_cian_auth( + db: Annotated[Session, Depends(get_db)], +) -> dict: + """Проверить что текущие сохранённые Cian cookies ещё валидны. + + Returns: {"authenticated": bool, "userId": , "reason": } + """ + if not settings.cookie_encryption_key: + return {"authenticated": False, "userId": None, "reason": "encryption_key_not_configured"} + + cookies = cian_session_svc.load_session(db) + if cookies is None: + return {"authenticated": False, "userId": None, "reason": "no_session_in_db"} + + state = await cian_session_svc.verify_session(cookies) + if state is None: + return {"authenticated": False, "userId": None, "reason": "session_expired_or_invalid"} + + user_id = state.get("user", {}).get("userId") + return {"authenticated": True, "userId": user_id, "reason": None} diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index dcf7b42c..890bad97 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -29,5 +29,9 @@ class Settings(BaseSettings): # Пусто = мониторинг выключен (dev). В prod — env GLITCHTIP_DSN из .env.runtime. glitchtip_dsn: str | None = None + # Ключ шифрования для pgp_sym_encrypt (Cian session cookies). + # Задаётся через env COOKIE_ENCRYPTION_KEY. Пусто = шифрование не работает. + cookie_encryption_key: str = "" + settings = Settings() diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py new file mode 100644 index 00000000..616fcbb5 --- /dev/null +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -0,0 +1,196 @@ +"""Cian session cookie management — load/save/verify encrypted cookies. + +Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API. +Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table. +""" +from __future__ import annotations + +import json +import logging +from typing import Any + +import httpx +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.config import settings +from app.services.scrapers.cian_state_parser import extract_state + +logger = logging.getLogger(__name__) + +# Cookies критичные для Cian auth — фильтр перед сохранением. +# Conservative set подтверждён в Stage 4 (probe через DevTools на live session). +CIAN_REQUIRED_COOKIES: set[str] = { + "_ym_uid", + "_ym_d", + "_ym_isad", + "_cian_visitor_session_id", + "_cian_app_session_id", + "_cian_uid", + "_ga", + "tlsr_id", + "cf_clearance", + "session-id", + "anti_bot", + "csrftoken", +} + +_VALUATION_TEST_URL = ( + "https://www.cian.ru/kalkulator-nedvizhimosti/" + "?address=%D0%A1%D0%B2%D0%B5%D1%80%D0%B4%D0%BB%D0%BE%D0%B2%D1%81%D0%BA%D0%B0%D1%8F" + "%20%D0%BE%D0%B1%D0%BB%2C%20%D0%95%D0%BA%D0%B0%D1%82%D0%B5%D1%80%D0%B8%D0%BD%D0%B1%D1%83%D1%80%D0%B3" + "&totalArea=40&roomsCount=1&valuationType=sale" + "&floor%5B0%5D=floorOther&repairType%5B0%5D=repairTypeCosmetic" +) + +_MFE_VALUATION = "valuation-for-agent-frontend" + + +async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None: + """Hit Cian Valuation Calculator with cookies — return parsed state if authenticated. + + Returns state dict containing user.isAuthenticated + userId, or None if + cookies are invalid/expired or state extraction fails. + Никогда не логирует сырые значения cookies. + """ + try: + async with httpx.AsyncClient( + cookies=cookies, + timeout=20.0, + follow_redirects=True, + ) as client: + resp = await client.get(_VALUATION_TEST_URL) + resp.raise_for_status() + state = extract_state(resp.text, mfe=_MFE_VALUATION, key="initialState") + if state is None: + logger.warning( + "Cian cookies verify: state extraction failed (mfe=%s)", _MFE_VALUATION + ) + return None + user = state.get("user", {}) + if not user.get("isAuthenticated"): + logger.warning("Cian cookies verify: user.isAuthenticated=false") + return None + logger.info("Cian cookies verified — userId=%s", user.get("userId")) + return state + except httpx.HTTPStatusError as exc: + logger.warning( + "Cian cookies verify HTTP error: %s %s", + exc.response.status_code, + exc.request.url, + ) + return None + except Exception as exc: + logger.warning("Cian cookies verify failed: %s", exc) + return None + + +def save_session( + db: Session, + account_user_id: int, + cookies: dict[str, str], + ttl_days: int = 30, +) -> None: + """Encrypt cookies via pgp_sym_encrypt and UPSERT into cian_session_cookies. + + Uses settings.cookie_encryption_key as the encryption secret. + Никогда не логирует сырые значения cookies. + """ + cookies_json = json.dumps(cookies) + db.execute( + text(""" + INSERT INTO cian_session_cookies ( + account_user_id, + cookies_encrypted, + expires_at_estimate, + uploaded_at + ) VALUES ( + CAST(:uid AS bigint), + pgp_sym_encrypt(:cookies_json, :key), + NOW() + (CAST(:ttl_days AS int) || ' days')::interval, + NOW() + ) + ON CONFLICT (account_user_id) DO UPDATE SET + cookies_encrypted = EXCLUDED.cookies_encrypted, + expires_at_estimate = EXCLUDED.expires_at_estimate, + uploaded_at = NOW(), + last_invalid_at = NULL + """), + { + "uid": account_user_id, + "cookies_json": cookies_json, + "key": settings.cookie_encryption_key, + "ttl_days": ttl_days, + }, + ) + db.commit() + logger.info( + "Cian cookies saved for userId=%s (count=%d, ttl=%d days)", + account_user_id, + len(cookies), + ttl_days, + ) + + +def load_session(db: Session) -> dict[str, str] | None: + """Load most-recently-uploaded valid Cian cookies (decrypt). + + Returns dict[cookie_name, cookie_value] или None если нет валидной session. + Выбирает только записи где expires_at_estimate > NOW() и сессия не + была инвалидирована после последнего upload'а. + """ + row = db.execute( + text(""" + SELECT + account_user_id, + pgp_sym_decrypt(cookies_encrypted, :key)::text AS cookies_json, + expires_at_estimate + FROM cian_session_cookies + WHERE expires_at_estimate > NOW() + AND (last_invalid_at IS NULL OR last_invalid_at < uploaded_at) + ORDER BY uploaded_at DESC + LIMIT 1 + """), + {"key": settings.cookie_encryption_key}, + ).mappings().first() + + if row is None: + logger.warning("No valid Cian session cookies in DB") + return None + + cookies: dict[str, str] = json.loads(row["cookies_json"]) + + # Обновляем last_used_at — не критично, игнорируем ошибки. + try: + db.execute( + text( + "UPDATE cian_session_cookies SET last_used_at = NOW()" + " WHERE account_user_id = CAST(:uid AS bigint)" + ), + {"uid": row["account_user_id"]}, + ) + db.commit() + except Exception as exc: + logger.warning( + "Failed to update last_used_at for userId=%s: %s", row["account_user_id"], exc + ) + + logger.info( + "Cian cookies loaded for userId=%s (count=%d)", + row["account_user_id"], + len(cookies), + ) + return cookies + + +def mark_session_invalid(db: Session, account_user_id: int) -> None: + """Flag session как expired/invalid (например после 401 во время scrape).""" + db.execute( + text( + "UPDATE cian_session_cookies SET last_invalid_at = NOW()" + " WHERE account_user_id = CAST(:uid AS bigint)" + ), + {"uid": account_user_id}, + ) + db.commit() + logger.warning("Cian session marked invalid for userId=%s", account_user_id) diff --git a/tradein-mvp/backend/app/services/scrapers/cian_detail.py b/tradein-mvp/backend/app/services/scrapers/cian_detail.py new file mode 100644 index 00000000..59d281cf --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/cian_detail.py @@ -0,0 +1,324 @@ +"""Cian.ru detail page scraper — single offer enrichment. + +Different from SERP (cian.py): +- URL: https://ekb.cian.ru/sale/flat// or https://www.cian.ru/sale/flat// +- MFE: 'frontend-offer-card' +- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState) +- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject + +Parses ~88 offer fields + sister data → DetailEnrichment dataclass. +Stage 5 of CianScraper v1. +""" +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Any + +from curl_cffi.requests import AsyncSession +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.scrapers.cian_state_parser import extract_all_states, extract_state + +logger = logging.getLogger(__name__) + + +@dataclass +class DetailEnrichment: + """Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows.""" + + # Core listing enrichment (extend existing listings row) + cian_id: int | None = None + windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet' + separate_wcs_count: int | None = None + combined_wcs_count: int | None = None + ceiling_height: float | None = None # meters + repair_type: str | None = None # 'cosmetic' / 'design' / 'no' + views_total: int | None = None # Cian stats.totalViewsFormattedString → int + views_today: int | None = None + + # BTI (вторичка only — primary buildings don't have БТИ) + bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot + + # Price history (offer.priceChanges) + price_changes: list[dict[str, Any]] = field(default_factory=list) + + # Agent profile (offer.agent.* expanded) + agent_profile: dict[str, Any] | None = None + + # Newbuilding link (if offer is from a newbuilding) + newbuilding_data: dict[str, Any] | None = None + + # Raw payload для debugging / future-extraction + raw_offer: dict[str, Any] | None = None + raw_sister_states: dict[str, Any] = field(default_factory=dict) + + +async def fetch_detail( + offer_url: str, + *, + session: AsyncSession | None = None, +) -> DetailEnrichment | None: + """Fetch detail page and extract all containers. + + Args: + offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/' + session: optional shared curl_cffi session (для batched scrapes) + + Returns: DetailEnrichment, or None если fetch / parse failed. + """ + close_session = False + if session is None: + session = AsyncSession( + impersonate="chrome120", + timeout=25.0, + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", + }, + ) + close_session = True + + try: + resp = await session.get(offer_url, allow_redirects=True) + if resp.status_code != 200: + logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code) + return None + html = resp.text + + # Primary state container — defaultState in frontend-offer-card MFE + # NOTE: detail pages use 'defaultState', SERP uses 'initialState' + offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState") + if offer_state is None: + logger.warning("Cian detail %s: defaultState extraction failed", offer_url) + return None + + offer_data = offer_state.get("offerData", {}) + offer = offer_data.get("offer", {}) + + result = DetailEnrichment( + cian_id=offer.get("cianId") or offer.get("id"), + windows_view_type=_extract_windows_view(offer), + separate_wcs_count=offer.get("separateWcsCount"), + combined_wcs_count=offer.get("combinedWcsCount"), + ceiling_height=_parse_float(offer.get("ceilingHeight")), + repair_type=_extract_repair_type(offer), + raw_offer=offer, + ) + + # Stats (views) — from stats key in offerData or top-level state + stats = offer_data.get("stats") or offer_state.get("stats") or {} + result.views_total = _parse_views(stats.get("totalViewsFormattedString")) + result.views_today = _parse_views(stats.get("todayViewsFormattedString")) + + # Price changes — from offer.priceChanges or state-level priceChanges + result.price_changes = _extract_price_changes(offer, offer_state) + + # Agent profile expansion + result.agent_profile = _extract_agent_profile(offer) + + # Sister state containers: bti, newObject, plus raw stash of everything else + all_states = extract_all_states(html) + offer_card_states = all_states.get("frontend-offer-card", {}) + + bti_state = offer_card_states.get("bti") + if bti_state: + result.bti_data = bti_state.get("houseData") + result.raw_sister_states["bti"] = bti_state + + # Newbuilding link (if applicable) + newbuilding = offer.get("newbuilding") + if newbuilding and newbuilding.get("id"): + result.newbuilding_data = newbuilding + + # Stash all non-defaultState sister containers for debugging / future-extraction + for k, v in offer_card_states.items(): + if k != "defaultState": + result.raw_sister_states[k] = v + + return result + + finally: + if close_session: + await session.close() + + +# ── Field extractors ────────────────────────────────────────────────────────── + + +def _extract_windows_view(offer: dict[str, Any]) -> str | None: + return offer.get("windowsViewType") + + +def _extract_repair_type(offer: dict[str, Any]) -> str | None: + # Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ... + return offer.get("repairType") + + +def _parse_float(value: Any) -> float | None: + if value is None: + return None + try: + return float(value) + except (ValueError, TypeError): + return None + + +def _parse_views(formatted_str: str | None) -> int | None: + """Cian's stats.totalViewsFormattedString — '1 234' → 1234.""" + if not formatted_str: + return None + try: + return int(str(formatted_str).replace(" ", "").replace(",", "")) + except (ValueError, TypeError): + return None + + +def _extract_price_changes(offer: dict[str, Any], state: dict[str, Any]) -> list[dict[str, Any]]: + """offer.priceChanges or state.priceChanges — normalize to common form.""" + raw = offer.get("priceChanges") or state.get("priceChanges") or [] + if not isinstance(raw, list): + return [] + result = [] + for entry in raw: + if not isinstance(entry, dict): + continue + result.append( + { + "change_time": entry.get("changeTime") or entry.get("date"), + "price_rub": entry.get("price") or entry.get("priceRub"), + "diff_percent": entry.get("diffPercent") or entry.get("diff_percent"), + } + ) + return result + + +def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None: + """offer.agent or offer.agency or offer.user — extract canonical profile shape.""" + agent = offer.get("agent") or {} + user = offer.get("user") or {} + if not agent and not user: + return None + return { + "ext_agent_id": agent.get("id") or user.get("cianUserId"), + "company_name": agent.get("companyName") or user.get("companyName"), + "is_pro": agent.get("isPro") or user.get("isPro"), + "skills": agent.get("skills") or [], + "account_type": agent.get("accountType") or user.get("accountType"), + } + + +# ── Save helpers ────────────────────────────────────────────────────────────── + + +def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None: + """Persist DetailEnrichment to DB: + + - UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id + - INSERT INTO offer_price_history rows (if price_changes non-empty) + - INSERT INTO agents (if agent_profile) and link via agent_id_fk + - bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id + + Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics). + """ + # Extend listings row with detail-enriched fields + db.execute( + text(""" + UPDATE listings SET + windows_view_type = COALESCE(:wvt, windows_view_type), + separate_wcs_count = COALESCE(:swc, separate_wcs_count), + combined_wcs_count = COALESCE(:cwc, combined_wcs_count), + ceiling_height = COALESCE(:ch, ceiling_height), + repair_type = COALESCE(:rt, repair_type), + views_total = COALESCE(:vt, views_total), + views_today = COALESCE(:vd, views_today) + WHERE id = CAST(:lid AS bigint) + """), + { + "lid": listing_id, + "wvt": enrichment.windows_view_type, + "swc": enrichment.separate_wcs_count, + "cwc": enrichment.combined_wcs_count, + "ch": enrichment.ceiling_height, + "rt": enrichment.repair_type, + "vt": enrichment.views_total, + "vd": enrichment.views_today, + }, + ) + + # Price changes — INSERT new rows, de-dup by listing_id + change_time + for change in enrichment.price_changes: + if not change.get("change_time") or not change.get("price_rub"): + continue + db.execute( + text(""" + INSERT INTO offer_price_history + (listing_id, change_time, price_rub, diff_percent, source) + VALUES ( + CAST(:lid AS bigint), + CAST(:ct AS timestamptz), + CAST(:price AS numeric), + CAST(:diff AS numeric), + 'cian' + ) + ON CONFLICT DO NOTHING + """), + { + "lid": listing_id, + "ct": change["change_time"], + "price": change["price_rub"], + "diff": change.get("diff_percent"), + }, + ) + + # Agent profile UPSERT — store agent row and link to listing + if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"): + ap = enrichment.agent_profile + row = db.execute( + text(""" + INSERT INTO agents + (ext_source, ext_agent_id, company_name, is_pro, skills, account_type) + VALUES ( + 'cian', + :eai, + :cn, + :ip, + CAST(:sk AS jsonb), + :at + ) + ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET + company_name = COALESCE(EXCLUDED.company_name, agents.company_name), + is_pro = COALESCE(EXCLUDED.is_pro, agents.is_pro), + skills = COALESCE(EXCLUDED.skills, agents.skills), + account_type = COALESCE(EXCLUDED.account_type, agents.account_type) + RETURNING id + """), + { + "eai": str(ap["ext_agent_id"]), + "cn": ap.get("company_name"), + "ip": ap.get("is_pro"), + "sk": json.dumps(ap.get("skills") or []), + "at": ap.get("account_type"), + }, + ) + agent_db_id = row.scalar_one_or_none() + if agent_db_id is not None: + db.execute( + text(""" + UPDATE listings + SET agent_id_fk = CAST(:aid AS bigint) + WHERE id = CAST(:lid AS bigint) + AND agent_id_fk IS NULL + """), + {"aid": agent_db_id, "lid": listing_id}, + ) + + db.commit() + logger.info( + "Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s)", + listing_id, + len(enrichment.price_changes), + bool(enrichment.agent_profile), + ) diff --git a/tradein-mvp/backend/tests/test_cian_detail.py b/tradein-mvp/backend/tests/test_cian_detail.py new file mode 100644 index 00000000..22fc8dc2 --- /dev/null +++ b/tradein-mvp/backend/tests/test_cian_detail.py @@ -0,0 +1,324 @@ +"""Tests for cian_detail.py — Stage 5 of CianScraper v1.""" +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from app.services.scrapers.cian_detail import ( + DetailEnrichment, + _extract_agent_profile, + _extract_price_changes, + _parse_views, + fetch_detail, +) + + +# ── _parse_views ────────────────────────────────────────────────────────────── + + +def test_parse_views_space_formatted(): + assert _parse_views("1 234") == 1234 + + +def test_parse_views_plain(): + assert _parse_views("567") == 567 + + +def test_parse_views_none(): + assert _parse_views(None) is None + + +def test_parse_views_invalid(): + assert _parse_views("invalid") is None + + +def test_parse_views_comma_formatted(): + assert _parse_views("1,234") == 1234 + + +# ── _extract_price_changes ──────────────────────────────────────────────────── + + +def test_extract_price_changes_normalizes(): + offer = { + "priceChanges": [ + {"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5}, + {"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0}, + ] + } + result = _extract_price_changes(offer, {}) + assert len(result) == 2 + assert result[0]["change_time"] == "2026-04-01T00:00:00Z" + assert result[0]["price_rub"] == 5000000 + assert result[0]["diff_percent"] == -2.5 + + +def test_extract_price_changes_handles_empty_offer(): + assert _extract_price_changes({}, {}) == [] + + +def test_extract_price_changes_handles_none_value(): + assert _extract_price_changes({"priceChanges": None}, {}) == [] + + +def test_extract_price_changes_falls_back_to_state(): + state = { + "priceChanges": [ + {"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0}, + ] + } + result = _extract_price_changes({}, state) + assert len(result) == 1 + assert result[0]["price_rub"] == 6000000 + + +def test_extract_price_changes_skips_non_dict_entries(): + offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]} + result = _extract_price_changes(offer, {}) + assert len(result) == 1 + + +# ── _extract_agent_profile ──────────────────────────────────────────────────── + + +def test_extract_agent_profile_from_agent_key(): + offer = { + "agent": { + "id": 12345, + "companyName": "Новосёл", + "isPro": True, + "skills": [{"id": 1, "name": "Продажа"}], + "accountType": "specialist", + } + } + profile = _extract_agent_profile(offer) + assert profile is not None + assert profile["ext_agent_id"] == 12345 + assert profile["company_name"] == "Новосёл" + assert profile["is_pro"] is True + assert profile["account_type"] == "specialist" + + +def test_extract_agent_profile_from_user_key(): + offer = { + "user": { + "cianUserId": 99999, + "companyName": "Агентство Икс", + "isPro": False, + "accountType": "agency", + } + } + profile = _extract_agent_profile(offer) + assert profile is not None + assert profile["ext_agent_id"] == 99999 + assert profile["company_name"] == "Агентство Икс" + + +def test_extract_agent_profile_returns_none_when_no_agent(): + assert _extract_agent_profile({}) is None + assert _extract_agent_profile({"agent": {}, "user": {}}) is None + + +# ── DetailEnrichment defaults ───────────────────────────────────────────────── + + +def test_dataclass_defaults(): + enrich = DetailEnrichment() + assert enrich.cian_id is None + assert enrich.price_changes == [] + assert enrich.raw_sister_states == {} + assert enrich.bti_data is None + assert enrich.agent_profile is None + assert enrich.newbuilding_data is None + + +# ── fetch_detail integration (mocked) ──────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_fetch_detail_returns_none_on_no_state(monkeypatch): + """If state extraction returns None → fetch_detail returns None gracefully.""" + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: None, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session) + assert result is None + + +@pytest.mark.asyncio +async def test_fetch_detail_returns_none_on_non_200(monkeypatch): + """Non-200 HTTP response → returns None, no exception raised.""" + response = MagicMock() + response.status_code = 403 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session) + assert result is None + + +@pytest.mark.asyncio +async def test_fetch_detail_extracts_core_fields(monkeypatch): + """Verify core fields are extracted correctly from mocked state.""" + fake_state = { + "offerData": { + "offer": { + "cianId": 327830237, + "windowsViewType": "yardAndStreet", + "separateWcsCount": 1, + "combinedWcsCount": 0, + "ceilingHeight": 2.65, + "repairType": "cosmetic", + "priceChanges": [ + { + "changeTime": "2026-04-01T00:00:00Z", + "price": 5500000, + "diffPercent": -1.8, + } + ], + "agent": { + "id": 7654321, + "companyName": "Первичка Плюс", + "isPro": True, + "skills": [], + "accountType": "specialist", + }, + }, + "stats": { + "totalViewsFormattedString": "1 042", + "todayViewsFormattedString": "7", + }, + } + } + + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: fake_state if key == "defaultState" else None, + ) + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_all_states", + lambda html: {}, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session) + assert result is not None + assert result.cian_id == 327830237 + assert result.windows_view_type == "yardAndStreet" + assert result.separate_wcs_count == 1 + assert result.combined_wcs_count == 0 + assert result.ceiling_height == 2.65 + assert result.repair_type == "cosmetic" + assert result.views_total == 1042 + assert result.views_today == 7 + assert len(result.price_changes) == 1 + assert result.price_changes[0]["price_rub"] == 5500000 + assert result.agent_profile is not None + assert result.agent_profile["ext_agent_id"] == 7654321 + assert result.agent_profile["company_name"] == "Первичка Плюс" + + +@pytest.mark.asyncio +async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch): + """BTI sister state is captured into bti_data.""" + fake_state = { + "offerData": { + "offer": { + "cianId": 111111, + } + } + } + fake_all_states = { + "frontend-offer-card": { + "bti": { + "houseData": { + "seriesName": "П-44Т", + "floorsCount": 17, + "flatCount": 120, + } + } + } + } + + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: fake_state if key == "defaultState" else None, + ) + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_all_states", + lambda html: fake_all_states, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session) + assert result is not None + assert result.bti_data is not None + assert result.bti_data["seriesName"] == "П-44Т" + assert "bti" in result.raw_sister_states + + +@pytest.mark.asyncio +async def test_fetch_detail_extracts_newbuilding_link(monkeypatch): + """Newbuilding reference is captured from offer.newbuilding.""" + fake_state = { + "offerData": { + "offer": { + "cianId": 222222, + "newbuilding": { + "id": 54016, + "name": "Екатерининский парк", + "url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/", + }, + } + } + } + + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_state", + lambda html, mfe, key: fake_state if key == "defaultState" else None, + ) + monkeypatch.setattr( + "app.services.scrapers.cian_detail.extract_all_states", + lambda html: {}, + ) + + response = MagicMock() + response.status_code = 200 + response.text = "" + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session) + assert result is not None + assert result.newbuilding_data is not None + assert result.newbuilding_data["id"] == 54016 diff --git a/tradein-mvp/backend/tests/test_cian_session.py b/tradein-mvp/backend/tests/test_cian_session.py new file mode 100644 index 00000000..82906028 --- /dev/null +++ b/tradein-mvp/backend/tests/test_cian_session.py @@ -0,0 +1,238 @@ +"""Tests for cian_session — cookie management service.""" +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from app.services.cian_session import ( + CIAN_REQUIRED_COOKIES, + load_session, + mark_session_invalid, + save_session, + verify_session, +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def mock_db() -> MagicMock: + db = MagicMock() + # Default: no row found (load_session → None) + db.execute.return_value.mappings.return_value.first.return_value = None + return db + + +# --------------------------------------------------------------------------- +# CIAN_REQUIRED_COOKIES +# --------------------------------------------------------------------------- + + +def test_required_cookies_set_not_empty() -> None: + assert isinstance(CIAN_REQUIRED_COOKIES, set) + assert len(CIAN_REQUIRED_COOKIES) >= 5 + + +def test_required_cookies_contains_key_names() -> None: + assert "_cian_visitor_session_id" in CIAN_REQUIRED_COOKIES + assert "_ym_uid" in CIAN_REQUIRED_COOKIES + assert "_cian_uid" in CIAN_REQUIRED_COOKIES + assert "tlsr_id" in CIAN_REQUIRED_COOKIES + assert "cf_clearance" in CIAN_REQUIRED_COOKIES + + +# --------------------------------------------------------------------------- +# save_session +# --------------------------------------------------------------------------- + + +def test_save_session_calls_pgp_sym_encrypt(mock_db: MagicMock) -> None: + save_session(mock_db, account_user_id=12345, cookies={"_ym_uid": "abc"}) + args, _ = mock_db.execute.call_args + sql_text = str(args[0]) + assert "pgp_sym_encrypt" in sql_text + assert "cian_session_cookies" in sql_text + + +def test_save_session_uses_cast_not_colon_colon(mock_db: MagicMock) -> None: + """Verify psycopg v3 compatibility — no ::bigint syntax in SQL.""" + save_session(mock_db, account_user_id=99, cookies={"csrftoken": "x"}) + args, _ = mock_db.execute.call_args + sql_text = str(args[0]) + # CAST(:x AS type) is required; ::type would fail with psycopg v3 named params + assert "CAST(" in sql_text + assert "::bigint" not in sql_text + + +def test_save_session_commits(mock_db: MagicMock) -> None: + save_session(mock_db, account_user_id=12345, cookies={"_ym_uid": "abc"}) + assert mock_db.commit.called + + +def test_save_session_on_conflict_do_update(mock_db: MagicMock) -> None: + save_session(mock_db, account_user_id=1, cookies={"_ym_uid": "v"}) + args, _ = mock_db.execute.call_args + sql_text = str(args[0]) + assert "ON CONFLICT" in sql_text + assert "DO UPDATE" in sql_text + + +def test_save_session_passes_correct_params(mock_db: MagicMock) -> None: + save_session(mock_db, account_user_id=777, cookies={"_ga": "GA1.2.x"}, ttl_days=14) + _, kwargs = mock_db.execute.call_args + params = kwargs if kwargs else mock_db.execute.call_args[0][1] + # params могут передаваться как второй позиционный аргумент + call_args = mock_db.execute.call_args + params = call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("params", {}) + assert params["uid"] == 777 + assert params["ttl_days"] == 14 + cookies_payload = json.loads(params["cookies_json"]) + assert cookies_payload == {"_ga": "GA1.2.x"} + + +# --------------------------------------------------------------------------- +# load_session +# --------------------------------------------------------------------------- + + +def test_load_session_returns_none_when_empty(mock_db: MagicMock) -> None: + result = load_session(mock_db) + assert result is None + + +def test_load_session_decodes_json(mock_db: MagicMock) -> None: + mock_row = { + "account_user_id": 12345, + "cookies_json": json.dumps({"_ym_uid": "abc", "_cian_uid": "def"}), + "expires_at_estimate": None, + } + mock_db.execute.return_value.mappings.return_value.first.return_value = mock_row + result = load_session(mock_db) + assert result == {"_ym_uid": "abc", "_cian_uid": "def"} + + +def test_load_session_updates_last_used_at(mock_db: MagicMock) -> None: + mock_row = { + "account_user_id": 42, + "cookies_json": json.dumps({"session-id": "s"}), + "expires_at_estimate": None, + } + mock_db.execute.return_value.mappings.return_value.first.return_value = mock_row + load_session(mock_db) + # Должно быть минимум 2 вызова execute: SELECT + UPDATE last_used_at + assert mock_db.execute.call_count >= 2 + + +# --------------------------------------------------------------------------- +# mark_session_invalid +# --------------------------------------------------------------------------- + + +def test_mark_session_invalid_updates_table(mock_db: MagicMock) -> None: + mark_session_invalid(mock_db, account_user_id=12345) + args, _ = mock_db.execute.call_args + sql = str(args[0]) + assert "last_invalid_at" in sql + assert "cian_session_cookies" in sql + + +def test_mark_session_invalid_commits(mock_db: MagicMock) -> None: + mark_session_invalid(mock_db, account_user_id=99) + assert mock_db.commit.called + + +def test_mark_session_invalid_uses_cast(mock_db: MagicMock) -> None: + mark_session_invalid(mock_db, account_user_id=5) + args, _ = mock_db.execute.call_args + sql = str(args[0]) + assert "CAST(" in sql + assert "::bigint" not in sql + + +# --------------------------------------------------------------------------- +# verify_session (async) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_verify_session_returns_none_when_state_missing(monkeypatch: pytest.MonkeyPatch) -> None: + """extract_state returns None → verify_session returns None.""" + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: None, + ) + + async def fake_get(url: str, **kwargs): # noqa: ARG001 + class FakeResp: + text = "" + def raise_for_status(self) -> None: + pass + return FakeResp() + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(return_value=_make_fake_resp("")) + mock_client_cls.return_value = mock_client + + result = await verify_session({"_ym_uid": "x"}) + assert result is None + + +@pytest.mark.asyncio +async def test_verify_session_returns_none_when_not_authenticated(monkeypatch: pytest.MonkeyPatch) -> None: + """state.user.isAuthenticated is false → verify_session returns None.""" + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}}, + ) + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(return_value=_make_fake_resp("")) + mock_client_cls.return_value = mock_client + + result = await verify_session({"_ym_uid": "x"}) + assert result is None + + +@pytest.mark.asyncio +async def test_verify_session_returns_state_when_authenticated(monkeypatch: pytest.MonkeyPatch) -> None: + """state.user.isAuthenticated is true → returns state dict.""" + expected_state = {"user": {"isAuthenticated": True, "userId": 102963817}} + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: expected_state, + ) + + with patch("httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.get = AsyncMock(return_value=_make_fake_resp("state here")) + mock_client_cls.return_value = mock_client + + result = await verify_session({"_ym_uid": "x", "_cian_uid": "y"}) + assert result is not None + assert result["user"]["isAuthenticated"] is True + assert result["user"]["userId"] == 102963817 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_fake_resp(text: str) -> MagicMock: + resp = MagicMock() + resp.text = text + resp.raise_for_status = MagicMock() + return resp diff --git a/tradein-mvp/scripts/upload-cian-cookies.sh b/tradein-mvp/scripts/upload-cian-cookies.sh new file mode 100644 index 00000000..57be0858 --- /dev/null +++ b/tradein-mvp/scripts/upload-cian-cookies.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Upload Cian session cookies to backend for Valuation Calculator auth. +# +# Usage: ./scripts/upload-cian-cookies.sh [cookies.json] +# +# cookies.json — JSON object { cookie_name: cookie_value, ... } +# Экспортируется из Chrome DevTools → Application → Cookies → www.cian.ru +# или через расширение Cookie-Editor → Export → JSON (object format). +# +# Переменные окружения: +# TRADEIN_ADMIN_TOKEN — admin token (или берётся из .env.runtime) +# TRADEIN_BASE_URL — base URL (default: https://gendsgn.ru/trade-in) + +set -euo pipefail + +COOKIES_FILE="${1:-cookies.json}" + +if [[ ! -f "$COOKIES_FILE" ]]; then + echo "Error: cookies file not found: $COOKIES_FILE" >&2 + echo "Usage: $0 cookies.json" >&2 + exit 1 +fi + +# Загружаем токен из env или .env.runtime +ADMIN_TOKEN="${TRADEIN_ADMIN_TOKEN:-}" +if [[ -z "$ADMIN_TOKEN" ]] && [[ -f ".env.runtime" ]]; then + ADMIN_TOKEN="$(grep '^TRADEIN_ADMIN_TOKEN=' .env.runtime 2>/dev/null | cut -d= -f2- || true)" +fi + +if [[ -z "$ADMIN_TOKEN" ]]; then + echo "Error: TRADEIN_ADMIN_TOKEN not set (env or .env.runtime)" >&2 + exit 1 +fi + +BASE_URL="${TRADEIN_BASE_URL:-https://gendsgn.ru/trade-in}" + +echo "Uploading Cian cookies from $COOKIES_FILE to $BASE_URL ..." +echo + +curl -fsSL -X POST "$BASE_URL/api/v1/admin/scrape/cian/upload-cookies" \ + -H "X-Admin-Token: $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d "@$COOKIES_FILE" \ + | python3 -m json.tool + +echo +echo "Verifying session:" +curl -fsSL "$BASE_URL/api/v1/admin/scrape/cian/test-auth" \ + -H "X-Admin-Token: $ADMIN_TOKEN" \ + | python3 -m json.tool