feat(tradein): cian_session cookies management + admin endpoints (Wave 4 Worker A) (#457)

tradein-mvp/backend/app/api/v1/admin.py

@@ -16,6 +16,7 @@ from sqlalchemy.orm import Session
 
 from app.core.config import settings
 from app.core.db import get_db
+from app.services import cian_session as cian_session_svc
 from app.services.geocoder import geocode
 from app.services.scrapers.avito import AvitoScraper
 from app.services.scrapers.base import save_listings
@@ -233,3 +234,72 @@ async def geocode_missing(
         "skipped": skipped,
         "remaining": int(remaining or 0),
     }
+
+
+@router.post("/scrape/cian/upload-cookies", status_code=200, dependencies=[Depends(require_admin)])
+async def upload_cian_cookies(
+    cookies: dict[str, str],
+    db: Annotated[Session, Depends(get_db)],
+) -> dict:
+    """Upload Cian session cookies для аутентификации Valuation Calculator.
+
+    Body: JSON object вида { "_ym_uid": "...", "_cian_visitor_session_id": "...", ... }
+    Cookies экспортируются из Chrome DevTools → Application → Cookies → www.cian.ru
+    или через расширение Cookie-Editor → Export → JSON (object format).
+
+    Шаги:
+      1. Фильтрует payload по CIAN_REQUIRED_COOKIES.
+      2. Верифицирует через GET /kalkulator-nedvizhimosti/ — проверяет isAuthenticated.
+      3. Сохраняет зашифрованно (pgp_sym_encrypt) в cian_session_cookies.
+
+    Returns: {"ok": true, "userId": <int>, "cookieCount": <int>}
+    """
+    if not settings.cookie_encryption_key:
+        raise HTTPException(status_code=503, detail="COOKIE_ENCRYPTION_KEY not configured")
+
+    cleaned = {k: v for k, v in cookies.items() if k in cian_session_svc.CIAN_REQUIRED_COOKIES}
+    if not cleaned:
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                f"No recognized Cian cookies in payload. "
+                f"Expected any of: {sorted(cian_session_svc.CIAN_REQUIRED_COOKIES)}"
+            ),
+        )
+
+    state = await cian_session_svc.verify_session(cleaned)
+    if state is None:
+        raise HTTPException(
+            status_code=401,
+            detail="Cookies invalid or session not authenticated on cian.ru",
+        )
+
+    user_id = state.get("user", {}).get("userId")
+    if not user_id:
+        raise HTTPException(status_code=400, detail="Authenticated state missing userId")
+
+    cian_session_svc.save_session(db, account_user_id=int(user_id), cookies=cleaned)
+    return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)}
+
+
+@router.get("/scrape/cian/test-auth", status_code=200, dependencies=[Depends(require_admin)])
+async def test_cian_auth(
+    db: Annotated[Session, Depends(get_db)],
+) -> dict:
+    """Проверить что текущие сохранённые Cian cookies ещё валидны.
+
+    Returns: {"authenticated": bool, "userId": <int|null>, "reason": <str|null>}
+    """
+    if not settings.cookie_encryption_key:
+        return {"authenticated": False, "userId": None, "reason": "encryption_key_not_configured"}
+
+    cookies = cian_session_svc.load_session(db)
+    if cookies is None:
+        return {"authenticated": False, "userId": None, "reason": "no_session_in_db"}
+
+    state = await cian_session_svc.verify_session(cookies)
+    if state is None:
+        return {"authenticated": False, "userId": None, "reason": "session_expired_or_invalid"}
+
+    user_id = state.get("user", {}).get("userId")
+    return {"authenticated": True, "userId": user_id, "reason": None}

tradein-mvp/backend/app/core/config.py

@@ -29,5 +29,9 @@ class Settings(BaseSettings):
     # Пусто = мониторинг выключен (dev). В prod — env GLITCHTIP_DSN из .env.runtime.
     glitchtip_dsn: str | None = None
 
+    # Ключ шифрования для pgp_sym_encrypt (Cian session cookies).
+    # Задаётся через env COOKIE_ENCRYPTION_KEY. Пусто = шифрование не работает.
+    cookie_encryption_key: str = ""
+
 
 settings = Settings()

tradein-mvp/backend/app/services/cian_session.py

@@ -0,0 +1,196 @@
+"""Cian session cookie management — load/save/verify encrypted cookies.
+
+Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API.
+Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+import httpx
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.core.config import settings
+from app.services.scrapers.cian_state_parser import extract_state
+
+logger = logging.getLogger(__name__)
+
+# Cookies критичные для Cian auth — фильтр перед сохранением.
+# Conservative set подтверждён в Stage 4 (probe через DevTools на live session).
+CIAN_REQUIRED_COOKIES: set[str] = {
+    "_ym_uid",
+    "_ym_d",
+    "_ym_isad",
+    "_cian_visitor_session_id",
+    "_cian_app_session_id",
+    "_cian_uid",
+    "_ga",
+    "tlsr_id",
+    "cf_clearance",
+    "session-id",
+    "anti_bot",
+    "csrftoken",
+}
+
+_VALUATION_TEST_URL = (
+    "https://www.cian.ru/kalkulator-nedvizhimosti/"
+    "?address=%D0%A1%D0%B2%D0%B5%D1%80%D0%B4%D0%BB%D0%BE%D0%B2%D1%81%D0%BA%D0%B0%D1%8F"
+    "%20%D0%BE%D0%B1%D0%BB%2C%20%D0%95%D0%BA%D0%B0%D1%82%D0%B5%D1%80%D0%B8%D0%BD%D0%B1%D1%83%D1%80%D0%B3"
+    "&totalArea=40&roomsCount=1&valuationType=sale"
+    "&floor%5B0%5D=floorOther&repairType%5B0%5D=repairTypeCosmetic"
+)
+
+_MFE_VALUATION = "valuation-for-agent-frontend"
+
+
+async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
+    """Hit Cian Valuation Calculator with cookies — return parsed state if authenticated.
+
+    Returns state dict containing user.isAuthenticated + userId, or None if
+    cookies are invalid/expired or state extraction fails.
+    Никогда не логирует сырые значения cookies.
+    """
+    try:
+        async with httpx.AsyncClient(
+            cookies=cookies,
+            timeout=20.0,
+            follow_redirects=True,
+        ) as client:
+            resp = await client.get(_VALUATION_TEST_URL)
+            resp.raise_for_status()
+            state = extract_state(resp.text, mfe=_MFE_VALUATION, key="initialState")
+            if state is None:
+                logger.warning(
+                    "Cian cookies verify: state extraction failed (mfe=%s)", _MFE_VALUATION
+                )
+                return None
+            user = state.get("user", {})
+            if not user.get("isAuthenticated"):
+                logger.warning("Cian cookies verify: user.isAuthenticated=false")
+                return None
+            logger.info("Cian cookies verified — userId=%s", user.get("userId"))
+            return state
+    except httpx.HTTPStatusError as exc:
+        logger.warning(
+            "Cian cookies verify HTTP error: %s %s",
+            exc.response.status_code,
+            exc.request.url,
+        )
+        return None
+    except Exception as exc:
+        logger.warning("Cian cookies verify failed: %s", exc)
+        return None
+
+
+def save_session(
+    db: Session,
+    account_user_id: int,
+    cookies: dict[str, str],
+    ttl_days: int = 30,
+) -> None:
+    """Encrypt cookies via pgp_sym_encrypt and UPSERT into cian_session_cookies.
+
+    Uses settings.cookie_encryption_key as the encryption secret.
+    Никогда не логирует сырые значения cookies.
+    """
+    cookies_json = json.dumps(cookies)
+    db.execute(
+        text("""
+            INSERT INTO cian_session_cookies (
+                account_user_id,
+                cookies_encrypted,
+                expires_at_estimate,
+                uploaded_at
+            ) VALUES (
+                CAST(:uid AS bigint),
+                pgp_sym_encrypt(:cookies_json, :key),
+                NOW() + (CAST(:ttl_days AS int) || ' days')::interval,
+                NOW()
+            )
+            ON CONFLICT (account_user_id) DO UPDATE SET
+                cookies_encrypted    = EXCLUDED.cookies_encrypted,
+                expires_at_estimate  = EXCLUDED.expires_at_estimate,
+                uploaded_at          = NOW(),
+                last_invalid_at      = NULL
+        """),
+        {
+            "uid": account_user_id,
+            "cookies_json": cookies_json,
+            "key": settings.cookie_encryption_key,
+            "ttl_days": ttl_days,
+        },
+    )
+    db.commit()
+    logger.info(
+        "Cian cookies saved for userId=%s (count=%d, ttl=%d days)",
+        account_user_id,
+        len(cookies),
+        ttl_days,
+    )
+
+
+def load_session(db: Session) -> dict[str, str] | None:
+    """Load most-recently-uploaded valid Cian cookies (decrypt).
+
+    Returns dict[cookie_name, cookie_value] или None если нет валидной session.
+    Выбирает только записи где expires_at_estimate > NOW() и сессия не
+    была инвалидирована после последнего upload'а.
+    """
+    row = db.execute(
+        text("""
+            SELECT
+                account_user_id,
+                pgp_sym_decrypt(cookies_encrypted, :key)::text AS cookies_json,
+                expires_at_estimate
+            FROM cian_session_cookies
+            WHERE expires_at_estimate > NOW()
+              AND (last_invalid_at IS NULL OR last_invalid_at < uploaded_at)
+            ORDER BY uploaded_at DESC
+            LIMIT 1
+        """),
+        {"key": settings.cookie_encryption_key},
+    ).mappings().first()
+
+    if row is None:
+        logger.warning("No valid Cian session cookies in DB")
+        return None
+
+    cookies: dict[str, str] = json.loads(row["cookies_json"])
+
+    # Обновляем last_used_at — не критично, игнорируем ошибки.
+    try:
+        db.execute(
+            text(
+                "UPDATE cian_session_cookies SET last_used_at = NOW()"
+                " WHERE account_user_id = CAST(:uid AS bigint)"
+            ),
+            {"uid": row["account_user_id"]},
+        )
+        db.commit()
+    except Exception as exc:
+        logger.warning(
+            "Failed to update last_used_at for userId=%s: %s", row["account_user_id"], exc
+        )
+
+    logger.info(
+        "Cian cookies loaded for userId=%s (count=%d)",
+        row["account_user_id"],
+        len(cookies),
+    )
+    return cookies
+
+
+def mark_session_invalid(db: Session, account_user_id: int) -> None:
+    """Flag session как expired/invalid (например после 401 во время scrape)."""
+    db.execute(
+        text(
+            "UPDATE cian_session_cookies SET last_invalid_at = NOW()"
+            " WHERE account_user_id = CAST(:uid AS bigint)"
+        ),
+        {"uid": account_user_id},
+    )
+    db.commit()
+    logger.warning("Cian session marked invalid for userId=%s", account_user_id)

tradein-mvp/backend/app/services/scrapers/cian_detail.py

@@ -0,0 +1,324 @@
+"""Cian.ru detail page scraper — single offer enrichment.
+
+Different from SERP (cian.py):
+- URL: https://ekb.cian.ru/sale/flat/<offer_id>/ or https://www.cian.ru/sale/flat/<offer_id>/
+- MFE: 'frontend-offer-card'
+- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState)
+- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject
+
+Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
+Stage 5 of CianScraper v1.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+from curl_cffi.requests import AsyncSession
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DetailEnrichment:
+    """Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows."""
+
+    # Core listing enrichment (extend existing listings row)
+    cian_id: int | None = None
+    windows_view_type: str | None = None        # 'yard' / 'street' / 'yardAndStreet'
+    separate_wcs_count: int | None = None
+    combined_wcs_count: int | None = None
+    ceiling_height: float | None = None         # meters
+    repair_type: str | None = None              # 'cosmetic' / 'design' / 'no'
+    views_total: int | None = None              # Cian stats.totalViewsFormattedString → int
+    views_today: int | None = None
+
+    # BTI (вторичка only — primary buildings don't have БТИ)
+    bti_data: dict[str, Any] | None = None      # raw bti.houseData snapshot
+
+    # Price history (offer.priceChanges)
+    price_changes: list[dict[str, Any]] = field(default_factory=list)
+
+    # Agent profile (offer.agent.* expanded)
+    agent_profile: dict[str, Any] | None = None
+
+    # Newbuilding link (if offer is from a newbuilding)
+    newbuilding_data: dict[str, Any] | None = None
+
+    # Raw payload для debugging / future-extraction
+    raw_offer: dict[str, Any] | None = None
+    raw_sister_states: dict[str, Any] = field(default_factory=dict)
+
+
+async def fetch_detail(
+    offer_url: str,
+    *,
+    session: AsyncSession | None = None,
+) -> DetailEnrichment | None:
+    """Fetch detail page and extract all containers.
+
+    Args:
+        offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/'
+        session: optional shared curl_cffi session (для batched scrapes)
+
+    Returns: DetailEnrichment, or None если fetch / parse failed.
+    """
+    close_session = False
+    if session is None:
+        session = AsyncSession(
+            impersonate="chrome120",
+            timeout=25.0,
+            headers={
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
+            },
+        )
+        close_session = True
+
+    try:
+        resp = await session.get(offer_url, allow_redirects=True)
+        if resp.status_code != 200:
+            logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code)
+            return None
+        html = resp.text
+
+        # Primary state container — defaultState in frontend-offer-card MFE
+        # NOTE: detail pages use 'defaultState', SERP uses 'initialState'
+        offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
+        if offer_state is None:
+            logger.warning("Cian detail %s: defaultState extraction failed", offer_url)
+            return None
+
+        offer_data = offer_state.get("offerData", {})
+        offer = offer_data.get("offer", {})
+
+        result = DetailEnrichment(
+            cian_id=offer.get("cianId") or offer.get("id"),
+            windows_view_type=_extract_windows_view(offer),
+            separate_wcs_count=offer.get("separateWcsCount"),
+            combined_wcs_count=offer.get("combinedWcsCount"),
+            ceiling_height=_parse_float(offer.get("ceilingHeight")),
+            repair_type=_extract_repair_type(offer),
+            raw_offer=offer,
+        )
+
+        # Stats (views) — from stats key in offerData or top-level state
+        stats = offer_data.get("stats") or offer_state.get("stats") or {}
+        result.views_total = _parse_views(stats.get("totalViewsFormattedString"))
+        result.views_today = _parse_views(stats.get("todayViewsFormattedString"))
+
+        # Price changes — from offer.priceChanges or state-level priceChanges
+        result.price_changes = _extract_price_changes(offer, offer_state)
+
+        # Agent profile expansion
+        result.agent_profile = _extract_agent_profile(offer)
+
+        # Sister state containers: bti, newObject, plus raw stash of everything else
+        all_states = extract_all_states(html)
+        offer_card_states = all_states.get("frontend-offer-card", {})
+
+        bti_state = offer_card_states.get("bti")
+        if bti_state:
+            result.bti_data = bti_state.get("houseData")
+            result.raw_sister_states["bti"] = bti_state
+
+        # Newbuilding link (if applicable)
+        newbuilding = offer.get("newbuilding")
+        if newbuilding and newbuilding.get("id"):
+            result.newbuilding_data = newbuilding
+
+        # Stash all non-defaultState sister containers for debugging / future-extraction
+        for k, v in offer_card_states.items():
+            if k != "defaultState":
+                result.raw_sister_states[k] = v
+
+        return result
+
+    finally:
+        if close_session:
+            await session.close()
+
+
+# ── Field extractors ──────────────────────────────────────────────────────────
+
+
+def _extract_windows_view(offer: dict[str, Any]) -> str | None:
+    return offer.get("windowsViewType")
+
+
+def _extract_repair_type(offer: dict[str, Any]) -> str | None:
+    # Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ...
+    return offer.get("repairType")
+
+
+def _parse_float(value: Any) -> float | None:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def _parse_views(formatted_str: str | None) -> int | None:
+    """Cian's stats.totalViewsFormattedString — '1 234' → 1234."""
+    if not formatted_str:
+        return None
+    try:
+        return int(str(formatted_str).replace(" ", "").replace(",", ""))
+    except (ValueError, TypeError):
+        return None
+
+
+def _extract_price_changes(offer: dict[str, Any], state: dict[str, Any]) -> list[dict[str, Any]]:
+    """offer.priceChanges or state.priceChanges — normalize to common form."""
+    raw = offer.get("priceChanges") or state.get("priceChanges") or []
+    if not isinstance(raw, list):
+        return []
+    result = []
+    for entry in raw:
+        if not isinstance(entry, dict):
+            continue
+        result.append(
+            {
+                "change_time": entry.get("changeTime") or entry.get("date"),
+                "price_rub": entry.get("price") or entry.get("priceRub"),
+                "diff_percent": entry.get("diffPercent") or entry.get("diff_percent"),
+            }
+        )
+    return result
+
+
+def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
+    """offer.agent or offer.agency or offer.user — extract canonical profile shape."""
+    agent = offer.get("agent") or {}
+    user = offer.get("user") or {}
+    if not agent and not user:
+        return None
+    return {
+        "ext_agent_id": agent.get("id") or user.get("cianUserId"),
+        "company_name": agent.get("companyName") or user.get("companyName"),
+        "is_pro": agent.get("isPro") or user.get("isPro"),
+        "skills": agent.get("skills") or [],
+        "account_type": agent.get("accountType") or user.get("accountType"),
+    }
+
+
+# ── Save helpers ──────────────────────────────────────────────────────────────
+
+
+def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None:
+    """Persist DetailEnrichment to DB:
+
+    - UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
+    - INSERT INTO offer_price_history rows (if price_changes non-empty)
+    - INSERT INTO agents (if agent_profile) and link via agent_id_fk
+    - bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id
+
+    Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
+    """
+    # Extend listings row with detail-enriched fields
+    db.execute(
+        text("""
+            UPDATE listings SET
+                windows_view_type  = COALESCE(:wvt, windows_view_type),
+                separate_wcs_count = COALESCE(:swc, separate_wcs_count),
+                combined_wcs_count = COALESCE(:cwc, combined_wcs_count),
+                ceiling_height     = COALESCE(:ch,  ceiling_height),
+                repair_type        = COALESCE(:rt,  repair_type),
+                views_total        = COALESCE(:vt,  views_total),
+                views_today        = COALESCE(:vd,  views_today)
+            WHERE id = CAST(:lid AS bigint)
+        """),
+        {
+            "lid": listing_id,
+            "wvt": enrichment.windows_view_type,
+            "swc": enrichment.separate_wcs_count,
+            "cwc": enrichment.combined_wcs_count,
+            "ch": enrichment.ceiling_height,
+            "rt": enrichment.repair_type,
+            "vt": enrichment.views_total,
+            "vd": enrichment.views_today,
+        },
+    )
+
+    # Price changes — INSERT new rows, de-dup by listing_id + change_time
+    for change in enrichment.price_changes:
+        if not change.get("change_time") or not change.get("price_rub"):
+            continue
+        db.execute(
+            text("""
+                INSERT INTO offer_price_history
+                    (listing_id, change_time, price_rub, diff_percent, source)
+                VALUES (
+                    CAST(:lid AS bigint),
+                    CAST(:ct AS timestamptz),
+                    CAST(:price AS numeric),
+                    CAST(:diff AS numeric),
+                    'cian'
+                )
+                ON CONFLICT DO NOTHING
+            """),
+            {
+                "lid": listing_id,
+                "ct": change["change_time"],
+                "price": change["price_rub"],
+                "diff": change.get("diff_percent"),
+            },
+        )
+
+    # Agent profile UPSERT — store agent row and link to listing
+    if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"):
+        ap = enrichment.agent_profile
+        row = db.execute(
+            text("""
+                INSERT INTO agents
+                    (ext_source, ext_agent_id, company_name, is_pro, skills, account_type)
+                VALUES (
+                    'cian',
+                    :eai,
+                    :cn,
+                    :ip,
+                    CAST(:sk AS jsonb),
+                    :at
+                )
+                ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET
+                    company_name = COALESCE(EXCLUDED.company_name, agents.company_name),
+                    is_pro       = COALESCE(EXCLUDED.is_pro,       agents.is_pro),
+                    skills       = COALESCE(EXCLUDED.skills,       agents.skills),
+                    account_type = COALESCE(EXCLUDED.account_type, agents.account_type)
+                RETURNING id
+            """),
+            {
+                "eai": str(ap["ext_agent_id"]),
+                "cn": ap.get("company_name"),
+                "ip": ap.get("is_pro"),
+                "sk": json.dumps(ap.get("skills") or []),
+                "at": ap.get("account_type"),
+            },
+        )
+        agent_db_id = row.scalar_one_or_none()
+        if agent_db_id is not None:
+            db.execute(
+                text("""
+                    UPDATE listings
+                    SET agent_id_fk = CAST(:aid AS bigint)
+                    WHERE id = CAST(:lid AS bigint)
+                      AND agent_id_fk IS NULL
+                """),
+                {"aid": agent_db_id, "lid": listing_id},
+            )
+
+    db.commit()
+    logger.info(
+        "Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s)",
+        listing_id,
+        len(enrichment.price_changes),
+        bool(enrichment.agent_profile),
+    )

tradein-mvp/backend/tests/test_cian_detail.py

@@ -0,0 +1,324 @@
+"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from app.services.scrapers.cian_detail import (
+    DetailEnrichment,
+    _extract_agent_profile,
+    _extract_price_changes,
+    _parse_views,
+    fetch_detail,
+)
+
+
+# ── _parse_views ──────────────────────────────────────────────────────────────
+
+
+def test_parse_views_space_formatted():
+    assert _parse_views("1 234") == 1234
+
+
+def test_parse_views_plain():
+    assert _parse_views("567") == 567
+
+
+def test_parse_views_none():
+    assert _parse_views(None) is None
+
+
+def test_parse_views_invalid():
+    assert _parse_views("invalid") is None
+
+
+def test_parse_views_comma_formatted():
+    assert _parse_views("1,234") == 1234
+
+
+# ── _extract_price_changes ────────────────────────────────────────────────────
+
+
+def test_extract_price_changes_normalizes():
+    offer = {
+        "priceChanges": [
+            {"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5},
+            {"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0},
+        ]
+    }
+    result = _extract_price_changes(offer, {})
+    assert len(result) == 2
+    assert result[0]["change_time"] == "2026-04-01T00:00:00Z"
+    assert result[0]["price_rub"] == 5000000
+    assert result[0]["diff_percent"] == -2.5
+
+
+def test_extract_price_changes_handles_empty_offer():
+    assert _extract_price_changes({}, {}) == []
+
+
+def test_extract_price_changes_handles_none_value():
+    assert _extract_price_changes({"priceChanges": None}, {}) == []
+
+
+def test_extract_price_changes_falls_back_to_state():
+    state = {
+        "priceChanges": [
+            {"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0},
+        ]
+    }
+    result = _extract_price_changes({}, state)
+    assert len(result) == 1
+    assert result[0]["price_rub"] == 6000000
+
+
+def test_extract_price_changes_skips_non_dict_entries():
+    offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]}
+    result = _extract_price_changes(offer, {})
+    assert len(result) == 1
+
+
+# ── _extract_agent_profile ────────────────────────────────────────────────────
+
+
+def test_extract_agent_profile_from_agent_key():
+    offer = {
+        "agent": {
+            "id": 12345,
+            "companyName": "Новосёл",
+            "isPro": True,
+            "skills": [{"id": 1, "name": "Продажа"}],
+            "accountType": "specialist",
+        }
+    }
+    profile = _extract_agent_profile(offer)
+    assert profile is not None
+    assert profile["ext_agent_id"] == 12345
+    assert profile["company_name"] == "Новосёл"
+    assert profile["is_pro"] is True
+    assert profile["account_type"] == "specialist"
+
+
+def test_extract_agent_profile_from_user_key():
+    offer = {
+        "user": {
+            "cianUserId": 99999,
+            "companyName": "Агентство Икс",
+            "isPro": False,
+            "accountType": "agency",
+        }
+    }
+    profile = _extract_agent_profile(offer)
+    assert profile is not None
+    assert profile["ext_agent_id"] == 99999
+    assert profile["company_name"] == "Агентство Икс"
+
+
+def test_extract_agent_profile_returns_none_when_no_agent():
+    assert _extract_agent_profile({}) is None
+    assert _extract_agent_profile({"agent": {}, "user": {}}) is None
+
+
+# ── DetailEnrichment defaults ─────────────────────────────────────────────────
+
+
+def test_dataclass_defaults():
+    enrich = DetailEnrichment()
+    assert enrich.cian_id is None
+    assert enrich.price_changes == []
+    assert enrich.raw_sister_states == {}
+    assert enrich.bti_data is None
+    assert enrich.agent_profile is None
+    assert enrich.newbuilding_data is None
+
+
+# ── fetch_detail integration (mocked) ────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_fetch_detail_returns_none_on_no_state(monkeypatch):
+    """If state extraction returns None → fetch_detail returns None gracefully."""
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_detail.extract_state",
+        lambda html, mfe, key: None,
+    )
+
+    response = MagicMock()
+    response.status_code = 200
+    response.text = "<html></html>"
+
+    session = MagicMock()
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+
+    result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session)
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_fetch_detail_returns_none_on_non_200(monkeypatch):
+    """Non-200 HTTP response → returns None, no exception raised."""
+    response = MagicMock()
+    response.status_code = 403
+    response.text = ""
+
+    session = MagicMock()
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+
+    result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session)
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_fetch_detail_extracts_core_fields(monkeypatch):
+    """Verify core fields are extracted correctly from mocked state."""
+    fake_state = {
+        "offerData": {
+            "offer": {
+                "cianId": 327830237,
+                "windowsViewType": "yardAndStreet",
+                "separateWcsCount": 1,
+                "combinedWcsCount": 0,
+                "ceilingHeight": 2.65,
+                "repairType": "cosmetic",
+                "priceChanges": [
+                    {
+                        "changeTime": "2026-04-01T00:00:00Z",
+                        "price": 5500000,
+                        "diffPercent": -1.8,
+                    }
+                ],
+                "agent": {
+                    "id": 7654321,
+                    "companyName": "Первичка Плюс",
+                    "isPro": True,
+                    "skills": [],
+                    "accountType": "specialist",
+                },
+            },
+            "stats": {
+                "totalViewsFormattedString": "1 042",
+                "todayViewsFormattedString": "7",
+            },
+        }
+    }
+
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_detail.extract_state",
+        lambda html, mfe, key: fake_state if key == "defaultState" else None,
+    )
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_detail.extract_all_states",
+        lambda html: {},
+    )
+
+    response = MagicMock()
+    response.status_code = 200
+    response.text = "<html></html>"
+
+    session = MagicMock()
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+
+    result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session)
+    assert result is not None
+    assert result.cian_id == 327830237
+    assert result.windows_view_type == "yardAndStreet"
+    assert result.separate_wcs_count == 1
+    assert result.combined_wcs_count == 0
+    assert result.ceiling_height == 2.65
+    assert result.repair_type == "cosmetic"
+    assert result.views_total == 1042
+    assert result.views_today == 7
+    assert len(result.price_changes) == 1
+    assert result.price_changes[0]["price_rub"] == 5500000
+    assert result.agent_profile is not None
+    assert result.agent_profile["ext_agent_id"] == 7654321
+    assert result.agent_profile["company_name"] == "Первичка Плюс"
+
+
+@pytest.mark.asyncio
+async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch):
+    """BTI sister state is captured into bti_data."""
+    fake_state = {
+        "offerData": {
+            "offer": {
+                "cianId": 111111,
+            }
+        }
+    }
+    fake_all_states = {
+        "frontend-offer-card": {
+            "bti": {
+                "houseData": {
+                    "seriesName": "П-44Т",
+                    "floorsCount": 17,
+                    "flatCount": 120,
+                }
+            }
+        }
+    }
+
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_detail.extract_state",
+        lambda html, mfe, key: fake_state if key == "defaultState" else None,
+    )
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_detail.extract_all_states",
+        lambda html: fake_all_states,
+    )
+
+    response = MagicMock()
+    response.status_code = 200
+    response.text = "<html></html>"
+
+    session = MagicMock()
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+
+    result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session)
+    assert result is not None
+    assert result.bti_data is not None
+    assert result.bti_data["seriesName"] == "П-44Т"
+    assert "bti" in result.raw_sister_states
+
+
+@pytest.mark.asyncio
+async def test_fetch_detail_extracts_newbuilding_link(monkeypatch):
+    """Newbuilding reference is captured from offer.newbuilding."""
+    fake_state = {
+        "offerData": {
+            "offer": {
+                "cianId": 222222,
+                "newbuilding": {
+                    "id": 54016,
+                    "name": "Екатерининский парк",
+                    "url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/",
+                },
+            }
+        }
+    }
+
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_detail.extract_state",
+        lambda html, mfe, key: fake_state if key == "defaultState" else None,
+    )
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_detail.extract_all_states",
+        lambda html: {},
+    )
+
+    response = MagicMock()
+    response.status_code = 200
+    response.text = "<html></html>"
+
+    session = MagicMock()
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+
+    result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session)
+    assert result is not None
+    assert result.newbuilding_data is not None
+    assert result.newbuilding_data["id"] == 54016

tradein-mvp/backend/tests/test_cian_session.py

@@ -0,0 +1,238 @@
+"""Tests for cian_session — cookie management service."""
+from __future__ import annotations
+
+import json
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from app.services.cian_session import (
+    CIAN_REQUIRED_COOKIES,
+    load_session,
+    mark_session_invalid,
+    save_session,
+    verify_session,
+)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def mock_db() -> MagicMock:
+    db = MagicMock()
+    # Default: no row found (load_session → None)
+    db.execute.return_value.mappings.return_value.first.return_value = None
+    return db
+
+
+# ---------------------------------------------------------------------------
+# CIAN_REQUIRED_COOKIES
+# ---------------------------------------------------------------------------
+
+
+def test_required_cookies_set_not_empty() -> None:
+    assert isinstance(CIAN_REQUIRED_COOKIES, set)
+    assert len(CIAN_REQUIRED_COOKIES) >= 5
+
+
+def test_required_cookies_contains_key_names() -> None:
+    assert "_cian_visitor_session_id" in CIAN_REQUIRED_COOKIES
+    assert "_ym_uid" in CIAN_REQUIRED_COOKIES
+    assert "_cian_uid" in CIAN_REQUIRED_COOKIES
+    assert "tlsr_id" in CIAN_REQUIRED_COOKIES
+    assert "cf_clearance" in CIAN_REQUIRED_COOKIES
+
+
+# ---------------------------------------------------------------------------
+# save_session
+# ---------------------------------------------------------------------------
+
+
+def test_save_session_calls_pgp_sym_encrypt(mock_db: MagicMock) -> None:
+    save_session(mock_db, account_user_id=12345, cookies={"_ym_uid": "abc"})
+    args, _ = mock_db.execute.call_args
+    sql_text = str(args[0])
+    assert "pgp_sym_encrypt" in sql_text
+    assert "cian_session_cookies" in sql_text
+
+
+def test_save_session_uses_cast_not_colon_colon(mock_db: MagicMock) -> None:
+    """Verify psycopg v3 compatibility — no ::bigint syntax in SQL."""
+    save_session(mock_db, account_user_id=99, cookies={"csrftoken": "x"})
+    args, _ = mock_db.execute.call_args
+    sql_text = str(args[0])
+    # CAST(:x AS type) is required; ::type would fail with psycopg v3 named params
+    assert "CAST(" in sql_text
+    assert "::bigint" not in sql_text
+
+
+def test_save_session_commits(mock_db: MagicMock) -> None:
+    save_session(mock_db, account_user_id=12345, cookies={"_ym_uid": "abc"})
+    assert mock_db.commit.called
+
+
+def test_save_session_on_conflict_do_update(mock_db: MagicMock) -> None:
+    save_session(mock_db, account_user_id=1, cookies={"_ym_uid": "v"})
+    args, _ = mock_db.execute.call_args
+    sql_text = str(args[0])
+    assert "ON CONFLICT" in sql_text
+    assert "DO UPDATE" in sql_text
+
+
+def test_save_session_passes_correct_params(mock_db: MagicMock) -> None:
+    save_session(mock_db, account_user_id=777, cookies={"_ga": "GA1.2.x"}, ttl_days=14)
+    _, kwargs = mock_db.execute.call_args
+    params = kwargs if kwargs else mock_db.execute.call_args[0][1]
+    # params могут передаваться как второй позиционный аргумент
+    call_args = mock_db.execute.call_args
+    params = call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("params", {})
+    assert params["uid"] == 777
+    assert params["ttl_days"] == 14
+    cookies_payload = json.loads(params["cookies_json"])
+    assert cookies_payload == {"_ga": "GA1.2.x"}
+
+
+# ---------------------------------------------------------------------------
+# load_session
+# ---------------------------------------------------------------------------
+
+
+def test_load_session_returns_none_when_empty(mock_db: MagicMock) -> None:
+    result = load_session(mock_db)
+    assert result is None
+
+
+def test_load_session_decodes_json(mock_db: MagicMock) -> None:
+    mock_row = {
+        "account_user_id": 12345,
+        "cookies_json": json.dumps({"_ym_uid": "abc", "_cian_uid": "def"}),
+        "expires_at_estimate": None,
+    }
+    mock_db.execute.return_value.mappings.return_value.first.return_value = mock_row
+    result = load_session(mock_db)
+    assert result == {"_ym_uid": "abc", "_cian_uid": "def"}
+
+
+def test_load_session_updates_last_used_at(mock_db: MagicMock) -> None:
+    mock_row = {
+        "account_user_id": 42,
+        "cookies_json": json.dumps({"session-id": "s"}),
+        "expires_at_estimate": None,
+    }
+    mock_db.execute.return_value.mappings.return_value.first.return_value = mock_row
+    load_session(mock_db)
+    # Должно быть минимум 2 вызова execute: SELECT + UPDATE last_used_at
+    assert mock_db.execute.call_count >= 2
+
+
+# ---------------------------------------------------------------------------
+# mark_session_invalid
+# ---------------------------------------------------------------------------
+
+
+def test_mark_session_invalid_updates_table(mock_db: MagicMock) -> None:
+    mark_session_invalid(mock_db, account_user_id=12345)
+    args, _ = mock_db.execute.call_args
+    sql = str(args[0])
+    assert "last_invalid_at" in sql
+    assert "cian_session_cookies" in sql
+
+
+def test_mark_session_invalid_commits(mock_db: MagicMock) -> None:
+    mark_session_invalid(mock_db, account_user_id=99)
+    assert mock_db.commit.called
+
+
+def test_mark_session_invalid_uses_cast(mock_db: MagicMock) -> None:
+    mark_session_invalid(mock_db, account_user_id=5)
+    args, _ = mock_db.execute.call_args
+    sql = str(args[0])
+    assert "CAST(" in sql
+    assert "::bigint" not in sql
+
+
+# ---------------------------------------------------------------------------
+# verify_session (async)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_verify_session_returns_none_when_state_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+    """extract_state returns None → verify_session returns None."""
+    monkeypatch.setattr(
+        "app.services.cian_session.extract_state",
+        lambda html, mfe, key: None,
+    )
+
+    async def fake_get(url: str, **kwargs):  # noqa: ARG001
+        class FakeResp:
+            text = "<html></html>"
+            def raise_for_status(self) -> None:
+                pass
+        return FakeResp()
+
+    with patch("httpx.AsyncClient") as mock_client_cls:
+        mock_client = AsyncMock()
+        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+        mock_client.__aexit__ = AsyncMock(return_value=None)
+        mock_client.get = AsyncMock(return_value=_make_fake_resp("<html></html>"))
+        mock_client_cls.return_value = mock_client
+
+        result = await verify_session({"_ym_uid": "x"})
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_verify_session_returns_none_when_not_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
+    """state.user.isAuthenticated is false → verify_session returns None."""
+    monkeypatch.setattr(
+        "app.services.cian_session.extract_state",
+        lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}},
+    )
+
+    with patch("httpx.AsyncClient") as mock_client_cls:
+        mock_client = AsyncMock()
+        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+        mock_client.__aexit__ = AsyncMock(return_value=None)
+        mock_client.get = AsyncMock(return_value=_make_fake_resp("<html></html>"))
+        mock_client_cls.return_value = mock_client
+
+        result = await verify_session({"_ym_uid": "x"})
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_verify_session_returns_state_when_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
+    """state.user.isAuthenticated is true → returns state dict."""
+    expected_state = {"user": {"isAuthenticated": True, "userId": 102963817}}
+    monkeypatch.setattr(
+        "app.services.cian_session.extract_state",
+        lambda html, mfe, key: expected_state,
+    )
+
+    with patch("httpx.AsyncClient") as mock_client_cls:
+        mock_client = AsyncMock()
+        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
+        mock_client.__aexit__ = AsyncMock(return_value=None)
+        mock_client.get = AsyncMock(return_value=_make_fake_resp("<html>state here</html>"))
+        mock_client_cls.return_value = mock_client
+
+        result = await verify_session({"_ym_uid": "x", "_cian_uid": "y"})
+    assert result is not None
+    assert result["user"]["isAuthenticated"] is True
+    assert result["user"]["userId"] == 102963817
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_fake_resp(text: str) -> MagicMock:
+    resp = MagicMock()
+    resp.text = text
+    resp.raise_for_status = MagicMock()
+    return resp

tradein-mvp/scripts/upload-cian-cookies.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Upload Cian session cookies to backend for Valuation Calculator auth.
+#
+# Usage: ./scripts/upload-cian-cookies.sh [cookies.json]
+#
+# cookies.json — JSON object { cookie_name: cookie_value, ... }
+# Экспортируется из Chrome DevTools → Application → Cookies → www.cian.ru
+# или через расширение Cookie-Editor → Export → JSON (object format).
+#
+# Переменные окружения:
+#   TRADEIN_ADMIN_TOKEN  — admin token (или берётся из .env.runtime)
+#   TRADEIN_BASE_URL     — base URL (default: https://gendsgn.ru/trade-in)
+
+set -euo pipefail
+
+COOKIES_FILE="${1:-cookies.json}"
+
+if [[ ! -f "$COOKIES_FILE" ]]; then
+  echo "Error: cookies file not found: $COOKIES_FILE" >&2
+  echo "Usage: $0 cookies.json" >&2
+  exit 1
+fi
+
+# Загружаем токен из env или .env.runtime
+ADMIN_TOKEN="${TRADEIN_ADMIN_TOKEN:-}"
+if [[ -z "$ADMIN_TOKEN" ]] && [[ -f ".env.runtime" ]]; then
+  ADMIN_TOKEN="$(grep '^TRADEIN_ADMIN_TOKEN=' .env.runtime 2>/dev/null | cut -d= -f2- || true)"
+fi
+
+if [[ -z "$ADMIN_TOKEN" ]]; then
+  echo "Error: TRADEIN_ADMIN_TOKEN not set (env or .env.runtime)" >&2
+  exit 1
+fi
+
+BASE_URL="${TRADEIN_BASE_URL:-https://gendsgn.ru/trade-in}"
+
+echo "Uploading Cian cookies from $COOKIES_FILE to $BASE_URL ..."
+echo
+
+curl -fsSL -X POST "$BASE_URL/api/v1/admin/scrape/cian/upload-cookies" \
+  -H "X-Admin-Token: $ADMIN_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d "@$COOKIES_FILE" \
+  | python3 -m json.tool
+
+echo
+echo "Verifying session:"
+curl -fsSL "$BASE_URL/api/v1/admin/scrape/cian/test-auth" \
+  -H "X-Admin-Token: $ADMIN_TOKEN" \
+  | python3 -m json.tool