gendesign/tradein-mvp/backend/app/services/cian_session.py

"""Cian session cookie management — load/save/verify encrypted cookies.

Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API.
Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table.
"""

from __future__ import annotations

import json
import logging
from typing import Any

from curl_cffi.requests import AsyncSession

# #2306: extract_state migrated to scraper_kit (byte-identical to legacy
# app.services.scrapers.cian_state_parser, см. tests/test_scraper_kit_cian_golden_parity.py
# + tests/test_scraper_kit_pricehistory_session_parity.py).
from scraper_kit.cian_state_parser import extract_state
from sqlalchemy import text
from sqlalchemy.orm import Session

from app.core.config import settings

logger = logging.getLogger(__name__)

# Cookies критичные для Cian auth — фильтр перед сохранением.
# Список обновлён по реальному DevTools-дампу из logged-in сессии cian.ru (2026-05-23).
# Старые записи оставлены как fallback (backward compat).
CIAN_REQUIRED_COOKIES: set[str] = {
    # --- Cian auth & session (critical) ---
    "DMIR_AUTH",  # Cian auth token (httpOnly) — основной auth-сигнал
    "_CIAN_GK",  # Cian session GUID
    # --- Yandex Metrika (обычно присутствуют, не critical) ---
    "_ym_uid",
    "_ym_d",
    "_ym_isad",
    "_ym_visorc",
    "_yasc",  # Yandex anti-spam
    # --- Cian UX state ---
    "uxfb_card_satisfaction",
    # --- Cian session IDs (fallback / legacy deployments) ---
    "_cian_visitor_session_id",
    "_cian_app_session_id",
    "_cian_uid",
    # --- Misc (legacy / optional) ---
    "_ga",  # Google Analytics
    "tlsr_id",  # legacy fingerprint
    "cf_clearance",  # Cloudflare bot check
    "session-id",  # generic session cookie
    "anti_bot",
    "csrftoken",
}

_VALUATION_TEST_URL = (
    "https://www.cian.ru/kalkulator-nedvizhimosti/"
    "?address=%D0%A1%D0%B2%D0%B5%D1%80%D0%B4%D0%BB%D0%BE%D0%B2%D1%81%D0%BA%D0%B0%D1%8F"
    "%20%D0%BE%D0%B1%D0%BB%2C%20%D0%95%D0%BA%D0%B0%D1%82%D0%B5%D1%80%D0%B8%D0%BD%D0%B1%D1%83%D1%80%D0%B3"
    "&totalArea=40&roomsCount=1&valuationType=sale"
    "&floor%5B0%5D=floorOther&repairType%5B0%5D=repairTypeCosmetic"
)

_MFE_VALUATION = "valuation-for-agent-frontend"
# #639 (2026-05): auth-сигнал (user.isAuthenticated/userId) уехал из valuation-MFE
# (там user теперь всегда anonymous) в site-header MFE `header-frontend`, который
# присутствует на любой cian-странице. Verify читает auth именно оттуда.
_MFE_AUTH = "header-frontend"

# Returned by verify_session when a TLS/bot-fingerprint ban (HTTP 403) is detected.
# Callers that check `if state is None` will NOT treat this as "expired cookies" —
# the sentinel is truthy (non-None), so cookie-refresh alerts are suppressed.
# Callers that need to distinguish ban from valid auth should check state.get("_ban").
VERIFY_BAN_SENTINEL: dict[str, Any] = {"_ban": True}


def _classify_verify_response(
    status_code: int,
    html: str | None,
) -> dict[str, Any] | None:
    """Pure classifier — maps (status_code, html) to verify_session outcome.

    Returns:
        VERIFY_BAN_SENTINEL  — 403/TLS ban (cookies may be fine, server is blocking)
        None                 — 401 or isAuthenticated=false (cookies genuinely expired)
        state dict           — authenticated successfully
    """
    if status_code == 403:
        return VERIFY_BAN_SENTINEL
    if status_code == 401:
        return None
    if html is None:
        return None
    state = extract_state(html, mfe=_MFE_AUTH, key="initialState")
    if state is None:
        return None
    user = state.get("user", {}) or {}
    if not user.get("isAuthenticated"):
        return None
    return state


async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
    """Hit Cian Valuation Calculator with cookies — return parsed state if authenticated.

    Uses curl_cffi with impersonate='chrome120' (same as prod scrapers) to avoid
    TLS-fingerprint bans that httpx would trigger.

    Returns:
        state dict           — authenticated (contains user.isAuthenticated + userId)
        VERIFY_BAN_SENTINEL  — HTTP 403 TLS/bot ban; cookies may still be valid —
                               callers should NOT trigger a cookie-refresh alert
        None                 — HTTP 401 or isAuthenticated=false; cookies expired

    Никогда не логирует сырые значения cookies.
    """
    try:
        # proxies: mobile-proxy egress (#806) — Cian блокирует datacenter-IP даже
        # при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403.
        # Пусто (env не задан) → прямое подключение (dev/no-op).
        _proxy_url = settings.cian_proxy_url
        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
        async with AsyncSession(
            impersonate="chrome120",
            cookies=cookies,
            timeout=20.0,
            proxies=_proxies,
        ) as session:
            resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True)
            status = resp.status_code
            html: str | None = resp.text if status == 200 else None

            result = _classify_verify_response(status, html)

            if result is VERIFY_BAN_SENTINEL:
                logger.warning(
                    "Cian cookies verify: HTTP 403 TLS/bot ban — cookies NOT marked expired"
                )
            elif result is None:
                logger.warning("Cian cookies verify: expired/unauthenticated (status=%d)", status)
            else:
                user = result.get("user", {}) or {}
                logger.info("Cian cookies verified — userId=%s", user.get("userId"))

            return result
    except Exception as exc:
        logger.warning("Cian cookies verify failed: %s", exc)
        return None


def save_session(
    db: Session,
    account_user_id: int,
    cookies: dict[str, str],
    ttl_days: int = 30,
) -> None:
    """Encrypt cookies via pgp_sym_encrypt and UPSERT into cian_session_cookies.

    Uses settings.cookie_encryption_key as the encryption secret.
    Никогда не логирует сырые значения cookies.
    """
    cookies_json = json.dumps(cookies)
    db.execute(
        text("""
            INSERT INTO cian_session_cookies (
                account_user_id,
                cookies_encrypted,
                expires_at_estimate,
                uploaded_at
            ) VALUES (
                CAST(:uid AS bigint),
                pgp_sym_encrypt(:cookies_json, :key),
                NOW() + (CAST(:ttl_days AS int) || ' days')::interval,
                NOW()
            )
            ON CONFLICT (account_user_id) DO UPDATE SET
                cookies_encrypted    = EXCLUDED.cookies_encrypted,
                expires_at_estimate  = EXCLUDED.expires_at_estimate,
                uploaded_at          = NOW(),
                last_invalid_at      = NULL
        """),
        {
            "uid": account_user_id,
            "cookies_json": cookies_json,
            "key": settings.cookie_encryption_key,
            "ttl_days": ttl_days,
        },
    )
    db.commit()
    logger.info(
        "Cian cookies saved for userId=%s (count=%d, ttl=%d days)",
        account_user_id,
        len(cookies),
        ttl_days,
    )


def load_session(db: Session) -> dict[str, str] | None:
    """Load most-recently-uploaded valid Cian cookies (decrypt).

    Returns dict[cookie_name, cookie_value] или None если нет валидной session.
    Выбирает только записи где expires_at_estimate > NOW() и сессия не
    была инвалидирована после последнего upload'а.
    """
    row = (
        db.execute(
            text("""
            SELECT
                account_user_id,
                pgp_sym_decrypt(cookies_encrypted, :key)::text AS cookies_json,
                expires_at_estimate
            FROM cian_session_cookies
            WHERE expires_at_estimate > NOW()
              AND (last_invalid_at IS NULL OR last_invalid_at < uploaded_at)
            ORDER BY uploaded_at DESC
            LIMIT 1
        """),
            {"key": settings.cookie_encryption_key},
        )
        .mappings()
        .first()
    )

    if row is None:
        logger.warning("No valid Cian session cookies in DB")
        return None

    cookies: dict[str, str] = json.loads(row["cookies_json"])

    # Обновляем last_used_at — не критично, игнорируем ошибки.
    try:
        db.execute(
            text(
                "UPDATE cian_session_cookies SET last_used_at = NOW()"
                " WHERE account_user_id = CAST(:uid AS bigint)"
            ),
            {"uid": row["account_user_id"]},
        )
        db.commit()
    except Exception as exc:
        logger.warning(
            "Failed to update last_used_at for userId=%s: %s", row["account_user_id"], exc
        )

    logger.info(
        "Cian cookies loaded for userId=%s (count=%d)",
        row["account_user_id"],
        len(cookies),
    )
    return cookies


def mark_session_invalid(db: Session, account_user_id: int) -> None:
    """Flag session как expired/invalid (например после 401 во время scrape)."""
    db.execute(
        text(
            "UPDATE cian_session_cookies SET last_invalid_at = NOW()"
            " WHERE account_user_id = CAST(:uid AS bigint)"
        ),
        {"uid": account_user_id},
    )
    db.commit()
    logger.warning("Cian session marked invalid for userId=%s", account_user_id)