fix(cian): verify_session на curl_cffi impersonate — ban != cookies expired (Refs #768) (#785)

Co-authored-by: bot-backend <bot-backend@gendsgn.local>
Co-committed-by: bot-backend <bot-backend@gendsgn.local>

tradein-mvp/backend/app/services/cian_session.py

@@ -3,13 +3,14 @@
 Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API.
 Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table.
 """
+
 from __future__ import annotations
 
 import json
 import logging
 from typing import Any
 
-import httpx
+from curl_cffi.requests import AsyncSession
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
@@ -23,29 +24,25 @@ logger = logging.getLogger(__name__)
 # Старые записи оставлены как fallback (backward compat).
 CIAN_REQUIRED_COOKIES: set[str] = {
     # --- Cian auth & session (critical) ---
-    "DMIR_AUTH",                   # Cian auth token (httpOnly) — основной auth-сигнал
-    "_CIAN_GK",                    # Cian session GUID
-
+    "DMIR_AUTH",  # Cian auth token (httpOnly) — основной auth-сигнал
+    "_CIAN_GK",  # Cian session GUID
     # --- Yandex Metrika (обычно присутствуют, не critical) ---
     "_ym_uid",
     "_ym_d",
     "_ym_isad",
     "_ym_visorc",
-    "_yasc",                       # Yandex anti-spam
-
+    "_yasc",  # Yandex anti-spam
     # --- Cian UX state ---
     "uxfb_card_satisfaction",
-
     # --- Cian session IDs (fallback / legacy deployments) ---
     "_cian_visitor_session_id",
     "_cian_app_session_id",
     "_cian_uid",
-
     # --- Misc (legacy / optional) ---
-    "_ga",                         # Google Analytics
-    "tlsr_id",                     # legacy fingerprint
-    "cf_clearance",                # Cloudflare bot check
-    "session-id",                  # generic session cookie
+    "_ga",  # Google Analytics
+    "tlsr_id",  # legacy fingerprint
+    "cf_clearance",  # Cloudflare bot check
+    "session-id",  # generic session cookie
     "anti_bot",
     "csrftoken",
 }
@@ -60,41 +57,76 @@ _VALUATION_TEST_URL = (
 
 _MFE_VALUATION = "valuation-for-agent-frontend"
 
+# Returned by verify_session when a TLS/bot-fingerprint ban (HTTP 403) is detected.
+# Callers that check `if state is None` will NOT treat this as "expired cookies" —
+# the sentinel is truthy (non-None), so cookie-refresh alerts are suppressed.
+# Callers that need to distinguish ban from valid auth should check state.get("_ban").
+VERIFY_BAN_SENTINEL: dict[str, Any] = {"_ban": True}
+
+
+def _classify_verify_response(
+    status_code: int,
+    html: str | None,
+) -> dict[str, Any] | None:
+    """Pure classifier — maps (status_code, html) to verify_session outcome.
+
+    Returns:
+        VERIFY_BAN_SENTINEL  — 403/TLS ban (cookies may be fine, server is blocking)
+        None                 — 401 or isAuthenticated=false (cookies genuinely expired)
+        state dict           — authenticated successfully
+    """
+    if status_code == 403:
+        return VERIFY_BAN_SENTINEL
+    if status_code == 401:
+        return None
+    if html is None:
+        return None
+    state = extract_state(html, mfe=_MFE_VALUATION, key="initialState")
+    if state is None:
+        return None
+    user = state.get("user", {}) or {}
+    if not user.get("isAuthenticated"):
+        return None
+    return state
+
 
 async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
     """Hit Cian Valuation Calculator with cookies — return parsed state if authenticated.
 
-    Returns state dict containing user.isAuthenticated + userId, or None if
-    cookies are invalid/expired or state extraction fails.
+    Uses curl_cffi with impersonate='chrome120' (same as prod scrapers) to avoid
+    TLS-fingerprint bans that httpx would trigger.
+
+    Returns:
+        state dict           — authenticated (contains user.isAuthenticated + userId)
+        VERIFY_BAN_SENTINEL  — HTTP 403 TLS/bot ban; cookies may still be valid —
+                               callers should NOT trigger a cookie-refresh alert
+        None                 — HTTP 401 or isAuthenticated=false; cookies expired
+
     Никогда не логирует сырые значения cookies.
     """
     try:
-        async with httpx.AsyncClient(
+        async with AsyncSession(
+            impersonate="chrome120",
             cookies=cookies,
             timeout=20.0,
-            follow_redirects=True,
-        ) as client:
-            resp = await client.get(_VALUATION_TEST_URL)
-            resp.raise_for_status()
-            state = extract_state(resp.text, mfe=_MFE_VALUATION, key="initialState")
-            if state is None:
+        ) as session:
+            resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True)
+            status = resp.status_code
+            html: str | None = resp.text if status == 200 else None
+
+            result = _classify_verify_response(status, html)
+
+            if result is VERIFY_BAN_SENTINEL:
                 logger.warning(
-                    "Cian cookies verify: state extraction failed (mfe=%s)", _MFE_VALUATION
+                    "Cian cookies verify: HTTP 403 TLS/bot ban — cookies NOT marked expired"
                 )
-                return None
-            user = state.get("user", {})
-            if not user.get("isAuthenticated"):
-                logger.warning("Cian cookies verify: user.isAuthenticated=false")
-                return None
-            logger.info("Cian cookies verified — userId=%s", user.get("userId"))
-            return state
-    except httpx.HTTPStatusError as exc:
-        logger.warning(
-            "Cian cookies verify HTTP error: %s %s",
-            exc.response.status_code,
-            exc.request.url,
-        )
-        return None
+            elif result is None:
+                logger.warning("Cian cookies verify: expired/unauthenticated (status=%d)", status)
+            else:
+                user = result.get("user", {}) or {}
+                logger.info("Cian cookies verified — userId=%s", user.get("userId"))
+
+            return result
     except Exception as exc:
         logger.warning("Cian cookies verify failed: %s", exc)
         return None
@@ -154,8 +186,9 @@ def load_session(db: Session) -> dict[str, str] | None:
     Выбирает только записи где expires_at_estimate > NOW() и сессия не
     была инвалидирована после последнего upload'а.
     """
-    row = db.execute(
-        text("""
+    row = (
+        db.execute(
+            text("""
             SELECT
                 account_user_id,
                 pgp_sym_decrypt(cookies_encrypted, :key)::text AS cookies_json,
@@ -166,8 +199,11 @@ def load_session(db: Session) -> dict[str, str] | None:
             ORDER BY uploaded_at DESC
             LIMIT 1
         """),
-        {"key": settings.cookie_encryption_key},
-    ).mappings().first()
+            {"key": settings.cookie_encryption_key},
+        )
+        .mappings()
+        .first()
+    )
 
     if row is None:
         logger.warning("No valid Cian session cookies in DB")

tradein-mvp/backend/tests/test_cian_session.py

@@ -1,4 +1,5 @@
 """Tests for cian_session — cookie management service."""
+
 from __future__ import annotations
 
 import json
@@ -8,13 +9,14 @@ import pytest
 
 from app.services.cian_session import (
     CIAN_REQUIRED_COOKIES,
+    VERIFY_BAN_SENTINEL,
+    _classify_verify_response,
     load_session,
     mark_session_invalid,
     save_session,
     verify_session,
 )
 
-
 # ---------------------------------------------------------------------------
 # Fixtures
 # ---------------------------------------------------------------------------
@@ -171,84 +173,180 @@ def test_mark_session_invalid_uses_cast(mock_db: MagicMock) -> None:
 
 
 # ---------------------------------------------------------------------------
-# verify_session (async)
+# _classify_verify_response (pure unit tests — no I/O)
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.asyncio
-async def test_verify_session_returns_none_when_state_missing(monkeypatch: pytest.MonkeyPatch) -> None:
-    """extract_state returns None → verify_session returns None."""
-    monkeypatch.setattr(
-        "app.services.cian_session.extract_state",
-        lambda html, mfe, key: None,
-    )
+def test_classify_403_returns_ban_sentinel() -> None:
+    """HTTP 403 TLS/bot ban → VERIFY_BAN_SENTINEL (not None, not 'expired')."""
+    result = _classify_verify_response(403, None)
+    assert result is VERIFY_BAN_SENTINEL
+    assert result is not None  # must NOT collapse into expired-cookies signal
 
-    async def fake_get(url: str, **kwargs):  # noqa: ARG001
-        class FakeResp:
-            text = "<html></html>"
-            def raise_for_status(self) -> None:
-                pass
-        return FakeResp()
 
-    with patch("httpx.AsyncClient") as mock_client_cls:
-        mock_client = AsyncMock()
-        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
-        mock_client.__aexit__ = AsyncMock(return_value=None)
-        mock_client.get = AsyncMock(return_value=_make_fake_resp("<html></html>"))
-        mock_client_cls.return_value = mock_client
-
-        result = await verify_session({"_ym_uid": "x"})
+def test_classify_401_returns_none() -> None:
+    """HTTP 401 Unauthorized → None (genuinely expired cookies)."""
+    result = _classify_verify_response(401, None)
     assert result is None
 
 
-@pytest.mark.asyncio
-async def test_verify_session_returns_none_when_not_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
-    """state.user.isAuthenticated is false → verify_session returns None."""
+def test_classify_200_not_authenticated_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """200 with isAuthenticated=false → None (expired)."""
     monkeypatch.setattr(
         "app.services.cian_session.extract_state",
         lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}},
     )
+    result = _classify_verify_response(200, "<html></html>")
+    assert result is None
 
-    with patch("httpx.AsyncClient") as mock_client_cls:
-        mock_client = AsyncMock()
-        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
-        mock_client.__aexit__ = AsyncMock(return_value=None)
-        mock_client.get = AsyncMock(return_value=_make_fake_resp("<html></html>"))
-        mock_client_cls.return_value = mock_client
 
-        result = await verify_session({"_ym_uid": "x"})
+def test_classify_200_authenticated_returns_state(monkeypatch: pytest.MonkeyPatch) -> None:
+    """200 with isAuthenticated=true → state dict."""
+    expected = {"user": {"isAuthenticated": True, "userId": 99}}
+    monkeypatch.setattr(
+        "app.services.cian_session.extract_state",
+        lambda html, mfe, key: expected,
+    )
+    result = _classify_verify_response(200, "<html>x</html>")
+    assert result == expected
+
+
+def test_classify_200_state_missing_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+    """200 but extract_state returns None → None."""
+    monkeypatch.setattr(
+        "app.services.cian_session.extract_state",
+        lambda html, mfe, key: None,
+    )
+    result = _classify_verify_response(200, "<html></html>")
+    assert result is None
+
+
+def test_classify_403_is_distinct_from_401() -> None:
+    """Ban and expired must not collapse to the same return value."""
+    ban = _classify_verify_response(403, None)
+    expired = _classify_verify_response(401, None)
+    assert ban is not expired
+    assert ban is not None
+    assert expired is None
+
+
+# ---------------------------------------------------------------------------
+# verify_session (async) — integration with curl_cffi mock
+# ---------------------------------------------------------------------------
+
+
+def _make_cffi_resp(status_code: int, text: str = "") -> MagicMock:
+    resp = MagicMock()
+    resp.status_code = status_code
+    resp.text = text
+    return resp
+
+
+@pytest.mark.asyncio
+async def test_verify_session_403_ban_not_treated_as_expired(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """HTTP 403 from curl_cffi → VERIFY_BAN_SENTINEL, never None."""
+    mock_session = AsyncMock()
+    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+    mock_session.get = AsyncMock(return_value=_make_cffi_resp(403))
+
+    with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
+        result = await verify_session({"DMIR_AUTH": "x"})
+
+    assert result is VERIFY_BAN_SENTINEL
+    assert result is not None  # must NOT trigger cookie-refresh alert
+
+
+@pytest.mark.asyncio
+async def test_verify_session_401_is_expired(monkeypatch: pytest.MonkeyPatch) -> None:
+    """HTTP 401 → None (genuine expiry)."""
+    mock_session = AsyncMock()
+    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+    mock_session.get = AsyncMock(return_value=_make_cffi_resp(401))
+
+    with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
+        result = await verify_session({"DMIR_AUTH": "x"})
+
     assert result is None
 
 
 @pytest.mark.asyncio
-async def test_verify_session_returns_state_when_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
-    """state.user.isAuthenticated is true → returns state dict."""
+async def test_verify_session_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
+    """200 + isAuthenticated=true → returns state dict."""
     expected_state = {"user": {"isAuthenticated": True, "userId": 102963817}}
     monkeypatch.setattr(
         "app.services.cian_session.extract_state",
         lambda html, mfe, key: expected_state,
     )
 
-    with patch("httpx.AsyncClient") as mock_client_cls:
-        mock_client = AsyncMock()
-        mock_client.__aenter__ = AsyncMock(return_value=mock_client)
-        mock_client.__aexit__ = AsyncMock(return_value=None)
-        mock_client.get = AsyncMock(return_value=_make_fake_resp("<html>state here</html>"))
-        mock_client_cls.return_value = mock_client
+    mock_session = AsyncMock()
+    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+    mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "<html>x</html>"))
+
+    with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
+        result = await verify_session({"DMIR_AUTH": "x", "_CIAN_GK": "y"})
 
-        result = await verify_session({"_ym_uid": "x", "_cian_uid": "y"})
     assert result is not None
     assert result["user"]["isAuthenticated"] is True
     assert result["user"]["userId"] == 102963817
 
 
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
+@pytest.mark.asyncio
+async def test_verify_session_not_authenticated_returns_none(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """200 + isAuthenticated=false → None (expired)."""
+    monkeypatch.setattr(
+        "app.services.cian_session.extract_state",
+        lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}},
+    )
+
+    mock_session = AsyncMock()
+    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+    mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "<html></html>"))
+
+    with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
+        result = await verify_session({"DMIR_AUTH": "x"})
+
+    assert result is None
 
 
-def _make_fake_resp(text: str) -> MagicMock:
-    resp = MagicMock()
-    resp.text = text
-    resp.raise_for_status = MagicMock()
-    return resp
+@pytest.mark.asyncio
+async def test_verify_session_state_missing_returns_none(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """200 + extract_state returns None → None."""
+    monkeypatch.setattr(
+        "app.services.cian_session.extract_state",
+        lambda html, mfe, key: None,
+    )
+
+    mock_session = AsyncMock()
+    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+    mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "<html></html>"))
+
+    with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
+        result = await verify_session({"DMIR_AUTH": "x"})
+
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_verify_session_uses_chrome120_impersonate() -> None:
+    """curl_cffi AsyncSession must be constructed with impersonate='chrome120'."""
+    mock_session = AsyncMock()
+    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+    mock_session.get = AsyncMock(return_value=_make_cffi_resp(403))
+
+    with patch("app.services.cian_session.AsyncSession", return_value=mock_session) as mock_cls:
+        await verify_session({"DMIR_AUTH": "x"})
+
+    _, kwargs = mock_cls.call_args
+    assert kwargs.get("impersonate") == "chrome120"