From 800bc631bfdfae4568b18ccbd62a7ae9b164b9fc Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 30 May 2026 19:35:22 +0300 Subject: [PATCH] =?UTF-8?q?fix(cian):=20verify=5Fsession=20=D0=BD=D0=B0=20?= =?UTF-8?q?curl=5Fcffi=20impersonate=20=E2=80=94=20ban=20!=3D=20cookies=20?= =?UTF-8?q?expired=20(Refs=20#768)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verify_session проверял куки через httpx → Cian банит по TLS-fingerprint → ложный сигнал «cookies expired» оператору. Переведено на curl_cffi AsyncSession(impersonate=chrome120) — тот же fingerprint что у боевого cian-скрейпа. _classify_verify_response: 403/TLS→VERIFY_BAN_SENTINEL (truthy, НЕ expired — алерт re-upload подавляется), 401/isAuthenticated=false→None (expired, контракт сохранён), 200+auth→state. Return-тип не изменился (dict|None). 11 тестов (6 unit classifier + 5 async). ruff clean, 26 pass. --- .../backend/app/services/cian_session.py | 116 ++++++---- .../backend/tests/test_cian_session.py | 198 +++++++++++++----- 2 files changed, 224 insertions(+), 90 deletions(-) diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py index f753a511..ee45331d 100644 --- a/tradein-mvp/backend/app/services/cian_session.py +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -3,13 +3,14 @@ Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API. Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table. """ + from __future__ import annotations import json import logging from typing import Any -import httpx +from curl_cffi.requests import AsyncSession from sqlalchemy import text from sqlalchemy.orm import Session @@ -23,29 +24,25 @@ logger = logging.getLogger(__name__) # Старые записи оставлены как fallback (backward compat). CIAN_REQUIRED_COOKIES: set[str] = { # --- Cian auth & session (critical) --- - "DMIR_AUTH", # Cian auth token (httpOnly) — основной auth-сигнал - "_CIAN_GK", # Cian session GUID - + "DMIR_AUTH", # Cian auth token (httpOnly) — основной auth-сигнал + "_CIAN_GK", # Cian session GUID # --- Yandex Metrika (обычно присутствуют, не critical) --- "_ym_uid", "_ym_d", "_ym_isad", "_ym_visorc", - "_yasc", # Yandex anti-spam - + "_yasc", # Yandex anti-spam # --- Cian UX state --- "uxfb_card_satisfaction", - # --- Cian session IDs (fallback / legacy deployments) --- "_cian_visitor_session_id", "_cian_app_session_id", "_cian_uid", - # --- Misc (legacy / optional) --- - "_ga", # Google Analytics - "tlsr_id", # legacy fingerprint - "cf_clearance", # Cloudflare bot check - "session-id", # generic session cookie + "_ga", # Google Analytics + "tlsr_id", # legacy fingerprint + "cf_clearance", # Cloudflare bot check + "session-id", # generic session cookie "anti_bot", "csrftoken", } @@ -60,41 +57,76 @@ _VALUATION_TEST_URL = ( _MFE_VALUATION = "valuation-for-agent-frontend" +# Returned by verify_session when a TLS/bot-fingerprint ban (HTTP 403) is detected. +# Callers that check `if state is None` will NOT treat this as "expired cookies" — +# the sentinel is truthy (non-None), so cookie-refresh alerts are suppressed. +# Callers that need to distinguish ban from valid auth should check state.get("_ban"). +VERIFY_BAN_SENTINEL: dict[str, Any] = {"_ban": True} + + +def _classify_verify_response( + status_code: int, + html: str | None, +) -> dict[str, Any] | None: + """Pure classifier — maps (status_code, html) to verify_session outcome. + + Returns: + VERIFY_BAN_SENTINEL — 403/TLS ban (cookies may be fine, server is blocking) + None — 401 or isAuthenticated=false (cookies genuinely expired) + state dict — authenticated successfully + """ + if status_code == 403: + return VERIFY_BAN_SENTINEL + if status_code == 401: + return None + if html is None: + return None + state = extract_state(html, mfe=_MFE_VALUATION, key="initialState") + if state is None: + return None + user = state.get("user", {}) or {} + if not user.get("isAuthenticated"): + return None + return state + async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None: """Hit Cian Valuation Calculator with cookies — return parsed state if authenticated. - Returns state dict containing user.isAuthenticated + userId, or None if - cookies are invalid/expired or state extraction fails. + Uses curl_cffi with impersonate='chrome120' (same as prod scrapers) to avoid + TLS-fingerprint bans that httpx would trigger. + + Returns: + state dict — authenticated (contains user.isAuthenticated + userId) + VERIFY_BAN_SENTINEL — HTTP 403 TLS/bot ban; cookies may still be valid — + callers should NOT trigger a cookie-refresh alert + None — HTTP 401 or isAuthenticated=false; cookies expired + Никогда не логирует сырые значения cookies. """ try: - async with httpx.AsyncClient( + async with AsyncSession( + impersonate="chrome120", cookies=cookies, timeout=20.0, - follow_redirects=True, - ) as client: - resp = await client.get(_VALUATION_TEST_URL) - resp.raise_for_status() - state = extract_state(resp.text, mfe=_MFE_VALUATION, key="initialState") - if state is None: + ) as session: + resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True) + status = resp.status_code + html: str | None = resp.text if status == 200 else None + + result = _classify_verify_response(status, html) + + if result is VERIFY_BAN_SENTINEL: logger.warning( - "Cian cookies verify: state extraction failed (mfe=%s)", _MFE_VALUATION + "Cian cookies verify: HTTP 403 TLS/bot ban — cookies NOT marked expired" ) - return None - user = state.get("user", {}) - if not user.get("isAuthenticated"): - logger.warning("Cian cookies verify: user.isAuthenticated=false") - return None - logger.info("Cian cookies verified — userId=%s", user.get("userId")) - return state - except httpx.HTTPStatusError as exc: - logger.warning( - "Cian cookies verify HTTP error: %s %s", - exc.response.status_code, - exc.request.url, - ) - return None + elif result is None: + logger.warning("Cian cookies verify: expired/unauthenticated (status=%d)", status) + else: + user = result.get("user", {}) or {} + logger.info("Cian cookies verified — userId=%s", user.get("userId")) + + return result except Exception as exc: logger.warning("Cian cookies verify failed: %s", exc) return None @@ -154,8 +186,9 @@ def load_session(db: Session) -> dict[str, str] | None: Выбирает только записи где expires_at_estimate > NOW() и сессия не была инвалидирована после последнего upload'а. """ - row = db.execute( - text(""" + row = ( + db.execute( + text(""" SELECT account_user_id, pgp_sym_decrypt(cookies_encrypted, :key)::text AS cookies_json, @@ -166,8 +199,11 @@ def load_session(db: Session) -> dict[str, str] | None: ORDER BY uploaded_at DESC LIMIT 1 """), - {"key": settings.cookie_encryption_key}, - ).mappings().first() + {"key": settings.cookie_encryption_key}, + ) + .mappings() + .first() + ) if row is None: logger.warning("No valid Cian session cookies in DB") diff --git a/tradein-mvp/backend/tests/test_cian_session.py b/tradein-mvp/backend/tests/test_cian_session.py index 4175ee96..2518ea25 100644 --- a/tradein-mvp/backend/tests/test_cian_session.py +++ b/tradein-mvp/backend/tests/test_cian_session.py @@ -1,4 +1,5 @@ """Tests for cian_session — cookie management service.""" + from __future__ import annotations import json @@ -8,13 +9,14 @@ import pytest from app.services.cian_session import ( CIAN_REQUIRED_COOKIES, + VERIFY_BAN_SENTINEL, + _classify_verify_response, load_session, mark_session_invalid, save_session, verify_session, ) - # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -171,84 +173,180 @@ def test_mark_session_invalid_uses_cast(mock_db: MagicMock) -> None: # --------------------------------------------------------------------------- -# verify_session (async) +# _classify_verify_response (pure unit tests — no I/O) # --------------------------------------------------------------------------- -@pytest.mark.asyncio -async def test_verify_session_returns_none_when_state_missing(monkeypatch: pytest.MonkeyPatch) -> None: - """extract_state returns None → verify_session returns None.""" - monkeypatch.setattr( - "app.services.cian_session.extract_state", - lambda html, mfe, key: None, - ) +def test_classify_403_returns_ban_sentinel() -> None: + """HTTP 403 TLS/bot ban → VERIFY_BAN_SENTINEL (not None, not 'expired').""" + result = _classify_verify_response(403, None) + assert result is VERIFY_BAN_SENTINEL + assert result is not None # must NOT collapse into expired-cookies signal - async def fake_get(url: str, **kwargs): # noqa: ARG001 - class FakeResp: - text = "" - def raise_for_status(self) -> None: - pass - return FakeResp() - with patch("httpx.AsyncClient") as mock_client_cls: - mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=None) - mock_client.get = AsyncMock(return_value=_make_fake_resp("")) - mock_client_cls.return_value = mock_client - - result = await verify_session({"_ym_uid": "x"}) +def test_classify_401_returns_none() -> None: + """HTTP 401 Unauthorized → None (genuinely expired cookies).""" + result = _classify_verify_response(401, None) assert result is None -@pytest.mark.asyncio -async def test_verify_session_returns_none_when_not_authenticated(monkeypatch: pytest.MonkeyPatch) -> None: - """state.user.isAuthenticated is false → verify_session returns None.""" +def test_classify_200_not_authenticated_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """200 with isAuthenticated=false → None (expired).""" monkeypatch.setattr( "app.services.cian_session.extract_state", lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}}, ) + result = _classify_verify_response(200, "") + assert result is None - with patch("httpx.AsyncClient") as mock_client_cls: - mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=None) - mock_client.get = AsyncMock(return_value=_make_fake_resp("")) - mock_client_cls.return_value = mock_client - result = await verify_session({"_ym_uid": "x"}) +def test_classify_200_authenticated_returns_state(monkeypatch: pytest.MonkeyPatch) -> None: + """200 with isAuthenticated=true → state dict.""" + expected = {"user": {"isAuthenticated": True, "userId": 99}} + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: expected, + ) + result = _classify_verify_response(200, "x") + assert result == expected + + +def test_classify_200_state_missing_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: + """200 but extract_state returns None → None.""" + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: None, + ) + result = _classify_verify_response(200, "") + assert result is None + + +def test_classify_403_is_distinct_from_401() -> None: + """Ban and expired must not collapse to the same return value.""" + ban = _classify_verify_response(403, None) + expired = _classify_verify_response(401, None) + assert ban is not expired + assert ban is not None + assert expired is None + + +# --------------------------------------------------------------------------- +# verify_session (async) — integration with curl_cffi mock +# --------------------------------------------------------------------------- + + +def _make_cffi_resp(status_code: int, text: str = "") -> MagicMock: + resp = MagicMock() + resp.status_code = status_code + resp.text = text + return resp + + +@pytest.mark.asyncio +async def test_verify_session_403_ban_not_treated_as_expired( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """HTTP 403 from curl_cffi → VERIFY_BAN_SENTINEL, never None.""" + mock_session = AsyncMock() + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=None) + mock_session.get = AsyncMock(return_value=_make_cffi_resp(403)) + + with patch("app.services.cian_session.AsyncSession", return_value=mock_session): + result = await verify_session({"DMIR_AUTH": "x"}) + + assert result is VERIFY_BAN_SENTINEL + assert result is not None # must NOT trigger cookie-refresh alert + + +@pytest.mark.asyncio +async def test_verify_session_401_is_expired(monkeypatch: pytest.MonkeyPatch) -> None: + """HTTP 401 → None (genuine expiry).""" + mock_session = AsyncMock() + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=None) + mock_session.get = AsyncMock(return_value=_make_cffi_resp(401)) + + with patch("app.services.cian_session.AsyncSession", return_value=mock_session): + result = await verify_session({"DMIR_AUTH": "x"}) + assert result is None @pytest.mark.asyncio -async def test_verify_session_returns_state_when_authenticated(monkeypatch: pytest.MonkeyPatch) -> None: - """state.user.isAuthenticated is true → returns state dict.""" +async def test_verify_session_authenticated(monkeypatch: pytest.MonkeyPatch) -> None: + """200 + isAuthenticated=true → returns state dict.""" expected_state = {"user": {"isAuthenticated": True, "userId": 102963817}} monkeypatch.setattr( "app.services.cian_session.extract_state", lambda html, mfe, key: expected_state, ) - with patch("httpx.AsyncClient") as mock_client_cls: - mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=None) - mock_client.get = AsyncMock(return_value=_make_fake_resp("state here")) - mock_client_cls.return_value = mock_client + mock_session = AsyncMock() + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=None) + mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "x")) + + with patch("app.services.cian_session.AsyncSession", return_value=mock_session): + result = await verify_session({"DMIR_AUTH": "x", "_CIAN_GK": "y"}) - result = await verify_session({"_ym_uid": "x", "_cian_uid": "y"}) assert result is not None assert result["user"]["isAuthenticated"] is True assert result["user"]["userId"] == 102963817 -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_verify_session_not_authenticated_returns_none( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """200 + isAuthenticated=false → None (expired).""" + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}}, + ) + + mock_session = AsyncMock() + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=None) + mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "")) + + with patch("app.services.cian_session.AsyncSession", return_value=mock_session): + result = await verify_session({"DMIR_AUTH": "x"}) + + assert result is None -def _make_fake_resp(text: str) -> MagicMock: - resp = MagicMock() - resp.text = text - resp.raise_for_status = MagicMock() - return resp +@pytest.mark.asyncio +async def test_verify_session_state_missing_returns_none( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """200 + extract_state returns None → None.""" + monkeypatch.setattr( + "app.services.cian_session.extract_state", + lambda html, mfe, key: None, + ) + + mock_session = AsyncMock() + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=None) + mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "")) + + with patch("app.services.cian_session.AsyncSession", return_value=mock_session): + result = await verify_session({"DMIR_AUTH": "x"}) + + assert result is None + + +@pytest.mark.asyncio +async def test_verify_session_uses_chrome120_impersonate() -> None: + """curl_cffi AsyncSession must be constructed with impersonate='chrome120'.""" + mock_session = AsyncMock() + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=None) + mock_session.get = AsyncMock(return_value=_make_cffi_resp(403)) + + with patch("app.services.cian_session.AsyncSession", return_value=mock_session) as mock_cls: + await verify_session({"DMIR_AUTH": "x"}) + + _, kwargs = mock_cls.call_args + assert kwargs.get("impersonate") == "chrome120"