fix(cian): verify_session на curl_cffi impersonate — ban != cookies expired (Refs #768)
verify_session проверял куки через httpx → Cian банит по TLS-fingerprint → ложный сигнал «cookies expired» оператору. Переведено на curl_cffi AsyncSession(impersonate=chrome120) — тот же fingerprint что у боевого cian-скрейпа. _classify_verify_response: 403/TLS→VERIFY_BAN_SENTINEL (truthy, НЕ expired — алерт re-upload подавляется), 401/isAuthenticated=false→None (expired, контракт сохранён), 200+auth→state. Return-тип не изменился (dict|None). 11 тестов (6 unit classifier + 5 async). ruff clean, 26 pass.
This commit is contained in:
parent
238f14a0d7
commit
800bc631bf
2 changed files with 224 additions and 90 deletions
|
|
@ -3,13 +3,14 @@
|
|||
Used by cian_valuation.py (Stage 7) to authenticate Valuation Calculator API.
|
||||
Cookies stored encrypted (pgp_sym_encrypt) in cian_session_cookies table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -25,22 +26,18 @@ CIAN_REQUIRED_COOKIES: set[str] = {
|
|||
# --- Cian auth & session (critical) ---
|
||||
"DMIR_AUTH", # Cian auth token (httpOnly) — основной auth-сигнал
|
||||
"_CIAN_GK", # Cian session GUID
|
||||
|
||||
# --- Yandex Metrika (обычно присутствуют, не critical) ---
|
||||
"_ym_uid",
|
||||
"_ym_d",
|
||||
"_ym_isad",
|
||||
"_ym_visorc",
|
||||
"_yasc", # Yandex anti-spam
|
||||
|
||||
# --- Cian UX state ---
|
||||
"uxfb_card_satisfaction",
|
||||
|
||||
# --- Cian session IDs (fallback / legacy deployments) ---
|
||||
"_cian_visitor_session_id",
|
||||
"_cian_app_session_id",
|
||||
"_cian_uid",
|
||||
|
||||
# --- Misc (legacy / optional) ---
|
||||
"_ga", # Google Analytics
|
||||
"tlsr_id", # legacy fingerprint
|
||||
|
|
@ -60,41 +57,76 @@ _VALUATION_TEST_URL = (
|
|||
|
||||
_MFE_VALUATION = "valuation-for-agent-frontend"
|
||||
|
||||
# Returned by verify_session when a TLS/bot-fingerprint ban (HTTP 403) is detected.
|
||||
# Callers that check `if state is None` will NOT treat this as "expired cookies" —
|
||||
# the sentinel is truthy (non-None), so cookie-refresh alerts are suppressed.
|
||||
# Callers that need to distinguish ban from valid auth should check state.get("_ban").
|
||||
VERIFY_BAN_SENTINEL: dict[str, Any] = {"_ban": True}
|
||||
|
||||
|
||||
def _classify_verify_response(
|
||||
status_code: int,
|
||||
html: str | None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pure classifier — maps (status_code, html) to verify_session outcome.
|
||||
|
||||
Returns:
|
||||
VERIFY_BAN_SENTINEL — 403/TLS ban (cookies may be fine, server is blocking)
|
||||
None — 401 or isAuthenticated=false (cookies genuinely expired)
|
||||
state dict — authenticated successfully
|
||||
"""
|
||||
if status_code == 403:
|
||||
return VERIFY_BAN_SENTINEL
|
||||
if status_code == 401:
|
||||
return None
|
||||
if html is None:
|
||||
return None
|
||||
state = extract_state(html, mfe=_MFE_VALUATION, key="initialState")
|
||||
if state is None:
|
||||
return None
|
||||
user = state.get("user", {}) or {}
|
||||
if not user.get("isAuthenticated"):
|
||||
return None
|
||||
return state
|
||||
|
||||
|
||||
async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
|
||||
"""Hit Cian Valuation Calculator with cookies — return parsed state if authenticated.
|
||||
|
||||
Returns state dict containing user.isAuthenticated + userId, or None if
|
||||
cookies are invalid/expired or state extraction fails.
|
||||
Uses curl_cffi with impersonate='chrome120' (same as prod scrapers) to avoid
|
||||
TLS-fingerprint bans that httpx would trigger.
|
||||
|
||||
Returns:
|
||||
state dict — authenticated (contains user.isAuthenticated + userId)
|
||||
VERIFY_BAN_SENTINEL — HTTP 403 TLS/bot ban; cookies may still be valid —
|
||||
callers should NOT trigger a cookie-refresh alert
|
||||
None — HTTP 401 or isAuthenticated=false; cookies expired
|
||||
|
||||
Никогда не логирует сырые значения cookies.
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
async with AsyncSession(
|
||||
impersonate="chrome120",
|
||||
cookies=cookies,
|
||||
timeout=20.0,
|
||||
follow_redirects=True,
|
||||
) as client:
|
||||
resp = await client.get(_VALUATION_TEST_URL)
|
||||
resp.raise_for_status()
|
||||
state = extract_state(resp.text, mfe=_MFE_VALUATION, key="initialState")
|
||||
if state is None:
|
||||
) as session:
|
||||
resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True)
|
||||
status = resp.status_code
|
||||
html: str | None = resp.text if status == 200 else None
|
||||
|
||||
result = _classify_verify_response(status, html)
|
||||
|
||||
if result is VERIFY_BAN_SENTINEL:
|
||||
logger.warning(
|
||||
"Cian cookies verify: state extraction failed (mfe=%s)", _MFE_VALUATION
|
||||
"Cian cookies verify: HTTP 403 TLS/bot ban — cookies NOT marked expired"
|
||||
)
|
||||
return None
|
||||
user = state.get("user", {})
|
||||
if not user.get("isAuthenticated"):
|
||||
logger.warning("Cian cookies verify: user.isAuthenticated=false")
|
||||
return None
|
||||
elif result is None:
|
||||
logger.warning("Cian cookies verify: expired/unauthenticated (status=%d)", status)
|
||||
else:
|
||||
user = result.get("user", {}) or {}
|
||||
logger.info("Cian cookies verified — userId=%s", user.get("userId"))
|
||||
return state
|
||||
except httpx.HTTPStatusError as exc:
|
||||
logger.warning(
|
||||
"Cian cookies verify HTTP error: %s %s",
|
||||
exc.response.status_code,
|
||||
exc.request.url,
|
||||
)
|
||||
return None
|
||||
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.warning("Cian cookies verify failed: %s", exc)
|
||||
return None
|
||||
|
|
@ -154,7 +186,8 @@ def load_session(db: Session) -> dict[str, str] | None:
|
|||
Выбирает только записи где expires_at_estimate > NOW() и сессия не
|
||||
была инвалидирована после последнего upload'а.
|
||||
"""
|
||||
row = db.execute(
|
||||
row = (
|
||||
db.execute(
|
||||
text("""
|
||||
SELECT
|
||||
account_user_id,
|
||||
|
|
@ -167,7 +200,10 @@ def load_session(db: Session) -> dict[str, str] | None:
|
|||
LIMIT 1
|
||||
"""),
|
||||
{"key": settings.cookie_encryption_key},
|
||||
).mappings().first()
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
)
|
||||
|
||||
if row is None:
|
||||
logger.warning("No valid Cian session cookies in DB")
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""Tests for cian_session — cookie management service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
|
@ -8,13 +9,14 @@ import pytest
|
|||
|
||||
from app.services.cian_session import (
|
||||
CIAN_REQUIRED_COOKIES,
|
||||
VERIFY_BAN_SENTINEL,
|
||||
_classify_verify_response,
|
||||
load_session,
|
||||
mark_session_invalid,
|
||||
save_session,
|
||||
verify_session,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -171,84 +173,180 @@ def test_mark_session_invalid_uses_cast(mock_db: MagicMock) -> None:
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# verify_session (async)
|
||||
# _classify_verify_response (pure unit tests — no I/O)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_returns_none_when_state_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""extract_state returns None → verify_session returns None."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: None,
|
||||
)
|
||||
def test_classify_403_returns_ban_sentinel() -> None:
|
||||
"""HTTP 403 TLS/bot ban → VERIFY_BAN_SENTINEL (not None, not 'expired')."""
|
||||
result = _classify_verify_response(403, None)
|
||||
assert result is VERIFY_BAN_SENTINEL
|
||||
assert result is not None # must NOT collapse into expired-cookies signal
|
||||
|
||||
async def fake_get(url: str, **kwargs): # noqa: ARG001
|
||||
class FakeResp:
|
||||
text = "<html></html>"
|
||||
def raise_for_status(self) -> None:
|
||||
pass
|
||||
return FakeResp()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client.get = AsyncMock(return_value=_make_fake_resp("<html></html>"))
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await verify_session({"_ym_uid": "x"})
|
||||
def test_classify_401_returns_none() -> None:
|
||||
"""HTTP 401 Unauthorized → None (genuinely expired cookies)."""
|
||||
result = _classify_verify_response(401, None)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_returns_none_when_not_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""state.user.isAuthenticated is false → verify_session returns None."""
|
||||
def test_classify_200_not_authenticated_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""200 with isAuthenticated=false → None (expired)."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}},
|
||||
)
|
||||
result = _classify_verify_response(200, "<html></html>")
|
||||
assert result is None
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client.get = AsyncMock(return_value=_make_fake_resp("<html></html>"))
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await verify_session({"_ym_uid": "x"})
|
||||
def test_classify_200_authenticated_returns_state(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""200 with isAuthenticated=true → state dict."""
|
||||
expected = {"user": {"isAuthenticated": True, "userId": 99}}
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: expected,
|
||||
)
|
||||
result = _classify_verify_response(200, "<html>x</html>")
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_classify_200_state_missing_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""200 but extract_state returns None → None."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: None,
|
||||
)
|
||||
result = _classify_verify_response(200, "<html></html>")
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_classify_403_is_distinct_from_401() -> None:
|
||||
"""Ban and expired must not collapse to the same return value."""
|
||||
ban = _classify_verify_response(403, None)
|
||||
expired = _classify_verify_response(401, None)
|
||||
assert ban is not expired
|
||||
assert ban is not None
|
||||
assert expired is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# verify_session (async) — integration with curl_cffi mock
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_cffi_resp(status_code: int, text: str = "") -> MagicMock:
|
||||
resp = MagicMock()
|
||||
resp.status_code = status_code
|
||||
resp.text = text
|
||||
return resp
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_403_ban_not_treated_as_expired(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""HTTP 403 from curl_cffi → VERIFY_BAN_SENTINEL, never None."""
|
||||
mock_session = AsyncMock()
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_session.get = AsyncMock(return_value=_make_cffi_resp(403))
|
||||
|
||||
with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
|
||||
result = await verify_session({"DMIR_AUTH": "x"})
|
||||
|
||||
assert result is VERIFY_BAN_SENTINEL
|
||||
assert result is not None # must NOT trigger cookie-refresh alert
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_401_is_expired(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""HTTP 401 → None (genuine expiry)."""
|
||||
mock_session = AsyncMock()
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_session.get = AsyncMock(return_value=_make_cffi_resp(401))
|
||||
|
||||
with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
|
||||
result = await verify_session({"DMIR_AUTH": "x"})
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_returns_state_when_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""state.user.isAuthenticated is true → returns state dict."""
|
||||
async def test_verify_session_authenticated(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""200 + isAuthenticated=true → returns state dict."""
|
||||
expected_state = {"user": {"isAuthenticated": True, "userId": 102963817}}
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: expected_state,
|
||||
)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client.get = AsyncMock(return_value=_make_fake_resp("<html>state here</html>"))
|
||||
mock_client_cls.return_value = mock_client
|
||||
mock_session = AsyncMock()
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "<html>x</html>"))
|
||||
|
||||
with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
|
||||
result = await verify_session({"DMIR_AUTH": "x", "_CIAN_GK": "y"})
|
||||
|
||||
result = await verify_session({"_ym_uid": "x", "_cian_uid": "y"})
|
||||
assert result is not None
|
||||
assert result["user"]["isAuthenticated"] is True
|
||||
assert result["user"]["userId"] == 102963817
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_not_authenticated_returns_none(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""200 + isAuthenticated=false → None (expired)."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: {"user": {"isAuthenticated": False, "userId": None}},
|
||||
)
|
||||
|
||||
mock_session = AsyncMock()
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "<html></html>"))
|
||||
|
||||
with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
|
||||
result = await verify_session({"DMIR_AUTH": "x"})
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def _make_fake_resp(text: str) -> MagicMock:
|
||||
resp = MagicMock()
|
||||
resp.text = text
|
||||
resp.raise_for_status = MagicMock()
|
||||
return resp
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_state_missing_returns_none(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""200 + extract_state returns None → None."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.cian_session.extract_state",
|
||||
lambda html, mfe, key: None,
|
||||
)
|
||||
|
||||
mock_session = AsyncMock()
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_session.get = AsyncMock(return_value=_make_cffi_resp(200, "<html></html>"))
|
||||
|
||||
with patch("app.services.cian_session.AsyncSession", return_value=mock_session):
|
||||
result = await verify_session({"DMIR_AUTH": "x"})
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_session_uses_chrome120_impersonate() -> None:
|
||||
"""curl_cffi AsyncSession must be constructed with impersonate='chrome120'."""
|
||||
mock_session = AsyncMock()
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_session.get = AsyncMock(return_value=_make_cffi_resp(403))
|
||||
|
||||
with patch("app.services.cian_session.AsyncSession", return_value=mock_session) as mock_cls:
|
||||
await verify_session({"DMIR_AUTH": "x"})
|
||||
|
||||
_, kwargs = mock_cls.call_args
|
||||
assert kwargs.get("impersonate") == "chrome120"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue