fix(tradein/admin): развести исходы проверки кук — бан не равен «плохим кукам» #2829
2 changed files with 376 additions and 14 deletions
|
|
@ -373,6 +373,56 @@ async def geocode_missing(
|
|||
}
|
||||
|
||||
|
||||
def _cian_verify_state_error(state: dict[str, Any] | None) -> HTTPException | None:
|
||||
"""Маппинг исхода cian_session_svc.verify_session() на HTTP-ответ админки.
|
||||
|
||||
verify_session() возвращает 4 разных исхода (см. докстринг сервиса) плюс успех —
|
||||
их нельзя схлопывать в один "cookies invalid", иначе бан по IP выглядит так же,
|
||||
как протухшие куки, и человек в момент инцидента перезаливает заведомо валидные
|
||||
куки вместо починки egress/прокси (инцидент 2026-08-10).
|
||||
|
||||
Sentinel'ы сравниваются через `is`, НЕ `==` — так требует докстринг verify_session.
|
||||
|
||||
Возвращает None, если state — это успешно распаршенный state dict (в т.ч. случай
|
||||
"успех, но userId не найден" — этот случай caller должен обработать отдельно).
|
||||
"""
|
||||
if state is cian_session_svc.VERIFY_BAN_SENTINEL:
|
||||
return HTTPException(
|
||||
status_code=503,
|
||||
detail=(
|
||||
"Cian заблокировал наш IP (HTTP 403, TLS/bot-fingerprint ban). "
|
||||
"Куки, скорее всего, валидны — блокировка не про них. "
|
||||
"Нужно чинить egress: проверить SCRAPER_PROXY_URL и баны в "
|
||||
"scrape_proxy_source_bans. Перезаливать куки бесполезно. "
|
||||
"(CIAN_PROXY_URL — мёртвая переменная, снята в #2616.)"
|
||||
),
|
||||
)
|
||||
if state is cian_session_svc.VERIFY_SOURCE_UNAVAILABLE_SENTINEL:
|
||||
return HTTPException(
|
||||
status_code=503,
|
||||
detail=(
|
||||
"Cian временно недоступен (5xx или сетевой сбой при проверке кук). "
|
||||
"Повторите проверку позже. Куки не трогать — источник просто не ответил."
|
||||
),
|
||||
)
|
||||
if state is cian_session_svc.VERIFY_MARKUP_CHANGED_SENTINEL:
|
||||
return HTTPException(
|
||||
status_code=500,
|
||||
detail=(
|
||||
"Cian изменил вёрстку/схему страницы — auth-state не найден/не "
|
||||
"распарсился (scraper_kit.cian_state_parser.extract_state, MFE "
|
||||
"header-frontend). Нужен инженерный фикс парсера, перезалив кук "
|
||||
"проблему НЕ решит."
|
||||
),
|
||||
)
|
||||
if state is None:
|
||||
return HTTPException(
|
||||
status_code=401,
|
||||
detail="Куки протухли или сессия разлогинена на cian.ru — перезалейте куки.",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/scrape/cian/upload-cookies", status_code=200)
|
||||
async def upload_cian_cookies(
|
||||
cookies: dict[str, str],
|
||||
|
|
@ -405,15 +455,21 @@ async def upload_cian_cookies(
|
|||
)
|
||||
|
||||
state = await cian_session_svc.verify_session(cleaned)
|
||||
if state is None:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Cookies invalid or session not authenticated on cian.ru",
|
||||
)
|
||||
verify_error = _cian_verify_state_error(state)
|
||||
if verify_error is not None:
|
||||
raise verify_error
|
||||
assert state is not None # narrowed by _cian_verify_state_error above
|
||||
|
||||
user_id = state.get("user", {}).get("userId")
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=400, detail="Authenticated state missing userId")
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"Cian подтвердил аутентификацию (state распарсился), но userId в "
|
||||
"ответе не найден — структура state неожиданная, куки тут ни при "
|
||||
"чём, смотрите server logs."
|
||||
),
|
||||
)
|
||||
|
||||
cian_session_svc.save_session(db, account_user_id=int(user_id), cookies=cleaned)
|
||||
return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)}
|
||||
|
|
@ -480,15 +536,21 @@ async def cian_auto_login(
|
|||
)
|
||||
|
||||
state = await cian_session_svc.verify_session(cleaned)
|
||||
if state is None:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Logged in but session not authenticated (cookies rejected by cian.ru)",
|
||||
)
|
||||
verify_error = _cian_verify_state_error(state)
|
||||
if verify_error is not None:
|
||||
raise verify_error
|
||||
assert state is not None # narrowed by _cian_verify_state_error above
|
||||
|
||||
user_id = state.get("user", {}).get("userId")
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=400, detail="Authenticated state missing userId")
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"Cian подтвердил аутентификацию (state распарсился), но userId в "
|
||||
"ответе не найден — структура state неожиданная, куки тут ни при "
|
||||
"чём, смотрите server logs."
|
||||
),
|
||||
)
|
||||
|
||||
cian_session_svc.save_session(db, account_user_id=int(user_id), cookies=cleaned)
|
||||
return {"ok": True, "userId": user_id, "cookieCount": len(cleaned)}
|
||||
|
|
@ -500,6 +562,12 @@ async def test_cian_auth(
|
|||
) -> dict:
|
||||
"""Проверить что текущие сохранённые Cian cookies ещё валидны.
|
||||
|
||||
reason различает 5 исходов (см. cian_session_svc.verify_session докстринг):
|
||||
"banned_403" — куки, вероятно, ОК, блокирован IP; "source_unavailable" —
|
||||
Cian недоступен, куки ни при чём; "markup_changed" — вёрстка Cian сменилась,
|
||||
нужен фикс парсера; "session_expired_or_invalid" — куки реально протухли;
|
||||
"no_session_in_db" / "encryption_key_not_configured" — конфигурация/данных нет.
|
||||
|
||||
Returns: {"authenticated": bool, "userId": <int|null>, "reason": <str|null>}
|
||||
"""
|
||||
if not settings.cookie_encryption_key:
|
||||
|
|
@ -510,10 +578,14 @@ async def test_cian_auth(
|
|||
return {"authenticated": False, "userId": None, "reason": "no_session_in_db"}
|
||||
|
||||
state = await cian_session_svc.verify_session(cookies)
|
||||
if state is cian_session_svc.VERIFY_BAN_SENTINEL:
|
||||
return {"authenticated": False, "userId": None, "reason": "banned_403"}
|
||||
if state is cian_session_svc.VERIFY_SOURCE_UNAVAILABLE_SENTINEL:
|
||||
return {"authenticated": False, "userId": None, "reason": "source_unavailable"}
|
||||
if state is cian_session_svc.VERIFY_MARKUP_CHANGED_SENTINEL:
|
||||
return {"authenticated": False, "userId": None, "reason": "markup_changed"}
|
||||
if state is None:
|
||||
return {"authenticated": False, "userId": None, "reason": "session_expired_or_invalid"}
|
||||
if state.get("_ban"):
|
||||
return {"authenticated": False, "userId": None, "reason": "banned_403"}
|
||||
|
||||
user_id = state.get("user", {}).get("userId")
|
||||
return {"authenticated": True, "userId": user_id, "reason": None}
|
||||
|
|
|
|||
290
tradein-mvp/backend/tests/test_admin_cian_session_endpoints.py
Normal file
290
tradein-mvp/backend/tests/test_admin_cian_session_endpoints.py
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
"""Offline tests для Cian cookie-session admin-эндпоинтов (#инцидент 2026-08-10).
|
||||
|
||||
cian_session_svc.verify_session() возвращает 5 разных исходов (state dict / None /
|
||||
VERIFY_BAN_SENTINEL / VERIFY_SOURCE_UNAVAILABLE_SENTINEL / VERIFY_MARKUP_CHANGED_SENTINEL)
|
||||
и раньше admin.py проверял только `if state is None`, из-за чего реальный бан по IP
|
||||
(403) и недоступность источника (5xx) выглядели как "куки протухли" — человек в
|
||||
момент инцидента перезаливал заведомо валидные куки вместо починки egress/прокси.
|
||||
|
||||
Покрытие 3 эндпоинтов (db/verify_session/BrowserFetcher мокаются, NO live network/DB),
|
||||
зеркалит паттерн test_domclick_admin_apis.py (dependency_overrides[get_db] + TestClient):
|
||||
- POST /api/v1/admin/scrape/cian/upload-cookies
|
||||
- POST /api/v1/admin/scrape/cian/auto-login
|
||||
- GET /api/v1/admin/scrape/cian/test-auth
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.services.cian_session import (
|
||||
VERIFY_BAN_SENTINEL,
|
||||
VERIFY_MARKUP_CHANGED_SENTINEL,
|
||||
VERIFY_SOURCE_UNAVAILABLE_SENTINEL,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
from app.api.v1 import admin as admin_module
|
||||
from app.core.db import get_db
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||||
|
||||
def fake_db():
|
||||
yield MagicMock()
|
||||
|
||||
app.dependency_overrides[get_db] = fake_db
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
_UPLOAD_URL = "/api/v1/admin/scrape/cian/upload-cookies"
|
||||
_AUTOLOGIN_URL = "/api/v1/admin/scrape/cian/auto-login"
|
||||
_TEST_AUTH_URL = "/api/v1/admin/scrape/cian/test-auth"
|
||||
|
||||
_RAW_COOKIES = {"DMIR_AUTH": "abc", "_CIAN_GK": "def"}
|
||||
_AUTHENTICATED_STATE = {"user": {"isAuthenticated": True, "userId": 102963817}}
|
||||
_AUTHENTICATED_STATE_NO_USERID = {"user": {"isAuthenticated": True}}
|
||||
|
||||
|
||||
# ── POST /scrape/cian/upload-cookies — пять исходов verify_session ────────────
|
||||
|
||||
|
||||
def test_upload_cookies_ban_returns_503(client: TestClient) -> None:
|
||||
"""403 TLS/bot ban → 503, текст говорит чинить прокси, НЕ перезаливать куки."""
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=VERIFY_BAN_SENTINEL),
|
||||
),
|
||||
):
|
||||
resp = client.post(_UPLOAD_URL, json=_RAW_COOKIES)
|
||||
assert resp.status_code == 503
|
||||
detail = resp.json()["detail"]
|
||||
assert "заблокировал" in detail
|
||||
assert "прокси" in detail or "egress" in detail
|
||||
assert "бесполезно" in detail
|
||||
# Текст обязан называть ЖИВУЮ переменную: инцидент 2026-08-10 — правка мёртвой
|
||||
# CIAN_PROXY_URL не давала эффекта, реальный egress задаётся SCRAPER_PROXY_URL.
|
||||
assert "SCRAPER_PROXY_URL" in detail
|
||||
|
||||
|
||||
def test_upload_cookies_source_unavailable_returns_503(client: TestClient) -> None:
|
||||
"""5xx/сеть → 503, текст говорит повторить позже, куки не трогать."""
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=VERIFY_SOURCE_UNAVAILABLE_SENTINEL),
|
||||
),
|
||||
):
|
||||
resp = client.post(_UPLOAD_URL, json=_RAW_COOKIES)
|
||||
assert resp.status_code == 503
|
||||
detail = resp.json()["detail"]
|
||||
assert "недоступен" in detail
|
||||
assert "позже" in detail
|
||||
|
||||
|
||||
def test_upload_cookies_markup_changed_returns_500(client: TestClient) -> None:
|
||||
"""200 но auth-state не распарсился → 500, текст указывает на фикс парсера."""
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=VERIFY_MARKUP_CHANGED_SENTINEL),
|
||||
),
|
||||
):
|
||||
resp = client.post(_UPLOAD_URL, json=_RAW_COOKIES)
|
||||
assert resp.status_code == 500
|
||||
detail = resp.json()["detail"]
|
||||
assert "вёрстк" in detail
|
||||
assert "перезалив" in detail
|
||||
|
||||
|
||||
def test_upload_cookies_expired_returns_401(client: TestClient) -> None:
|
||||
"""None → 401, текст просит перезалить куки (единственный случай re-upload)."""
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=None),
|
||||
),
|
||||
):
|
||||
resp = client.post(_UPLOAD_URL, json=_RAW_COOKIES)
|
||||
assert resp.status_code == 401
|
||||
assert "перезалейте" in resp.json()["detail"]
|
||||
|
||||
|
||||
def test_upload_cookies_success_missing_userid_returns_400(client: TestClient) -> None:
|
||||
"""Успешный state, но без userId → 400 с уточнением что куки тут ни при чём."""
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=_AUTHENTICATED_STATE_NO_USERID),
|
||||
),
|
||||
):
|
||||
resp = client.post(_UPLOAD_URL, json=_RAW_COOKIES)
|
||||
assert resp.status_code == 400
|
||||
detail = resp.json()["detail"]
|
||||
assert "userId" in detail
|
||||
assert "куки тут ни при" in detail
|
||||
|
||||
|
||||
def test_upload_cookies_success_saves_and_returns_200(client: TestClient) -> None:
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=_AUTHENTICATED_STATE),
|
||||
),
|
||||
patch("app.api.v1.admin.cian_session_svc.save_session") as mock_save,
|
||||
):
|
||||
resp = client.post(_UPLOAD_URL, json=_RAW_COOKIES)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["ok"] is True
|
||||
assert body["userId"] == 102963817
|
||||
mock_save.assert_called_once()
|
||||
_, kwargs = mock_save.call_args
|
||||
assert kwargs["account_user_id"] == 102963817
|
||||
|
||||
|
||||
# ── POST /scrape/cian/auto-login — тот же маппинг после успешного browser-login ──
|
||||
|
||||
|
||||
def _mock_browser_fetcher(raw_cookies: dict[str, str]) -> MagicMock:
|
||||
fetcher = AsyncMock()
|
||||
fetcher.__aenter__ = AsyncMock(return_value=fetcher)
|
||||
fetcher.__aexit__ = AsyncMock(return_value=None)
|
||||
fetcher.login = AsyncMock(return_value=raw_cookies)
|
||||
return fetcher
|
||||
|
||||
|
||||
def test_auto_login_ban_returns_503(client: TestClient) -> None:
|
||||
"""Browser login прошёл, но verify_session ловит 403 ban → 503, не "куки протухли"."""
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch("app.api.v1.admin.settings.cian_login_email", "user@example.com"),
|
||||
patch("app.api.v1.admin.settings.cian_login_password", "secret"),
|
||||
patch(
|
||||
"app.api.v1.admin.BrowserFetcher",
|
||||
return_value=_mock_browser_fetcher(_RAW_COOKIES),
|
||||
),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=VERIFY_BAN_SENTINEL),
|
||||
),
|
||||
):
|
||||
resp = client.post(_AUTOLOGIN_URL)
|
||||
assert resp.status_code == 503
|
||||
assert "заблокировал" in resp.json()["detail"]
|
||||
|
||||
|
||||
def test_auto_login_success_saves_and_returns_200(client: TestClient) -> None:
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch("app.api.v1.admin.settings.cian_login_email", "user@example.com"),
|
||||
patch("app.api.v1.admin.settings.cian_login_password", "secret"),
|
||||
patch(
|
||||
"app.api.v1.admin.BrowserFetcher",
|
||||
return_value=_mock_browser_fetcher(_RAW_COOKIES),
|
||||
),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=_AUTHENTICATED_STATE),
|
||||
),
|
||||
patch("app.api.v1.admin.cian_session_svc.save_session") as mock_save,
|
||||
):
|
||||
resp = client.post(_AUTOLOGIN_URL)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["userId"] == 102963817
|
||||
mock_save.assert_called_once()
|
||||
|
||||
|
||||
# ── GET /scrape/cian/test-auth — reason различает все пять исходов ────────────
|
||||
|
||||
|
||||
def test_test_auth_ban_reason(client: TestClient) -> None:
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch("app.api.v1.admin.cian_session_svc.load_session", return_value=_RAW_COOKIES),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=VERIFY_BAN_SENTINEL),
|
||||
),
|
||||
):
|
||||
resp = client.get(_TEST_AUTH_URL)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["authenticated"] is False
|
||||
assert body["reason"] == "banned_403"
|
||||
|
||||
|
||||
def test_test_auth_source_unavailable_reason(client: TestClient) -> None:
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch("app.api.v1.admin.cian_session_svc.load_session", return_value=_RAW_COOKIES),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=VERIFY_SOURCE_UNAVAILABLE_SENTINEL),
|
||||
),
|
||||
):
|
||||
resp = client.get(_TEST_AUTH_URL)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["reason"] == "source_unavailable"
|
||||
|
||||
|
||||
def test_test_auth_markup_changed_reason(client: TestClient) -> None:
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch("app.api.v1.admin.cian_session_svc.load_session", return_value=_RAW_COOKIES),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=VERIFY_MARKUP_CHANGED_SENTINEL),
|
||||
),
|
||||
):
|
||||
resp = client.get(_TEST_AUTH_URL)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["reason"] == "markup_changed"
|
||||
|
||||
|
||||
def test_test_auth_expired_reason(client: TestClient) -> None:
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch("app.api.v1.admin.cian_session_svc.load_session", return_value=_RAW_COOKIES),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=None),
|
||||
),
|
||||
):
|
||||
resp = client.get(_TEST_AUTH_URL)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["reason"] == "session_expired_or_invalid"
|
||||
|
||||
|
||||
def test_test_auth_success(client: TestClient) -> None:
|
||||
with (
|
||||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||||
patch("app.api.v1.admin.cian_session_svc.load_session", return_value=_RAW_COOKIES),
|
||||
patch(
|
||||
"app.api.v1.admin.cian_session_svc.verify_session",
|
||||
new=AsyncMock(return_value=_AUTHENTICATED_STATE),
|
||||
),
|
||||
):
|
||||
resp = client.get(_TEST_AUTH_URL)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["authenticated"] is True
|
||||
assert body["userId"] == 102963817
|
||||
assert body["reason"] is None
|
||||
Loading…
Add table
Reference in a new issue