All checks were successful
CI / changes (pull_request) Successful in 7s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI Trade-In / changes (pull_request) Successful in 7s
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 52s
Эмпирически подтверждено вживую 2026-07-04: инъекция cookies валидной аутентифицированной test-аккаунт сессии (Sber ID) полностью обходит QRATOR reputation-блок DomClick, даже на уже подозрительном прокси-IP (проверено: проход через camoufox context.addCookies() → 200 + полный __SSR_STATE__ на IP, который секундами ранее отдавал anti-bot страницу без cookies). - domclick_session_cookies (миграция 174) + app/services/domclick_session.py — зеркалит существующий cian_session.py (pgp_sym_encrypt, account_cas_id вместо account_user_id), без verify_session (DomClick-верификация требует реального browser-фетча, не curl_cffi — future enhancement). - POST /scrape/domclick/upload-cookies — ручная загрузка (зеркалит upload_cian_cookies). - POST /scrape/domclick/debug/detail-fetch — единственный сейчас живой путь прогнать связку session→cookies→fetch_detail (боевой backfill-оркестратор для DomClick ещё не смёржен). source="domclick" — выделенный резидентный прокси-пул (id=1, provider_affinity='domclick'), построенный и verified в этой же сессии, а не устаревший source="cian" из старой рекомендации. - _assert_domclick_host — SSRF-guard для *.domclick.ru поддоменов (реальные карточки живут на региональных поддоменах, не на голом domclick.ru). - server.py/browser_fetcher.py: cookies-параметр расширяет origin-механизм (#2430) той же схемой — keyword-only, default None → byte-identical для avito/cian/yandex. Домен cookie выводится из hostname целевого URL (провайдер-агностично), не захардкожен под domclick.
174 lines
6.7 KiB
Python
174 lines
6.7 KiB
Python
"""Offline tests для DomClick cookie-injection admin-эндпоинтов (#2000 MVP).
|
||
|
||
Покрытие 2 эндпоинтов (db/BrowserFetcher.fetch мокаются, NO live network/DB), зеркалит
|
||
паттерн test_scraper_admin_apis.py (dependency_overrides[get_db] + TestClient):
|
||
- POST /api/v1/admin/scrape/domclick/upload-cookies
|
||
- POST /api/v1/admin/scrape/domclick/debug/detail-fetch
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
from scraper_kit.providers.domclick.detail import DomClickDetailEnrichment
|
||
|
||
|
||
@pytest.fixture
|
||
def client() -> TestClient:
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
def fake_db():
|
||
yield MagicMock()
|
||
|
||
app.dependency_overrides[get_db] = fake_db
|
||
return TestClient(app)
|
||
|
||
|
||
_UPLOAD_URL = "/api/v1/admin/scrape/domclick/upload-cookies"
|
||
_DEBUG_URL = "/api/v1/admin/scrape/domclick/debug/detail-fetch"
|
||
|
||
|
||
# ── POST /scrape/domclick/upload-cookies ──────────────────────────────────────
|
||
|
||
|
||
def test_upload_cookies_503_when_no_encryption_key(client: TestClient) -> None:
|
||
with patch("app.api.v1.admin.settings.cookie_encryption_key", ""):
|
||
resp = client.post(_UPLOAD_URL, json={"CAS_ID": "12345"})
|
||
assert resp.status_code == 503
|
||
|
||
|
||
def test_upload_cookies_400_when_no_recognized_cookies(client: TestClient) -> None:
|
||
with patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"):
|
||
resp = client.post(_UPLOAD_URL, json={"unrelated_cookie": "x"})
|
||
assert resp.status_code == 400
|
||
assert "No recognized DomClick cookies" in resp.json()["detail"]
|
||
|
||
|
||
def test_upload_cookies_400_when_cas_id_missing(client: TestClient) -> None:
|
||
with patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"):
|
||
resp = client.post(_UPLOAD_URL, json={"qrator_jsid2": "abc"})
|
||
assert resp.status_code == 400
|
||
assert "CAS_ID" in resp.json()["detail"]
|
||
|
||
|
||
def test_upload_cookies_400_when_cas_id_non_numeric(client: TestClient) -> None:
|
||
with patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"):
|
||
resp = client.post(_UPLOAD_URL, json={"CAS_ID": "not-a-number", "qrator_jsid2": "abc"})
|
||
assert resp.status_code == 400
|
||
assert "not numeric" in resp.json()["detail"]
|
||
|
||
|
||
def test_upload_cookies_success_filters_and_calls_save_session(client: TestClient) -> None:
|
||
with (
|
||
patch("app.api.v1.admin.settings.cookie_encryption_key", "test-key"),
|
||
patch("app.api.v1.admin.domclick_session_svc.save_session") as mock_save,
|
||
):
|
||
resp = client.post(
|
||
_UPLOAD_URL,
|
||
json={"CAS_ID": "12345", "qrator_jsid2": "abc", "unrelated_field": "y"},
|
||
)
|
||
assert resp.status_code == 200
|
||
assert resp.json() == {"ok": True, "accountCasId": 12345, "cookieCount": 2}
|
||
|
||
mock_save.assert_called_once()
|
||
_, kwargs = mock_save.call_args
|
||
assert kwargs["account_cas_id"] == 12345
|
||
assert kwargs["cookies"] == {"CAS_ID": "12345", "qrator_jsid2": "abc"}
|
||
assert "unrelated_field" not in kwargs["cookies"]
|
||
|
||
|
||
# ── POST /scrape/domclick/debug/detail-fetch ──────────────────────────────────
|
||
|
||
_CARD_URL = "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321"
|
||
|
||
|
||
def test_debug_detail_fetch_400_when_host_not_allowed(client: TestClient) -> None:
|
||
"""SSRF guard: хост не *.domclick.ru → 400, никакого fetch не происходит."""
|
||
resp = client.post(_DEBUG_URL, json={"card_url": "https://evil.example.com/x"})
|
||
assert resp.status_code == 400
|
||
|
||
|
||
def test_debug_detail_fetch_success_without_session_cookies(client: TestClient) -> None:
|
||
"""load_session()→None (нет сохранённой сессии) → fetch без инъекции, cookies_injected=False."""
|
||
enrichment = DomClickDetailEnrichment(
|
||
item_id="2075729321",
|
||
source_url=_CARD_URL,
|
||
repair_state="good",
|
||
living_area_m2=45.5,
|
||
year_built=2015,
|
||
price_changes=[{"change_time": "x", "price_rub": 5_000_000}],
|
||
raw_extra={"wall_type": "brick"},
|
||
)
|
||
with (
|
||
patch("app.api.v1.admin.domclick_session_svc.load_session", return_value=None),
|
||
patch(
|
||
"scraper_kit.providers.domclick.detail.fetch_detail",
|
||
new=AsyncMock(return_value=enrichment),
|
||
) as mock_fetch,
|
||
):
|
||
resp = client.post(_DEBUG_URL, json={"card_url": _CARD_URL})
|
||
|
||
assert resp.status_code == 200
|
||
body = resp.json()
|
||
assert body["ok"] is True
|
||
assert body["cookies_injected"] is False
|
||
assert body["item_id"] == "2075729321"
|
||
assert body["repair_state"] == "good"
|
||
assert body["price_changes_count"] == 1
|
||
assert body["raw_extra"] == {"wall_type": "brick"}
|
||
|
||
mock_fetch.assert_called_once()
|
||
_, kwargs = mock_fetch.call_args
|
||
assert kwargs["cookies"] is None
|
||
|
||
|
||
def test_debug_detail_fetch_success_with_session_cookies(client: TestClient) -> None:
|
||
"""load_session() возвращает сессию → cookies_injected=True, cookies проброшены в fetch_detail.
|
||
|
||
Не завязан на конкретные HTML/enrichment-детали — cookies-проброс проверяется
|
||
напрямую через call_args мока fetch_detail.
|
||
"""
|
||
session_cookies = {"CAS_ID": "12345", "qrator_jsid2": "abc"}
|
||
enrichment = DomClickDetailEnrichment(item_id="2075729321", source_url=_CARD_URL)
|
||
with (
|
||
patch(
|
||
"app.api.v1.admin.domclick_session_svc.load_session",
|
||
return_value=session_cookies,
|
||
),
|
||
patch(
|
||
"scraper_kit.providers.domclick.detail.fetch_detail",
|
||
new=AsyncMock(return_value=enrichment),
|
||
) as mock_fetch,
|
||
):
|
||
resp = client.post(_DEBUG_URL, json={"card_url": _CARD_URL})
|
||
|
||
assert resp.status_code == 200
|
||
assert resp.json()["cookies_injected"] is True
|
||
|
||
mock_fetch.assert_called_once()
|
||
_, kwargs = mock_fetch.call_args
|
||
assert kwargs["cookies"] == session_cookies
|
||
|
||
|
||
def test_debug_detail_fetch_502_on_fetch_failure(client: TestClient) -> None:
|
||
with (
|
||
patch("app.api.v1.admin.domclick_session_svc.load_session", return_value=None),
|
||
patch(
|
||
"scraper_kit.providers.domclick.detail.fetch_detail",
|
||
new=AsyncMock(side_effect=RuntimeError("blocked")),
|
||
),
|
||
):
|
||
resp = client.post(_DEBUG_URL, json={"card_url": _CARD_URL})
|
||
|
||
assert resp.status_code == 502
|