lekss361
ced616d150
All checks were successful
Deploy Trade-In / test (push) Successful in 49s
Deploy Trade-In / changes (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Successful in 2m13s
Deploy Trade-In / build-backend (push) Successful in 1m32s
Deploy Trade-In / deploy (push) Successful in 7m16s
451 lines
17 KiB
Python
451 lines
17 KiB
Python
"""test_server_smoke.py — смоук для resilience-логики tradein-browser.
|
||
|
||
Полный браузер (camoufox subprocess) юнит-тестить непрактично, поэтому здесь
|
||
только смоук: (1) модуль импортируется; (2) /fetch и /login отдают 503 когда
|
||
браузер поставщика не поднят (прокси лежит) — критичный resilience-инвариант,
|
||
защищающий от crash-storm. camoufox в окружении теста НЕ нужен: launch замокан
|
||
no-op'ом, имитируя «прокси всё ещё недоступен».
|
||
|
||
Per-provider модель (#1793): состояние keyed по provider (avito/cian/yandex/
|
||
generic), доступ к инстансу сериализован _locks[provider]; разные поставщики
|
||
работают параллельно. Конкурентность проверяется в test_server.py.
|
||
|
||
Запуск (из tradein-mvp/browser/)::
|
||
|
||
python -m pytest test_server_smoke.py -q
|
||
|
||
aiohttp.test_utils.make_mocked_request не поднимает реальный сервер/сокет —
|
||
хендлер вызывается напрямую, без event-loop-сервера.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import importlib.util
|
||
import json
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from aiohttp.test_utils import make_mocked_request
|
||
|
||
# server.py — не пакет (отдельный сервис без __init__/pyproject). Грузим по пути.
|
||
_SERVER_PATH = Path(__file__).resolve().parent / "server.py"
|
||
_spec = importlib.util.spec_from_file_location("tradein_browser_server", _SERVER_PATH)
|
||
assert _spec is not None and _spec.loader is not None
|
||
server = importlib.util.module_from_spec(_spec)
|
||
_spec.loader.exec_module(server)
|
||
|
||
|
||
def _json_body(response: Any) -> dict[str, Any]:
|
||
"""Достаёт JSON-тело из aiohttp Response (body — bytes после json_response)."""
|
||
return json.loads(response.body.decode())
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _reset_state(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Чистое per-provider состояние + инициализированный _locks_guard на каждый тест."""
|
||
monkeypatch.setattr(server, "_browsers", {})
|
||
monkeypatch.setattr(server, "_browser_cms", {})
|
||
monkeypatch.setattr(server, "_page_counters", {})
|
||
monkeypatch.setattr(server, "_locks", {})
|
||
monkeypatch.setattr(server, "_retry_tasks", {})
|
||
monkeypatch.setattr(server, "_locks_guard", asyncio.Lock())
|
||
|
||
|
||
def test_module_imports() -> None:
|
||
"""Смоук: модуль импортируется и экспонирует ключевые хендлеры."""
|
||
assert callable(server.fetch_handler)
|
||
assert callable(server.login_handler)
|
||
assert callable(server.health_handler)
|
||
|
||
|
||
def test_health_reports_browsers_not_ready() -> None:
|
||
"""/health всегда 200, browsers[<provider>]=False когда инстансы не подняты."""
|
||
request = make_mocked_request("GET", "/health")
|
||
response = asyncio.run(server.health_handler(request))
|
||
assert response.status == 200
|
||
body = _json_body(response)
|
||
assert body["status"] == "ok"
|
||
assert body["browsers"] == {"avito": False, "cian": False, "yandex": False, "generic": False}
|
||
|
||
|
||
def test_health_reports_partial_readiness() -> None:
|
||
"""/health отражает per-provider readiness: один поднят, остальные нет."""
|
||
server._browsers["avito"] = object()
|
||
request = make_mocked_request("GET", "/health")
|
||
response = asyncio.run(server.health_handler(request))
|
||
body = _json_body(response)
|
||
assert body["browsers"]["avito"] is True
|
||
assert body["browsers"]["cian"] is False
|
||
|
||
|
||
def test_fetch_returns_503_when_browser_unavailable(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""/fetch отдаёт 503 (не 500/краш) когда инстанс поставщика не поднят.
|
||
|
||
Имитирует «прокси лежит»: _ensure_browser no-op, оставляющий инстанс None.
|
||
Никакого реального camoufox-launch'а / direct-fetch.
|
||
"""
|
||
|
||
async def _no_ensure(provider: str) -> bool:
|
||
return False # прокси всё ещё недоступен → браузер остаётся None
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _no_ensure)
|
||
|
||
request = make_mocked_request("POST", "/fetch")
|
||
|
||
async def _run() -> Any:
|
||
request.json = lambda: _coro({"url": "https://www.avito.ru/x"}) # type: ignore[method-assign]
|
||
return await server.fetch_handler(request)
|
||
|
||
response = asyncio.run(_run())
|
||
assert response.status == 503
|
||
body = _json_body(response)
|
||
assert "browser unavailable" in body["error"]
|
||
|
||
|
||
def test_login_returns_503_when_browser_unavailable(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""/login тоже отдаёт 503 когда инстанс не поднят (без credential-логирования)."""
|
||
|
||
async def _no_ensure(provider: str) -> bool:
|
||
return False
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _no_ensure)
|
||
|
||
login_body = {
|
||
"url": "https://www.cian.ru/login",
|
||
"email": "u@example.com",
|
||
"password": "secret",
|
||
"email_selector": "#email",
|
||
"password_selector": "#password",
|
||
"submit_selector": "#submit",
|
||
}
|
||
request = make_mocked_request("POST", "/login")
|
||
|
||
async def _run() -> Any:
|
||
request.json = lambda: _coro(login_body) # type: ignore[method-assign]
|
||
return await server.login_handler(request)
|
||
|
||
response = asyncio.run(_run())
|
||
assert response.status == 503
|
||
body = _json_body(response)
|
||
assert "browser unavailable" in body["error"]
|
||
|
||
|
||
# ── per-provider state isolation ─────────────────────────────────────────────────
|
||
|
||
|
||
class _FakePage:
|
||
"""Поддельная page: считает open/close и отмечает пик параллелизма."""
|
||
|
||
def __init__(self, tracker: "_Tracker") -> None:
|
||
self._tracker = tracker
|
||
|
||
async def route(self, pattern: str, handler: Any) -> None:
|
||
return None
|
||
|
||
async def goto(self, url: str, **kwargs: Any) -> None:
|
||
self._tracker.active += 1
|
||
self._tracker.peak = max(self._tracker.peak, self._tracker.active)
|
||
await asyncio.sleep(0.02)
|
||
self._tracker.active -= 1
|
||
|
||
async def wait_for_timeout(self, ms: int) -> None:
|
||
return None
|
||
|
||
async def content(self) -> str:
|
||
return "<html>ok</html>"
|
||
|
||
async def close(self) -> None:
|
||
self._tracker.closed += 1
|
||
|
||
|
||
class _FakeBrowser:
|
||
def __init__(self, tracker: "_Tracker") -> None:
|
||
self._tracker = tracker
|
||
|
||
async def new_page(self) -> _FakePage:
|
||
self._tracker.opened += 1
|
||
return _FakePage(self._tracker)
|
||
|
||
|
||
class _Tracker:
|
||
def __init__(self) -> None:
|
||
self.opened = 0
|
||
self.closed = 0
|
||
self.active = 0
|
||
self.peak = 0
|
||
|
||
|
||
def _make_fetch_request(url: str = "https://www.avito.ru/x") -> Any:
|
||
request = make_mocked_request("POST", "/fetch")
|
||
request.json = lambda: _coro({"url": url}) # type: ignore[method-assign]
|
||
return request
|
||
|
||
|
||
def test_fetch_opens_and_closes_own_page(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Один /fetch открывает ровно одну собственную страницу и закрывает её."""
|
||
tracker = _Tracker()
|
||
server._browsers["avito"] = _FakeBrowser(tracker)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
response = asyncio.run(server.fetch_handler(_make_fetch_request()))
|
||
assert response.status == 200
|
||
assert _json_body(response)["html"] == "<html>ok</html>"
|
||
assert tracker.opened == 1
|
||
assert tracker.closed == 1
|
||
|
||
|
||
# ── _relaunch resets per-provider page counter (recycle) ─────────────────────────
|
||
|
||
|
||
def test_fetch_recycles_after_threshold(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Достижение BROWSER_RECYCLE_PAGES → _relaunch_browser + сброс счётчика."""
|
||
tracker = _Tracker()
|
||
server._browsers["avito"] = _FakeBrowser(tracker)
|
||
server._page_counters["avito"] = 0
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 1)
|
||
|
||
relaunched = {"n": 0}
|
||
|
||
async def _fake_relaunch(provider: str) -> None:
|
||
relaunched["n"] += 1
|
||
server._page_counters[provider] = 0
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_relaunch_browser", _fake_relaunch)
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
asyncio.run(server.fetch_handler(_make_fetch_request()))
|
||
assert relaunched["n"] == 1, "recycle должен сработать на пороге"
|
||
assert server._page_counters["avito"] == 0
|
||
|
||
|
||
# ── _login_once closes page on CancelledError (BaseException leak fix) ────────────
|
||
|
||
|
||
class _CancellingLoginPage:
|
||
"""Page для /login, которая бросает CancelledError внутри логина."""
|
||
|
||
def __init__(self, tracker: "_Tracker") -> None:
|
||
self._tracker = tracker
|
||
self.url = "https://www.cian.ru/login"
|
||
|
||
async def goto(self, url: str, **kwargs: Any) -> None:
|
||
return None
|
||
|
||
async def wait_for_timeout(self, ms: int) -> None:
|
||
return None
|
||
|
||
async def fill(self, selector: str, value: str, **kwargs: Any) -> None:
|
||
raise asyncio.CancelledError()
|
||
|
||
async def close(self) -> None:
|
||
self._tracker.closed += 1
|
||
|
||
|
||
class _CancellingLoginBrowser:
|
||
def __init__(self, tracker: "_Tracker") -> None:
|
||
self._tracker = tracker
|
||
|
||
async def new_page(self) -> _CancellingLoginPage:
|
||
self._tracker.opened += 1
|
||
return _CancellingLoginPage(self._tracker)
|
||
|
||
|
||
def test_login_closes_page_on_cancelled_error(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""_login_once закрывает страницу даже при CancelledError (finally, не except)."""
|
||
tracker = _Tracker()
|
||
server._browsers["cian"] = _CancellingLoginBrowser(tracker)
|
||
|
||
params = {
|
||
"url": "https://www.cian.ru/login",
|
||
"email": "u@example.com",
|
||
"password": "secret",
|
||
"email_selector": "#email",
|
||
"password_selector": "#password",
|
||
"submit_selector": "#submit",
|
||
}
|
||
|
||
with pytest.raises(asyncio.CancelledError):
|
||
asyncio.run(server._login_once("cian", params))
|
||
|
||
assert tracker.opened == 1
|
||
assert tracker.closed == 1, "страница должна закрыться в finally при CancelledError"
|
||
|
||
|
||
# ── _fetch_once: origin-goto ТОЛЬКО когда передан (DomClick anti-bot, #2429-ish) ──
|
||
# Зеркалит _fetch_json_once (#1917): когда origin передан, камуфокс сначала заходит
|
||
# на origin (same-site якорь, напр. SERP), затем на целевой url — органическая
|
||
# навигация вместо холодного goto. origin=None (дефолт, все providers кроме
|
||
# domclick detail) должен остаться РОВНО одним goto(url), без регрессии.
|
||
|
||
|
||
class _FakeCookieContext:
|
||
"""Поддельный BrowserContext: фиксирует add_cookies-вызовы (порядок + payload)."""
|
||
|
||
def __init__(self) -> None:
|
||
self.add_cookies_calls: list[list[dict]] = []
|
||
|
||
async def add_cookies(self, cookies: list[dict]) -> None:
|
||
self.add_cookies_calls.append(cookies)
|
||
|
||
|
||
class _OriginTrackingPage:
|
||
"""Поддельная page: фиксирует ВСЕ goto-URL по порядку вызова + cookie-инъекции."""
|
||
|
||
def __init__(self) -> None:
|
||
self.goto_urls: list[str] = []
|
||
self.closed = 0
|
||
self.context = _FakeCookieContext()
|
||
|
||
async def route(self, pattern: str, handler: Any) -> None:
|
||
return None
|
||
|
||
async def goto(self, url: str, **kwargs: Any) -> None:
|
||
self.goto_urls.append(url)
|
||
|
||
async def wait_for_timeout(self, ms: int) -> None:
|
||
return None
|
||
|
||
async def content(self) -> str:
|
||
return "<html>ok</html>"
|
||
|
||
async def close(self) -> None:
|
||
self.closed += 1
|
||
|
||
|
||
class _OriginTrackingBrowser:
|
||
def __init__(self, page: _OriginTrackingPage) -> None:
|
||
self._page = page
|
||
|
||
async def new_page(self) -> _OriginTrackingPage:
|
||
return self._page
|
||
|
||
|
||
def test_fetch_once_navigates_origin_then_url_when_provided(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""origin передан → ДВА goto по порядку: origin первым, затем url."""
|
||
page = _OriginTrackingPage()
|
||
server._browsers["avito"] = _OriginTrackingBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
html = asyncio.run(
|
||
server._fetch_once(
|
||
"avito", "https://www.avito.ru/card/1", origin="https://www.avito.ru/serp"
|
||
)
|
||
)
|
||
|
||
assert html == "<html>ok</html>"
|
||
assert page.goto_urls == ["https://www.avito.ru/serp", "https://www.avito.ru/card/1"]
|
||
assert page.closed == 1
|
||
|
||
|
||
def test_fetch_once_single_goto_when_origin_none(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""origin не передан (дефолт, avito/cian/yandex) → РОВНО один goto(url), без regressии."""
|
||
page = _OriginTrackingPage()
|
||
server._browsers["avito"] = _OriginTrackingBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
html = asyncio.run(server._fetch_once("avito", "https://www.avito.ru/card/1"))
|
||
|
||
assert html == "<html>ok</html>"
|
||
assert page.goto_urls == ["https://www.avito.ru/card/1"]
|
||
|
||
|
||
# ── _fetch_once: cookies-инъекция ТОЛЬКО когда передана (DomClick QRATOR-bypass) ──
|
||
# Провайдер-агностично: domain НЕ захардкожен ".domclick.ru" — выводится из
|
||
# urlparse(url).hostname (с ведущей точкой) прямо в _fetch_once, поэтому механизм
|
||
# годится для будущих caller'ов (Cian/Avito/Yandex), не только DomClick.
|
||
# cookies=None (дефолт, все providers сейчас) должен остаться без add_cookies-вызова.
|
||
|
||
|
||
def test_fetch_once_injects_cookies_before_goto_with_derived_domain(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""cookies передан → context.add_cookies() ДО goto(url), domain = "." + hostname."""
|
||
page = _OriginTrackingPage()
|
||
server._browsers["cian"] = _OriginTrackingBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
html = asyncio.run(
|
||
server._fetch_once(
|
||
"cian",
|
||
"https://ekaterinburg.domclick.ru/card/sale__flat__1",
|
||
cookies={"CAS_ID": "12345", "qrator_jsid2": "abc"},
|
||
)
|
||
)
|
||
|
||
assert html == "<html>ok</html>"
|
||
assert len(page.context.add_cookies_calls) == 1
|
||
injected = page.context.add_cookies_calls[0]
|
||
assert {
|
||
"name": "CAS_ID",
|
||
"value": "12345",
|
||
"domain": ".ekaterinburg.domclick.ru",
|
||
"path": "/",
|
||
} in injected
|
||
assert {
|
||
"name": "qrator_jsid2",
|
||
"value": "abc",
|
||
"domain": ".ekaterinburg.domclick.ru",
|
||
"path": "/",
|
||
} in injected
|
||
# Инъекция ДО навигации на целевой url — единственный goto остаётся ровно url.
|
||
assert page.goto_urls == ["https://ekaterinburg.domclick.ru/card/sale__flat__1"]
|
||
|
||
|
||
def test_fetch_once_no_cookie_injection_when_cookies_none(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""cookies=None (дефолт) → add_cookies НЕ вызывается, поведение не меняется."""
|
||
page = _OriginTrackingPage()
|
||
server._browsers["avito"] = _OriginTrackingBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
asyncio.run(server._fetch_once("avito", "https://www.avito.ru/card/1"))
|
||
|
||
assert page.context.add_cookies_calls == []
|
||
|
||
|
||
def test_fetch_once_cookie_domain_derivation_is_generic_not_domclick_hardcoded(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Domain derivation работает для ЛЮБОГО хоста, не только domclick.ru.
|
||
|
||
Доказывает, что механизм провайдер-агностичен: hardcode ".domclick.ru" НЕ
|
||
используется — тот же код с avito.ru url выводит ".www.avito.ru".
|
||
"""
|
||
page = _OriginTrackingPage()
|
||
server._browsers["avito"] = _OriginTrackingBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
asyncio.run(
|
||
server._fetch_once(
|
||
"avito", "https://www.avito.ru/items/1", cookies={"sessionid": "xyz"}
|
||
)
|
||
)
|
||
|
||
injected = page.context.add_cookies_calls[0]
|
||
assert injected == [
|
||
{"name": "sessionid", "value": "xyz", "domain": ".www.avito.ru", "path": "/"}
|
||
]
|
||
|
||
|
||
async def _coro(value: Any) -> Any:
|
||
"""Хелпер: оборачивает значение в awaitable для подмены request.json()."""
|
||
return value
|