Light1YT
2687f75b92
First XHR after goto(origin) intermittently hit "NetworkError when attempting to fetch resource" (page net stack/anti-bot not ready), healed only by the outer re-navigation retry (~30-45s/house in the house_imv_backfill, #562). _fetch_json_once now: - settles FETCH_JSON_SETTLE_MS (default 1200, was hardcoded 500) — fewer first-fails; - wraps the in-page fetch() in a JS retry loop (FETCH_JSON_INPAGE_RETRIES default 1, FETCH_JSON_RETRY_DELAY_MS default 800) that retries ONLY on network throw, never on HTTP status (4xx/5xx short-circuit, caller decides). An in-page retry costs ~retryDelayMs vs the ~30-45s outer re-navigation. Last error re-thrown — outer crash-retry contract intact. /fetch (SERP) path untouched. +2 tests (settle ms, retry params). Refs #1917, #562
367 lines
15 KiB
Python
367 lines
15 KiB
Python
"""test_server_fetch_json.py — юниты для аддитивного /fetch-json (#915 Stage 2).
|
||
|
||
Проверяет новый in-page fetch путь, НЕ трогая существующий /fetch:
|
||
1. fetch_json_handler возвращает {"status","body"} из page.evaluate;
|
||
2. 400 при отсутствии url;
|
||
3. 503 когда _ensure_browser отдаёт False (прокси/браузер недоступны);
|
||
4. навигация идёт на origin (same-origin якорь), не на url;
|
||
5. page закрывается после in-page fetch (finally).
|
||
|
||
camoufox НЕ запускается: _ensure_browser мокается / провайдер-браузер подделан
|
||
фейковой page с AsyncMock на evaluate. Хендлер вызывается напрямую
|
||
(make_mocked_request не поднимает сокет) — как в test_server_smoke.py.
|
||
|
||
Запуск (из tradein-mvp/browser/)::
|
||
|
||
python -m pytest test_server_fetch_json.py -q
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import importlib.util
|
||
import json
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock
|
||
|
||
import pytest
|
||
from aiohttp.test_utils import make_mocked_request
|
||
|
||
# server.py — не пакет (отдельный сервис без __init__/pyproject). Грузим по пути.
|
||
_SERVER_PATH = Path(__file__).resolve().parent / "server.py"
|
||
_spec = importlib.util.spec_from_file_location("tradein_browser_server", _SERVER_PATH)
|
||
assert _spec is not None and _spec.loader is not None
|
||
server = importlib.util.module_from_spec(_spec)
|
||
_spec.loader.exec_module(server)
|
||
|
||
|
||
def _json_body(response: Any) -> dict[str, Any]:
|
||
"""Достаёт JSON-тело из aiohttp Response (body — bytes после json_response)."""
|
||
return json.loads(response.body.decode())
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _reset_state(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Чистое per-provider состояние + инициализированный _locks_guard на каждый тест."""
|
||
monkeypatch.setattr(server, "_browsers", {})
|
||
monkeypatch.setattr(server, "_browser_cms", {})
|
||
monkeypatch.setattr(server, "_page_counters", {})
|
||
monkeypatch.setattr(server, "_locks", {})
|
||
monkeypatch.setattr(server, "_retry_tasks", {})
|
||
monkeypatch.setattr(server, "_last_goto_at", {})
|
||
monkeypatch.setattr(server, "_locks_guard", asyncio.Lock())
|
||
|
||
|
||
async def _coro(value: Any) -> Any:
|
||
"""Хелпер: оборачивает значение в awaitable для подмены request.json()."""
|
||
return value
|
||
|
||
|
||
# ── фейковые page/browser с AsyncMock на evaluate ─────────────────────────────────
|
||
|
||
|
||
class _FakePage:
|
||
"""Поддельная page: фиксирует goto-цель, отдаёт фиктивный результат evaluate."""
|
||
|
||
def __init__(self, evaluate_result: dict[str, Any]) -> None:
|
||
self.goto_urls: list[str] = []
|
||
self.waits: list[int] = [] # записанные wait_for_timeout(ms) — settle-проверка #1917
|
||
self.closed = 0
|
||
# evaluate — AsyncMock, чтобы проверять как сам результат, так и аргументы.
|
||
self.evaluate = AsyncMock(return_value=evaluate_result)
|
||
|
||
async def route(self, pattern: str, handler: Any) -> None:
|
||
return None
|
||
|
||
async def goto(self, url: str, **kwargs: Any) -> None:
|
||
self.goto_urls.append(url)
|
||
|
||
async def wait_for_timeout(self, ms: int) -> None:
|
||
self.waits.append(ms)
|
||
|
||
async def close(self) -> None:
|
||
self.closed += 1
|
||
|
||
|
||
class _FakeBrowser:
|
||
def __init__(self, page: _FakePage) -> None:
|
||
self._page = page
|
||
self.opened = 0
|
||
|
||
async def new_page(self) -> _FakePage:
|
||
self.opened += 1
|
||
return self._page
|
||
|
||
|
||
def _make_request(body: dict[str, Any]) -> Any:
|
||
request = make_mocked_request("POST", "/fetch-json")
|
||
request.json = lambda: _coro(body) # type: ignore[method-assign]
|
||
return request
|
||
|
||
|
||
# ── happy path ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_fetch_json_returns_evaluate_result(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""fetch_json_handler возвращает {"status","body"} ровно как отдал page.evaluate."""
|
||
page = _FakePage({"status": 200, "body": '{"ok": true}'})
|
||
server._browsers["avito"] = _FakeBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
response = asyncio.run(
|
||
server.fetch_json_handler(
|
||
_make_request({"url": "https://www.avito.ru/api/x", "source": "avito"})
|
||
)
|
||
)
|
||
assert response.status == 200
|
||
body = _json_body(response)
|
||
assert body == {"status": 200, "body": '{"ok": true}'}
|
||
# page.evaluate был вызван ровно один раз с url+method+headers+body.
|
||
page.evaluate.assert_awaited_once()
|
||
call_args = page.evaluate.await_args
|
||
assert call_args.args[1]["url"] == "https://www.avito.ru/api/x"
|
||
assert call_args.args[1]["method"] == "GET"
|
||
# Страница открылась и закрылась.
|
||
assert page.closed == 1
|
||
|
||
|
||
def test_fetch_json_navigates_to_origin_not_url(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Навигация идёт на origin (same-origin якорь), а fetch — уже на url."""
|
||
page = _FakePage({"status": 200, "body": "ok"})
|
||
server._browsers["avito"] = _FakeBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
asyncio.run(
|
||
server.fetch_json_handler(
|
||
_make_request({"url": "https://www.avito.ru/api/listings?q=1", "source": "avito"})
|
||
)
|
||
)
|
||
# Дефолтный origin выводится из url: scheme://netloc/.
|
||
assert page.goto_urls == ["https://www.avito.ru/"]
|
||
|
||
|
||
def test_fetch_json_uses_explicit_origin(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Явный body["origin"] перебивает выведенный из url."""
|
||
page = _FakePage({"status": 200, "body": "ok"})
|
||
server._browsers["avito"] = _FakeBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
asyncio.run(
|
||
server.fetch_json_handler(
|
||
_make_request(
|
||
{
|
||
"url": "https://www.avito.ru/api/x",
|
||
"origin": "https://www.avito.ru/moskva/kvartiry",
|
||
"source": "avito",
|
||
}
|
||
)
|
||
)
|
||
)
|
||
assert page.goto_urls == ["https://www.avito.ru/moskva/kvartiry"]
|
||
|
||
|
||
def test_fetch_json_passes_post_method_and_body(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""method/headers/body прокидываются в page.evaluate без искажений."""
|
||
page = _FakePage({"status": 201, "body": "{}"})
|
||
server._browsers["avito"] = _FakeBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
asyncio.run(
|
||
server.fetch_json_handler(
|
||
_make_request(
|
||
{
|
||
"url": "https://www.avito.ru/api/x",
|
||
"method": "POST",
|
||
"headers": {"content-type": "application/json"},
|
||
"body": '{"a": 1}',
|
||
"source": "avito",
|
||
}
|
||
)
|
||
)
|
||
)
|
||
payload = page.evaluate.await_args.args[1]
|
||
assert payload["method"] == "POST"
|
||
assert payload["headers"] == {"content-type": "application/json"}
|
||
assert payload["body"] == '{"a": 1}'
|
||
|
||
|
||
# ── settle + in-page retry (#1917) ─────────────────────────────────────────────────
|
||
|
||
|
||
def test_fetch_json_settle_uses_configured_ms(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Settle перед in-page fetch = FETCH_JSON_SETTLE_MS (а не хардкод 500)."""
|
||
page = _FakePage({"status": 200, "body": "ok"})
|
||
server._browsers["avito"] = _FakeBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
monkeypatch.setattr(server, "FETCH_JSON_SETTLE_MS", 1777)
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
asyncio.run(
|
||
server.fetch_json_handler(
|
||
_make_request({"url": "https://www.avito.ru/api/x", "source": "avito"})
|
||
)
|
||
)
|
||
assert page.waits == [1777]
|
||
|
||
|
||
def test_fetch_json_passes_inpage_retry_params(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""В page.evaluate прокидываются retries/retryDelayMs из конфигурации (#1917)."""
|
||
page = _FakePage({"status": 200, "body": "ok", "attempts": 1})
|
||
server._browsers["avito"] = _FakeBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
monkeypatch.setattr(server, "FETCH_JSON_INPAGE_RETRIES", 2)
|
||
monkeypatch.setattr(server, "FETCH_JSON_RETRY_DELAY_MS", 654)
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
asyncio.run(
|
||
server.fetch_json_handler(
|
||
_make_request({"url": "https://www.avito.ru/api/x", "source": "avito"})
|
||
)
|
||
)
|
||
payload = page.evaluate.await_args.args[1]
|
||
assert payload["retries"] == 2
|
||
assert payload["retryDelayMs"] == 654
|
||
|
||
|
||
# ── валидация / resilience ────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_fetch_json_400_on_missing_url() -> None:
|
||
"""Отсутствие url → 400 (без обращения к браузеру)."""
|
||
response = asyncio.run(server.fetch_json_handler(_make_request({"source": "avito"})))
|
||
assert response.status == 400
|
||
assert "missing 'url'" in _json_body(response)["error"]
|
||
|
||
|
||
def test_fetch_json_400_on_invalid_json() -> None:
|
||
"""Невалидное JSON-тело → 400."""
|
||
request = make_mocked_request("POST", "/fetch-json")
|
||
|
||
async def _raise() -> Any:
|
||
raise ValueError("bad json")
|
||
|
||
request.json = _raise # type: ignore[method-assign]
|
||
response = asyncio.run(server.fetch_json_handler(request))
|
||
assert response.status == 400
|
||
assert "invalid JSON body" in _json_body(response)["error"]
|
||
|
||
|
||
def test_fetch_json_503_when_browser_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""/fetch-json отдаёт 503 (не 500/краш) когда инстанс поставщика не поднят."""
|
||
|
||
async def _no_ensure(provider: str) -> bool:
|
||
return False # прокси всё ещё недоступен → браузер остаётся None
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _no_ensure)
|
||
|
||
response = asyncio.run(
|
||
server.fetch_json_handler(
|
||
_make_request({"url": "https://www.avito.ru/api/x", "source": "avito"})
|
||
)
|
||
)
|
||
assert response.status == 503
|
||
assert "browser unavailable" in _json_body(response)["error"]
|
||
|
||
|
||
def test_fetch_json_500_on_evaluate_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Ошибка page.evaluate (не краш браузера) → 500 с типом исключения."""
|
||
page = _FakePage({"status": 200, "body": "ok"})
|
||
page.evaluate = AsyncMock(side_effect=RuntimeError("eval boom"))
|
||
server._browsers["avito"] = _FakeBrowser(page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
async def _ensure(provider: str) -> bool:
|
||
return True
|
||
|
||
monkeypatch.setattr(server, "_ensure_browser", _ensure)
|
||
|
||
response = asyncio.run(
|
||
server.fetch_json_handler(
|
||
_make_request({"url": "https://www.avito.ru/api/x", "source": "avito"})
|
||
)
|
||
)
|
||
assert response.status == 500
|
||
assert "RuntimeError" in _json_body(response)["error"]
|
||
# Даже при ошибке evaluate страница должна закрыться (finally).
|
||
assert page.closed == 1
|
||
|
||
|
||
# ── crash-retry (#1915 deep-review nice-to-have) ──────────────────────────────────
|
||
|
||
|
||
class TargetClosedError(Exception):
|
||
"""Имя класса распознаётся _is_browser_crash (по type(exc).__name__) как краш."""
|
||
|
||
|
||
def test_do_fetch_json_relaunch_on_browser_crash(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""_do_fetch_json: первый _fetch_json_once крашит браузер → relaunch + второй успех.
|
||
|
||
Проверяем именно crash-retry ветку: первый page.evaluate бросает
|
||
TargetClosedError (его type().__name__ ловит _is_browser_crash), что триггерит
|
||
_relaunch_browser + повторный _fetch_json_once на свежей странице. Результат
|
||
должен прийти из ВТОРОЙ попытки.
|
||
"""
|
||
crashing_page = _FakePage({"status": 0, "body": ""})
|
||
crashing_page.evaluate = AsyncMock(side_effect=TargetClosedError("Target page crashed"))
|
||
healthy_page = _FakePage({"status": 200, "body": '{"recovered": true}'})
|
||
|
||
# Первый инстанс — крашащийся; после relaunch подставляем здоровый.
|
||
server._browsers["avito"] = _FakeBrowser(crashing_page)
|
||
monkeypatch.setattr(server, "BROWSER_RECYCLE_PAGES", 10_000)
|
||
|
||
relaunched: list[str] = []
|
||
|
||
async def _fake_relaunch(provider: str) -> None:
|
||
relaunched.append(provider)
|
||
server._browsers[provider] = _FakeBrowser(healthy_page)
|
||
|
||
monkeypatch.setattr(server, "_relaunch_browser", _fake_relaunch)
|
||
|
||
result = asyncio.run(
|
||
server._do_fetch_json(
|
||
"avito",
|
||
"https://www.avito.ru/api/x",
|
||
method="GET",
|
||
headers={},
|
||
body=None,
|
||
origin="https://www.avito.ru/",
|
||
)
|
||
)
|
||
|
||
# Результат — из ВТОРОЙ (после relaunch) попытки, не из крашнувшей.
|
||
assert result == {"status": 200, "body": '{"recovered": true}'}
|
||
assert relaunched == ["avito"]
|
||
# Обе страницы успели открыться/закрыться (crashing — в finally, healthy — норм).
|
||
crashing_page.evaluate.assert_awaited_once()
|
||
healthy_page.evaluate.assert_awaited_once()
|
||
assert crashing_page.closed == 1
|
||
assert healthy_page.closed == 1
|