bot-backend
b9eb478c8f
Route each scraper source (avito/cian/yandex/domclick) to its own camoufox browser+proxy so they no longer wedge each other through a single global egress. Feature-flagged (FEATURE_BROWSER_POOL_ENABLED, default OFF): with the flag off the /fetch and /login paths are byte-for-byte the existing single-browser behavior. When on, /fetch routes by body["source"] to a per-proxy browser via BROWSER_PROXY_MAP (BROWSER_PROXY_AVITO/CIAN/YANDEX/DOMCLICK with legacy fallbacks), each guarded by its own lazy-launched lock. /login stays single-browser in Phase 1. BrowserFetcher gains a source arg (default avito) and sends it in the /fetch body; all scraper callsites pass their source. No docker-compose/.env.runtime changes (Phase 2, owner-gated).
240 lines
9.9 KiB
Python
240 lines
9.9 KiB
Python
"""test_browser_pool_routing.py — per-source proxy pool routing (Phase 1, #browser-pool).
|
||
|
||
Юнит-тесты server.fetch_handler с включённым/выключенным FEATURE_BROWSER_POOL_ENABLED.
|
||
camoufox не нужен: launch/_do_fetch_pool замоканы.
|
||
|
||
server.py живёт в tradein-mvp/browser/ (отдельный сервис) и импортит `from aiohttp
|
||
import web`. aiohttp в backend-venv НЕТ (только в browser-образе), поэтому здесь в
|
||
sys.modules инжектится минимальный стаб aiohttp.web ДО загрузки server.py — ровно те
|
||
имена, что использует модуль (Application/Request/Response/json_response). Так
|
||
routing-логика гоняется прямо в backend pytest-gate без тяжёлой aiohttp-зависимости.
|
||
|
||
Проверяем:
|
||
- флаг ON → fetch_handler роутит по body["source"] на нужный proxy_url и держит
|
||
его lock во время _do_fetch_pool (замоканного);
|
||
- неизвестный/отсутствующий source → fallback на avito-прокси;
|
||
- launch недоступного прокси → 503;
|
||
- флаг OFF → берётся одиночный путь (_fetch_via_single), source игнорируется;
|
||
- _build_proxy_map отбрасывает пустые env и применяет legacy-fallback.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import importlib.util
|
||
import json
|
||
import sys
|
||
import types
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
|
||
# ── минимальный стаб aiohttp.web (server.py юзает только эти имена) ──────────────
|
||
|
||
|
||
class _StubResponse:
|
||
"""Имитация web.Response от json_response: хранит status + JSON-байты в .body."""
|
||
|
||
def __init__(self, payload: dict[str, Any], status: int = 200) -> None:
|
||
self.status = status
|
||
self.body = json.dumps(payload).encode()
|
||
|
||
|
||
def _json_response(payload: dict[str, Any], status: int = 200) -> _StubResponse:
|
||
return _StubResponse(payload, status=status)
|
||
|
||
|
||
def _install_aiohttp_stub() -> None:
|
||
"""Кладёт фейковый aiohttp + aiohttp.web в sys.modules, если настоящего нет."""
|
||
try:
|
||
import aiohttp # noqa: F401
|
||
|
||
return # настоящий aiohttp доступен — стаб не нужен
|
||
except ModuleNotFoundError:
|
||
pass
|
||
|
||
web = types.ModuleType("aiohttp.web")
|
||
web.Application = type("Application", (), {}) # type: ignore[attr-defined]
|
||
web.Request = type("Request", (), {}) # type: ignore[attr-defined]
|
||
web.Response = _StubResponse # type: ignore[attr-defined]
|
||
web.json_response = _json_response # type: ignore[attr-defined]
|
||
|
||
aiohttp_mod = types.ModuleType("aiohttp")
|
||
aiohttp_mod.web = web # type: ignore[attr-defined]
|
||
sys.modules["aiohttp"] = aiohttp_mod
|
||
sys.modules["aiohttp.web"] = web
|
||
|
||
|
||
_install_aiohttp_stub()
|
||
|
||
# server.py — отдельный сервис вне backend-пакета. Грузим по относительному пути.
|
||
_SERVER_PATH = Path(__file__).resolve().parents[2] / "browser" / "server.py"
|
||
_spec = importlib.util.spec_from_file_location("tradein_browser_server_pool", _SERVER_PATH)
|
||
assert _spec is not None and _spec.loader is not None
|
||
server = importlib.util.module_from_spec(_spec)
|
||
_spec.loader.exec_module(server)
|
||
|
||
|
||
class _StubRequest:
|
||
"""Минимальный stub aiohttp-request: handler дёргает только .json()."""
|
||
|
||
def __init__(self, body: dict[str, Any]) -> None:
|
||
self._body = body
|
||
|
||
async def json(self) -> dict[str, Any]:
|
||
return self._body
|
||
|
||
|
||
def _json_body(response: Any) -> dict[str, Any]:
|
||
return json.loads(response.body.decode())
|
||
|
||
|
||
def _make_fetch_request(body: dict[str, Any]) -> Any:
|
||
return _StubRequest(body)
|
||
|
||
|
||
# ── флаг ON: routing по source ───────────────────────────────────────────────────
|
||
|
||
|
||
def test_pool_routes_by_source_and_holds_proxy_lock(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""ON: source=yandex → fetch на yandex-прокси, держа именно его lock."""
|
||
proxy_map = {
|
||
"avito": "http://avito-proxy:1",
|
||
"cian": "http://cian-proxy:2",
|
||
"yandex": "http://yandex-proxy:3",
|
||
}
|
||
monkeypatch.setattr(server, "FEATURE_BROWSER_POOL_ENABLED", True)
|
||
monkeypatch.setattr(server, "BROWSER_PROXY_MAP", proxy_map)
|
||
monkeypatch.setattr(server, "_pool_locks", {})
|
||
monkeypatch.setattr(server, "_pool_locks_guard", asyncio.Lock())
|
||
|
||
seen: dict[str, Any] = {}
|
||
|
||
async def _fake_get_or_launch(proxy_url: str) -> object:
|
||
seen["launched_proxy"] = proxy_url
|
||
return object() # «браузер поднят»
|
||
|
||
async def _fake_do_fetch_pool(url: str, proxy_url: str) -> str:
|
||
seen["fetch_proxy"] = proxy_url
|
||
# lock этого proxy_url должен быть удержан вызывающим (handler).
|
||
seen["lock_held"] = server._pool_locks[proxy_url].locked()
|
||
return "<html>yandex</html>"
|
||
|
||
monkeypatch.setattr(server, "_get_or_launch_pool_browser", _fake_get_or_launch)
|
||
monkeypatch.setattr(server, "_do_fetch_pool", _fake_do_fetch_pool)
|
||
|
||
request = _make_fetch_request({"url": "https://realty.yandex.ru/x", "source": "yandex"})
|
||
response = asyncio.run(server.fetch_handler(request))
|
||
|
||
assert response.status == 200
|
||
assert _json_body(response)["html"] == "<html>yandex</html>"
|
||
assert seen["launched_proxy"] == "http://yandex-proxy:3"
|
||
assert seen["fetch_proxy"] == "http://yandex-proxy:3"
|
||
assert seen["lock_held"] is True
|
||
assert "http://yandex-proxy:3" in server._pool_locks
|
||
|
||
|
||
def test_pool_unknown_source_falls_back_to_avito(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""ON: domclick без своего прокси → fallback на avito-прокси."""
|
||
proxy_map = {"avito": "http://avito-proxy:1"} # нет domclick
|
||
monkeypatch.setattr(server, "FEATURE_BROWSER_POOL_ENABLED", True)
|
||
monkeypatch.setattr(server, "BROWSER_PROXY_MAP", proxy_map)
|
||
monkeypatch.setattr(server, "_pool_locks", {})
|
||
monkeypatch.setattr(server, "_pool_locks_guard", asyncio.Lock())
|
||
|
||
seen: dict[str, Any] = {}
|
||
|
||
async def _fake_get_or_launch(proxy_url: str) -> object:
|
||
return object()
|
||
|
||
async def _fake_do_fetch_pool(url: str, proxy_url: str) -> str:
|
||
seen["fetch_proxy"] = proxy_url
|
||
return "<html>fallback</html>"
|
||
|
||
monkeypatch.setattr(server, "_get_or_launch_pool_browser", _fake_get_or_launch)
|
||
monkeypatch.setattr(server, "_do_fetch_pool", _fake_do_fetch_pool)
|
||
|
||
request = _make_fetch_request({"url": "https://domclick.ru/x", "source": "domclick"})
|
||
response = asyncio.run(server.fetch_handler(request))
|
||
|
||
assert response.status == 200
|
||
assert seen["fetch_proxy"] == "http://avito-proxy:1"
|
||
|
||
|
||
def test_pool_returns_503_when_browser_unavailable(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""ON: launch недоступного прокси → None → 503 (та же форма, что одиночный путь)."""
|
||
monkeypatch.setattr(server, "FEATURE_BROWSER_POOL_ENABLED", True)
|
||
monkeypatch.setattr(server, "BROWSER_PROXY_MAP", {"avito": "http://avito-proxy:1"})
|
||
monkeypatch.setattr(server, "_pool_locks", {})
|
||
monkeypatch.setattr(server, "_pool_locks_guard", asyncio.Lock())
|
||
|
||
async def _fake_get_or_launch(proxy_url: str) -> None:
|
||
return None # прокси лежит
|
||
|
||
monkeypatch.setattr(server, "_get_or_launch_pool_browser", _fake_get_or_launch)
|
||
|
||
request = _make_fetch_request({"url": "https://avito.ru/x", "source": "avito"})
|
||
response = asyncio.run(server.fetch_handler(request))
|
||
|
||
assert response.status == 503
|
||
assert "browser unavailable" in _json_body(response)["error"]
|
||
|
||
|
||
# ── флаг OFF: одиночный путь, source игнорируется ────────────────────────────────
|
||
|
||
|
||
def test_flag_off_uses_single_path_and_ignores_source(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""OFF: берётся _fetch_via_single, pool-путь НЕ вызывается, source неважен."""
|
||
monkeypatch.setattr(server, "FEATURE_BROWSER_POOL_ENABLED", False)
|
||
|
||
single_called = {"n": 0}
|
||
|
||
async def _fake_single(url: str) -> Any:
|
||
single_called["n"] += 1
|
||
return server.web.json_response({"html": "<html>single</html>"})
|
||
|
||
async def _fail_pool(url: str, source: str | None) -> Any:
|
||
raise AssertionError("pool path не должен вызываться при OFF-флаге")
|
||
|
||
monkeypatch.setattr(server, "_fetch_via_single", _fake_single)
|
||
monkeypatch.setattr(server, "_fetch_via_pool", _fail_pool)
|
||
|
||
request = _make_fetch_request({"url": "https://avito.ru/x", "source": "yandex"})
|
||
response = asyncio.run(server.fetch_handler(request))
|
||
|
||
assert response.status == 200
|
||
assert _json_body(response)["html"] == "<html>single</html>"
|
||
assert single_called["n"] == 1
|
||
|
||
|
||
def test_build_proxy_map_drops_empties(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""_build_proxy_map: пустые/None env не попадают в map; legacy-fallback работает."""
|
||
for var in (
|
||
"BROWSER_PROXY_AVITO",
|
||
"AVITO_PROXY_URL",
|
||
"SCRAPER_PROXY_URL",
|
||
"BROWSER_PROXY_CIAN",
|
||
"CIAN_PROXY_URL",
|
||
"BROWSER_PROXY_YANDEX",
|
||
"YANDEX_PROXY_URL",
|
||
"BROWSER_PROXY_DOMCLICK",
|
||
):
|
||
monkeypatch.delenv(var, raising=False)
|
||
|
||
monkeypatch.setenv("AVITO_PROXY_URL", "http://avito:1") # avito через legacy-fallback
|
||
monkeypatch.setenv("BROWSER_PROXY_CIAN", "http://cian:2")
|
||
# yandex/domclick не заданы → отсутствуют в map.
|
||
|
||
result = server._build_proxy_map()
|
||
assert result == {"avito": "http://avito:1", "cian": "http://cian:2"}
|
||
assert "yandex" not in result
|
||
assert "domclick" not in result
|