Light1YT
8eec6a752c
Add an in-page fetch() path so backend code can run a same-origin fetch from
inside the warm camoufox page (real fingerprint + context cookies + per-provider
proxy) and get back {status, body}. Purely additive: /fetch, _fetch_once,
_do_fetch, fetch_handler and BrowserFetcher.fetch are untouched. Nothing in the
prod flow calls it yet — a later PR wires avito IMV to it.
Sidecar: _fetch_json_once navigates to origin (anchor page, short 500ms settle
instead of full BROWSER_WAIT_MS), runs fetch via page.evaluate, mirrors the
_fetch_once page lifecycle + recycle/relaunch. _do_fetch_json mirrors _do_fetch
crash-retry. fetch_json_handler mirrors fetch_handler (400/503/500 handling).
Client: BrowserFetcher.fetch_json posts to /fetch-json with the same
single-retry-on-HTTPError/TransportError pattern as fetch.
Tests: browser/test_server_fetch_json.py (handler happy path, origin navigation,
POST payload passthrough, 400 missing url, 400 bad json, 503 unavailable, 500 on
evaluate error) and backend fetch_json client tests (payload, defaults, retry,
raise-after-two-errors).
Refs #915
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
253 lines
11 KiB
Python
253 lines
11 KiB
Python
"""browser_fetcher.py — HTTP-клиент к tradein-browser сервису (#884/#905).
|
||
|
||
Тонкий клиент над tradein-browser контейнером, который запускает AsyncCamoufox
|
||
локально и экспонирует HTTP API (POST /fetch).
|
||
|
||
Публичный интерфейс не изменился:
|
||
|
||
async with BrowserFetcher() as fetcher:
|
||
html = await fetcher.fetch("https://example.com")
|
||
|
||
Внутреннее устройство: httpx.AsyncClient + POST к settings.browser_http_endpoint.
|
||
Recycle, crash-recovery и управление браузером живут на стороне сервера (server.py).
|
||
При HTTPError / ConnectError делает одну повторную попытку после короткой паузы,
|
||
затем пробрасывает исключение.
|
||
|
||
Архитектура выбрана потому, что playwright WS-сервер (launch_server) несовместим
|
||
с playwright >=1.45: ``browserServerImpl.js`` отсутствует → MODULE_NOT_FOUND.
|
||
Локальный AsyncCamoufox + HTTP — работающая альтернатива.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
|
||
import httpx
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
_RETRY_SLEEP_S: float = 1.0
|
||
_HTTP_TIMEOUT_S: float = 120.0 # навигация медленная → щедрый таймаут
|
||
|
||
|
||
class BrowserFetcher:
|
||
"""Async context manager: HTTP-клиент к tradein-browser HTTP-сервису.
|
||
|
||
Использование::
|
||
|
||
async with BrowserFetcher() as fetcher:
|
||
html = await fetcher.fetch("https://example.com")
|
||
"""
|
||
|
||
def __init__(self, source: str = "avito", fetch_timeout_s: float = _HTTP_TIMEOUT_S) -> None:
|
||
# source — логический источник ("avito"/"cian"/"yandex"/"domclick"). Сервер
|
||
# роутит /fetch по нему на отдельный браузер+прокси, когда включён
|
||
# FEATURE_BROWSER_POOL_ENABLED (Phase 1). При выключенном флаге source
|
||
# игнорируется — поведение не меняется.
|
||
# fetch_timeout_s — таймаут httpx-клиента для POST /fetch. Yandex-путь передаёт
|
||
# 30s чтобы один проблемный combo занимал ≤30s×retries вместо 120s×retries.
|
||
self._source = source
|
||
self._fetch_timeout_s = fetch_timeout_s
|
||
self._client: httpx.AsyncClient | None = None
|
||
self._endpoint: str | None = None
|
||
|
||
# ── lifecycle ──────────────────────────────────────────────────────────────
|
||
|
||
async def __aenter__(self) -> BrowserFetcher:
|
||
from app.core.config import settings
|
||
|
||
self._endpoint = settings.browser_http_endpoint
|
||
self._client = httpx.AsyncClient(timeout=self._fetch_timeout_s)
|
||
logger.info("BrowserFetcher: клиент создан, endpoint=%s", self._endpoint)
|
||
return self
|
||
|
||
async def __aexit__(self, *_: object) -> None:
|
||
if self._client is not None:
|
||
await self._client.aclose()
|
||
self._client = None
|
||
logger.debug("BrowserFetcher: клиент закрыт")
|
||
|
||
# ── public API ─────────────────────────────────────────────────────────────
|
||
|
||
async def fetch(self, url: str) -> str:
|
||
"""Запрашивает HTML страницы через tradein-browser HTTP-сервис.
|
||
|
||
При HTTPError или ConnectError делает одну повторную попытку после
|
||
короткой паузы. Остальные исключения всплывают к вызывающему коду.
|
||
|
||
Returns:
|
||
Полный HTML-контент страницы.
|
||
"""
|
||
assert self._client is not None, "BrowserFetcher: используй как async context manager"
|
||
assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
|
||
|
||
try:
|
||
return await self._post_fetch(url)
|
||
except (httpx.HTTPError, httpx.TransportError) as exc:
|
||
logger.warning(
|
||
"BrowserFetcher: ошибка запроса (%s), retry через %.1fs: %s",
|
||
type(exc).__name__,
|
||
_RETRY_SLEEP_S,
|
||
url,
|
||
)
|
||
await asyncio.sleep(_RETRY_SLEEP_S)
|
||
return await self._post_fetch(url)
|
||
|
||
async def fetch_json(
|
||
self,
|
||
url: str,
|
||
*,
|
||
method: str = "GET",
|
||
headers: dict[str, str] | None = None,
|
||
body: str | None = None,
|
||
origin: str | None = None,
|
||
) -> dict:
|
||
"""In-page fetch() через sidecar /fetch-json. Возвращает {"status": int, "body": str}.
|
||
|
||
body — уже сериализованная строка (caller делает json.dumps для POST).
|
||
origin — same-origin страница, на которую перейдёт камуфокс перед fetch.
|
||
|
||
При HTTPError / TransportError делает одну повторную попытку после короткой
|
||
паузы. Остальные исключения всплывают к вызывающему коду.
|
||
"""
|
||
assert self._client is not None, "BrowserFetcher: используй как async context manager"
|
||
assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
|
||
|
||
try:
|
||
return await self._post_fetch_json(url, method, headers, body, origin)
|
||
except (httpx.HTTPError, httpx.TransportError) as exc:
|
||
logger.warning(
|
||
"BrowserFetcher: ошибка fetch-json запроса (%s), retry через %.1fs: %s",
|
||
type(exc).__name__,
|
||
_RETRY_SLEEP_S,
|
||
url,
|
||
)
|
||
await asyncio.sleep(_RETRY_SLEEP_S)
|
||
return await self._post_fetch_json(url, method, headers, body, origin)
|
||
|
||
async def login(
|
||
self,
|
||
*,
|
||
url: str,
|
||
email: str,
|
||
password: str,
|
||
email_selector: str,
|
||
password_selector: str,
|
||
submit_selector: str,
|
||
success_cookie: str,
|
||
pre_click_selectors: list[str] | None = None,
|
||
wait_ms: int | None = None,
|
||
) -> dict[str, str]:
|
||
"""Логинится через tradein-browser /login и возвращает cookies как dict name→value.
|
||
|
||
Использует увеличенный таймаут (60s) — логин медленнее обычного fetch.
|
||
При HTTPError / TransportError делает одну повторную попытку.
|
||
|
||
Returns:
|
||
Плоский словарь {cookie_name: cookie_value}.
|
||
"""
|
||
assert self._client is not None, "BrowserFetcher: используй как async context manager"
|
||
assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
|
||
|
||
body: dict[str, object] = {
|
||
"url": url,
|
||
"email": email,
|
||
"password": password,
|
||
"email_selector": email_selector,
|
||
"password_selector": password_selector,
|
||
"submit_selector": submit_selector,
|
||
"success_cookie": success_cookie,
|
||
"pre_click_selectors": pre_click_selectors or [],
|
||
}
|
||
if wait_ms is not None:
|
||
body["wait_ms"] = wait_ms
|
||
|
||
try:
|
||
return await self._post_login(body)
|
||
except (httpx.HTTPError, httpx.TransportError) as exc:
|
||
logger.warning(
|
||
"BrowserFetcher: ошибка login-запроса (%s), retry через %.1fs",
|
||
type(exc).__name__,
|
||
_RETRY_SLEEP_S,
|
||
)
|
||
await asyncio.sleep(_RETRY_SLEEP_S)
|
||
return await self._post_login(body)
|
||
|
||
# ── internal ───────────────────────────────────────────────────────────────
|
||
|
||
async def _post_fetch(self, url: str) -> str:
|
||
"""Один HTTP POST к /fetch эндпоинту сервиса."""
|
||
assert self._client is not None
|
||
assert self._endpoint is not None
|
||
|
||
resp = await self._client.post(
|
||
f"{self._endpoint}/fetch",
|
||
json={"url": url, "source": self._source},
|
||
)
|
||
resp.raise_for_status()
|
||
data: dict[str, str] = resp.json()
|
||
html = data["html"]
|
||
logger.debug("BrowserFetcher: fetch OK url=%r html_len=%d", url, len(html))
|
||
return html
|
||
|
||
async def _post_fetch_json(
|
||
self,
|
||
url: str,
|
||
method: str,
|
||
headers: dict[str, str] | None,
|
||
body: str | None,
|
||
origin: str | None,
|
||
) -> dict:
|
||
"""Один HTTP POST к /fetch-json эндпоинту сервиса."""
|
||
assert self._client is not None
|
||
assert self._endpoint is not None
|
||
|
||
resp = await self._client.post(
|
||
f"{self._endpoint}/fetch-json",
|
||
json={
|
||
"url": url,
|
||
"source": self._source,
|
||
"method": method,
|
||
"headers": headers or {},
|
||
"body": body,
|
||
"origin": origin,
|
||
},
|
||
)
|
||
resp.raise_for_status()
|
||
data: dict = resp.json()
|
||
logger.debug(
|
||
"BrowserFetcher: fetch-json OK url=%r status=%s body_len=%d",
|
||
url,
|
||
data.get("status"),
|
||
len(data.get("body") or ""),
|
||
)
|
||
return data
|
||
|
||
async def _post_login(self, body: dict[str, object]) -> dict[str, str]:
|
||
"""Один HTTP POST к /login эндпоинту сервиса."""
|
||
assert self._client is not None
|
||
assert self._endpoint is not None
|
||
|
||
resp = await self._client.post(
|
||
f"{self._endpoint}/login",
|
||
json=body,
|
||
timeout=httpx.Timeout(60.0),
|
||
)
|
||
if resp.status_code == 502:
|
||
data = resp.json()
|
||
has_screenshot = bool(data.get("screenshot_b64"))
|
||
logger.debug(
|
||
"BrowserFetcher: login 502 — has_screenshot=%s page_url=%r",
|
||
has_screenshot,
|
||
data.get("page_url"),
|
||
)
|
||
raise RuntimeError(
|
||
f"browser login failed: {data.get('error')} url={data.get('page_url')}"
|
||
)
|
||
resp.raise_for_status()
|
||
data = resp.json()
|
||
raw: list[dict[str, object]] = data["cookies"]
|
||
result = {str(c["name"]): str(c["value"]) for c in raw}
|
||
logger.info("BrowserFetcher: login OK cookie_count=%d", len(result))
|
||
return result
|