bot-backend
6044a1fe3c
The playwright WS-server path (camoufox.server.launch_server) is incompatible
with playwright >=1.45: camoufox 0.4.11's launchServer.js requires
playwright/driver/package/lib/browserServerImpl.js, which no longer exists in
any playwright 1.45-1.60 → tradein-browser crash-looped with MODULE_NOT_FOUND
('Server process terminated unexpectedly'). Verified by running the built image.
Pivot (no playwright protocol between containers):
- browser/server.py: aiohttp service holding a local AsyncCamoufox (headless,
os/locale/geoip/humanize, proxy from SCRAPER_PROXY_URL). POST /fetch {url} →
new_page/goto/content/close → {html}; GET /health. Page-recycle every N +
crash-recovery (relaunch + 1 retry), serialized via asyncio.Lock.
- browser/Dockerfile: +aiohttp (camoufox fetch-as-root/#908 block untouched).
- backend BrowserFetcher: playwright.connect → httpx POST to the browser service
(one retry on transport/HTTP error). All playwright imports dropped.
- config: browser_ws_endpoint → browser_http_endpoint (http://tradein-browser:3000).
Verified by a REAL local image build + run: image builds, container stays Up
(Restarts=0, no crash-loop), GET /health=200, POST /fetch launches camoufox and
drives Firefox (only NS_ERROR_UNKNOWN_HOST = no DNS egress in the local build env,
proving the full HTTP→camoufox→Firefox chain; real fetch validates on prod with
network+proxy). 7 unit tests + ruff clean.
Dormant: scraper_fetch_mode stays curl_cffi. Refs #905, #883, #884
257 lines
10 KiB
Python
257 lines
10 KiB
Python
"""server.py — tradein-browser service main process.
|
||
|
||
Этот модуль запускается как точка входа контейнера ``tradein-browser``.
|
||
Он запускает camoufox **локально** внутри контейнера (AsyncCamoufox) и
|
||
экспонирует простой HTTP API на базе aiohttp:
|
||
|
||
GET /health → {"status": "ok"}
|
||
POST /fetch → {"url": "..."} → {"html": "..."}
|
||
|
||
Такой подход выбран потому, что ``camoufox.server.launch_server`` (Playwright
|
||
WS-сервер) несовместим с современными версиями playwright (1.45–1.60):
|
||
``browserServerImpl.js`` отсутствует в дистрибутиве → контейнер краш-лупится
|
||
с MODULE_NOT_FOUND / RuntimeError при запуске через ``launch_server``.
|
||
Локальный запуск ``AsyncCamoufox`` работает стабильно — проверено.
|
||
|
||
Переменные окружения:
|
||
BROWSER_PORT — TCP-порт HTTP-сервера (default: 3000)
|
||
BROWSER_RECYCLE_PAGES — страниц в одном сеансе браузера до перезапуска (default: 15)
|
||
BROWSER_NAV_TIMEOUT_MS — таймаут page.goto в мс (default: 60000)
|
||
BROWSER_WAIT_MS — ожидание после DOMContentLoaded в мс (default: 2500)
|
||
SCRAPER_PROXY_URL — прокси URL формата ``http://user:pass@host:port`` (опционально)
|
||
|
||
Эндпоинт /fetch доступен из Docker-сети как ``http://tradein-browser:3000/fetch``.
|
||
"""
|
||
|
||
import asyncio
|
||
import logging
|
||
import os
|
||
from urllib.parse import urlparse
|
||
|
||
from aiohttp import web
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ── конфигурация из env ────────────────────────────────────────────────────────
|
||
|
||
BROWSER_PORT: int = int(os.environ.get("BROWSER_PORT", "3000"))
|
||
BROWSER_RECYCLE_PAGES: int = int(os.environ.get("BROWSER_RECYCLE_PAGES", "15"))
|
||
BROWSER_NAV_TIMEOUT_MS: int = int(os.environ.get("BROWSER_NAV_TIMEOUT_MS", "60000"))
|
||
BROWSER_WAIT_MS: int = int(os.environ.get("BROWSER_WAIT_MS", "2500"))
|
||
SCRAPER_PROXY_URL: str | None = os.environ.get("SCRAPER_PROXY_URL")
|
||
|
||
|
||
def _parse_proxy(proxy_url: str | None) -> dict[str, str] | None:
|
||
"""Парсит proxy URL → camoufox proxy dict.
|
||
|
||
Формат входного URL: ``http://user:pass@host:port`` или ``http://host:port``.
|
||
Возвращает None если proxy_url пуст.
|
||
|
||
Returns:
|
||
``{"server": "http://host:port", "username": "...", "password": "..."}``
|
||
или None.
|
||
"""
|
||
if not proxy_url:
|
||
return None
|
||
|
||
parsed = urlparse(proxy_url)
|
||
port_part = f":{parsed.port}" if parsed.port else ""
|
||
server = f"{parsed.scheme}://{parsed.hostname}{port_part}"
|
||
|
||
result: dict[str, str] = {"server": server}
|
||
if parsed.username:
|
||
result["username"] = parsed.username
|
||
if parsed.password:
|
||
result["password"] = parsed.password
|
||
|
||
return result
|
||
|
||
|
||
# ── состояние браузера ─────────────────────────────────────────────────────────
|
||
|
||
_browser: object | None = None # текущий экземпляр Browser
|
||
_browser_cm: object | None = None # AsyncCamoufox context manager (для __aexit__)
|
||
_page_counter: int = 0 # страниц с момента последнего (пере)запуска
|
||
_fetch_lock: asyncio.Lock | None = None # сериализация fetch-запросов
|
||
|
||
|
||
async def _launch_browser() -> None:
|
||
"""Запускает AsyncCamoufox и сохраняет browser + CM в модульных переменных."""
|
||
global _browser, _browser_cm, _page_counter
|
||
|
||
from camoufox.async_api import AsyncCamoufox # noqa: PLC0415
|
||
|
||
proxy = _parse_proxy(SCRAPER_PROXY_URL)
|
||
kwargs: dict[str, object] = {
|
||
"headless": True,
|
||
"os": "windows",
|
||
"locale": "ru-RU",
|
||
"geoip": True,
|
||
"humanize": True,
|
||
}
|
||
if proxy is not None:
|
||
kwargs["proxy"] = proxy
|
||
|
||
logger.info(
|
||
"tradein-browser: запуск AsyncCamoufox (proxy=%s, recycle_pages=%d)",
|
||
proxy is not None,
|
||
BROWSER_RECYCLE_PAGES,
|
||
)
|
||
cm = AsyncCamoufox(**kwargs) # type: ignore[arg-type]
|
||
browser = await cm.__aenter__()
|
||
_browser_cm = cm
|
||
_browser = browser
|
||
_page_counter = 0
|
||
logger.info("tradein-browser: браузер запущен")
|
||
|
||
|
||
async def _close_browser() -> None:
|
||
"""Закрывает текущий экземпляр браузера."""
|
||
global _browser, _browser_cm, _page_counter
|
||
|
||
cm = _browser_cm
|
||
if cm is not None:
|
||
try:
|
||
await cm.__aexit__(None, None, None) # type: ignore[attr-defined]
|
||
logger.info("tradein-browser: браузер закрыт")
|
||
except Exception as exc:
|
||
logger.warning("tradein-browser: ошибка при закрытии браузера: %s", exc)
|
||
finally:
|
||
_browser = None
|
||
_browser_cm = None
|
||
_page_counter = 0
|
||
|
||
|
||
async def _relaunch_browser() -> None:
|
||
"""Закрывает текущий браузер и запускает новый (recycle / crash-recovery)."""
|
||
logger.info("tradein-browser: перезапуск браузера")
|
||
await _close_browser()
|
||
await _launch_browser()
|
||
|
||
|
||
# ── aiohttp lifecycle hooks ────────────────────────────────────────────────────
|
||
|
||
|
||
async def _on_startup(app: web.Application) -> None:
|
||
global _fetch_lock
|
||
_fetch_lock = asyncio.Lock()
|
||
await _launch_browser()
|
||
|
||
|
||
async def _on_cleanup(app: web.Application) -> None:
|
||
await _close_browser()
|
||
|
||
|
||
# ── handlers ───────────────────────────────────────────────────────────────────
|
||
|
||
|
||
async def health_handler(request: web.Request) -> web.Response:
|
||
return web.json_response({"status": "ok"})
|
||
|
||
|
||
async def fetch_handler(request: web.Request) -> web.Response:
|
||
"""POST /fetch {"url": "..."} → {"html": "..."}
|
||
|
||
Выполняет навигацию в браузере и возвращает полный HTML страницы.
|
||
Запросы сериализованы через lock (один за раз).
|
||
"""
|
||
global _page_counter
|
||
|
||
try:
|
||
body = await request.json()
|
||
except Exception:
|
||
return web.json_response({"error": "invalid JSON body"}, status=400)
|
||
|
||
url: str | None = body.get("url")
|
||
if not url:
|
||
return web.json_response({"error": "missing 'url' field"}, status=400)
|
||
|
||
assert _fetch_lock is not None, "_fetch_lock not initialised"
|
||
|
||
async with _fetch_lock:
|
||
try:
|
||
html = await _do_fetch(url)
|
||
except Exception as exc:
|
||
logger.error("tradein-browser: fetch error url=%r: %s: %s", url, type(exc).__name__, exc)
|
||
return web.json_response({"error": f"{type(exc).__name__}: {exc}"}, status=500)
|
||
|
||
return web.json_response({"html": html})
|
||
|
||
|
||
async def _do_fetch(url: str) -> str:
|
||
"""Одна попытка навигации; при краше браузера — перезапускает и повторяет один раз."""
|
||
global _page_counter
|
||
|
||
try:
|
||
return await _fetch_once(url)
|
||
except Exception as exc:
|
||
if _is_browser_crash(exc):
|
||
logger.warning(
|
||
"tradein-browser: краш браузера (%s), перезапуск + retry: %s",
|
||
type(exc).__name__,
|
||
url,
|
||
)
|
||
await _relaunch_browser()
|
||
return await _fetch_once(url)
|
||
raise
|
||
|
||
|
||
async def _fetch_once(url: str) -> str:
|
||
"""Открывает страницу, переходит по URL, ждёт JS, возвращает HTML."""
|
||
global _page_counter
|
||
|
||
browser = _browser
|
||
assert browser is not None, "browser not launched"
|
||
|
||
page = await browser.new_page() # type: ignore[attr-defined]
|
||
try:
|
||
await page.goto(url, timeout=BROWSER_NAV_TIMEOUT_MS, wait_until="domcontentloaded") # type: ignore[attr-defined]
|
||
if BROWSER_WAIT_MS > 0:
|
||
await page.wait_for_timeout(BROWSER_WAIT_MS) # type: ignore[attr-defined]
|
||
html: str = await page.content() # type: ignore[attr-defined]
|
||
finally:
|
||
await page.close() # type: ignore[attr-defined]
|
||
|
||
_page_counter += 1
|
||
logger.debug("tradein-browser: fetch OK url=%r pages_since_launch=%d", url, _page_counter)
|
||
|
||
if _page_counter >= BROWSER_RECYCLE_PAGES:
|
||
logger.info(
|
||
"tradein-browser: recycle threshold (%d) достигнут, перезапуск браузера",
|
||
BROWSER_RECYCLE_PAGES,
|
||
)
|
||
await _relaunch_browser()
|
||
|
||
return html
|
||
|
||
|
||
def _is_browser_crash(exc: BaseException) -> bool:
|
||
"""Проверяет является ли исключение признаком краша / разрыва браузера."""
|
||
cls_name = type(exc).__name__
|
||
if cls_name in ("TargetClosedError", "BrowserClosedError", "ConnectionClosedError"):
|
||
return True
|
||
msg = str(exc).lower()
|
||
return (
|
||
"browser has been closed" in msg
|
||
or "target closed" in msg
|
||
or "connection closed" in msg
|
||
or "browser disconnected" in msg
|
||
)
|
||
|
||
|
||
# ── entrypoint ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def build_app() -> web.Application:
|
||
app = web.Application()
|
||
app.on_startup.append(_on_startup)
|
||
app.on_cleanup.append(_on_cleanup)
|
||
app.router.add_get("/health", health_handler)
|
||
app.router.add_post("/fetch", fetch_handler)
|
||
return app
|
||
|
||
|
||
if __name__ == "__main__":
|
||
logger.info("tradein-browser: старт HTTP-сервера на порту %d", BROWSER_PORT)
|
||
web.run_app(build_app(), host="0.0.0.0", port=BROWSER_PORT)
|