bot-backend
5fb367295a
Phase 0.5 — revise #884's in-backend camoufox into a dedicated browser service. camoufox.server.launch_server exposes a Playwright WS endpoint (fingerprint/geoip/ proxy configured server-side); the backend connects remotely via playwright, so it no longer ships Firefox. - NEW tradein-mvp/browser/{server.py,Dockerfile}: camoufox WS server on :3000; Firefox fetch under app-user + ENV HOME (carries the #899 cache-path fix) - BrowserFetcher rewritten launch-in-process → playwright.firefox.connect(ws_endpoint); recycle (close+reconnect every N) + crash-recovery (TargetClosed → reconnect + 1 retry) - backend Dockerfile slimmed: drop 5 Firefox libs + camoufox fetch layer - pyproject: remove camoufox[geoip] from backend, promote playwright to main deps - docker-compose.prod.yml: add browser service (mem_limit 2.5g moves here), remove mem_limit from backend, backend depends_on browser (service_started) - config: browser_ws_endpoint default ws://tradein-browser:3000/playwright - tests/scrapers/test_browser_fetcher.py rewritten for connect-mode (6 pass) Dormant: scraper_fetch_mode stays curl_cffi → no prod behavior change. launch_server signature verified (**kwargs→launch_options, port passes through). Refs #905, #883, #884, #899
82 lines
3 KiB
Python
82 lines
3 KiB
Python
"""server.py — tradein-browser service main process.
|
|
|
|
Этот модуль запускается как точка входа контейнера ``tradein-browser``.
|
|
Он читает конфигурацию из переменных окружения и вызывает
|
|
``camoufox.server.launch_server`` для экспозиции Playwright WS-эндпоинта,
|
|
к которому подключается ``tradein-backend`` через ``playwright.firefox.connect``.
|
|
|
|
Переменные окружения:
|
|
BROWSER_PORT — TCP-порт WS-сервера (default: 3000)
|
|
SCRAPER_PROXY_URL — прокси URL формата ``http://user:pass@host:port`` (опционально)
|
|
|
|
WS-эндпоинт доступен внутри Docker-сети как
|
|
``ws://tradein-browser:<BROWSER_PORT>/playwright``.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
from urllib.parse import urlparse
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _parse_proxy(proxy_url: str | None) -> dict[str, str] | None:
|
|
"""Парсит proxy URL → playwright proxy dict.
|
|
|
|
Формат входного URL: ``http://user:pass@host:port`` или ``http://host:port``.
|
|
Возвращает None если proxy_url пуст.
|
|
|
|
Returns:
|
|
``{"server": "http://host:port", "username": "...", "password": "..."}``
|
|
или None.
|
|
"""
|
|
if not proxy_url:
|
|
return None
|
|
|
|
parsed = urlparse(proxy_url)
|
|
port_part = f":{parsed.port}" if parsed.port else ""
|
|
server = f"{parsed.scheme}://{parsed.hostname}{port_part}"
|
|
|
|
result: dict[str, str] = {"server": server}
|
|
if parsed.username:
|
|
result["username"] = parsed.username
|
|
if parsed.password:
|
|
result["password"] = parsed.password
|
|
|
|
return result
|
|
|
|
|
|
def main() -> None:
|
|
port = int(os.environ.get("BROWSER_PORT", "3000"))
|
|
proxy_url = os.environ.get("SCRAPER_PROXY_URL")
|
|
proxy = _parse_proxy(proxy_url)
|
|
|
|
logger.info(
|
|
"tradein-browser: запуск camoufox WS-сервера на порту %d (proxy=%s)",
|
|
port,
|
|
proxy is not None,
|
|
)
|
|
|
|
# Импортируем camoufox.server здесь — в контейнере Firefox уже установлен.
|
|
from camoufox.server import launch_server # noqa: PLC0415
|
|
|
|
kwargs: dict[str, object] = {
|
|
"headless": True,
|
|
"os": "windows",
|
|
"locale": "ru-RU",
|
|
"geoip": True,
|
|
"humanize": True,
|
|
"port": port,
|
|
# port передаётся через **launch_options в launch_options() → прокидывается
|
|
# в Playwright launchServer({ port: N }) — сервер слушает на заданном порту.
|
|
}
|
|
if proxy is not None:
|
|
kwargs["proxy"] = proxy
|
|
|
|
# launch_server блокируется — это основной процесс контейнера.
|
|
launch_server(**kwargs) # type: ignore[arg-type]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|