From 57acec215f5857427e124a24233157fdc6be00b3 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sun, 31 May 2026 18:49:45 +0300 Subject: [PATCH 1/2] fix(browser): read AVITO_PROXY_URL for camoufox proxy (#905) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prod smoke: tradein-browser fetched avito but got the ~400KB soft-block stub (preloadedState present but priceDetailed=0, firewall=0) — because camoufox ran with NO proxy (datacenter IP). Root cause: server.py read SCRAPER_PROXY_URL, but the proxy in backend/.env.runtime is named AVITO_PROXY_URL (mobileproxy, #623) — the same var curl_cffi-avito uses. So the var was empty in the browser container. Fix: read AVITO_PROXY_URL first, SCRAPER_PROXY_URL as a generic fallback. One-line env-name swap; the value is the working mobileproxy egress already on the host. After deploy, re-run tradein-mvp/scripts/browser-smoke.sh — expect real data (priceDetailed>0, ~1.4MB, firewall 0). IP rotation (AVITO_PROXY_ROTATE_URL) stays backend-side and keeps working (same proxy endpoint, changing egress IP). Refs #905, #883, #623 --- tradein-mvp/browser/server.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tradein-mvp/browser/server.py b/tradein-mvp/browser/server.py index c268ec7c..2b62d186 100644 --- a/tradein-mvp/browser/server.py +++ b/tradein-mvp/browser/server.py @@ -18,7 +18,8 @@ WS-сервер) несовместим с современными версия BROWSER_RECYCLE_PAGES — страниц в одном сеансе браузера до перезапуска (default: 15) BROWSER_NAV_TIMEOUT_MS — таймаут page.goto в мс (default: 60000) BROWSER_WAIT_MS — ожидание после DOMContentLoaded в мс (default: 2500) - SCRAPER_PROXY_URL — прокси URL формата ``http://user:pass@host:port`` (опционально) + AVITO_PROXY_URL — прокси ``http://user:pass@host:port`` (mobileproxy, #623); + SCRAPER_PROXY_URL — generic-fallback. Без него — soft-block. Эндпоинт /fetch доступен из Docker-сети как ``http://tradein-browser:3000/fetch``. """ @@ -39,7 +40,13 @@ BROWSER_PORT: int = int(os.environ.get("BROWSER_PORT", "3000")) BROWSER_RECYCLE_PAGES: int = int(os.environ.get("BROWSER_RECYCLE_PAGES", "15")) BROWSER_NAV_TIMEOUT_MS: int = int(os.environ.get("BROWSER_NAV_TIMEOUT_MS", "60000")) BROWSER_WAIT_MS: int = int(os.environ.get("BROWSER_WAIT_MS", "2500")) -SCRAPER_PROXY_URL: str | None = os.environ.get("SCRAPER_PROXY_URL") +# Прокси: основной источник — AVITO_PROXY_URL (mobileproxy, #623, лежит в +# backend/.env.runtime, тот же что у curl_cffi-avito). SCRAPER_PROXY_URL — +# опциональный generic-fallback на будущее (cian/yandex). Без прокси camoufox +# фетчит с датацентр-IP → avito отдаёт soft-block заглушку. +SCRAPER_PROXY_URL: str | None = os.environ.get("AVITO_PROXY_URL") or os.environ.get( + "SCRAPER_PROXY_URL" +) def _parse_proxy(proxy_url: str | None) -> dict[str, str] | None: -- 2.45.3 From 3527a6bc8dd6fde13675fe8792c326a8d9c385b1 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sun, 31 May 2026 18:50:35 +0300 Subject: [PATCH 2/2] style(browser): fix pre-existing ruff lints in server.py (RUF100, E501) Drive-by: browser/server.py was never linted (pre-commit ruff is scoped to backend/), so #909 left an unused noqa + a 101-char line. Cleaned up while editing this file for the proxy fix. --- tradein-mvp/browser/server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tradein-mvp/browser/server.py b/tradein-mvp/browser/server.py index 2b62d186..d071a0ba 100644 --- a/tradein-mvp/browser/server.py +++ b/tradein-mvp/browser/server.py @@ -87,7 +87,7 @@ async def _launch_browser() -> None: """Запускает AsyncCamoufox и сохраняет browser + CM в модульных переменных.""" global _browser, _browser_cm, _page_counter - from camoufox.async_api import AsyncCamoufox # noqa: PLC0415 + from camoufox.async_api import AsyncCamoufox proxy = _parse_proxy(SCRAPER_PROXY_URL) kwargs: dict[str, object] = { @@ -180,7 +180,9 @@ async def fetch_handler(request: web.Request) -> web.Response: try: html = await _do_fetch(url) except Exception as exc: - logger.error("tradein-browser: fetch error url=%r: %s: %s", url, type(exc).__name__, exc) + logger.error( + "tradein-browser: fetch error url=%r: %s: %s", url, type(exc).__name__, exc + ) return web.json_response({"error": f"{type(exc).__name__}: {exc}"}, status=500) return web.json_response({"html": html}) -- 2.45.3