Merge remote-tracking branch 'forgejo/main' into feat/915-pipeline-browser-routing
This commit is contained in:
commit
e19e45b3f4
22 changed files with 1869 additions and 464 deletions
|
|
@ -18,6 +18,7 @@ concurrency:
|
|||
env:
|
||||
IMAGE_BACKEND: ghcr.io/lekss361/gendesign-tradein-backend
|
||||
IMAGE_FRONTEND: ghcr.io/lekss361/gendesign-tradein-frontend
|
||||
IMAGE_BROWSER: ghcr.io/lekss361/gendesign-tradein-browser
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
|
|
@ -25,6 +26,7 @@ jobs:
|
|||
outputs:
|
||||
backend: ${{ steps.filter.outputs.backend }}
|
||||
frontend: ${{ steps.filter.outputs.frontend }}
|
||||
browser: ${{ steps.filter.outputs.browser }}
|
||||
infra: ${{ steps.filter.outputs.infra }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -36,6 +38,8 @@ jobs:
|
|||
- 'tradein-mvp/backend/**'
|
||||
frontend:
|
||||
- 'tradein-mvp/frontend/**'
|
||||
browser:
|
||||
- 'tradein-mvp/browser/**'
|
||||
infra:
|
||||
- 'tradein-mvp/docker-compose.prod.yml'
|
||||
- 'tradein-mvp/deploy/**'
|
||||
|
|
@ -149,9 +153,41 @@ jobs:
|
|||
${{ env.IMAGE_FRONTEND }}:latest
|
||||
${{ env.IMAGE_FRONTEND }}:${{ github.sha }}
|
||||
|
||||
build-browser:
|
||||
runs-on: ubuntu-latest
|
||||
needs: changes
|
||||
# tradein-browser несёт camoufox + Firefox-build (#905). Триггерится на
|
||||
# изменения browser/ или infra (compose ссылается на образ) или вручную.
|
||||
if: |
|
||||
needs.changes.outputs.browser == 'true' ||
|
||||
needs.changes.outputs.infra == 'true' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
|
||||
env:
|
||||
GHCR_PAT: ${{ secrets.GHCR_PAT }}
|
||||
run: |
|
||||
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build & push tradein-browser
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./tradein-mvp/browser
|
||||
push: true
|
||||
cache-from: type=registry,ref=${{ env.IMAGE_BROWSER }}:buildcache
|
||||
cache-to: type=registry,ref=${{ env.IMAGE_BROWSER }}:buildcache,mode=max
|
||||
tags: |
|
||||
${{ env.IMAGE_BROWSER }}:latest
|
||||
${{ env.IMAGE_BROWSER }}:${{ github.sha }}
|
||||
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [changes, test, build-backend, build-frontend]
|
||||
needs: [changes, test, build-backend, build-frontend, build-browser]
|
||||
# NB: a failed `test` skips build-backend (result='skipped', not 'failure'),
|
||||
# so we must block deploy on test failure explicitly (#666 quality gate).
|
||||
if: |
|
||||
|
|
@ -159,7 +195,8 @@ jobs:
|
|||
!cancelled() &&
|
||||
needs.test.result != 'failure' &&
|
||||
needs.build-backend.result != 'failure' &&
|
||||
needs.build-frontend.result != 'failure'
|
||||
needs.build-frontend.result != 'failure' &&
|
||||
needs.build-browser.result != 'failure'
|
||||
steps:
|
||||
- name: Deploy via SSH
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
|
|
|
|||
|
|
@ -43,11 +43,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
|||
libpangoft2-1.0-0 \
|
||||
fonts-dejavu-core \
|
||||
curl \
|
||||
libgtk-3-0 \
|
||||
libasound2 \
|
||||
libdbus-glib-1-2 \
|
||||
libx11-xcb1 \
|
||||
libxtst6 \
|
||||
&& useradd --create-home --uid 1000 app
|
||||
|
||||
WORKDIR /app
|
||||
|
|
@ -58,16 +53,10 @@ COPY --from=builder --chown=app:app /app/scripts /app/scripts
|
|||
|
||||
USER app
|
||||
|
||||
# HOME должен быть явным: Docker НЕ выставляет $HOME по USER, а camoufox/platformdirs
|
||||
# резолвит кэш Firefox как $HOME/.cache/camoufox. Без этого fetch (под root) клал браузер
|
||||
# в /root/.cache, а рантайм-app искал в /home/app/.cache → "browser not found" в browser-mode.
|
||||
# (подтверждено сборкой+прогоном образа на деплой-хосте 2026-05-31, баг из #884.)
|
||||
# HOME должен быть явным: Docker НЕ выставляет $HOME по USER, а некоторые
|
||||
# платформенные библиотеки резолвят $HOME/.cache. Безвредно оставить.
|
||||
ENV HOME=/home/app
|
||||
|
||||
# Загружаем Firefox-сборку для camoufox (#884) — ПОД app-юзером, чтобы кэш совпал с рантаймом.
|
||||
# Слой кешируется: пересобирается только при изменении .venv (обновление camoufox). ~960MB.
|
||||
RUN python -m camoufox fetch
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
|
||||
|
|
|
|||
|
|
@ -12,6 +12,12 @@ class Settings(BaseSettings):
|
|||
# required — задаётся через env DATABASE_URL. Нет дефолта: fail-fast при старте
|
||||
# если переменная не задана (C-3 security audit).
|
||||
database_url: str
|
||||
|
||||
# In-app asyncio scheduler enable flag (#581).
|
||||
# Set SCHEDULER_ENABLE=false when using the systemd timer trigger instead.
|
||||
# Default true preserves existing behaviour.
|
||||
scheduler_enable: bool = Field(default=True, validation_alias="SCHEDULER_ENABLE")
|
||||
|
||||
cors_origins: list[str] = ["http://localhost", "http://localhost:3000", "http://localhost:8080"]
|
||||
environment: str = "dev"
|
||||
|
||||
|
|
@ -157,6 +163,8 @@ class Settings(BaseSettings):
|
|||
"realty.yandex.ru",
|
||||
"ekaterinburg.n1.ru",
|
||||
"n1.ru",
|
||||
"domclick.ru",
|
||||
"www.domclick.ru",
|
||||
}
|
||||
|
||||
# ── Scraper mobile proxy (#806) ──────────────────────────────────────────
|
||||
|
|
@ -205,19 +213,25 @@ class Settings(BaseSettings):
|
|||
# ENV: YANDEX_COOKIES_FILE.
|
||||
yandex_cookies_file: str | None = None
|
||||
|
||||
# ── #884: BrowserFetcher (camoufox) — Phase 0 foundation ─────────────────
|
||||
# ── #884/#905: BrowserFetcher — HTTP-клиент к tradein-browser контейнеру ────
|
||||
# scraper_fetch_mode: "curl_cffi" (дефолт, текущее поведение) или "browser"
|
||||
# (AsyncCamoufox headless Firefox). Переключение в "browser" — Phase 1+.
|
||||
# (HTTP POST к tradein-browser /fetch эндпоинту). Phase 1+.
|
||||
# Пока дефолт "curl_cffi" — никаких изменений в рантайме.
|
||||
# ENV: SCRAPER_FETCH_MODE.
|
||||
scraper_fetch_mode: Literal["curl_cffi", "browser"] = "curl_cffi"
|
||||
# Сколько страниц обработать в одном browser-процессе перед перезапуском
|
||||
# (ограничение утечек памяти). ENV: BROWSER_RECYCLE_PAGES.
|
||||
# HTTP-эндпоинт tradein-browser сервиса. В Docker-сети — имя сервиса из compose.
|
||||
# ENV: BROWSER_HTTP_ENDPOINT.
|
||||
browser_http_endpoint: str = "http://tradein-browser:3000"
|
||||
# Сколько страниц обработать в одном browser-сеансе перед перезапуском браузера
|
||||
# (ограничение утечек памяти). Читается сервером из env BROWSER_RECYCLE_PAGES.
|
||||
# Оставлено для справки / compat. ENV: BROWSER_RECYCLE_PAGES.
|
||||
browser_recycle_pages: int = 15
|
||||
# Таймаут навигации (page.goto) в мс. ENV: BROWSER_NAV_TIMEOUT_MS.
|
||||
# Таймаут навигации (page.goto) в мс. Читается сервером из env BROWSER_NAV_TIMEOUT_MS.
|
||||
# ENV: BROWSER_NAV_TIMEOUT_MS.
|
||||
browser_nav_timeout_ms: int = 60000
|
||||
# Ожидание после DOMContentLoaded для JS-гидрации в мс. ENV: BROWSER_WAIT_MS.
|
||||
browser_wait_ms: int = 2500
|
||||
# Ожидание после DOMContentLoaded для JS-гидрации в мс. Читается сервером.
|
||||
# ENV: BROWSER_WAIT_MS.
|
||||
browser_wait_ms: int = 6000
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
|
|
|||
|
|
@ -74,17 +74,23 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||
except Exception:
|
||||
logger.exception("FDW user mapping bootstrap failed — cadastral queries may fail")
|
||||
|
||||
# Startup: launch scheduler background task
|
||||
scheduler_task = asyncio.create_task(scheduler_loop())
|
||||
logger.info("FastAPI lifespan: scheduler task spawned")
|
||||
# Startup: launch scheduler background task (disabled when SCHEDULER_ENABLE=false,
|
||||
# e.g. when using the systemd timer trigger — see tradein-mvp/ops/systemd/ #581).
|
||||
scheduler_task = None
|
||||
if settings.scheduler_enable:
|
||||
scheduler_task = asyncio.create_task(scheduler_loop())
|
||||
logger.info("FastAPI lifespan: scheduler task spawned")
|
||||
else:
|
||||
logger.info("FastAPI lifespan: in-app scheduler disabled (SCHEDULER_ENABLE=false)")
|
||||
yield
|
||||
# Shutdown: cancel scheduler
|
||||
scheduler_task.cancel()
|
||||
try:
|
||||
await scheduler_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
logger.info("FastAPI lifespan: scheduler cancelled cleanly")
|
||||
# Shutdown: cancel scheduler if it was started
|
||||
if scheduler_task is not None:
|
||||
scheduler_task.cancel()
|
||||
try:
|
||||
await scheduler_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
logger.info("FastAPI lifespan: scheduler cancelled cleanly")
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ _CACHE_TTL_SEC = 60.0
|
|||
_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
|
||||
"avito": 7.0,
|
||||
"cian": 5.0,
|
||||
"domklik": 8.0,
|
||||
"n1": 5.0,
|
||||
"yandex": 5.0,
|
||||
"yandex_detail": 5.0,
|
||||
|
|
@ -82,9 +83,7 @@ def _get_setting_cached(key: str) -> float:
|
|||
try:
|
||||
with _open_session() as db:
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT request_delay_sec FROM scraper_settings WHERE source = :s"
|
||||
),
|
||||
text("SELECT request_delay_sec FROM scraper_settings WHERE source = :s"),
|
||||
{"s": key},
|
||||
).first()
|
||||
if row is not None:
|
||||
|
|
@ -96,8 +95,8 @@ def _get_setting_cached(key: str) -> float:
|
|||
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
||||
except Exception as e:
|
||||
logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
|
||||
value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(
|
||||
key, _GLOBAL_DEFAULT_DELAY
|
||||
value = (
|
||||
0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
||||
)
|
||||
|
||||
with _CACHE_LOCK:
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ from app.services.scrapers.avito_exceptions import (
|
|||
AvitoRateLimitedError,
|
||||
)
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -215,10 +216,16 @@ class AvitoScraper(BaseScraper):
|
|||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
self._cffi: AsyncSession | None = None
|
||||
self._browser: BrowserFetcher | None = None
|
||||
# #823: счётчик карточек, которые не удалось распарсить из-за неожиданной структуры DOM.
|
||||
self.parse_failures: int = 0
|
||||
|
||||
async def __aenter__(self) -> AvitoScraper:
|
||||
if settings.scraper_fetch_mode == "browser":
|
||||
self._browser = BrowserFetcher()
|
||||
await self._browser.__aenter__()
|
||||
logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901")
|
||||
return self
|
||||
await super().__aenter__()
|
||||
proxies = None
|
||||
if settings.scraper_proxy_url:
|
||||
|
|
@ -242,6 +249,8 @@ class AvitoScraper(BaseScraper):
|
|||
return self
|
||||
|
||||
async def __aexit__(self, *args: Any) -> None:
|
||||
if self._browser is not None:
|
||||
await self._browser.__aexit__(*args)
|
||||
if self._cffi is not None:
|
||||
await self._cffi.close()
|
||||
await super().__aexit__(*args)
|
||||
|
|
@ -277,9 +286,31 @@ class AvitoScraper(BaseScraper):
|
|||
|
||||
Returns:
|
||||
HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации).
|
||||
В browser-mode HTTP-статус недоступен: метод возвращает HTML или
|
||||
поднимает ``AvitoBlockedError`` (pagination ends via empty-parse in callers).
|
||||
Raises:
|
||||
AvitoBlockedError / AvitoRateLimitedError — если бан не снят ротацией.
|
||||
"""
|
||||
if self._browser is not None:
|
||||
# Browser-mode (#901): no HTTP status code — soft firewall-detect + rotate.
|
||||
max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
|
||||
rot_done = 0
|
||||
while True:
|
||||
html = await self._browser.fetch(url)
|
||||
if _is_firewall_page(html):
|
||||
if rot_done < max_rot and await self._rotate_ip():
|
||||
rot_done += 1
|
||||
logger.info(
|
||||
"avito page=%d browser firewall — retry after rotation #%d",
|
||||
page,
|
||||
rot_done,
|
||||
)
|
||||
continue
|
||||
logger.error("avito SERP firewall in browser-mode page=%d url=%s", page, url)
|
||||
raise AvitoBlockedError(
|
||||
f"Avito SERP firewall (browser-mode) at page={page} — IP banned"
|
||||
)
|
||||
return html
|
||||
assert self._cffi is not None
|
||||
max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
|
||||
rot_done = 0
|
||||
|
|
|
|||
|
|
@ -1,29 +1,38 @@
|
|||
"""browser_fetcher.py — camoufox wrapper для browser-mode scraping (Phase 0, #884).
|
||||
"""browser_fetcher.py — HTTP-клиент к tradein-browser сервису (#884/#905).
|
||||
|
||||
Провайдер-агностичная основа: не затрагивает существующую avito/cian-логику.
|
||||
Активируется только когда settings.scraper_fetch_mode == "browser" (пока
|
||||
дефолт "curl_cffi" — поведение идентично старому до явного переключения).
|
||||
Тонкий клиент над tradein-browser контейнером, который запускает AsyncCamoufox
|
||||
локально и экспонирует HTTP API (POST /fetch).
|
||||
|
||||
Особенности:
|
||||
- Ленивые импорты camoufox/playwright: не ломают app-импорт если Firefox не
|
||||
установлен (аналогично curl_cffi в avito_imv.py).
|
||||
- Page recycle: каждые settings.browser_recycle_pages страниц браузер
|
||||
перезапускается (ограничение утечек памяти).
|
||||
- Crash-recovery: TargetClosedError → relaunch → one retry.
|
||||
- Proxy: парсит settings.scraper_proxy_url → playwright dict формат
|
||||
{"server": "http://host:port", "username": ..., "password": ...}.
|
||||
Публичный интерфейс не изменился:
|
||||
|
||||
async with BrowserFetcher() as fetcher:
|
||||
html = await fetcher.fetch("https://example.com")
|
||||
|
||||
Внутреннее устройство: httpx.AsyncClient + POST к settings.browser_http_endpoint.
|
||||
Recycle, crash-recovery и управление браузером живут на стороне сервера (server.py).
|
||||
При HTTPError / ConnectError делает одну повторную попытку после короткой паузы,
|
||||
затем пробрасывает исключение.
|
||||
|
||||
Архитектура выбрана потому, что playwright WS-сервер (launch_server) несовместим
|
||||
с playwright >=1.45: ``browserServerImpl.js`` отсутствует → MODULE_NOT_FOUND.
|
||||
Локальный AsyncCamoufox + HTTP — работающая альтернатива.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_RETRY_SLEEP_S: float = 1.0
|
||||
_HTTP_TIMEOUT_S: float = 120.0 # навигация медленная → щедрый таймаут
|
||||
|
||||
|
||||
class BrowserFetcher:
|
||||
"""Async context manager: camoufox headless Firefox с рециклингом страниц.
|
||||
"""Async context manager: HTTP-клиент к tradein-browser HTTP-сервису.
|
||||
|
||||
Использование::
|
||||
|
||||
|
|
@ -32,188 +41,64 @@ class BrowserFetcher:
|
|||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._browser: object | None = None
|
||||
self._page_count: int = 0
|
||||
# Lazy-loaded настройки чтобы не импортировать settings на верхнем уровне
|
||||
self._recycle_after: int | None = None
|
||||
self._nav_timeout_ms: int | None = None
|
||||
self._wait_ms: int | None = None
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
self._endpoint: str | None = None
|
||||
|
||||
# ── lifecycle ──────────────────────────────────────────────────────────────
|
||||
|
||||
async def __aenter__(self) -> BrowserFetcher:
|
||||
from app.core.config import settings
|
||||
|
||||
self._recycle_after = settings.browser_recycle_pages
|
||||
self._nav_timeout_ms = settings.browser_nav_timeout_ms
|
||||
self._wait_ms = settings.browser_wait_ms
|
||||
await self._launch()
|
||||
self._endpoint = settings.browser_http_endpoint
|
||||
self._client = httpx.AsyncClient(timeout=_HTTP_TIMEOUT_S)
|
||||
logger.info("BrowserFetcher: клиент создан, endpoint=%s", self._endpoint)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_: object) -> None:
|
||||
await self._close_browser()
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
logger.debug("BrowserFetcher: клиент закрыт")
|
||||
|
||||
# ── public API ─────────────────────────────────────────────────────────────
|
||||
|
||||
async def fetch(self, url: str) -> str:
|
||||
"""Открывает новую страницу, переходит по URL, ждёт JS, возвращает HTML.
|
||||
"""Запрашивает HTML страницы через tradein-browser HTTP-сервис.
|
||||
|
||||
При TargetClosedError (crash) перезапускает браузер и делает одну
|
||||
повторную попытку. Остальные исключения всплывают к вызывающему коду.
|
||||
При HTTPError или ConnectError делает одну повторную попытку после
|
||||
короткой паузы. Остальные исключения всплывают к вызывающему коду.
|
||||
|
||||
Returns:
|
||||
Полный HTML-контент страницы (innerHTML documentElement).
|
||||
Полный HTML-контент страницы.
|
||||
"""
|
||||
assert self._client is not None, "BrowserFetcher: используй как async context manager"
|
||||
assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
|
||||
|
||||
try:
|
||||
return await self._fetch_once(url)
|
||||
except Exception as exc:
|
||||
# Определяем crash vs обычная ошибка через имя класса (lazy import)
|
||||
if _is_target_closed(exc) or _is_crash_exception(exc):
|
||||
logger.warning(
|
||||
"BrowserFetcher: браузер упал (%s), перезапуск + retry: %s",
|
||||
type(exc).__name__,
|
||||
url,
|
||||
)
|
||||
await self._relaunch()
|
||||
return await self._fetch_once(url)
|
||||
raise
|
||||
return await self._post_fetch(url)
|
||||
except (httpx.HTTPError, httpx.TransportError) as exc:
|
||||
logger.warning(
|
||||
"BrowserFetcher: ошибка запроса (%s), retry через %.1fs: %s",
|
||||
type(exc).__name__,
|
||||
_RETRY_SLEEP_S,
|
||||
url,
|
||||
)
|
||||
await asyncio.sleep(_RETRY_SLEEP_S)
|
||||
return await self._post_fetch(url)
|
||||
|
||||
# ── internal ───────────────────────────────────────────────────────────────
|
||||
|
||||
async def _fetch_once(self, url: str) -> str:
|
||||
"""Одна попытка: new_page → goto → wait → content → close."""
|
||||
# Ленивый импорт: не нужен при curl_cffi mode (не ломает app при
|
||||
# отсутствии установленного Firefox)
|
||||
try:
|
||||
from camoufox.async_api import AsyncCamoufox # noqa: F401 (type check)
|
||||
except ImportError:
|
||||
pass # импорт нужен только для type-check; browser уже запущен через _launch
|
||||
|
||||
assert self._browser is not None, "BrowserFetcher: браузер не запущен"
|
||||
|
||||
# Тип browser — AsyncCamoufox-инстанс (playwright.async_api.Browser-like)
|
||||
browser = self._browser # type: ignore[assignment]
|
||||
|
||||
page = await browser.new_page() # type: ignore[attr-defined]
|
||||
try:
|
||||
await page.goto(url, timeout=self._nav_timeout_ms, wait_until="domcontentloaded")
|
||||
# Даём JS-гидрации отработать
|
||||
if self._wait_ms and self._wait_ms > 0:
|
||||
await page.wait_for_timeout(self._wait_ms)
|
||||
html: str = await page.content()
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
self._page_count += 1
|
||||
logger.debug("BrowserFetcher: fetch OK url=%r pages_since_launch=%d", url, self._page_count)
|
||||
|
||||
# Recycle: перезапускаем браузер каждые N страниц
|
||||
assert self._recycle_after is not None
|
||||
if self._page_count >= self._recycle_after:
|
||||
logger.info(
|
||||
"BrowserFetcher: recycle threshold (%d) достигнут, перезапуск браузера",
|
||||
self._recycle_after,
|
||||
)
|
||||
await self._relaunch()
|
||||
async def _post_fetch(self, url: str) -> str:
|
||||
"""Один HTTP POST к /fetch эндпоинту сервиса."""
|
||||
assert self._client is not None
|
||||
assert self._endpoint is not None
|
||||
|
||||
resp = await self._client.post(
|
||||
f"{self._endpoint}/fetch",
|
||||
json={"url": url},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data: dict[str, str] = resp.json()
|
||||
html = data["html"]
|
||||
logger.debug("BrowserFetcher: fetch OK url=%r html_len=%d", url, len(html))
|
||||
return html
|
||||
|
||||
async def _launch(self) -> None:
|
||||
"""Запускает AsyncCamoufox браузер."""
|
||||
try:
|
||||
from camoufox.async_api import AsyncCamoufox
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"camoufox не установлен. Добавь 'camoufox[geoip]>=0.4.0' в pyproject.toml "
|
||||
"и выполни 'python -m camoufox fetch' для загрузки Firefox-сборки."
|
||||
) from exc
|
||||
|
||||
proxy = self._proxy_dict()
|
||||
kwargs: dict[str, object] = {
|
||||
"headless": True,
|
||||
"os": "windows",
|
||||
"locale": "ru-RU",
|
||||
"geoip": True,
|
||||
"humanize": True,
|
||||
}
|
||||
if proxy is not None:
|
||||
kwargs["proxy"] = proxy
|
||||
|
||||
# AsyncCamoufox — async context manager; входим в него чтобы получить browser
|
||||
ctx = AsyncCamoufox(**kwargs) # type: ignore[call-arg]
|
||||
self._browser = await ctx.__aenter__()
|
||||
self._browser_ctx = ctx # сохраняем для корректного __aexit__
|
||||
self._page_count = 0
|
||||
logger.info("BrowserFetcher: браузер запущен (proxy=%s)", proxy is not None)
|
||||
|
||||
async def _close_browser(self) -> None:
|
||||
"""Закрывает браузер если запущен."""
|
||||
ctx = getattr(self, "_browser_ctx", None)
|
||||
if ctx is not None:
|
||||
try:
|
||||
await ctx.__aexit__(None, None, None)
|
||||
logger.info("BrowserFetcher: браузер закрыт")
|
||||
except Exception as exc:
|
||||
logger.warning("BrowserFetcher: ошибка при закрытии браузера: %s", exc)
|
||||
finally:
|
||||
self._browser = None
|
||||
self._browser_ctx = None
|
||||
self._page_count = 0
|
||||
|
||||
async def _relaunch(self) -> None:
|
||||
"""Закрывает текущий браузер и запускает новый."""
|
||||
await self._close_browser()
|
||||
await self._launch()
|
||||
|
||||
def _proxy_dict(self) -> dict[str, str] | None:
|
||||
"""Парсит settings.scraper_proxy_url → playwright proxy dict.
|
||||
|
||||
Формат входного URL: ``http://user:pass@host:port`` или ``http://host:port``.
|
||||
Возвращает None если proxy не настроен.
|
||||
|
||||
Returns:
|
||||
``{"server": "http://host:port", "username": "...", "password": "..."}``
|
||||
или None.
|
||||
"""
|
||||
from app.core.config import settings
|
||||
|
||||
proxy_url = settings.scraper_proxy_url
|
||||
if not proxy_url:
|
||||
return None
|
||||
|
||||
parsed = urlparse(proxy_url)
|
||||
# Собираем server без credentials (playwright требует отдельных полей)
|
||||
port_part = f":{parsed.port}" if parsed.port else ""
|
||||
server = f"{parsed.scheme}://{parsed.hostname}{port_part}"
|
||||
|
||||
result: dict[str, str] = {"server": server}
|
||||
if parsed.username:
|
||||
result["username"] = parsed.username
|
||||
if parsed.password:
|
||||
result["password"] = parsed.password
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── helpers ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _is_target_closed(exc: BaseException) -> bool:
|
||||
"""Проверяет является ли исключение playwright TargetClosedError.
|
||||
|
||||
Проверка по имени класса (не import) — playwright не установлен обязательно.
|
||||
"""
|
||||
cls_name = type(exc).__name__
|
||||
return cls_name == "TargetClosedError"
|
||||
|
||||
|
||||
def _is_crash_exception(exc: BaseException) -> bool:
|
||||
"""Проверяет является ли исключение признаком краша браузера.
|
||||
|
||||
Ловим широкий набор playwright crash-сигналов по имени/сообщению.
|
||||
"""
|
||||
cls_name = type(exc).__name__
|
||||
if cls_name in ("BrowserClosedError", "ConnectionClosedError"):
|
||||
return True
|
||||
msg = str(exc).lower()
|
||||
return "browser has been closed" in msg or "target closed" in msg
|
||||
|
|
|
|||
406
tradein-mvp/backend/app/services/scrapers/domclick.py
Normal file
406
tradein-mvp/backend/app/services/scrapers/domclick.py
Normal file
|
|
@ -0,0 +1,406 @@
|
|||
"""DomClick.ru scraper — вторичка через headless BrowserFetcher (#796).
|
||||
|
||||
Стратегия: HTML scrape через camoufox (AsyncCamoufox) — единственный
|
||||
рабочий путь: DataDome пропускает headless Playwright, curl_cffi -> 401.
|
||||
|
||||
URL шаблон:
|
||||
https://domclick.ru/search?deal_type=sale&category=living&offer_type=flat
|
||||
&city_id={city_id}&rooms={r}&p={page}
|
||||
|
||||
Оффер-карточки: `a[href*="/card/"]` с href-паттерном `/card/sale__flat__<id>`.
|
||||
Координаты SERP не отдаёт (lat = lon = None).
|
||||
|
||||
Важно: selectolax `.text()` объединяет все дочерние текстовые узлы без
|
||||
разделителей. Числа из адреса (номер дома) могут слипнуться с ценой.
|
||||
Поэтому цену извлекаем из каждого дочернего элемента отдельно (а не из
|
||||
суммарного card_text), где элемент содержит символ рубля.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── DataDome block detection ─────────────────────────────────────────────────
|
||||
_DATADOME_MARKERS = ("datadome", "blocked", "access denied", "bot detected")
|
||||
|
||||
|
||||
def _is_blocked_page(html: str) -> bool:
|
||||
head = html[:2048].lower()
|
||||
return any(m in head for m in _DATADOME_MARKERS)
|
||||
|
||||
|
||||
# ── Regex helpers ────────────────────────────────────────────────────────────
|
||||
# _RE_ROOMS расширен относительно avito.py: DomClick пишет «2-комн. квартира»
|
||||
# (с «комн.») — avito.py-вариант ловит только «N-к. кв.» / «N-к квартира».
|
||||
_RE_ROOMS = re.compile(
|
||||
r"(\d)-(?:комн\.?\s*(?:квартира|кв\.?)?|к\.?\s*(?:квартира|кв\.?))",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_RE_STUDIO = re.compile(r"\bстуди[яиюей]\b", re.IGNORECASE)
|
||||
_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[2²]", re.IGNORECASE)
|
||||
_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE)
|
||||
|
||||
# Цена: парсим из одного элемента DOM (не из суммарного card_text) —
|
||||
# иначе число дома может слипнуться с ценой. Цена всегда в элементе с «₽».
|
||||
_RE_PRICE_EL = re.compile(r"([\d\s ]+)\s*[₽р](?:уб\.?)?", re.IGNORECASE)
|
||||
|
||||
# source_id из href вида /card/sale__flat__2075671636
|
||||
_RE_SOURCE_ID = re.compile(r"sale__flat__(\d+)")
|
||||
|
||||
# Адресные ключевые слова (эвристика)
|
||||
_RE_ADDR_KW = re.compile(
|
||||
r"ул\.|улица|пер\.|переулок|пр-т|проспект"
|
||||
r"|бул\.|бульвар|шоссе|наб\.|набережная"
|
||||
r"|пл\.|площадь|тракт|д\.\s*\d",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
# ── Вспомогательные функции ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _extract_rooms(text: str) -> int | None:
|
||||
if _RE_STUDIO.search(text):
|
||||
return 0
|
||||
m = _RE_ROOMS.search(text)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _extract_area(text: str) -> float | None:
|
||||
m = _RE_AREA.search(text)
|
||||
return float(m.group(1).replace(",", ".")) if m else None
|
||||
|
||||
|
||||
def _extract_floor(text: str) -> tuple[int | None, int | None]:
|
||||
m = _RE_FLOOR.search(text)
|
||||
if m:
|
||||
return int(m.group(1)), int(m.group(2))
|
||||
return None, None
|
||||
|
||||
|
||||
def _extract_price_from_element(text: str) -> int | None:
|
||||
"""Извлекаем цену из текста ОДНОГО элемента DOM.
|
||||
|
||||
Парсим из одного span/div — не из суммарного card_text. Это исключает
|
||||
случай, когда адресный номер дома стоит вплотную перед ценой в
|
||||
объединённом тексте карточки («Ленина, 503 100 000 руб.»).
|
||||
"""
|
||||
m = _RE_PRICE_EL.search(text)
|
||||
if m:
|
||||
# Нормализуем пробелы любого вида (обычный, неразрывный, узкий)
|
||||
raw = re.sub(r"[\s ]", "", m.group(1))
|
||||
if raw.isdigit():
|
||||
val = int(raw)
|
||||
if val > 0:
|
||||
return val
|
||||
return None
|
||||
|
||||
|
||||
def _extract_source_id(href: str) -> str | None:
|
||||
"""Числовой ID из href типа /card/sale__flat__2075671636."""
|
||||
m = _RE_SOURCE_ID.search(href)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
# ── DomClickScraper ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class DomClickScraper(BaseScraper):
|
||||
"""DomClick вторичка parser. Источник = 'domklik'.
|
||||
|
||||
Использует BrowserFetcher (camoufox headless Firefox) — DataDome пропускает
|
||||
его без дополнительной stealth-настройки. curl_cffi -> 401 (DataDome блок).
|
||||
|
||||
Основной метод: fetch_city(city_id, rooms, pages).
|
||||
fetch_around() не реализован: DomClick не поддерживает geo-radius в URL.
|
||||
"""
|
||||
|
||||
name = "domklik"
|
||||
source = "domklik"
|
||||
base_url = "https://domclick.ru"
|
||||
# DataDome — консервативная задержка между страницами (fallback до init)
|
||||
request_delay_sec = 8.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
# Счётчик карточек с неожиданной DOM-структурой (для observability)
|
||||
self.parse_failures: int = 0
|
||||
|
||||
async def __aenter__(self) -> DomClickScraper:
|
||||
await super().__aenter__()
|
||||
return self
|
||||
|
||||
# ── fetch_around — stub ───────────────────────────────────────────────────
|
||||
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list[ScrapedLot]:
|
||||
"""DomClick не поддерживает geo-radius. Используй fetch_city()."""
|
||||
raise NotImplementedError(
|
||||
"DomClick не поддерживает geo-radius; используй fetch_city(city_id=...)"
|
||||
)
|
||||
|
||||
# ── Основной метод ────────────────────────────────────────────────────────
|
||||
|
||||
async def fetch_city(
|
||||
self,
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 20,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Citywide sweep: все страницы SERP для city_id.
|
||||
|
||||
Args:
|
||||
city_id: числовой ID города в DomClick (например, 4 = Екатеринбург).
|
||||
rooms: список значений комнатности (0=студия, 1, 2, 3, 4, ...).
|
||||
None -> без фильтра комнатности (все квартиры сразу).
|
||||
pages: максимальное число страниц на один sweep. Break on empty.
|
||||
|
||||
Returns:
|
||||
Дедуплицированный по source_id список ScrapedLot.
|
||||
"""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
all_lots: list[ScrapedLot] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
# Формируем список room-значений для sweep'ов
|
||||
room_values: list[int | None] = [None] if not rooms else list(rooms)
|
||||
|
||||
async with BrowserFetcher() as fetcher:
|
||||
for room_val in room_values:
|
||||
room_label = f"rooms={room_val}" if room_val is not None else "all_rooms"
|
||||
logger.info(
|
||||
"domklik: city_id=%d %s sweep (max %d pages)", city_id, room_label, pages
|
||||
)
|
||||
|
||||
for page_num in range(1, pages + 1):
|
||||
url = self._build_url(city_id, room_val, page_num)
|
||||
logger.debug("domklik: fetch %s", url)
|
||||
|
||||
try:
|
||||
html = await fetcher.fetch(url)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"domklik: fetch failed city_id=%d page=%d: %s",
|
||||
city_id,
|
||||
page_num,
|
||||
exc,
|
||||
)
|
||||
break
|
||||
|
||||
try:
|
||||
lots = self._parse_html(html)
|
||||
except DomClickBlockedError:
|
||||
logger.warning(
|
||||
"domklik: DataDome block at city_id=%d page=%d — stopping sweep",
|
||||
city_id,
|
||||
page_num,
|
||||
)
|
||||
break
|
||||
|
||||
if not lots:
|
||||
logger.info(
|
||||
"domklik: empty page city_id=%d %s page=%d — stopping sweep",
|
||||
city_id,
|
||||
room_label,
|
||||
page_num,
|
||||
)
|
||||
break
|
||||
|
||||
# Дедупликация по source_id (кросс-sweep и кросс-страница)
|
||||
new_lots: list[ScrapedLot] = []
|
||||
for lot in lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key not in seen_ids:
|
||||
seen_ids.add(key)
|
||||
new_lots.append(lot)
|
||||
|
||||
all_lots.extend(new_lots)
|
||||
logger.info(
|
||||
"domklik: city_id=%d %s page=%d -> %d new (total %d)",
|
||||
city_id,
|
||||
room_label,
|
||||
page_num,
|
||||
len(new_lots),
|
||||
len(all_lots),
|
||||
)
|
||||
|
||||
# Пауза между страницами (anti-DataDome)
|
||||
if page_num < pages:
|
||||
await self.sleep_between_requests()
|
||||
|
||||
logger.info(
|
||||
"domklik: fetch_city done city_id=%d total=%d parse_failures=%d",
|
||||
city_id,
|
||||
len(all_lots),
|
||||
self.parse_failures,
|
||||
)
|
||||
return all_lots
|
||||
|
||||
# ── URL builder ───────────────────────────────────────────────────────────
|
||||
|
||||
def _build_url(self, city_id: int, rooms: int | None, page: int) -> str:
|
||||
"""Строит URL SERP DomClick."""
|
||||
url = (
|
||||
f"{self.base_url}/search"
|
||||
f"?deal_type=sale&category=living&offer_type=flat"
|
||||
f"&city_id={city_id}"
|
||||
)
|
||||
if rooms is not None:
|
||||
url += f"&rooms={rooms}"
|
||||
url += f"&p={page}"
|
||||
return url
|
||||
|
||||
# ── HTML parsing ──────────────────────────────────────────────────────────
|
||||
|
||||
def _parse_html(self, html: str) -> list[ScrapedLot]:
|
||||
"""Парсим карточки из HTML через selectolax.
|
||||
|
||||
Селектор: `a[href*="/card/"]` с паттерном `/card/sale__flat__<id>`.
|
||||
"""
|
||||
if _is_blocked_page(html):
|
||||
logger.warning("domklik: DataDome block page detected, 0 cards returned")
|
||||
raise DomClickBlockedError("DomClick returned DataDome block page")
|
||||
tree = HTMLParser(html)
|
||||
card_links = tree.css('a[href*="/card/"]')
|
||||
|
||||
seen_hrefs: set[str] = set()
|
||||
lots: list[ScrapedLot] = []
|
||||
|
||||
for link in card_links:
|
||||
href = link.attributes.get("href", "")
|
||||
# Только карточки вторичного жилья
|
||||
if "sale__flat__" not in href:
|
||||
continue
|
||||
if href in seen_hrefs:
|
||||
continue
|
||||
seen_hrefs.add(href)
|
||||
|
||||
lot = self._card_link_to_lot(link, href)
|
||||
if lot is not None:
|
||||
lots.append(lot)
|
||||
|
||||
return lots
|
||||
|
||||
def _card_link_to_lot(self, link: Node, href: str) -> ScrapedLot | None:
|
||||
"""Парсинг одной карточки-ссылки -> ScrapedLot.
|
||||
|
||||
DomClick SERP оборачивает карточку в `<a href="/card/...">` — вся
|
||||
информация (title, цена, адрес) внутри этого элемента.
|
||||
|
||||
Цену извлекаем из отдельных дочерних элементов (не из объединённого
|
||||
card_text) чтобы исключить слипание числа дома с ценой.
|
||||
"""
|
||||
try:
|
||||
source_id = _extract_source_id(href)
|
||||
source_url = urljoin(self.base_url, href)
|
||||
|
||||
# Суммарный текст — для rooms/area/floor (эти поля не подвержены
|
||||
# проблеме слипания, т.к. используют специфичные маркеры: «м²», «эт.»)
|
||||
card_text = link.text(strip=True)
|
||||
if not card_text:
|
||||
return None
|
||||
|
||||
rooms = _extract_rooms(card_text)
|
||||
area = _extract_area(card_text)
|
||||
floor, total_floors = _extract_floor(card_text)
|
||||
|
||||
# Цена: ищем в каждом дочернем элементе отдельно чтобы не слипались
|
||||
# числа дома с ценой при конкатенации card_text
|
||||
price = self._extract_price_from_children(link)
|
||||
if not price or price <= 0:
|
||||
return None
|
||||
|
||||
address = self._extract_address(link)
|
||||
|
||||
return ScrapedLot(
|
||||
source="domklik",
|
||||
source_url=source_url,
|
||||
source_id=source_id,
|
||||
address=address,
|
||||
lat=None, # DomClick SERP координаты не отдаёт
|
||||
lon=None,
|
||||
rooms=rooms,
|
||||
area_m2=area,
|
||||
floor=floor,
|
||||
total_floors=total_floors,
|
||||
price_rub=price,
|
||||
listing_segment="vtorichka",
|
||||
raw_payload={"card_text": card_text[:500]},
|
||||
)
|
||||
except Exception:
|
||||
self.parse_failures += 1
|
||||
logger.warning(
|
||||
"domclick _card_link_to_lot: parse failed href=%r (parse_failures=%d)",
|
||||
href,
|
||||
self.parse_failures,
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
|
||||
def _extract_price_from_children(self, link: Node) -> int | None:
|
||||
"""Ищем цену в дочерних элементах карточки по символу рубля.
|
||||
|
||||
Итерируем по всем потомкам: первый элемент с рублёвым символом
|
||||
в тексте — источник цены.
|
||||
"""
|
||||
for selector in ("span", "div", "p", "strong", "b"):
|
||||
for el in link.css(selector):
|
||||
text = el.text(strip=True)
|
||||
price = _extract_price_from_element(text)
|
||||
if price:
|
||||
return price
|
||||
# Fallback: суммарный текст карточки (если структура нестандартная)
|
||||
return _extract_price_from_element(link.text(strip=True))
|
||||
|
||||
def _extract_address(self, link: Node) -> str | None:
|
||||
"""Эвристика извлечения адреса из DOM карточки.
|
||||
|
||||
DomClick помещает адрес в отдельный текстовый блок внутри карточки.
|
||||
Обходим вложенные элементы и ищем первый текст с адресными ключевыми
|
||||
словами, пропуская строки с площадью/этажом/ценой.
|
||||
|
||||
Возвращает None если адрес не найден.
|
||||
"""
|
||||
for selector in ("span", "p", "div"):
|
||||
for el in link.css(selector):
|
||||
text = el.text(strip=True)
|
||||
if not text or len(text) < 5 or len(text) > 200:
|
||||
continue
|
||||
# Пропускаем блоки с площадью / этажом / ценой
|
||||
if _RE_AREA.search(text) or _RE_FLOOR.search(text):
|
||||
continue
|
||||
if _RE_PRICE_EL.search(text):
|
||||
continue
|
||||
if _RE_ADDR_KW.search(text):
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
# ── Convenience runner (для Celery tasks) ────────────────────────────────────
|
||||
|
||||
|
||||
async def scrape_domclick_city(
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 20,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Удобная точка входа для вызова из Celery tasks.
|
||||
|
||||
Пример::
|
||||
|
||||
import asyncio
|
||||
lots = asyncio.run(scrape_domclick_city(city_id=4, rooms=[1, 2, 3]))
|
||||
"""
|
||||
async with DomClickScraper() as scraper:
|
||||
return await scraper.fetch_city(city_id=city_id, rooms=rooms, pages=pages)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
"""DomClick-specific exceptions для anti-bot detection."""
|
||||
|
||||
|
||||
class DomClickBlockedError(Exception):
|
||||
"""DomClick вернул DataDome block-страницу (HTTP 200 + block HTML)."""
|
||||
|
|
@ -23,7 +23,7 @@ dependencies = [
|
|||
"sentry-sdk>=2.0.0", # мониторинг ошибок → GlitchTip (#396)
|
||||
"redis>=5.0.0", # async hot cache для /api/v1/search (Phase 3.2)
|
||||
"pyyaml>=6.0.0", # RBAC roles.yaml loader (app/core/auth.py)
|
||||
"camoufox[geoip]>=0.4.0", # headless Firefox wrapper для browser-mode scraping (#884)
|
||||
"playwright>=1.45", # Playwright client для connect к tradein-browser (#905)
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
|
|
@ -31,7 +31,6 @@ dev = [
|
|||
"pytest>=8.0.0",
|
||||
"pytest-asyncio>=0.24.0",
|
||||
"ruff>=0.5.0",
|
||||
"playwright>=1.45", # address-mismatch audit fallback (issue #582 Phase 1)
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
"""Tests for BrowserFetcher (#884, Phase 0).
|
||||
"""Tests for BrowserFetcher (#884/#905, HTTP-клиент).
|
||||
|
||||
Полностью замоканы через monkeypatch на httpx.AsyncClient.post:
|
||||
не требуют установленного Firefox, camoufox или реального HTTP-сервера.
|
||||
|
||||
Полностью замоканы: не требуют установленного Firefox или camoufox.
|
||||
Покрывают:
|
||||
- _proxy_dict(): парсинг proxy URL с credentials и без
|
||||
- fetch(): new_page → goto → wait_for_timeout → content → close
|
||||
- recycle: счётчик страниц → relaunch браузера при достижении порога
|
||||
- crash-recovery: TargetClosedError → relaunch → retry
|
||||
- fetch(): POST к правильному endpoint + разбор {"html": "..."} из ответа
|
||||
- HTTPError триггерит один retry, затем поднимает исключение
|
||||
- URL строится из settings.browser_http_endpoint
|
||||
- импорт модуля не ломается без playwright/camoufox
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -14,6 +16,7 @@ import os
|
|||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
|
@ -23,273 +26,181 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:
|
|||
|
||||
|
||||
def _mock_settings(
|
||||
proxy_url: str | None = None,
|
||||
recycle_pages: int = 15,
|
||||
nav_timeout_ms: int = 60000,
|
||||
wait_ms: int = 2500,
|
||||
browser_http_endpoint: str = "http://fake-browser:3000",
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
scraper_proxy_url=proxy_url,
|
||||
browser_recycle_pages=recycle_pages,
|
||||
browser_nav_timeout_ms=nav_timeout_ms,
|
||||
browser_wait_ms=wait_ms,
|
||||
return SimpleNamespace(browser_http_endpoint=browser_http_endpoint)
|
||||
|
||||
|
||||
def _make_ok_response(html: str = "<html>ok</html>") -> MagicMock:
|
||||
"""Создаёт мок httpx.Response с JSON {"html": html} и статусом 200."""
|
||||
resp = MagicMock(spec=httpx.Response)
|
||||
resp.status_code = 200
|
||||
resp.json = MagicMock(return_value={"html": html})
|
||||
resp.raise_for_status = MagicMock() # нет исключения
|
||||
return resp
|
||||
|
||||
|
||||
def _make_error_response(status: int = 500) -> MagicMock:
|
||||
"""Создаёт мок httpx.Response который кидает HTTPStatusError при raise_for_status."""
|
||||
resp = MagicMock(spec=httpx.Response)
|
||||
resp.status_code = status
|
||||
resp.raise_for_status = MagicMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
f"Server error {status}",
|
||||
request=MagicMock(),
|
||||
response=resp,
|
||||
)
|
||||
)
|
||||
return resp
|
||||
|
||||
|
||||
def _make_mock_page(html: str = "<html>ok</html>") -> MagicMock:
|
||||
"""Создаёт мок страницы playwright с нужными async-методами."""
|
||||
page = MagicMock()
|
||||
page.goto = AsyncMock()
|
||||
page.wait_for_timeout = AsyncMock()
|
||||
page.content = AsyncMock(return_value=html)
|
||||
page.close = AsyncMock()
|
||||
return page
|
||||
# ── импорт не ломается ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_mock_browser(page: MagicMock | None = None) -> MagicMock:
|
||||
"""Создаёт мок browser-контекста camoufox."""
|
||||
if page is None:
|
||||
page = _make_mock_page()
|
||||
browser = MagicMock()
|
||||
browser.new_page = AsyncMock(return_value=page)
|
||||
return browser
|
||||
def test_import_browser_fetcher() -> None:
|
||||
"""import app.services.scrapers.browser_fetcher не поднимает исключений."""
|
||||
import app.services.scrapers.browser_fetcher # noqa: F401
|
||||
|
||||
|
||||
def _make_mock_camoufox_ctx(browser: MagicMock) -> MagicMock:
|
||||
"""Мок AsyncCamoufox(...) context manager → возвращает browser из __aenter__."""
|
||||
ctx = MagicMock()
|
||||
ctx.__aenter__ = AsyncMock(return_value=browser)
|
||||
ctx.__aexit__ = AsyncMock(return_value=None)
|
||||
return ctx
|
||||
|
||||
|
||||
# ── _proxy_dict() ──────────────────────────────────────────────────────────────
|
||||
# _proxy_dict() импортирует settings через `from app.core.config import settings`
|
||||
# внутри метода, поэтому патчим app.core.config.settings (источник).
|
||||
|
||||
|
||||
def test_proxy_dict_parses_full_url():
|
||||
"""_proxy_dict() разбирает URL с username/password → playwright dict."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
fetcher = BrowserFetcher()
|
||||
ms = _mock_settings(proxy_url="http://user123:pass456@proxy.example.com:8080")
|
||||
with patch("app.core.config.settings", ms):
|
||||
result = fetcher._proxy_dict()
|
||||
|
||||
assert result == {
|
||||
"server": "http://proxy.example.com:8080",
|
||||
"username": "user123",
|
||||
"password": "pass456",
|
||||
}
|
||||
|
||||
|
||||
def test_proxy_dict_no_credentials():
|
||||
"""_proxy_dict() для URL без credentials возвращает только server."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
fetcher = BrowserFetcher()
|
||||
ms = _mock_settings(proxy_url="http://proxy.example.com:3128")
|
||||
with patch("app.core.config.settings", ms):
|
||||
result = fetcher._proxy_dict()
|
||||
|
||||
assert result == {"server": "http://proxy.example.com:3128"}
|
||||
assert "username" not in result
|
||||
assert "password" not in result
|
||||
|
||||
|
||||
def test_proxy_dict_returns_none_when_no_proxy():
|
||||
"""_proxy_dict() → None когда scraper_proxy_url не задан."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
fetcher = BrowserFetcher()
|
||||
ms = _mock_settings(proxy_url=None)
|
||||
with patch("app.core.config.settings", ms):
|
||||
result = fetcher._proxy_dict()
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
# ── fetch(): new_page → goto → content → close ────────────────────────────────
|
||||
# ── fetch(): возвращает HTML из JSON-ответа ────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_calls_page_lifecycle():
|
||||
"""fetch() вызывает new_page, goto, wait_for_timeout, content, close."""
|
||||
async def test_fetch_returns_html_from_json_response() -> None:
|
||||
"""fetch() разбирает {'html': ...} из 200-ответа и возвращает строку."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
expected_html = "<html><body>test</body></html>"
|
||||
page = _make_mock_page(html=expected_html)
|
||||
browser = _make_mock_browser(page=page)
|
||||
ctx = _make_mock_camoufox_ctx(browser)
|
||||
|
||||
ms = _mock_settings(recycle_pages=15, nav_timeout_ms=60000, wait_ms=2500)
|
||||
expected_html = "<html><body>avito</body></html>"
|
||||
ok_resp = _make_ok_response(html=expected_html)
|
||||
ms = _mock_settings()
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
result = await fetcher.fetch("https://example.com")
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
assert fetcher._client is not None
|
||||
fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign]
|
||||
result = await fetcher.fetch("https://avito.ru/test")
|
||||
|
||||
assert result == expected_html
|
||||
browser.new_page.assert_called_once()
|
||||
page.goto.assert_called_once_with(
|
||||
"https://example.com", timeout=60000, wait_until="domcontentloaded"
|
||||
)
|
||||
page.wait_for_timeout.assert_called_once_with(2500)
|
||||
page.content.assert_called_once()
|
||||
page.close.assert_called_once()
|
||||
assert isinstance(result, str)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_returns_html_string():
|
||||
"""fetch() возвращает строку HTML от page.content()."""
|
||||
async def test_fetch_posts_to_correct_endpoint() -> None:
|
||||
"""fetch() делает POST к {browser_http_endpoint}/fetch с {"url": ...}."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
html = "<html><title>Avito</title></html>"
|
||||
page = _make_mock_page(html=html)
|
||||
browser = _make_mock_browser(page=page)
|
||||
ctx = _make_mock_camoufox_ctx(browser)
|
||||
endpoint = "http://fake-browser:3000"
|
||||
ok_resp = _make_ok_response()
|
||||
ms = _mock_settings(browser_http_endpoint=endpoint)
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
assert fetcher._client is not None
|
||||
post_mock = AsyncMock(return_value=ok_resp)
|
||||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||||
await fetcher.fetch("https://example.com/page")
|
||||
|
||||
post_mock.assert_called_once_with(
|
||||
f"{endpoint}/fetch",
|
||||
json={"url": "https://example.com/page"},
|
||||
)
|
||||
|
||||
|
||||
# ── retry при HTTPError ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_retries_once_on_http_error() -> None:
|
||||
"""При HTTPError первый вызов → retry → успешный второй вызов."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
html = "<html>retry-ok</html>"
|
||||
ok_resp = _make_ok_response(html=html)
|
||||
error_resp = _make_error_response(500)
|
||||
ms = _mock_settings()
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
assert fetcher._client is not None
|
||||
# Первый вызов кидает HTTPStatusError, второй возвращает OK
|
||||
post_mock = AsyncMock(side_effect=[error_resp, ok_resp])
|
||||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||||
|
||||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||||
result = await fetcher.fetch("https://avito.ru/retry")
|
||||
|
||||
assert result == html
|
||||
assert post_mock.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_raises_after_two_http_errors() -> None:
|
||||
"""Если оба вызова кидают HTTPError — исключение пробрасывается."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
error_resp_1 = _make_error_response(500)
|
||||
error_resp_2 = _make_error_response(500)
|
||||
ms = _mock_settings()
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
assert fetcher._client is not None
|
||||
post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2])
|
||||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||||
|
||||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
await fetcher.fetch("https://avito.ru/fail")
|
||||
|
||||
assert post_mock.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_retries_on_connect_error() -> None:
|
||||
"""ConnectError (подкласс TransportError) тоже триггерит retry."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
html = "<html>connect-retry-ok</html>"
|
||||
ok_resp = _make_ok_response(html=html)
|
||||
connect_err = httpx.ConnectError("Connection refused")
|
||||
ms = _mock_settings()
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
assert fetcher._client is not None
|
||||
post_mock = AsyncMock(side_effect=[connect_err, ok_resp])
|
||||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||||
|
||||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||||
result = await fetcher.fetch("https://avito.ru/connect")
|
||||
|
||||
assert result == html
|
||||
assert post_mock.await_count == 2
|
||||
|
||||
|
||||
# ── non-http exception пробрасывается без перехвата ───────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_propagates_unexpected_exception() -> None:
|
||||
"""Не-HTTP исключение (ValueError и т.п.) не подавляется и не вызывает retry."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
ms = _mock_settings()
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
result = await fetcher.fetch("https://avito.ru/test")
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
assert fetcher._client is not None
|
||||
post_mock = AsyncMock(side_effect=ValueError("unexpected"))
|
||||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||||
|
||||
assert result == html
|
||||
assert isinstance(result, str)
|
||||
with pytest.raises(ValueError, match="unexpected"):
|
||||
await fetcher.fetch("https://avito.ru/bad")
|
||||
|
||||
|
||||
# ── recycle: relaunch at threshold ────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recycle_triggers_relaunch_at_threshold():
|
||||
"""После recycle_pages страниц браузер перезапускается (relaunch)."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
recycle_after = 3
|
||||
|
||||
page = _make_mock_page()
|
||||
browser = _make_mock_browser(page=page)
|
||||
ctx = _make_mock_camoufox_ctx(browser)
|
||||
|
||||
ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0)
|
||||
|
||||
launch_count = 0
|
||||
original_launch = BrowserFetcher._launch
|
||||
|
||||
async def counting_launch(self: BrowserFetcher) -> None:
|
||||
nonlocal launch_count
|
||||
launch_count += 1
|
||||
await original_launch(self)
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||||
with patch.object(BrowserFetcher, "_launch", counting_launch):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
# Первый запуск через __aenter__
|
||||
assert launch_count == 1
|
||||
# Отправляем recycle_after страниц — на последней сработает relaunch
|
||||
for _ in range(recycle_after):
|
||||
await fetcher.fetch("https://example.com")
|
||||
# После recycle браузер должен был перезапуститься ещё раз
|
||||
assert launch_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_page_counter_resets_after_relaunch():
|
||||
"""После recycle page_count сбрасывается в 0."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
recycle_after = 2
|
||||
|
||||
page = _make_mock_page()
|
||||
browser = _make_mock_browser(page=page)
|
||||
ctx = _make_mock_camoufox_ctx(browser)
|
||||
|
||||
ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0)
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
for _ in range(recycle_after):
|
||||
await fetcher.fetch("https://example.com")
|
||||
# После recycle счётчик сброшен
|
||||
assert fetcher._page_count == 0
|
||||
|
||||
|
||||
# ── crash-recovery ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _FakeTargetClosedError(Exception):
|
||||
"""Имитирует playwright TargetClosedError (проверка по имени класса)."""
|
||||
|
||||
|
||||
_FakeTargetClosedError.__name__ = "TargetClosedError"
|
||||
_FakeTargetClosedError.__qualname__ = "TargetClosedError"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_retries_once_on_target_closed_error():
|
||||
"""fetch() при TargetClosedError перезапускает браузер и повторяет запрос."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
html = "<html>retry-ok</html>"
|
||||
|
||||
# Первая страница бросает краш, вторая отвечает нормально
|
||||
crash_page = MagicMock()
|
||||
crash_page.goto = AsyncMock(side_effect=_FakeTargetClosedError("target closed"))
|
||||
crash_page.close = AsyncMock()
|
||||
|
||||
ok_page = _make_mock_page(html=html)
|
||||
browser = _make_mock_browser()
|
||||
|
||||
call_num = 0
|
||||
|
||||
async def new_page_side_effect() -> MagicMock:
|
||||
nonlocal call_num
|
||||
call_num += 1
|
||||
return crash_page if call_num == 1 else ok_page
|
||||
|
||||
browser.new_page = AsyncMock(side_effect=new_page_side_effect)
|
||||
ctx = _make_mock_camoufox_ctx(browser)
|
||||
|
||||
ms = _mock_settings(wait_ms=0)
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
result = await fetcher.fetch("https://avito.ru/page")
|
||||
|
||||
assert result == html
|
||||
# 2 обращения к new_page: первое (crash) + второе (retry после relaunch)
|
||||
assert call_num == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_propagates_non_crash_exception():
|
||||
"""fetch() не подавляет обычные (не-crash) исключения."""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
page = MagicMock()
|
||||
page.goto = AsyncMock(side_effect=ValueError("unexpected parse error"))
|
||||
page.close = AsyncMock()
|
||||
|
||||
browser = _make_mock_browser(page=page)
|
||||
ctx = _make_mock_camoufox_ctx(browser)
|
||||
|
||||
ms = _mock_settings(wait_ms=0)
|
||||
|
||||
with patch("app.core.config.settings", ms):
|
||||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||||
fetcher = BrowserFetcher()
|
||||
async with fetcher:
|
||||
with pytest.raises(ValueError, match="unexpected parse error"):
|
||||
await fetcher.fetch("https://avito.ru/bad")
|
||||
assert post_mock.await_count == 1
|
||||
|
|
|
|||
252
tradein-mvp/backend/tests/scrapers/test_domclick.py
Normal file
252
tradein-mvp/backend/tests/scrapers/test_domclick.py
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
"""Unit-тесты для DomClick scraper (#796).
|
||||
|
||||
Тестируем:
|
||||
- Парсинг title-паттерна (rooms/area/floor) из текста карточки
|
||||
- Парсинг цены из отдельного DOM-элемента
|
||||
- Извлечение source_id из href
|
||||
- Полный цикл _parse_html -> list[ScrapedLot] на mocked HTML
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.domclick import (
|
||||
DomClickScraper,
|
||||
_extract_area,
|
||||
_extract_floor,
|
||||
_extract_price_from_element,
|
||||
_extract_rooms,
|
||||
_extract_source_id,
|
||||
)
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||||
|
||||
# ── Rooms / studio ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, expected",
|
||||
[
|
||||
("2-комн. квартира 53,5 м² 5/15 эт.", 2),
|
||||
("1-к. квартира 32,1 м² 3/9 эт.", 1),
|
||||
("3-комн. кв. 78 м² 10/12 эт.", 3),
|
||||
("4-комн. квартира 95 м² 2/5 эт.", 4),
|
||||
("студия 24 м² 1/10 эт.", 0),
|
||||
("Студия 30,5 м² 8/22 эт.", 0),
|
||||
("Нет комнат — просто текст", None),
|
||||
],
|
||||
)
|
||||
def test_extract_rooms(text: str, expected: int | None) -> None:
|
||||
assert _extract_rooms(text) == expected
|
||||
|
||||
|
||||
# ── Area ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, expected",
|
||||
[
|
||||
("2-комн. квартира 53,5 м² 5/15 эт.", 53.5),
|
||||
("студия 24 м² 1/10 эт.", 24.0),
|
||||
("78м2 на этаже", 78.0),
|
||||
("площадь 101.7 м²", 101.7),
|
||||
("Нет площади", None),
|
||||
],
|
||||
)
|
||||
def test_extract_area(text: str, expected: float | None) -> None:
|
||||
assert _extract_area(text) == expected
|
||||
|
||||
|
||||
# ── Floor ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, floor, total",
|
||||
[
|
||||
("2-комн. квартира 53,5 м² 5/15 эт.", 5, 15),
|
||||
("студия 24 м² 1/10 эт.", 1, 10),
|
||||
("12/24 эт.", 12, 24),
|
||||
("нет этажа", None, None),
|
||||
],
|
||||
)
|
||||
def test_extract_floor(text: str, floor: int | None, total: int | None) -> None:
|
||||
assert _extract_floor(text) == (floor, total)
|
||||
|
||||
|
||||
# ── Price (per-element) ───────────────────────────────────────────────────────
|
||||
# Цена парсится из отдельного DOM-элемента (не из суммарного card_text),
|
||||
# поэтому числа адреса не загрязняют парсинг цены.
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, expected",
|
||||
[
|
||||
("5 200 000 ₽", 5_200_000),
|
||||
("12 345 678 ₽", 12_345_678),
|
||||
("3 990 000 руб.", 3_990_000),
|
||||
("от 4 500 000 ₽", 4_500_000),
|
||||
("без цены", None),
|
||||
("0 ₽", None),
|
||||
],
|
||||
)
|
||||
def test_extract_price_from_element(text: str, expected: int | None) -> None:
|
||||
assert _extract_price_from_element(text) == expected
|
||||
|
||||
|
||||
# ── source_id from href ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"href, expected",
|
||||
[
|
||||
("/card/sale__flat__2075671636", "2075671636"),
|
||||
("https://domclick.ru/card/sale__flat__123456789", "123456789"),
|
||||
("/card/sale__flat__1", "1"),
|
||||
("/card/something_else", None),
|
||||
("", None),
|
||||
],
|
||||
)
|
||||
def test_extract_source_id(href: str, expected: str | None) -> None:
|
||||
assert _extract_source_id(href) == expected
|
||||
|
||||
|
||||
# ── _parse_html -> list[ScrapedLot] ──────────────────────────────────────────
|
||||
# Цена в каждой карточке вынесена в отдельный <span> с символом рубля,
|
||||
# чтобы не сливаться с адресным номером дома при конкатенации card_text.
|
||||
|
||||
_MOCK_HTML = """\
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<!-- Карточка 1: комнатность, площадь, этаж, адрес, цена в отдельных span -->
|
||||
<a href="/card/sale__flat__2075671636">
|
||||
<span>2-комн. квартира 53,5 м² 5/15 эт.</span>
|
||||
<span>ул. Малышева, 1</span>
|
||||
<span>5 200 000 ₽</span>
|
||||
</a>
|
||||
|
||||
<!-- Карточка 2: студия -->
|
||||
<a href="/card/sale__flat__1111111111">
|
||||
<span>Студия 28 м² 3/10 эт.</span>
|
||||
<span>пр-т Ленина, 50</span>
|
||||
<span>3 100 000 ₽</span>
|
||||
</a>
|
||||
|
||||
<!-- Карточка 3: без цены -> должна быть отфильтрована -->
|
||||
<a href="/card/sale__flat__9999999999">
|
||||
<span>1-комн. квартира 32 м² 2/9 эт.</span>
|
||||
</a>
|
||||
|
||||
<!-- Не карточка (без sale__flat__) -> должна быть отфильтрована -->
|
||||
<a href="/card/sale__apartment__123">
|
||||
<span>что-то другое</span>
|
||||
</a>
|
||||
|
||||
<!-- Дубликат первой карточки -> один раз на страницу -->
|
||||
<a href="/card/sale__flat__2075671636">
|
||||
<span>2-комн. квартира 53,5 м² 5/15 эт.</span>
|
||||
<span>5 200 000 ₽</span>
|
||||
</a>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def test_parse_html_returns_lots() -> None:
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
# Карточки 1 и 2 парсятся; карточка 3 (без цены) отфильтрована; дубль — дедуплицирован
|
||||
assert len(lots) == 2
|
||||
|
||||
|
||||
def test_parse_html_card1_fields() -> None:
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
card1 = next(lot for lot in lots if lot.source_id == "2075671636")
|
||||
assert card1.source == "domklik"
|
||||
assert card1.source_url == "https://domclick.ru/card/sale__flat__2075671636"
|
||||
assert card1.rooms == 2
|
||||
assert card1.area_m2 == 53.5
|
||||
assert card1.floor == 5
|
||||
assert card1.total_floors == 15
|
||||
assert card1.price_rub == 5_200_000
|
||||
assert card1.lat is None
|
||||
assert card1.lon is None
|
||||
assert card1.listing_segment == "vtorichka"
|
||||
|
||||
|
||||
def test_parse_html_studio() -> None:
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
studio = next(lot for lot in lots if lot.source_id == "1111111111")
|
||||
assert studio.rooms == 0 # студия
|
||||
assert studio.area_m2 == 28.0
|
||||
assert studio.price_rub == 3_100_000
|
||||
|
||||
|
||||
def test_parse_html_no_price_filtered() -> None:
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
source_ids = {lot.source_id for lot in lots}
|
||||
assert "9999999999" not in source_ids
|
||||
|
||||
|
||||
def test_parse_html_dedup_same_href() -> None:
|
||||
"""Дубликат href на одной странице должен давать одну карточку."""
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
ids_2075 = [lot for lot in lots if lot.source_id == "2075671636"]
|
||||
assert len(ids_2075) == 1
|
||||
|
||||
|
||||
def test_scraper_source_name() -> None:
|
||||
scraper = DomClickScraper()
|
||||
assert scraper.name == "domklik"
|
||||
assert scraper.source == "domklik"
|
||||
assert scraper.base_url == "https://domclick.ru"
|
||||
# request_delay_sec установлен из get_scraper_delay (DB или fallback 8.0)
|
||||
assert scraper.request_delay_sec >= 0.0
|
||||
|
||||
|
||||
def test_fetch_around_raises() -> None:
|
||||
scraper = DomClickScraper()
|
||||
with pytest.raises(NotImplementedError, match="geo-radius"):
|
||||
asyncio.run(scraper.fetch_around(56.8, 60.6))
|
||||
|
||||
|
||||
def test_build_url_no_rooms() -> None:
|
||||
scraper = DomClickScraper()
|
||||
url = scraper._build_url(city_id=4, rooms=None, page=1)
|
||||
assert "city_id=4" in url
|
||||
assert "rooms" not in url
|
||||
assert "p=1" in url
|
||||
|
||||
|
||||
def test_build_url_with_rooms() -> None:
|
||||
scraper = DomClickScraper()
|
||||
url = scraper._build_url(city_id=4, rooms=2, page=3)
|
||||
assert "rooms=2" in url
|
||||
assert "p=3" in url
|
||||
|
||||
|
||||
def test_parse_failures_counter() -> None:
|
||||
"""parse_failures счётчик начинается с 0."""
|
||||
scraper = DomClickScraper()
|
||||
assert scraper.parse_failures == 0
|
||||
|
||||
|
||||
def test_blocked_page_raises_domclick_blocked_error() -> None:
|
||||
"""HTML с datadome в начале вызывает DomClickBlockedError."""
|
||||
scraper = DomClickScraper()
|
||||
blocked_html = (
|
||||
"<html><head><title>datadome protection</title></head><body>blocked</body></html>"
|
||||
)
|
||||
with pytest.raises(DomClickBlockedError):
|
||||
scraper._parse_html(blocked_html)
|
||||
225
tradein-mvp/backend/tests/test_avito_browser_mode.py
Normal file
225
tradein-mvp/backend/tests/test_avito_browser_mode.py
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
"""#901 — avito SERP fetch via BrowserFetcher behind scraper_fetch_mode flag.
|
||||
|
||||
Tests:
|
||||
1. browser branch chosen: mode="browser" → _browser set, _cffi is None, HTML from mock browser.
|
||||
2. curl branch unchanged: default mode → _browser is None, _cffi session used.
|
||||
3. firewall-detect in browser-mode: firewall HTML + no rotate URL → AvitoBlockedError.
|
||||
4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks.
|
||||
|
||||
Без сети, без БД, без curl_cffi.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
|
||||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||||
from app.services.scrapers.base import ScrapedLot
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
|
||||
_FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"
|
||||
|
||||
|
||||
def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
|
||||
"""Return a mock BrowserFetcher context manager that yields html from fetch()."""
|
||||
mock = MagicMock()
|
||||
mock.__aenter__ = AsyncMock(return_value=mock)
|
||||
mock.__aexit__ = AsyncMock(return_value=None)
|
||||
mock.fetch = AsyncMock(return_value=html)
|
||||
return mock
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. browser branch chosen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aenter_browser_mode_sets_browser_attribute() -> None:
|
||||
"""mode=browser → __aenter__ instantiates BrowserFetcher, _cffi stays None."""
|
||||
mock_browser = _make_mock_browser()
|
||||
scraper = AvitoScraper()
|
||||
|
||||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||||
mock_settings.scraper_fetch_mode = "browser"
|
||||
mock_settings.scraper_proxy_url = None
|
||||
mock_settings.avito_proxy_rotate_url = None
|
||||
mock_settings.avito_proxy_max_rotations = 2
|
||||
|
||||
with patch(
|
||||
"app.services.scrapers.avito.BrowserFetcher",
|
||||
return_value=mock_browser,
|
||||
):
|
||||
# Patch BaseScraper.__aenter__ so we skip its real setup (DB/network)
|
||||
with patch(
|
||||
"app.services.scrapers.base.BaseScraper.__aenter__",
|
||||
new=AsyncMock(return_value=scraper),
|
||||
):
|
||||
result = await scraper.__aenter__()
|
||||
|
||||
assert result is scraper
|
||||
assert scraper._browser is mock_browser
|
||||
assert scraper._cffi is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_serp_html_browser_mode_returns_html() -> None:
|
||||
"""_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""
|
||||
scraper = AvitoScraper()
|
||||
mock_browser = _make_mock_browser(_NORMAL_HTML)
|
||||
scraper._browser = mock_browser
|
||||
|
||||
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
||||
|
||||
assert result == _NORMAL_HTML
|
||||
mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
|
||||
# _cffi was never set — assert remains None
|
||||
assert scraper._cffi is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. curl branch unchanged
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_serp_html_curl_mode_uses_cffi() -> None:
|
||||
"""Default mode (_browser is None) → _fetch_serp_html goes through curl path."""
|
||||
scraper = AvitoScraper()
|
||||
|
||||
# Mock the curl session returning HTTP 200 with normal HTML
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = _NORMAL_HTML
|
||||
|
||||
mock_cffi = MagicMock()
|
||||
mock_cffi.get = AsyncMock(return_value=mock_response)
|
||||
scraper._cffi = mock_cffi
|
||||
|
||||
# _browser must be None (default curl path)
|
||||
assert scraper._browser is None
|
||||
|
||||
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
||||
|
||||
assert result == _NORMAL_HTML
|
||||
mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_mode_firewall_raises_blocked_error() -> None:
|
||||
"""Browser returns firewall page + no rotate URL configured → AvitoBlockedError."""
|
||||
scraper = AvitoScraper()
|
||||
mock_browser = _make_mock_browser(_FIREWALL_HTML)
|
||||
scraper._browser = mock_browser
|
||||
|
||||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||||
mock_settings.avito_proxy_rotate_url = None # no rotation configured
|
||||
mock_settings.avito_proxy_max_rotations = 2
|
||||
|
||||
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
||||
await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_mode_firewall_rotates_and_retries() -> None:
|
||||
"""Browser returns firewall on first call, normal HTML after rotation → returns HTML."""
|
||||
scraper = AvitoScraper()
|
||||
|
||||
# First call → firewall; second call → normal HTML
|
||||
mock_browser = MagicMock()
|
||||
mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML])
|
||||
scraper._browser = mock_browser
|
||||
|
||||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||||
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate"
|
||||
mock_settings.avito_proxy_max_rotations = 1
|
||||
|
||||
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)):
|
||||
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2)
|
||||
|
||||
assert result == _NORMAL_HTML
|
||||
assert mock_browser.fetch.await_count == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. pagination terminates via empty-parse
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
|
||||
"""fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops.
|
||||
|
||||
Verifies the documented invariant: callers break on `not lots` regardless of
|
||||
fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is
|
||||
already covered by unit tests above); here we confirm the caller loop behaviour.
|
||||
"""
|
||||
scraper = AvitoScraper()
|
||||
|
||||
html_p1 = "<html>page1</html>"
|
||||
html_p2 = "<html>page2</html>"
|
||||
|
||||
lot = ScrapedLot(
|
||||
source="avito",
|
||||
source_url="https://www.avito.ru/ekaterinburg/kvartiry/test",
|
||||
source_id="LOT_1",
|
||||
price_rub=5_000_000,
|
||||
)
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def mock_fetch_html(url: str, page: int) -> str:
|
||||
return html_p1 if page == 1 else html_p2
|
||||
|
||||
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return [lot] if "page1" in html else []
|
||||
|
||||
with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html):
|
||||
with patch.object(scraper, "_parse_html", side_effect=mock_parse):
|
||||
with patch.object(scraper, "sleep_between_requests", new=AsyncMock()):
|
||||
result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].source_id == "LOT_1"
|
||||
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
|
||||
assert call_count == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. rotation returns False → immediate AvitoBlockedError
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
|
||||
"""rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
|
||||
|
||||
Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
|
||||
(e.g. rotation API timeout/failure) → we cannot recover → blocked.
|
||||
"""
|
||||
scraper = AvitoScraper()
|
||||
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
||||
|
||||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||||
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
||||
mock_settings.avito_proxy_max_rotations = 2
|
||||
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
|
||||
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
||||
await scraper._fetch_serp_html("https://avito.ru/test", page=1)
|
||||
44
tradein-mvp/browser/Dockerfile
Normal file
44
tradein-mvp/browser/Dockerfile
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
# tradein-browser — camoufox Playwright WS server (изолированный контейнер Firefox).
|
||||
# Backend подключается к этому контейнеру через playwright.firefox.connect.
|
||||
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
# Firefox runtime libs + ca-certificates для TLS
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update && apt-get install -y --no-install-recommends \
|
||||
libgtk-3-0 \
|
||||
libasound2 \
|
||||
libdbus-glib-1-2 \
|
||||
libx11-xcb1 \
|
||||
libxtst6 \
|
||||
ca-certificates \
|
||||
&& useradd --create-home --uid 1000 app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# camoufox[geoip] тянет playwright как зависимость
|
||||
RUN pip install --no-cache-dir "camoufox[geoip]" aiohttp
|
||||
|
||||
COPY server.py ./server.py
|
||||
|
||||
# camoufox fetch качает (а) Firefox-сборку и (б) GeoLite2 mmdb (geoip=True).
|
||||
# Firefox идёт в $HOME/.cache → ENV HOME=/home/app кладёт его в кэш app-юзера,
|
||||
# куда рантайм его ищет (фикс #899). НО mmdb пишется в site-packages пакета
|
||||
# camoufox (root-owned после pip install) — под app это PermissionError на
|
||||
# GeoLite2-City.mmdb. Поэтому fetch выполняем под ROOT (пишет и mmdb, и —
|
||||
# благодаря ENV HOME — Firefox в /home/app/.cache), затем chown'им Firefox-кэш
|
||||
# на app. mmdb остаётся root-owned в site-packages (world-readable → app читает
|
||||
# на рантайме). Verified реальной сборкой образа 2026-05-31.
|
||||
ENV HOME=/home/app
|
||||
|
||||
RUN python -m camoufox fetch && chown -R app:app /home/app/.cache
|
||||
|
||||
USER app
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
CMD ["python", "server.py"]
|
||||
269
tradein-mvp/browser/server.py
Normal file
269
tradein-mvp/browser/server.py
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
"""server.py — tradein-browser service main process.
|
||||
|
||||
Этот модуль запускается как точка входа контейнера ``tradein-browser``.
|
||||
Он запускает camoufox **локально** внутри контейнера (AsyncCamoufox) и
|
||||
экспонирует простой HTTP API на базе aiohttp:
|
||||
|
||||
GET /health → {"status": "ok"}
|
||||
POST /fetch → {"url": "..."} → {"html": "..."}
|
||||
|
||||
Такой подход выбран потому, что ``camoufox.server.launch_server`` (Playwright
|
||||
WS-сервер) несовместим с современными версиями playwright (1.45–1.60):
|
||||
``browserServerImpl.js`` отсутствует в дистрибутиве → контейнер краш-лупится
|
||||
с MODULE_NOT_FOUND / RuntimeError при запуске через ``launch_server``.
|
||||
Локальный запуск ``AsyncCamoufox`` работает стабильно — проверено.
|
||||
|
||||
Переменные окружения:
|
||||
BROWSER_PORT — TCP-порт HTTP-сервера (default: 3000)
|
||||
BROWSER_RECYCLE_PAGES — страниц в одном сеансе браузера до перезапуска (default: 15)
|
||||
BROWSER_NAV_TIMEOUT_MS — таймаут page.goto в мс (default: 60000)
|
||||
BROWSER_WAIT_MS — ожидание гидрации listings после DOMContentLoaded, мс (default: 6000)
|
||||
AVITO_PROXY_URL — прокси ``http://user:pass@host:port`` (mobileproxy, #623);
|
||||
SCRAPER_PROXY_URL — generic-fallback. Без него — soft-block.
|
||||
|
||||
Эндпоинт /fetch доступен из Docker-сети как ``http://tradein-browser:3000/fetch``.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── конфигурация из env ────────────────────────────────────────────────────────
|
||||
|
||||
BROWSER_PORT: int = int(os.environ.get("BROWSER_PORT", "3000"))
|
||||
BROWSER_RECYCLE_PAGES: int = int(os.environ.get("BROWSER_RECYCLE_PAGES", "15"))
|
||||
BROWSER_NAV_TIMEOUT_MS: int = int(os.environ.get("BROWSER_NAV_TIMEOUT_MS", "60000"))
|
||||
# 6000 (не 2500): avito гидрирует listings client-side ПОСЛЕ domcontentloaded;
|
||||
# на 2.5с в HTML генерик-шелл без объявлений (0 listings), на 5-6с — полная
|
||||
# выдача (~50 карточек, 3.2МБ). Подтверждено прод-дебагом 2026-05-31.
|
||||
BROWSER_WAIT_MS: int = int(os.environ.get("BROWSER_WAIT_MS", "6000"))
|
||||
# Прокси: основной источник — AVITO_PROXY_URL (mobileproxy, #623, лежит в
|
||||
# backend/.env.runtime, тот же что у curl_cffi-avito). SCRAPER_PROXY_URL —
|
||||
# опциональный generic-fallback на будущее (cian/yandex). Без прокси camoufox
|
||||
# фетчит с датацентр-IP → avito отдаёт soft-block заглушку.
|
||||
SCRAPER_PROXY_URL: str | None = os.environ.get("AVITO_PROXY_URL") or os.environ.get(
|
||||
"SCRAPER_PROXY_URL"
|
||||
)
|
||||
|
||||
|
||||
def _parse_proxy(proxy_url: str | None) -> dict[str, str] | None:
|
||||
"""Парсит proxy URL → camoufox proxy dict.
|
||||
|
||||
Формат входного URL: ``http://user:pass@host:port`` или ``http://host:port``.
|
||||
Возвращает None если proxy_url пуст.
|
||||
|
||||
Returns:
|
||||
``{"server": "http://host:port", "username": "...", "password": "..."}``
|
||||
или None.
|
||||
"""
|
||||
if not proxy_url:
|
||||
return None
|
||||
|
||||
parsed = urlparse(proxy_url)
|
||||
port_part = f":{parsed.port}" if parsed.port else ""
|
||||
server = f"{parsed.scheme}://{parsed.hostname}{port_part}"
|
||||
|
||||
result: dict[str, str] = {"server": server}
|
||||
if parsed.username:
|
||||
result["username"] = parsed.username
|
||||
if parsed.password:
|
||||
result["password"] = parsed.password
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── состояние браузера ─────────────────────────────────────────────────────────
|
||||
|
||||
_browser: object | None = None # текущий экземпляр Browser
|
||||
_browser_cm: object | None = None # AsyncCamoufox context manager (для __aexit__)
|
||||
_page_counter: int = 0 # страниц с момента последнего (пере)запуска
|
||||
_fetch_lock: asyncio.Lock | None = None # сериализация fetch-запросов
|
||||
|
||||
|
||||
async def _launch_browser() -> None:
|
||||
"""Запускает AsyncCamoufox и сохраняет browser + CM в модульных переменных."""
|
||||
global _browser, _browser_cm, _page_counter
|
||||
|
||||
from camoufox.async_api import AsyncCamoufox
|
||||
|
||||
proxy = _parse_proxy(SCRAPER_PROXY_URL)
|
||||
kwargs: dict[str, object] = {
|
||||
"headless": True,
|
||||
"os": "windows",
|
||||
"locale": "ru-RU",
|
||||
"geoip": True,
|
||||
"humanize": True,
|
||||
}
|
||||
if proxy is not None:
|
||||
kwargs["proxy"] = proxy
|
||||
|
||||
logger.info(
|
||||
"tradein-browser: запуск AsyncCamoufox (proxy=%s, recycle_pages=%d)",
|
||||
proxy is not None,
|
||||
BROWSER_RECYCLE_PAGES,
|
||||
)
|
||||
cm = AsyncCamoufox(**kwargs) # type: ignore[arg-type]
|
||||
browser = await cm.__aenter__()
|
||||
_browser_cm = cm
|
||||
_browser = browser
|
||||
_page_counter = 0
|
||||
logger.info("tradein-browser: браузер запущен")
|
||||
|
||||
|
||||
async def _close_browser() -> None:
|
||||
"""Закрывает текущий экземпляр браузера."""
|
||||
global _browser, _browser_cm, _page_counter
|
||||
|
||||
cm = _browser_cm
|
||||
if cm is not None:
|
||||
try:
|
||||
await cm.__aexit__(None, None, None) # type: ignore[attr-defined]
|
||||
logger.info("tradein-browser: браузер закрыт")
|
||||
except Exception as exc:
|
||||
logger.warning("tradein-browser: ошибка при закрытии браузера: %s", exc)
|
||||
finally:
|
||||
_browser = None
|
||||
_browser_cm = None
|
||||
_page_counter = 0
|
||||
|
||||
|
||||
async def _relaunch_browser() -> None:
|
||||
"""Закрывает текущий браузер и запускает новый (recycle / crash-recovery)."""
|
||||
logger.info("tradein-browser: перезапуск браузера")
|
||||
await _close_browser()
|
||||
await _launch_browser()
|
||||
|
||||
|
||||
# ── aiohttp lifecycle hooks ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _on_startup(app: web.Application) -> None:
|
||||
global _fetch_lock
|
||||
_fetch_lock = asyncio.Lock()
|
||||
await _launch_browser()
|
||||
|
||||
|
||||
async def _on_cleanup(app: web.Application) -> None:
|
||||
await _close_browser()
|
||||
|
||||
|
||||
# ── handlers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def health_handler(request: web.Request) -> web.Response:
|
||||
return web.json_response({"status": "ok"})
|
||||
|
||||
|
||||
async def fetch_handler(request: web.Request) -> web.Response:
|
||||
"""POST /fetch {"url": "..."} → {"html": "..."}
|
||||
|
||||
Выполняет навигацию в браузере и возвращает полный HTML страницы.
|
||||
Запросы сериализованы через lock (один за раз).
|
||||
"""
|
||||
global _page_counter
|
||||
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return web.json_response({"error": "invalid JSON body"}, status=400)
|
||||
|
||||
url: str | None = body.get("url")
|
||||
if not url:
|
||||
return web.json_response({"error": "missing 'url' field"}, status=400)
|
||||
|
||||
assert _fetch_lock is not None, "_fetch_lock not initialised"
|
||||
|
||||
async with _fetch_lock:
|
||||
try:
|
||||
html = await _do_fetch(url)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"tradein-browser: fetch error url=%r: %s: %s", url, type(exc).__name__, exc
|
||||
)
|
||||
return web.json_response({"error": f"{type(exc).__name__}: {exc}"}, status=500)
|
||||
|
||||
return web.json_response({"html": html})
|
||||
|
||||
|
||||
async def _do_fetch(url: str) -> str:
|
||||
"""Одна попытка навигации; при краше браузера — перезапускает и повторяет один раз."""
|
||||
global _page_counter
|
||||
|
||||
try:
|
||||
return await _fetch_once(url)
|
||||
except Exception as exc:
|
||||
if _is_browser_crash(exc):
|
||||
logger.warning(
|
||||
"tradein-browser: краш браузера (%s), перезапуск + retry: %s",
|
||||
type(exc).__name__,
|
||||
url,
|
||||
)
|
||||
await _relaunch_browser()
|
||||
return await _fetch_once(url)
|
||||
raise
|
||||
|
||||
|
||||
async def _fetch_once(url: str) -> str:
|
||||
"""Открывает страницу, переходит по URL, ждёт JS, возвращает HTML."""
|
||||
global _page_counter
|
||||
|
||||
browser = _browser
|
||||
assert browser is not None, "browser not launched"
|
||||
|
||||
page = await browser.new_page() # type: ignore[attr-defined]
|
||||
try:
|
||||
await page.goto(url, timeout=BROWSER_NAV_TIMEOUT_MS, wait_until="domcontentloaded") # type: ignore[attr-defined]
|
||||
if BROWSER_WAIT_MS > 0:
|
||||
await page.wait_for_timeout(BROWSER_WAIT_MS) # type: ignore[attr-defined]
|
||||
html: str = await page.content() # type: ignore[attr-defined]
|
||||
finally:
|
||||
await page.close() # type: ignore[attr-defined]
|
||||
|
||||
_page_counter += 1
|
||||
logger.debug("tradein-browser: fetch OK url=%r pages_since_launch=%d", url, _page_counter)
|
||||
|
||||
if _page_counter >= BROWSER_RECYCLE_PAGES:
|
||||
logger.info(
|
||||
"tradein-browser: recycle threshold (%d) достигнут, перезапуск браузера",
|
||||
BROWSER_RECYCLE_PAGES,
|
||||
)
|
||||
await _relaunch_browser()
|
||||
|
||||
return html
|
||||
|
||||
|
||||
def _is_browser_crash(exc: BaseException) -> bool:
|
||||
"""Проверяет является ли исключение признаком краша / разрыва браузера."""
|
||||
cls_name = type(exc).__name__
|
||||
if cls_name in ("TargetClosedError", "BrowserClosedError", "ConnectionClosedError"):
|
||||
return True
|
||||
msg = str(exc).lower()
|
||||
return (
|
||||
"browser has been closed" in msg
|
||||
or "target closed" in msg
|
||||
or "connection closed" in msg
|
||||
or "browser disconnected" in msg
|
||||
)
|
||||
|
||||
|
||||
# ── entrypoint ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_app() -> web.Application:
|
||||
app = web.Application()
|
||||
app.on_startup.append(_on_startup)
|
||||
app.on_cleanup.append(_on_cleanup)
|
||||
app.router.add_get("/health", health_handler)
|
||||
app.router.add_post("/fetch", fetch_handler)
|
||||
return app
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger.info("tradein-browser: старт HTTP-сервера на порту %d", BROWSER_PORT)
|
||||
web.run_app(build_app(), host="0.0.0.0", port=BROWSER_PORT)
|
||||
|
|
@ -13,6 +13,22 @@
|
|||
# - Frontend строится с basePath=/trade-in (см. next.config.ts)
|
||||
|
||||
services:
|
||||
browser:
|
||||
image: ghcr.io/lekss361/gendesign-tradein-browser:${IMAGE_TAG:-latest}
|
||||
container_name: tradein-browser
|
||||
mem_limit: 2.5g # headless Firefox (camoufox) потребляет до ~1.5GB (#884/#905)
|
||||
env_file:
|
||||
- path: ./backend/.env.runtime
|
||||
required: false
|
||||
environment:
|
||||
BROWSER_PORT: "3000"
|
||||
# SCRAPER_PROXY_URL читается из .env.runtime (см. env_file выше)
|
||||
expose:
|
||||
- "3000"
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- tradein-net
|
||||
|
||||
postgres:
|
||||
image: postgis/postgis:16-3.4
|
||||
container_name: tradein-postgres
|
||||
|
|
@ -36,7 +52,6 @@ services:
|
|||
backend:
|
||||
image: ghcr.io/lekss361/gendesign-tradein-backend:${IMAGE_TAG:-latest}
|
||||
container_name: tradein-backend
|
||||
mem_limit: 2.5g # headless Firefox (camoufox) потребляет до ~1.5GB (#884)
|
||||
# Prod: uvicorn БЕЗ --reload (Dockerfile CMD несёт --reload только для dev hot-reload,
|
||||
# где app/ bind-mount'ится). В prod --reload = лишний WatchFiles-наблюдатель + риск
|
||||
# reload в середине запроса. Single worker ОБЯЗАТЕЛЕН: in-app планировщик
|
||||
|
|
@ -67,6 +82,8 @@ services:
|
|||
# cookies upload endpoint returns 503. Generate with `openssl rand -hex 32`.
|
||||
COOKIE_ENCRYPTION_KEY: "${COOKIE_ENCRYPTION_KEY:-}"
|
||||
depends_on:
|
||||
browser:
|
||||
condition: service_started
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
|
|
|
|||
57
tradein-mvp/ops/systemd/README.md
Normal file
57
tradein-mvp/ops/systemd/README.md
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# TradeIn systemd scrape trigger
|
||||
|
||||
Replaces the in-app asyncio scheduler with a systemd timer that calls the admin API every 60s.
|
||||
|
||||
## Install
|
||||
|
||||
Run as root on the VPS after deploy:
|
||||
|
||||
```bash
|
||||
cd /opt/gendesign/tradein-mvp/ops/systemd
|
||||
bash install.sh
|
||||
```
|
||||
|
||||
`install.sh` copies the unit files to `/etc/systemd/system/`, reloads systemd, and
|
||||
enables + starts the timer.
|
||||
|
||||
## Environment variables
|
||||
|
||||
Create `/etc/tradein/trigger.env` before enabling the timer:
|
||||
|
||||
```ini
|
||||
# psycopg v3 connection string (postgresql://... or postgresql+psycopg://...)
|
||||
TRADEIN_DATABASE_URL=postgresql://tradein:<password>@localhost:5432/tradein
|
||||
|
||||
# Admin username from roles.yaml with role=admin — passed as X-Authenticated-User header
|
||||
TRADEIN_ADMIN_TOKEN=admin
|
||||
|
||||
# Backend base URL (no trailing slash). Default: http://localhost:8000
|
||||
TRADEIN_API_BASE=http://localhost:8000
|
||||
```
|
||||
|
||||
## Diagnostic commands
|
||||
|
||||
```bash
|
||||
# Follow live timer logs
|
||||
journalctl -u tradein-scrape-trigger -f
|
||||
|
||||
# List all active timers and next trigger times
|
||||
systemctl list-timers
|
||||
|
||||
# Check timer status
|
||||
systemctl status tradein-scrape-trigger.timer
|
||||
|
||||
# Check last service run
|
||||
systemctl status tradein-scrape-trigger.service
|
||||
|
||||
# Manual one-shot trigger (for testing)
|
||||
systemctl start tradein-scrape-trigger.service
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- The script uses `flock` on `/var/run/tradein-trigger.lock` to prevent overlapping
|
||||
runs if the previous invocation is still running.
|
||||
- Sources without a dedicated admin endpoint (e.g. `rosreestr_dkp_import`,
|
||||
`listing_source_snapshot`) are logged as warnings and skipped. Set
|
||||
`SCHEDULER_ENABLE=true` in the backend env to handle them via the in-app scheduler.
|
||||
12
tradein-mvp/ops/systemd/install.sh
Normal file
12
tradein-mvp/ops/systemd/install.sh
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
#!/bin/bash
|
||||
# Run as root on VPS after deploy.
|
||||
# Installs (or updates) the systemd timer that triggers scrape schedules every 60s.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
cp "$SCRIPT_DIR/tradein-scrape-trigger.service" /etc/systemd/system/
|
||||
cp "$SCRIPT_DIR/tradein-scrape-trigger.timer" /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now tradein-scrape-trigger.timer
|
||||
systemctl status tradein-scrape-trigger.timer
|
||||
15
tradein-mvp/ops/systemd/tradein-scrape-trigger.service
Normal file
15
tradein-mvp/ops/systemd/tradein-scrape-trigger.service
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[Unit]
|
||||
Description=TradeIn Scrape Schedule Trigger
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
EnvironmentFile=/etc/tradein/trigger.env
|
||||
ExecStart=/usr/bin/python3 /opt/gendesign/tradein-mvp/scripts/trigger-schedules.py
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
TimeoutStartSec=55
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
11
tradein-mvp/ops/systemd/tradein-scrape-trigger.timer
Normal file
11
tradein-mvp/ops/systemd/tradein-scrape-trigger.timer
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=TradeIn Scrape Trigger — every 60s
|
||||
Requires=tradein-scrape-trigger.service
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
33
tradein-mvp/scripts/browser-smoke.sh
Normal file
33
tradein-mvp/scripts/browser-smoke.sh
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/sh
|
||||
# Smoke-test the tradein-browser service (camoufox HTTP wrapper, #905) from
|
||||
# inside the tradein docker network. Verifies /health and a real /fetch.
|
||||
#
|
||||
# Run from the deploy host (after `git pull`), piping the script into the
|
||||
# backend container (which shares the tradein-net with tradein-browser):
|
||||
#
|
||||
# docker exec -i tradein-backend sh < tradein-mvp/scripts/browser-smoke.sh
|
||||
#
|
||||
# Override the target URL:
|
||||
# URL=https://example.com docker exec -i -e URL tradein-backend sh < tradein-mvp/scripts/browser-smoke.sh
|
||||
set -eu
|
||||
|
||||
B="${BROWSER_HTTP_ENDPOINT:-http://tradein-browser:3000}"
|
||||
U="${URL:-https://www.avito.ru/ekaterinburg/kvartiry/prodam}"
|
||||
|
||||
printf 'health: '
|
||||
curl -s --max-time 10 "$B/health" || echo "(no response)"
|
||||
echo
|
||||
|
||||
echo "{\"url\":\"$U\"}" > /tmp/smoke_payload.json
|
||||
curl -s --max-time 120 -X POST "$B/fetch" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d @/tmp/smoke_payload.json \
|
||||
-o /tmp/smoke_result.html || true
|
||||
|
||||
echo "url: $U"
|
||||
echo "bytes: $(wc -c < /tmp/smoke_result.html 2>/dev/null || echo 0)"
|
||||
echo "preloaded: $(grep -c preloadedState /tmp/smoke_result.html 2>/dev/null || echo 0)"
|
||||
echo "firewall: $(grep -ci firewall-container /tmp/smoke_result.html 2>/dev/null || echo 0)"
|
||||
echo "--- first 200 bytes (error JSON shows here if /fetch failed) ---"
|
||||
head -c 200 /tmp/smoke_result.html 2>/dev/null || true
|
||||
echo
|
||||
188
tradein-mvp/scripts/trigger-schedules.py
Normal file
188
tradein-mvp/scripts/trigger-schedules.py
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Trigger due scrape schedules via admin API. Called by systemd timer every 60s."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import psycopg
|
||||
|
||||
LOCK_FILE = "/var/run/tradein-trigger.lock"
|
||||
DB_URL = os.environ["TRADEIN_DATABASE_URL"]
|
||||
API_BASE = os.environ.get("TRADEIN_API_BASE", "http://localhost:8000")
|
||||
# Username that maps to role=admin in roles.yaml — passed as X-Authenticated-User.
|
||||
ADMIN_TOKEN = os.environ["TRADEIN_ADMIN_TOKEN"]
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Mapping from scrape_schedules.source → admin API path.
|
||||
# Sources handled entirely inside the scheduler loop (no separate admin endpoint)
|
||||
# are listed in _NO_ENDPOINT_SOURCES: they are skipped when the systemd trigger
|
||||
# runs, and require SCHEDULER_ENABLE=true to fire.
|
||||
_SOURCE_TO_PATH: dict[str, str] = {
|
||||
"avito_city_sweep": "/api/v1/admin/scrape/avito-city-sweep",
|
||||
"yandex_city_sweep": "/api/v1/admin/scrape/yandex-city-sweep",
|
||||
"n1_city_sweep": "/api/v1/admin/scrape/n1",
|
||||
"cian_history_backfill": "/api/v1/admin/scrape/cian-backfill-history",
|
||||
"yandex_address_backfill": "/api/v1/admin/scrape/yandex-address-backfill",
|
||||
}
|
||||
|
||||
# Sources with no dedicated admin trigger endpoint.
|
||||
_NO_ENDPOINT_SOURCES = frozenset(
|
||||
{
|
||||
"rosreestr_dkp_import",
|
||||
"listing_source_snapshot",
|
||||
"asking_to_sold_ratio_refresh",
|
||||
"refresh_search_matview",
|
||||
"deactivate_stale_avito",
|
||||
"sber_index_pull",
|
||||
"rosreestr_quarter_poll",
|
||||
}
|
||||
)
|
||||
|
||||
_MAX_RETRIES = 3
|
||||
_RETRY_BASE_S = 2.0
|
||||
|
||||
|
||||
def _normalize_db_url(url: str) -> str:
|
||||
"""Strip SQLAlchemy driver prefix so psycopg v3 accepts the connection string."""
|
||||
if url.startswith("postgresql+psycopg://"):
|
||||
return "postgresql://" + url[len("postgresql+psycopg://"):]
|
||||
return url
|
||||
|
||||
|
||||
def get_due_schedules() -> list[dict[str, str]]:
|
||||
"""Fetch scrape_schedules rows that are due for execution."""
|
||||
conn_url = _normalize_db_url(DB_URL)
|
||||
with psycopg.connect(conn_url) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, source, schedule_type
|
||||
FROM scrape_schedules
|
||||
WHERE enabled = true
|
||||
AND (next_run_at IS NULL OR next_run_at <= NOW())
|
||||
""",
|
||||
).fetchall()
|
||||
description = conn.execute(
|
||||
"SELECT id, source, schedule_type FROM scrape_schedules LIMIT 0"
|
||||
).description or []
|
||||
col_names = [desc.name for desc in description]
|
||||
if not col_names:
|
||||
col_names = ["id", "source", "schedule_type"]
|
||||
return [
|
||||
{col_names[i]: (str(row[i]) if row[i] is not None else "") for i in range(len(col_names))}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def trigger_source(client: httpx.Client, source: str) -> None:
|
||||
"""POST to the admin trigger endpoint for a given source. Retries on connection error."""
|
||||
path = _SOURCE_TO_PATH.get(source)
|
||||
if path is None:
|
||||
if source in _NO_ENDPOINT_SOURCES:
|
||||
log.warning(
|
||||
"trigger: source=%s has no admin API endpoint — "
|
||||
"ensure SCHEDULER_ENABLE=true or add a dedicated admin endpoint",
|
||||
source,
|
||||
)
|
||||
else:
|
||||
log.warning("trigger: unknown source=%s, skipping", source)
|
||||
return
|
||||
|
||||
url = f"{API_BASE}{path}"
|
||||
last_exc: Exception | None = None
|
||||
|
||||
for attempt in range(1, _MAX_RETRIES + 1):
|
||||
try:
|
||||
resp = client.post(
|
||||
url,
|
||||
headers={"X-Authenticated-User": ADMIN_TOKEN},
|
||||
timeout=50.0,
|
||||
)
|
||||
log.info(
|
||||
"trigger: source=%s url=%s status=%d attempt=%d",
|
||||
source,
|
||||
url,
|
||||
resp.status_code,
|
||||
attempt,
|
||||
)
|
||||
if resp.status_code >= 500:
|
||||
log.warning(
|
||||
"trigger: source=%s got HTTP %d — body: %s",
|
||||
source,
|
||||
resp.status_code,
|
||||
resp.text[:200],
|
||||
)
|
||||
return
|
||||
except httpx.ConnectError as exc:
|
||||
last_exc = exc
|
||||
wait = _RETRY_BASE_S * (2 ** (attempt - 1))
|
||||
log.warning(
|
||||
"trigger: source=%s connection error (attempt %d/%d), "
|
||||
"retrying in %.1fs: %s",
|
||||
source,
|
||||
attempt,
|
||||
_MAX_RETRIES,
|
||||
wait,
|
||||
exc,
|
||||
)
|
||||
if attempt < _MAX_RETRIES:
|
||||
time.sleep(wait)
|
||||
|
||||
log.error(
|
||||
"trigger: source=%s failed after %d attempts: %s",
|
||||
source,
|
||||
_MAX_RETRIES,
|
||||
last_exc,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# Exclusive non-blocking flock: if a previous run is still active, exit immediately
|
||||
# with code 0 so systemd does not log a failure.
|
||||
lock_fd = open(LOCK_FILE, "w")
|
||||
try:
|
||||
fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
log.info("trigger: lock busy — previous run still active, exiting")
|
||||
lock_fd.close()
|
||||
return 0
|
||||
|
||||
try:
|
||||
due = get_due_schedules()
|
||||
if not due:
|
||||
log.info("trigger: no schedules due")
|
||||
return 0
|
||||
|
||||
log.info("trigger: %d schedule(s) due", len(due))
|
||||
|
||||
with httpx.Client() as client:
|
||||
for i, row in enumerate(due):
|
||||
source = row.get("source", "")
|
||||
trigger_source(client, source)
|
||||
if i < len(due) - 1:
|
||||
# Stagger requests to avoid thundering herd on the API
|
||||
time.sleep(random.uniform(1, 4))
|
||||
|
||||
return 0
|
||||
|
||||
except Exception:
|
||||
log.exception("trigger: unexpected error")
|
||||
return 1
|
||||
finally:
|
||||
try:
|
||||
fcntl.flock(lock_fd.fileno(), fcntl.LOCK_UN)
|
||||
except Exception:
|
||||
pass
|
||||
lock_fd.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Reference in a new issue