23 changed files with 495 additions and 1869 deletions
--- a/.forgejo/workflows/deploy-tradein.yml
+++ b/.forgejo/workflows/deploy-tradein.yml
@ -18,7 +18,6 @@ concurrency:
 env:
  IMAGE_BACKEND: ghcr.io/lekss361/gendesign-tradein-backend
  IMAGE_FRONTEND: ghcr.io/lekss361/gendesign-tradein-frontend
  IMAGE_BROWSER: ghcr.io/lekss361/gendesign-tradein-browser
 jobs:
  changes:
@ -26,7 +25,6 @@ jobs:
    outputs:
      backend: ${{ steps.filter.outputs.backend }}
      frontend: ${{ steps.filter.outputs.frontend }}
      browser: ${{ steps.filter.outputs.browser }}
      infra: ${{ steps.filter.outputs.infra }}
    steps:
      - uses: actions/checkout@v4
@ -38,8 +36,6 @@ jobs:
              - 'tradein-mvp/backend/**'
            frontend:
              - 'tradein-mvp/frontend/**'
            browser:
              - 'tradein-mvp/browser/**'
            infra:
              - 'tradein-mvp/docker-compose.prod.yml'
              - 'tradein-mvp/deploy/**'
@ -153,41 +149,9 @@ jobs:
            ${{ env.IMAGE_FRONTEND }}:latest
            ${{ env.IMAGE_FRONTEND }}:${{ github.sha }}
  build-browser:
    runs-on: ubuntu-latest
    needs: changes
    # tradein-browser несёт camoufox + Firefox-build (#905). Триггерится на
    # изменения browser/ или infra (compose ссылается на образ) или вручную.
    if: |
      needs.changes.outputs.browser == 'true' ||
      needs.changes.outputs.infra == 'true' ||
      github.event_name == 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v4
      - name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
        env:
          GHCR_PAT: ${{ secrets.GHCR_PAT }}
        run: |
          echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Build & push tradein-browser
        uses: docker/build-push-action@v6
        with:
          context: ./tradein-mvp/browser
          push: true
          cache-from: type=registry,ref=${{ env.IMAGE_BROWSER }}:buildcache
          cache-to: type=registry,ref=${{ env.IMAGE_BROWSER }}:buildcache,mode=max
          tags: |
            ${{ env.IMAGE_BROWSER }}:latest
            ${{ env.IMAGE_BROWSER }}:${{ github.sha }}
  deploy:
    runs-on: ubuntu-latest
-    needs: [changes, test, build-backend, build-frontend, build-browser]
+    needs: [changes, test, build-backend, build-frontend]
    # NB: a failed `test` skips build-backend (result='skipped', not 'failure'),
    # so we must block deploy on test failure explicitly (#666 quality gate).
    if: |
@ -195,8 +159,7 @@ jobs:
      !cancelled() &&
      needs.test.result != 'failure' &&
      needs.build-backend.result != 'failure' &&
-      needs.build-frontend.result != 'failure' &&
+      needs.build-frontend.result != 'failure'
      needs.build-browser.result != 'failure'
    steps:
      - name: Deploy via SSH
        uses: appleboy/ssh-action@v1.0.3
--- a/tradein-mvp/backend/Dockerfile
+++ b/tradein-mvp/backend/Dockerfile
@ -43,6 +43,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
    libpangoft2-1.0-0 \
    fonts-dejavu-core \
    curl \
    libgtk-3-0 \
    libasound2 \
    libdbus-glib-1-2 \
    libx11-xcb1 \
    libxtst6 \
    && useradd --create-home --uid 1000 app
 WORKDIR /app
@ -53,10 +58,16 @@ COPY --from=builder --chown=app:app /app/scripts /app/scripts
 USER app
-# HOME должен быть явным: Docker НЕ выставляет $HOME по USER, а некоторые
+# HOME должен быть явным: Docker НЕ выставляет $HOME по USER, а camoufox/platformdirs
-# платформенные библиотеки резолвят $HOME/.cache. Безвредно оставить.
+# резолвит кэш Firefox как $HOME/.cache/camoufox. Без этого fetch (под root) клал браузер
 # в /root/.cache, а рантайм-app искал в /home/app/.cache → "browser not found" в browser-mode.
 # (подтверждено сборкой+прогоном образа на деплой-хосте 2026-05-31, баг из #884.)
 ENV HOME=/home/app
 # Загружаем Firefox-сборку для camoufox (#884) — ПОД app-юзером, чтобы кэш совпал с рантаймом.
 # Слой кешируется: пересобирается только при изменении .venv (обновление camoufox). ~960MB.
 RUN python -m camoufox fetch
 EXPOSE 8000
 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
--- a/tradein-mvp/backend/app/core/config.py
+++ b/tradein-mvp/backend/app/core/config.py
@ -12,12 +12,6 @@ class Settings(BaseSettings):
    # required — задаётся через env DATABASE_URL. Нет дефолта: fail-fast при старте
    # если переменная не задана (C-3 security audit).
    database_url: str
    # In-app asyncio scheduler enable flag (#581).
    # Set SCHEDULER_ENABLE=false when using the systemd timer trigger instead.
    # Default true preserves existing behaviour.
    scheduler_enable: bool = Field(default=True, validation_alias="SCHEDULER_ENABLE")
    cors_origins: list[str] = ["http://localhost", "http://localhost:3000", "http://localhost:8080"]
    environment: str = "dev"
@ -163,8 +157,6 @@ class Settings(BaseSettings):
        "realty.yandex.ru",
        "ekaterinburg.n1.ru",
        "n1.ru",
        "domclick.ru",
        "www.domclick.ru",
    }
    # ── Scraper mobile proxy (#806) ──────────────────────────────────────────
@ -213,25 +205,19 @@ class Settings(BaseSettings):
    # ENV: YANDEX_COOKIES_FILE.
    yandex_cookies_file: str | None = None
-    # ── #884/#905: BrowserFetcher — HTTP-клиент к tradein-browser контейнеру ────
+    # ── #884: BrowserFetcher (camoufox) — Phase 0 foundation ─────────────────
    # scraper_fetch_mode: "curl_cffi" (дефолт, текущее поведение) или "browser"
-    # (HTTP POST к tradein-browser /fetch эндпоинту). Phase 1+.
+    # (AsyncCamoufox headless Firefox). Переключение в "browser" — Phase 1+.
    # Пока дефолт "curl_cffi" — никаких изменений в рантайме.
    # ENV: SCRAPER_FETCH_MODE.
    scraper_fetch_mode: Literal["curl_cffi", "browser"] = "curl_cffi"
-    # HTTP-эндпоинт tradein-browser сервиса. В Docker-сети — имя сервиса из compose.
+    # Сколько страниц обработать в одном browser-процессе перед перезапуском
-    # ENV: BROWSER_HTTP_ENDPOINT.
+    # (ограничение утечек памяти). ENV: BROWSER_RECYCLE_PAGES.
    browser_http_endpoint: str = "http://tradein-browser:3000"
    # Сколько страниц обработать в одном browser-сеансе перед перезапуском браузера
    # (ограничение утечек памяти). Читается сервером из env BROWSER_RECYCLE_PAGES.
    # Оставлено для справки / compat. ENV: BROWSER_RECYCLE_PAGES.
    browser_recycle_pages: int = 15
-    # Таймаут навигации (page.goto) в мс. Читается сервером из env BROWSER_NAV_TIMEOUT_MS.
+    # Таймаут навигации (page.goto) в мс. ENV: BROWSER_NAV_TIMEOUT_MS.
    # ENV: BROWSER_NAV_TIMEOUT_MS.
    browser_nav_timeout_ms: int = 60000
-    # Ожидание после DOMContentLoaded для JS-гидрации в мс. Читается сервером.
+    # Ожидание после DOMContentLoaded для JS-гидрации в мс. ENV: BROWSER_WAIT_MS.
-    # ENV: BROWSER_WAIT_MS.
+    browser_wait_ms: int = 2500
    browser_wait_ms: int = 6000
 settings = Settings()
--- a/tradein-mvp/backend/app/main.py
+++ b/tradein-mvp/backend/app/main.py
@ -74,23 +74,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
    except Exception:
        logger.exception("FDW user mapping bootstrap failed — cadastral queries may fail")
-    # Startup: launch scheduler background task (disabled when SCHEDULER_ENABLE=false,
+    # Startup: launch scheduler background task
-    # e.g. when using the systemd timer trigger — see tradein-mvp/ops/systemd/ #581).
+    scheduler_task = asyncio.create_task(scheduler_loop())
-    scheduler_task = None
+    logger.info("FastAPI lifespan: scheduler task spawned")
    if settings.scheduler_enable:
        scheduler_task = asyncio.create_task(scheduler_loop())
        logger.info("FastAPI lifespan: scheduler task spawned")
    else:
        logger.info("FastAPI lifespan: in-app scheduler disabled (SCHEDULER_ENABLE=false)")
    yield
-    # Shutdown: cancel scheduler if it was started
+    # Shutdown: cancel scheduler
-    if scheduler_task is not None:
+    scheduler_task.cancel()
-        scheduler_task.cancel()
+    try:
-        try:
+        await scheduler_task
-            await scheduler_task
+    except asyncio.CancelledError:
-        except asyncio.CancelledError:
+        pass
-            pass
+    logger.info("FastAPI lifespan: scheduler cancelled cleanly")
        logger.info("FastAPI lifespan: scheduler cancelled cleanly")
 app = FastAPI(
--- a/tradein-mvp/backend/app/services/scraper_settings.py
+++ b/tradein-mvp/backend/app/services/scraper_settings.py
@ -31,7 +31,6 @@ _CACHE_TTL_SEC = 60.0
 _DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
    "avito": 7.0,
    "cian": 5.0,
    "domklik": 8.0,
    "n1": 5.0,
    "yandex": 5.0,
    "yandex_detail": 5.0,
@ -83,7 +82,9 @@ def _get_setting_cached(key: str) -> float:
    try:
        with _open_session() as db:
            row = db.execute(
-                text("SELECT request_delay_sec FROM scraper_settings WHERE source = :s"),
+                text(
                    "SELECT request_delay_sec FROM scraper_settings WHERE source = :s"
                ),
                {"s": key},
            ).first()
        if row is not None:
@ -95,8 +96,8 @@ def _get_setting_cached(key: str) -> float:
            value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
    except Exception as e:
        logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
-        value = (
+        value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(
-            0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
+            key, _GLOBAL_DEFAULT_DELAY
        )
    with _CACHE_LOCK:
--- a/tradein-mvp/backend/app/services/scrapers/avito.py
+++ b/tradein-mvp/backend/app/services/scrapers/avito.py
@ -41,7 +41,6 @@ from app.services.scrapers.avito_exceptions import (
    AvitoRateLimitedError,
 )
 from app.services.scrapers.base import BaseScraper, ScrapedLot
 from app.services.scrapers.browser_fetcher import BrowserFetcher
 logger = logging.getLogger(__name__)
@ -216,16 +215,10 @@ class AvitoScraper(BaseScraper):
        super().__init__()
        self.request_delay_sec = get_scraper_delay(self.name)
        self._cffi: AsyncSession | None = None
        self._browser: BrowserFetcher | None = None
        # #823: счётчик карточек, которые не удалось распарсить из-за неожиданной структуры DOM.
        self.parse_failures: int = 0
    async def __aenter__(self) -> AvitoScraper:
        if settings.scraper_fetch_mode == "browser":
            self._browser = BrowserFetcher()
            await self._browser.__aenter__()
            logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901")
            return self
        await super().__aenter__()
        proxies = None
        if settings.scraper_proxy_url:
@ -249,8 +242,6 @@ class AvitoScraper(BaseScraper):
        return self
    async def __aexit__(self, *args: Any) -> None:
        if self._browser is not None:
            await self._browser.__aexit__(*args)
        if self._cffi is not None:
            await self._cffi.close()
        await super().__aexit__(*args)
@ -286,31 +277,9 @@ class AvitoScraper(BaseScraper):
        Returns:
            HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации).
            В browser-mode HTTP-статус недоступен: метод возвращает HTML или
            поднимает ``AvitoBlockedError`` (pagination ends via empty-parse in callers).
        Raises:
            AvitoBlockedError / AvitoRateLimitedError — если бан не снят ротацией.
        """
        if self._browser is not None:
            # Browser-mode (#901): no HTTP status code — soft firewall-detect + rotate.
            max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
            rot_done = 0
            while True:
                html = await self._browser.fetch(url)
                if _is_firewall_page(html):
                    if rot_done < max_rot and await self._rotate_ip():
                        rot_done += 1
                        logger.info(
                            "avito page=%d browser firewall — retry after rotation #%d",
                            page,
                            rot_done,
                        )
                        continue
                    logger.error("avito SERP firewall in browser-mode page=%d url=%s", page, url)
                    raise AvitoBlockedError(
                        f"Avito SERP firewall (browser-mode) at page={page} — IP banned"
                    )
                return html
        assert self._cffi is not None
        max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
        rot_done = 0
--- a/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py
+++ b/tradein-mvp/backend/app/services/scrapers/browser_fetcher.py
@ -1,38 +1,29 @@
-"""browser_fetcher.py — HTTP-клиент к tradein-browser сервису (#884/#905).
+"""browser_fetcher.py — camoufox wrapper для browser-mode scraping (Phase 0, #884).
-Тонкий клиент над tradein-browser контейнером, который запускает AsyncCamoufox
+Провайдер-агностичная основа: не затрагивает существующую avito/cian-логику.
-локально и экспонирует HTTP API (POST /fetch).
+Активируется только когда settings.scraper_fetch_mode == "browser" (пока
 дефолт "curl_cffi" — поведение идентично старому до явного переключения).
-Публичный интерфейс не изменился:
+Особенности:
-
+- Ленивые импорты camoufox/playwright: не ломают app-импорт если Firefox не
-    async with BrowserFetcher() as fetcher:
+  установлен (аналогично curl_cffi в avito_imv.py).
-        html = await fetcher.fetch("https://example.com")
+- Page recycle: каждые settings.browser_recycle_pages страниц браузер
-
+  перезапускается (ограничение утечек памяти).
-Внутреннее устройство: httpx.AsyncClient + POST к settings.browser_http_endpoint.
+- Crash-recovery: TargetClosedError → relaunch → one retry.
-Recycle, crash-recovery и управление браузером живут на стороне сервера (server.py).
+- Proxy: парсит settings.scraper_proxy_url → playwright dict формат
-При HTTPError / ConnectError делает одну повторную попытку после короткой паузы,
+  {"server": "http://host:port", "username": ..., "password": ...}.
 затем пробрасывает исключение.
 Архитектура выбрана потому, что playwright WS-сервер (launch_server) несовместим
 с playwright >=1.45: ``browserServerImpl.js`` отсутствует → MODULE_NOT_FOUND.
 Локальный AsyncCamoufox + HTTP — работающая альтернатива.
 """
 from __future__ import annotations
 import asyncio
 import logging
-
+from urllib.parse import urlparse
 import httpx
 logger = logging.getLogger(__name__)
 _RETRY_SLEEP_S: float = 1.0
 _HTTP_TIMEOUT_S: float = 120.0  # навигация медленная → щедрый таймаут
 class BrowserFetcher:
-    """Async context manager: HTTP-клиент к tradein-browser HTTP-сервису.
+    """Async context manager: camoufox headless Firefox с рециклингом страниц.
    Использование::
@ -41,64 +32,188 @@ class BrowserFetcher:
    """
    def __init__(self) -> None:
-        self._client: httpx.AsyncClient | None = None
+        self._browser: object | None = None
-        self._endpoint: str | None = None
+        self._page_count: int = 0
        # Lazy-loaded настройки чтобы не импортировать settings на верхнем уровне
        self._recycle_after: int | None = None
        self._nav_timeout_ms: int | None = None
        self._wait_ms: int | None = None
    # ── lifecycle ──────────────────────────────────────────────────────────────
    async def __aenter__(self) -> BrowserFetcher:
        from app.core.config import settings
-        self._endpoint = settings.browser_http_endpoint
+        self._recycle_after = settings.browser_recycle_pages
-        self._client = httpx.AsyncClient(timeout=_HTTP_TIMEOUT_S)
+        self._nav_timeout_ms = settings.browser_nav_timeout_ms
-        logger.info("BrowserFetcher: клиент создан, endpoint=%s", self._endpoint)
+        self._wait_ms = settings.browser_wait_ms
        await self._launch()
        return self
    async def __aexit__(self, *_: object) -> None:
-        if self._client is not None:
+        await self._close_browser()
            await self._client.aclose()
            self._client = None
        logger.debug("BrowserFetcher: клиент закрыт")
    # ── public API ─────────────────────────────────────────────────────────────
    async def fetch(self, url: str) -> str:
-        """Запрашивает HTML страницы через tradein-browser HTTP-сервис.
+        """Открывает новую страницу, переходит по URL, ждёт JS, возвращает HTML.
-        При HTTPError или ConnectError делает одну повторную попытку после
+        При TargetClosedError (crash) перезапускает браузер и делает одну
-        короткой паузы. Остальные исключения всплывают к вызывающему коду.
+        повторную попытку. Остальные исключения всплывают к вызывающему коду.
        Returns:
-            Полный HTML-контент страницы.
+            Полный HTML-контент страницы (innerHTML documentElement).
        """
        assert self._client is not None, "BrowserFetcher: используй как async context manager"
        assert self._endpoint is not None, "BrowserFetcher: endpoint не задан"
        try:
-            return await self._post_fetch(url)
+            return await self._fetch_once(url)
-        except (httpx.HTTPError, httpx.TransportError) as exc:
+        except Exception as exc:
-            logger.warning(
+            # Определяем crash vs обычная ошибка через имя класса (lazy import)
-                "BrowserFetcher: ошибка запроса (%s), retry через %.1fs: %s",
+            if _is_target_closed(exc) or _is_crash_exception(exc):
-                type(exc).__name__,
+                logger.warning(
-                _RETRY_SLEEP_S,
+                    "BrowserFetcher: браузер упал (%s), перезапуск + retry: %s",
-                url,
+                    type(exc).__name__,
-            )
+                    url,
-            await asyncio.sleep(_RETRY_SLEEP_S)
+                )
-            return await self._post_fetch(url)
+                await self._relaunch()
                return await self._fetch_once(url)
            raise
    # ── internal ───────────────────────────────────────────────────────────────
-    async def _post_fetch(self, url: str) -> str:
+    async def _fetch_once(self, url: str) -> str:
-        """Один HTTP POST к /fetch эндпоинту сервиса."""
+        """Одна попытка: new_page → goto → wait → content → close."""
-        assert self._client is not None
+        # Ленивый импорт: не нужен при curl_cffi mode (не ломает app при
-        assert self._endpoint is not None
+        # отсутствии установленного Firefox)
        try:
            from camoufox.async_api import AsyncCamoufox  # noqa: F401 (type check)
        except ImportError:
            pass  # импорт нужен только для type-check; browser уже запущен через _launch
        assert self._browser is not None, "BrowserFetcher: браузер не запущен"
        # Тип browser — AsyncCamoufox-инстанс (playwright.async_api.Browser-like)
        browser = self._browser  # type: ignore[assignment]
        page = await browser.new_page()  # type: ignore[attr-defined]
        try:
            await page.goto(url, timeout=self._nav_timeout_ms, wait_until="domcontentloaded")
            # Даём JS-гидрации отработать
            if self._wait_ms and self._wait_ms > 0:
                await page.wait_for_timeout(self._wait_ms)
            html: str = await page.content()
        finally:
            await page.close()
        self._page_count += 1
        logger.debug("BrowserFetcher: fetch OK url=%r pages_since_launch=%d", url, self._page_count)
        # Recycle: перезапускаем браузер каждые N страниц
        assert self._recycle_after is not None
        if self._page_count >= self._recycle_after:
            logger.info(
                "BrowserFetcher: recycle threshold (%d) достигнут, перезапуск браузера",
                self._recycle_after,
            )
            await self._relaunch()
        resp = await self._client.post(
            f"{self._endpoint}/fetch",
            json={"url": url},
        )
        resp.raise_for_status()
        data: dict[str, str] = resp.json()
        html = data["html"]
        logger.debug("BrowserFetcher: fetch OK url=%r html_len=%d", url, len(html))
        return html
    async def _launch(self) -> None:
        """Запускает AsyncCamoufox браузер."""
        try:
            from camoufox.async_api import AsyncCamoufox
        except ImportError as exc:
            raise RuntimeError(
                "camoufox не установлен. Добавь 'camoufox[geoip]>=0.4.0' в pyproject.toml "
                "и выполни 'python -m camoufox fetch' для загрузки Firefox-сборки."
            ) from exc
        proxy = self._proxy_dict()
        kwargs: dict[str, object] = {
            "headless": True,
            "os": "windows",
            "locale": "ru-RU",
            "geoip": True,
            "humanize": True,
        }
        if proxy is not None:
            kwargs["proxy"] = proxy
        # AsyncCamoufox — async context manager; входим в него чтобы получить browser
        ctx = AsyncCamoufox(**kwargs)  # type: ignore[call-arg]
        self._browser = await ctx.__aenter__()
        self._browser_ctx = ctx  # сохраняем для корректного __aexit__
        self._page_count = 0
        logger.info("BrowserFetcher: браузер запущен (proxy=%s)", proxy is not None)
    async def _close_browser(self) -> None:
        """Закрывает браузер если запущен."""
        ctx = getattr(self, "_browser_ctx", None)
        if ctx is not None:
            try:
                await ctx.__aexit__(None, None, None)
                logger.info("BrowserFetcher: браузер закрыт")
            except Exception as exc:
                logger.warning("BrowserFetcher: ошибка при закрытии браузера: %s", exc)
            finally:
                self._browser = None
                self._browser_ctx = None
                self._page_count = 0
    async def _relaunch(self) -> None:
        """Закрывает текущий браузер и запускает новый."""
        await self._close_browser()
        await self._launch()
    def _proxy_dict(self) -> dict[str, str] | None:
        """Парсит settings.scraper_proxy_url → playwright proxy dict.
        Формат входного URL: ``http://user:pass@host:port`` или ``http://host:port``.
        Возвращает None если proxy не настроен.
        Returns:
            ``{"server": "http://host:port", "username": "...", "password": "..."}``
            или None.
        """
        from app.core.config import settings
        proxy_url = settings.scraper_proxy_url
        if not proxy_url:
            return None
        parsed = urlparse(proxy_url)
        # Собираем server без credentials (playwright требует отдельных полей)
        port_part = f":{parsed.port}" if parsed.port else ""
        server = f"{parsed.scheme}://{parsed.hostname}{port_part}"
        result: dict[str, str] = {"server": server}
        if parsed.username:
            result["username"] = parsed.username
        if parsed.password:
            result["password"] = parsed.password
        return result
 # ── helpers ────────────────────────────────────────────────────────────────────
 def _is_target_closed(exc: BaseException) -> bool:
    """Проверяет является ли исключение playwright TargetClosedError.
    Проверка по имени класса (не import) — playwright не установлен обязательно.
    """
    cls_name = type(exc).__name__
    return cls_name == "TargetClosedError"
 def _is_crash_exception(exc: BaseException) -> bool:
    """Проверяет является ли исключение признаком краша браузера.
    Ловим широкий набор playwright crash-сигналов по имени/сообщению.
    """
    cls_name = type(exc).__name__
    if cls_name in ("BrowserClosedError", "ConnectionClosedError"):
        return True
    msg = str(exc).lower()
    return "browser has been closed" in msg or "target closed" in msg
--- a/tradein-mvp/backend/app/services/scrapers/domclick.py
+++ b/tradein-mvp/backend/app/services/scrapers/domclick.py
@ -1,406 +0,0 @@
 """DomClick.ru scraper — вторичка через headless BrowserFetcher (#796).
 Стратегия: HTML scrape через camoufox (AsyncCamoufox) — единственный
 рабочий путь: DataDome пропускает headless Playwright, curl_cffi -> 401.
 URL шаблон:
    https://domclick.ru/search?deal_type=sale&category=living&offer_type=flat
        &city_id={city_id}&rooms={r}&p={page}
 Оффер-карточки: `a[href*="/card/"]` с href-паттерном `/card/sale__flat__<id>`.
 Координаты SERP не отдаёт (lat = lon = None).
 Важно: selectolax `.text()` объединяет все дочерние текстовые узлы без
 разделителей. Числа из адреса (номер дома) могут слипнуться с ценой.
 Поэтому цену извлекаем из каждого дочернего элемента отдельно (а не из
 суммарного card_text), где элемент содержит символ рубля.
 """
 from __future__ import annotations
 import logging
 import re
 from urllib.parse import urljoin
 from selectolax.parser import HTMLParser, Node
 from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper, ScrapedLot
 from app.services.scrapers.domclick_exceptions import DomClickBlockedError
 logger = logging.getLogger(__name__)
 # ── DataDome block detection ─────────────────────────────────────────────────
 _DATADOME_MARKERS = ("datadome", "blocked", "access denied", "bot detected")
 def _is_blocked_page(html: str) -> bool:
    head = html[:2048].lower()
    return any(m in head for m in _DATADOME_MARKERS)
 # ── Regex helpers ────────────────────────────────────────────────────────────
 # _RE_ROOMS расширен относительно avito.py: DomClick пишет «2-комн. квартира»
 # (с «комн.») — avito.py-вариант ловит только «N-к. кв.» / «N-к квартира».
 _RE_ROOMS = re.compile(
    r"(\d)-(?:комн\.?\s*(?:квартира|кв\.?)?|к\.?\s*(?:квартира|кв\.?))",
    re.IGNORECASE,
 )
 _RE_STUDIO = re.compile(r"\bстуди[яиюей]\b", re.IGNORECASE)
 _RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[2²]", re.IGNORECASE)
 _RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE)
 # Цена: парсим из одного элемента DOM (не из суммарного card_text) —
 # иначе число дома может слипнуться с ценой. Цена всегда в элементе с «₽».
 _RE_PRICE_EL = re.compile(r"([\d\s  ]+)\s*[₽р](?:уб\.?)?", re.IGNORECASE)
 # source_id из href вида /card/sale__flat__2075671636
 _RE_SOURCE_ID = re.compile(r"sale__flat__(\d+)")
 # Адресные ключевые слова (эвристика)
 _RE_ADDR_KW = re.compile(
    r"ул\.|улица|пер\.|переулок|пр-т|проспект"
    r"|бул\.|бульвар|шоссе|наб\.|набережная"
    r"|пл\.|площадь|тракт|д\.\s*\d",
    re.IGNORECASE,
 )
 # ── Вспомогательные функции ──────────────────────────────────────────────────
 def _extract_rooms(text: str) -> int | None:
    if _RE_STUDIO.search(text):
        return 0
    m = _RE_ROOMS.search(text)
    return int(m.group(1)) if m else None
 def _extract_area(text: str) -> float | None:
    m = _RE_AREA.search(text)
    return float(m.group(1).replace(",", ".")) if m else None
 def _extract_floor(text: str) -> tuple[int | None, int | None]:
    m = _RE_FLOOR.search(text)
    if m:
        return int(m.group(1)), int(m.group(2))
    return None, None
 def _extract_price_from_element(text: str) -> int | None:
    """Извлекаем цену из текста ОДНОГО элемента DOM.
    Парсим из одного span/div — не из суммарного card_text. Это исключает
    случай, когда адресный номер дома стоит вплотную перед ценой в
    объединённом тексте карточки («Ленина, 503 100 000 руб.»).
    """
    m = _RE_PRICE_EL.search(text)
    if m:
        # Нормализуем пробелы любого вида (обычный, неразрывный, узкий)
        raw = re.sub(r"[\s  ]", "", m.group(1))
        if raw.isdigit():
            val = int(raw)
            if val > 0:
                return val
    return None
 def _extract_source_id(href: str) -> str | None:
    """Числовой ID из href типа /card/sale__flat__2075671636."""
    m = _RE_SOURCE_ID.search(href)
    return m.group(1) if m else None
 # ── DomClickScraper ──────────────────────────────────────────────────────────
 class DomClickScraper(BaseScraper):
    """DomClick вторичка parser. Источник = 'domklik'.
    Использует BrowserFetcher (camoufox headless Firefox) — DataDome пропускает
    его без дополнительной stealth-настройки. curl_cffi -> 401 (DataDome блок).
    Основной метод: fetch_city(city_id, rooms, pages).
    fetch_around() не реализован: DomClick не поддерживает geo-radius в URL.
    """
    name = "domklik"
    source = "domklik"
    base_url = "https://domclick.ru"
    # DataDome — консервативная задержка между страницами (fallback до init)
    request_delay_sec = 8.0
    def __init__(self) -> None:
        super().__init__()
        self.request_delay_sec = get_scraper_delay(self.name)
        # Счётчик карточек с неожиданной DOM-структурой (для observability)
        self.parse_failures: int = 0
    async def __aenter__(self) -> DomClickScraper:
        await super().__aenter__()
        return self
    # ── fetch_around — stub ───────────────────────────────────────────────────
    async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list[ScrapedLot]:
        """DomClick не поддерживает geo-radius. Используй fetch_city()."""
        raise NotImplementedError(
            "DomClick не поддерживает geo-radius; используй fetch_city(city_id=...)"
        )
    # ── Основной метод ────────────────────────────────────────────────────────
    async def fetch_city(
        self,
        city_id: int,
        rooms: list[int] | None = None,
        pages: int = 20,
    ) -> list[ScrapedLot]:
        """Citywide sweep: все страницы SERP для city_id.
        Args:
            city_id: числовой ID города в DomClick (например, 4 = Екатеринбург).
            rooms: список значений комнатности (0=студия, 1, 2, 3, 4, ...).
                   None -> без фильтра комнатности (все квартиры сразу).
            pages: максимальное число страниц на один sweep. Break on empty.
        Returns:
            Дедуплицированный по source_id список ScrapedLot.
        """
        from app.services.scrapers.browser_fetcher import BrowserFetcher
        all_lots: list[ScrapedLot] = []
        seen_ids: set[str] = set()
        # Формируем список room-значений для sweep'ов
        room_values: list[int | None] = [None] if not rooms else list(rooms)
        async with BrowserFetcher() as fetcher:
            for room_val in room_values:
                room_label = f"rooms={room_val}" if room_val is not None else "all_rooms"
                logger.info(
                    "domklik: city_id=%d %s sweep (max %d pages)", city_id, room_label, pages
                )
                for page_num in range(1, pages + 1):
                    url = self._build_url(city_id, room_val, page_num)
                    logger.debug("domklik: fetch %s", url)
                    try:
                        html = await fetcher.fetch(url)
                    except Exception as exc:
                        logger.error(
                            "domklik: fetch failed city_id=%d page=%d: %s",
                            city_id,
                            page_num,
                            exc,
                        )
                        break
                    try:
                        lots = self._parse_html(html)
                    except DomClickBlockedError:
                        logger.warning(
                            "domklik: DataDome block at city_id=%d page=%d — stopping sweep",
                            city_id,
                            page_num,
                        )
                        break
                    if not lots:
                        logger.info(
                            "domklik: empty page city_id=%d %s page=%d — stopping sweep",
                            city_id,
                            room_label,
                            page_num,
                        )
                        break
                    # Дедупликация по source_id (кросс-sweep и кросс-страница)
                    new_lots: list[ScrapedLot] = []
                    for lot in lots:
                        key = lot.source_id or lot.source_url
                        if key not in seen_ids:
                            seen_ids.add(key)
                            new_lots.append(lot)
                    all_lots.extend(new_lots)
                    logger.info(
                        "domklik: city_id=%d %s page=%d -> %d new (total %d)",
                        city_id,
                        room_label,
                        page_num,
                        len(new_lots),
                        len(all_lots),
                    )
                    # Пауза между страницами (anti-DataDome)
                    if page_num < pages:
                        await self.sleep_between_requests()
        logger.info(
            "domklik: fetch_city done city_id=%d total=%d parse_failures=%d",
            city_id,
            len(all_lots),
            self.parse_failures,
        )
        return all_lots
    # ── URL builder ───────────────────────────────────────────────────────────
    def _build_url(self, city_id: int, rooms: int | None, page: int) -> str:
        """Строит URL SERP DomClick."""
        url = (
            f"{self.base_url}/search"
            f"?deal_type=sale&category=living&offer_type=flat"
            f"&city_id={city_id}"
        )
        if rooms is not None:
            url += f"&rooms={rooms}"
        url += f"&p={page}"
        return url
    # ── HTML parsing ──────────────────────────────────────────────────────────
    def _parse_html(self, html: str) -> list[ScrapedLot]:
        """Парсим карточки из HTML через selectolax.
        Селектор: `a[href*="/card/"]` с паттерном `/card/sale__flat__<id>`.
        """
        if _is_blocked_page(html):
            logger.warning("domklik: DataDome block page detected, 0 cards returned")
            raise DomClickBlockedError("DomClick returned DataDome block page")
        tree = HTMLParser(html)
        card_links = tree.css('a[href*="/card/"]')
        seen_hrefs: set[str] = set()
        lots: list[ScrapedLot] = []
        for link in card_links:
            href = link.attributes.get("href", "")
            # Только карточки вторичного жилья
            if "sale__flat__" not in href:
                continue
            if href in seen_hrefs:
                continue
            seen_hrefs.add(href)
            lot = self._card_link_to_lot(link, href)
            if lot is not None:
                lots.append(lot)
        return lots
    def _card_link_to_lot(self, link: Node, href: str) -> ScrapedLot | None:
        """Парсинг одной карточки-ссылки -> ScrapedLot.
        DomClick SERP оборачивает карточку в `<a href="/card/...">` — вся
        информация (title, цена, адрес) внутри этого элемента.
        Цену извлекаем из отдельных дочерних элементов (не из объединённого
        card_text) чтобы исключить слипание числа дома с ценой.
        """
        try:
            source_id = _extract_source_id(href)
            source_url = urljoin(self.base_url, href)
            # Суммарный текст — для rooms/area/floor (эти поля не подвержены
            # проблеме слипания, т.к. используют специфичные маркеры: «м²», «эт.»)
            card_text = link.text(strip=True)
            if not card_text:
                return None
            rooms = _extract_rooms(card_text)
            area = _extract_area(card_text)
            floor, total_floors = _extract_floor(card_text)
            # Цена: ищем в каждом дочернем элементе отдельно чтобы не слипались
            # числа дома с ценой при конкатенации card_text
            price = self._extract_price_from_children(link)
            if not price or price <= 0:
                return None
            address = self._extract_address(link)
            return ScrapedLot(
                source="domklik",
                source_url=source_url,
                source_id=source_id,
                address=address,
                lat=None,  # DomClick SERP координаты не отдаёт
                lon=None,
                rooms=rooms,
                area_m2=area,
                floor=floor,
                total_floors=total_floors,
                price_rub=price,
                listing_segment="vtorichka",
                raw_payload={"card_text": card_text[:500]},
            )
        except Exception:
            self.parse_failures += 1
            logger.warning(
                "domclick _card_link_to_lot: parse failed href=%r (parse_failures=%d)",
                href,
                self.parse_failures,
                exc_info=True,
            )
            return None
    def _extract_price_from_children(self, link: Node) -> int | None:
        """Ищем цену в дочерних элементах карточки по символу рубля.
        Итерируем по всем потомкам: первый элемент с рублёвым символом
        в тексте — источник цены.
        """
        for selector in ("span", "div", "p", "strong", "b"):
            for el in link.css(selector):
                text = el.text(strip=True)
                price = _extract_price_from_element(text)
                if price:
                    return price
        # Fallback: суммарный текст карточки (если структура нестандартная)
        return _extract_price_from_element(link.text(strip=True))
    def _extract_address(self, link: Node) -> str | None:
        """Эвристика извлечения адреса из DOM карточки.
        DomClick помещает адрес в отдельный текстовый блок внутри карточки.
        Обходим вложенные элементы и ищем первый текст с адресными ключевыми
        словами, пропуская строки с площадью/этажом/ценой.
        Возвращает None если адрес не найден.
        """
        for selector in ("span", "p", "div"):
            for el in link.css(selector):
                text = el.text(strip=True)
                if not text or len(text) < 5 or len(text) > 200:
                    continue
                # Пропускаем блоки с площадью / этажом / ценой
                if _RE_AREA.search(text) or _RE_FLOOR.search(text):
                    continue
                if _RE_PRICE_EL.search(text):
                    continue
                if _RE_ADDR_KW.search(text):
                    return text
        return None
 # ── Convenience runner (для Celery tasks) ────────────────────────────────────
 async def scrape_domclick_city(
    city_id: int,
    rooms: list[int] | None = None,
    pages: int = 20,
 ) -> list[ScrapedLot]:
    """Удобная точка входа для вызова из Celery tasks.
    Пример::
        import asyncio
        lots = asyncio.run(scrape_domclick_city(city_id=4, rooms=[1, 2, 3]))
    """
    async with DomClickScraper() as scraper:
        return await scraper.fetch_city(city_id=city_id, rooms=rooms, pages=pages)
--- a/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py
+++ b/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py
@ -1,5 +0,0 @@
 """DomClick-specific exceptions для anti-bot detection."""
 class DomClickBlockedError(Exception):
    """DomClick вернул DataDome block-страницу (HTTP 200 + block HTML)."""
--- a/tradein-mvp/backend/pyproject.toml
+++ b/tradein-mvp/backend/pyproject.toml
@ -23,7 +23,7 @@ dependencies = [
    "sentry-sdk>=2.0.0",          # мониторинг ошибок → GlitchTip (#396)
    "redis>=5.0.0",               # async hot cache для /api/v1/search (Phase 3.2)
    "pyyaml>=6.0.0",              # RBAC roles.yaml loader (app/core/auth.py)
-    "playwright>=1.45",          # Playwright client для connect к tradein-browser (#905)
+    "camoufox[geoip]>=0.4.0",    # headless Firefox wrapper для browser-mode scraping (#884)
 ]
 [dependency-groups]
@ -31,6 +31,7 @@ dev = [
    "pytest>=8.0.0",
    "pytest-asyncio>=0.24.0",
    "ruff>=0.5.0",
    "playwright>=1.45",  # address-mismatch audit fallback (issue #582 Phase 1)
 ]
 [tool.pytest.ini_options]
--- a/tradein-mvp/backend/tests/scrapers/test_browser_fetcher.py
+++ b/tradein-mvp/backend/tests/scrapers/test_browser_fetcher.py
@ -1,13 +1,11 @@
-"""Tests for BrowserFetcher (#884/#905, HTTP-клиент).
+"""Tests for BrowserFetcher (#884, Phase 0).
 Полностью замоканы через monkeypatch на httpx.AsyncClient.post:
 не требуют установленного Firefox, camoufox или реального HTTP-сервера.
 Полностью замоканы: не требуют установленного Firefox или camoufox.
 Покрывают:
-  - fetch(): POST к правильному endpoint + разбор {"html": "..."} из ответа
+  - _proxy_dict(): парсинг proxy URL с credentials и без
-  - HTTPError триггерит один retry, затем поднимает исключение
+  - fetch(): new_page → goto → wait_for_timeout → content → close
-  - URL строится из settings.browser_http_endpoint
+  - recycle: счётчик страниц → relaunch браузера при достижении порога
-  - импорт модуля не ломается без playwright/camoufox
+  - crash-recovery: TargetClosedError → relaunch → retry
 """
 from __future__ import annotations
@ -16,7 +14,6 @@ import os
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 import httpx
 import pytest
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
@ -26,181 +23,273 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:
 def _mock_settings(
-    browser_http_endpoint: str = "http://fake-browser:3000",
+    proxy_url: str | None = None,
    recycle_pages: int = 15,
    nav_timeout_ms: int = 60000,
    wait_ms: int = 2500,
 ) -> SimpleNamespace:
-    return SimpleNamespace(browser_http_endpoint=browser_http_endpoint)
+    return SimpleNamespace(
-
+        scraper_proxy_url=proxy_url,
-
+        browser_recycle_pages=recycle_pages,
-def _make_ok_response(html: str = "<html>ok</html>") -> MagicMock:
+        browser_nav_timeout_ms=nav_timeout_ms,
-    """Создаёт мок httpx.Response с JSON {"html": html} и статусом 200."""
+        browser_wait_ms=wait_ms,
    resp = MagicMock(spec=httpx.Response)
    resp.status_code = 200
    resp.json = MagicMock(return_value={"html": html})
    resp.raise_for_status = MagicMock()  # нет исключения
    return resp
 def _make_error_response(status: int = 500) -> MagicMock:
    """Создаёт мок httpx.Response который кидает HTTPStatusError при raise_for_status."""
    resp = MagicMock(spec=httpx.Response)
    resp.status_code = status
    resp.raise_for_status = MagicMock(
        side_effect=httpx.HTTPStatusError(
            f"Server error {status}",
            request=MagicMock(),
            response=resp,
        )
    )
    return resp
-# ── импорт не ломается ─────────────────────────────────────────────────────────
+def _make_mock_page(html: str = "<html>ok</html>") -> MagicMock:
    """Создаёт мок страницы playwright с нужными async-методами."""
    page = MagicMock()
    page.goto = AsyncMock()
    page.wait_for_timeout = AsyncMock()
    page.content = AsyncMock(return_value=html)
    page.close = AsyncMock()
    return page
-def test_import_browser_fetcher() -> None:
+def _make_mock_browser(page: MagicMock | None = None) -> MagicMock:
-    """import app.services.scrapers.browser_fetcher не поднимает исключений."""
+    """Создаёт мок browser-контекста camoufox."""
-    import app.services.scrapers.browser_fetcher  # noqa: F401
+    if page is None:
        page = _make_mock_page()
    browser = MagicMock()
    browser.new_page = AsyncMock(return_value=page)
    return browser
-# ── fetch(): возвращает HTML из JSON-ответа ────────────────────────────────────
+def _make_mock_camoufox_ctx(browser: MagicMock) -> MagicMock:
    """Мок AsyncCamoufox(...) context manager → возвращает browser из __aenter__."""
    ctx = MagicMock()
    ctx.__aenter__ = AsyncMock(return_value=browser)
    ctx.__aexit__ = AsyncMock(return_value=None)
    return ctx
 # ── _proxy_dict() ──────────────────────────────────────────────────────────────
 # _proxy_dict() импортирует settings через `from app.core.config import settings`
 # внутри метода, поэтому патчим app.core.config.settings (источник).
 def test_proxy_dict_parses_full_url():
    """_proxy_dict() разбирает URL с username/password → playwright dict."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
    fetcher = BrowserFetcher()
    ms = _mock_settings(proxy_url="http://user123:pass456@proxy.example.com:8080")
    with patch("app.core.config.settings", ms):
        result = fetcher._proxy_dict()
    assert result == {
        "server": "http://proxy.example.com:8080",
        "username": "user123",
        "password": "pass456",
    }
 def test_proxy_dict_no_credentials():
    """_proxy_dict() для URL без credentials возвращает только server."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
    fetcher = BrowserFetcher()
    ms = _mock_settings(proxy_url="http://proxy.example.com:3128")
    with patch("app.core.config.settings", ms):
        result = fetcher._proxy_dict()
    assert result == {"server": "http://proxy.example.com:3128"}
    assert "username" not in result
    assert "password" not in result
 def test_proxy_dict_returns_none_when_no_proxy():
    """_proxy_dict() → None когда scraper_proxy_url не задан."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
    fetcher = BrowserFetcher()
    ms = _mock_settings(proxy_url=None)
    with patch("app.core.config.settings", ms):
        result = fetcher._proxy_dict()
    assert result is None
 # ── fetch(): new_page → goto → content → close ────────────────────────────────
@pytest.mark.asyncio
-async def test_fetch_returns_html_from_json_response() -> None:
+async def test_fetch_calls_page_lifecycle():
-    """fetch() разбирает {'html': ...} из 200-ответа и возвращает строку."""
+    """fetch() вызывает new_page, goto, wait_for_timeout, content, close."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
-    expected_html = "<html><body>avito</body></html>"
+    expected_html = "<html><body>test</body></html>"
-    ok_resp = _make_ok_response(html=expected_html)
+    page = _make_mock_page(html=expected_html)
    browser = _make_mock_browser(page=page)
    ctx = _make_mock_camoufox_ctx(browser)
    ms = _mock_settings(recycle_pages=15, nav_timeout_ms=60000, wait_ms=2500)
    with patch("app.core.config.settings", ms):
        with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
            fetcher = BrowserFetcher()
            async with fetcher:
                result = await fetcher.fetch("https://example.com")
    assert result == expected_html
    browser.new_page.assert_called_once()
    page.goto.assert_called_once_with(
        "https://example.com", timeout=60000, wait_until="domcontentloaded"
    )
    page.wait_for_timeout.assert_called_once_with(2500)
    page.content.assert_called_once()
    page.close.assert_called_once()
@pytest.mark.asyncio
 async def test_fetch_returns_html_string():
    """fetch() возвращает строку HTML от page.content()."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
    html = "<html><title>Avito</title></html>"
    page = _make_mock_page(html=html)
    browser = _make_mock_browser(page=page)
    ctx = _make_mock_camoufox_ctx(browser)
    ms = _mock_settings()
    with patch("app.core.config.settings", ms):
-        fetcher = BrowserFetcher()
+        with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
-        async with fetcher:
+            fetcher = BrowserFetcher()
-            assert fetcher._client is not None
+            async with fetcher:
-            fetcher._client.post = AsyncMock(return_value=ok_resp)  # type: ignore[method-assign]
+                result = await fetcher.fetch("https://avito.ru/test")
            result = await fetcher.fetch("https://avito.ru/test")
-    assert result == expected_html
+    assert result == html
    assert isinstance(result, str)
 # ── recycle: relaunch at threshold ────────────────────────────────────────────
@pytest.mark.asyncio
-async def test_fetch_posts_to_correct_endpoint() -> None:
+async def test_recycle_triggers_relaunch_at_threshold():
-    """fetch() делает POST к {browser_http_endpoint}/fetch с {"url": ...}."""
+    """После recycle_pages страниц браузер перезапускается (relaunch)."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
-    endpoint = "http://fake-browser:3000"
+    recycle_after = 3
-    ok_resp = _make_ok_response()
+
-    ms = _mock_settings(browser_http_endpoint=endpoint)
+    page = _make_mock_page()
    browser = _make_mock_browser(page=page)
    ctx = _make_mock_camoufox_ctx(browser)
    ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0)
    launch_count = 0
    original_launch = BrowserFetcher._launch
    async def counting_launch(self: BrowserFetcher) -> None:
        nonlocal launch_count
        launch_count += 1
        await original_launch(self)
    with patch("app.core.config.settings", ms):
-        fetcher = BrowserFetcher()
+        with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
-        async with fetcher:
+            with patch.object(BrowserFetcher, "_launch", counting_launch):
-            assert fetcher._client is not None
+                fetcher = BrowserFetcher()
-            post_mock = AsyncMock(return_value=ok_resp)
+                async with fetcher:
-            fetcher._client.post = post_mock  # type: ignore[method-assign]
+                    # Первый запуск через __aenter__
-            await fetcher.fetch("https://example.com/page")
+                    assert launch_count == 1
-
+                    # Отправляем recycle_after страниц — на последней сработает relaunch
-    post_mock.assert_called_once_with(
+                    for _ in range(recycle_after):
-        f"{endpoint}/fetch",
+                        await fetcher.fetch("https://example.com")
-        json={"url": "https://example.com/page"},
+                    # После recycle браузер должен был перезапуститься ещё раз
-    )
+                    assert launch_count == 2
 # ── retry при HTTPError ────────────────────────────────────────────────────────
@pytest.mark.asyncio
-async def test_fetch_retries_once_on_http_error() -> None:
+async def test_page_counter_resets_after_relaunch():
-    """При HTTPError первый вызов → retry → успешный второй вызов."""
+    """После recycle page_count сбрасывается в 0."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
    recycle_after = 2
    page = _make_mock_page()
    browser = _make_mock_browser(page=page)
    ctx = _make_mock_camoufox_ctx(browser)
    ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0)
    with patch("app.core.config.settings", ms):
        with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
            fetcher = BrowserFetcher()
            async with fetcher:
                for _ in range(recycle_after):
                    await fetcher.fetch("https://example.com")
                # После recycle счётчик сброшен
                assert fetcher._page_count == 0
 # ── crash-recovery ─────────────────────────────────────────────────────────────
 class _FakeTargetClosedError(Exception):
    """Имитирует playwright TargetClosedError (проверка по имени класса)."""
 _FakeTargetClosedError.__name__ = "TargetClosedError"
 _FakeTargetClosedError.__qualname__ = "TargetClosedError"
@pytest.mark.asyncio
 async def test_fetch_retries_once_on_target_closed_error():
    """fetch() при TargetClosedError перезапускает браузер и повторяет запрос."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
    html = "<html>retry-ok</html>"
-    ok_resp = _make_ok_response(html=html)
+
-    error_resp = _make_error_response(500)
+    # Первая страница бросает краш, вторая отвечает нормально
-    ms = _mock_settings()
+    crash_page = MagicMock()
    crash_page.goto = AsyncMock(side_effect=_FakeTargetClosedError("target closed"))
    crash_page.close = AsyncMock()
    ok_page = _make_mock_page(html=html)
    browser = _make_mock_browser()
    call_num = 0
    async def new_page_side_effect() -> MagicMock:
        nonlocal call_num
        call_num += 1
        return crash_page if call_num == 1 else ok_page
    browser.new_page = AsyncMock(side_effect=new_page_side_effect)
    ctx = _make_mock_camoufox_ctx(browser)
    ms = _mock_settings(wait_ms=0)
    with patch("app.core.config.settings", ms):
-        fetcher = BrowserFetcher()
+        with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
-        async with fetcher:
+            fetcher = BrowserFetcher()
-            assert fetcher._client is not None
+            async with fetcher:
-            # Первый вызов кидает HTTPStatusError, второй возвращает OK
+                result = await fetcher.fetch("https://avito.ru/page")
            post_mock = AsyncMock(side_effect=[error_resp, ok_resp])
            fetcher._client.post = post_mock  # type: ignore[method-assign]
            with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
                result = await fetcher.fetch("https://avito.ru/retry")
    assert result == html
-    assert post_mock.await_count == 2
+    # 2 обращения к new_page: первое (crash) + второе (retry после relaunch)
    assert call_num == 2
@pytest.mark.asyncio
-async def test_fetch_raises_after_two_http_errors() -> None:
+async def test_fetch_propagates_non_crash_exception():
-    """Если оба вызова кидают HTTPError — исключение пробрасывается."""
+    """fetch() не подавляет обычные (не-crash) исключения."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
-    error_resp_1 = _make_error_response(500)
+    page = MagicMock()
-    error_resp_2 = _make_error_response(500)
+    page.goto = AsyncMock(side_effect=ValueError("unexpected parse error"))
-    ms = _mock_settings()
+    page.close = AsyncMock()
    browser = _make_mock_browser(page=page)
    ctx = _make_mock_camoufox_ctx(browser)
    ms = _mock_settings(wait_ms=0)
    with patch("app.core.config.settings", ms):
-        fetcher = BrowserFetcher()
+        with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
-        async with fetcher:
+            fetcher = BrowserFetcher()
-            assert fetcher._client is not None
+            async with fetcher:
-            post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2])
+                with pytest.raises(ValueError, match="unexpected parse error"):
-            fetcher._client.post = post_mock  # type: ignore[method-assign]
+                    await fetcher.fetch("https://avito.ru/bad")
            with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
                with pytest.raises(httpx.HTTPStatusError):
                    await fetcher.fetch("https://avito.ru/fail")
    assert post_mock.await_count == 2
@pytest.mark.asyncio
 async def test_fetch_retries_on_connect_error() -> None:
    """ConnectError (подкласс TransportError) тоже триггерит retry."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
    html = "<html>connect-retry-ok</html>"
    ok_resp = _make_ok_response(html=html)
    connect_err = httpx.ConnectError("Connection refused")
    ms = _mock_settings()
    with patch("app.core.config.settings", ms):
        fetcher = BrowserFetcher()
        async with fetcher:
            assert fetcher._client is not None
            post_mock = AsyncMock(side_effect=[connect_err, ok_resp])
            fetcher._client.post = post_mock  # type: ignore[method-assign]
            with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
                result = await fetcher.fetch("https://avito.ru/connect")
    assert result == html
    assert post_mock.await_count == 2
 # ── non-http exception пробрасывается без перехвата ───────────────────────────
@pytest.mark.asyncio
 async def test_fetch_propagates_unexpected_exception() -> None:
    """Не-HTTP исключение (ValueError и т.п.) не подавляется и не вызывает retry."""
    from app.services.scrapers.browser_fetcher import BrowserFetcher
    ms = _mock_settings()
    with patch("app.core.config.settings", ms):
        fetcher = BrowserFetcher()
        async with fetcher:
            assert fetcher._client is not None
            post_mock = AsyncMock(side_effect=ValueError("unexpected"))
            fetcher._client.post = post_mock  # type: ignore[method-assign]
            with pytest.raises(ValueError, match="unexpected"):
                await fetcher.fetch("https://avito.ru/bad")
    assert post_mock.await_count == 1
--- a/tradein-mvp/backend/tests/scrapers/test_domclick.py
+++ b/tradein-mvp/backend/tests/scrapers/test_domclick.py
@ -1,252 +0,0 @@
 """Unit-тесты для DomClick scraper (#796).
 Тестируем:
 - Парсинг title-паттерна (rooms/area/floor) из текста карточки
 - Парсинг цены из отдельного DOM-элемента
 - Извлечение source_id из href
 - Полный цикл _parse_html -> list[ScrapedLot] на mocked HTML
 """
 from __future__ import annotations
 import asyncio
 import pytest
 from app.services.scrapers.domclick import (
    DomClickScraper,
    _extract_area,
    _extract_floor,
    _extract_price_from_element,
    _extract_rooms,
    _extract_source_id,
 )
 from app.services.scrapers.domclick_exceptions import DomClickBlockedError
 # ── Rooms / studio ───────────────────────────────────────────────────────────
@pytest.mark.parametrize(
    "text, expected",
    [
        ("2-комн. квартира 53,5 м² 5/15 эт.", 2),
        ("1-к. квартира 32,1 м² 3/9 эт.", 1),
        ("3-комн. кв. 78 м² 10/12 эт.", 3),
        ("4-комн. квартира 95 м² 2/5 эт.", 4),
        ("студия 24 м² 1/10 эт.", 0),
        ("Студия 30,5 м² 8/22 эт.", 0),
        ("Нет комнат — просто текст", None),
    ],
 )
 def test_extract_rooms(text: str, expected: int | None) -> None:
    assert _extract_rooms(text) == expected
 # ── Area ─────────────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
    "text, expected",
    [
        ("2-комн. квартира 53,5 м² 5/15 эт.", 53.5),
        ("студия 24 м² 1/10 эт.", 24.0),
        ("78м2 на этаже", 78.0),
        ("площадь 101.7 м²", 101.7),
        ("Нет площади", None),
    ],
 )
 def test_extract_area(text: str, expected: float | None) -> None:
    assert _extract_area(text) == expected
 # ── Floor ─────────────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
    "text, floor, total",
    [
        ("2-комн. квартира 53,5 м² 5/15 эт.", 5, 15),
        ("студия 24 м² 1/10 эт.", 1, 10),
        ("12/24 эт.", 12, 24),
        ("нет этажа", None, None),
    ],
 )
 def test_extract_floor(text: str, floor: int | None, total: int | None) -> None:
    assert _extract_floor(text) == (floor, total)
 # ── Price (per-element) ───────────────────────────────────────────────────────
 # Цена парсится из отдельного DOM-элемента (не из суммарного card_text),
 # поэтому числа адреса не загрязняют парсинг цены.
@pytest.mark.parametrize(
    "text, expected",
    [
        ("5 200 000 ₽", 5_200_000),
        ("12 345 678 ₽", 12_345_678),
        ("3 990 000 руб.", 3_990_000),
        ("от 4 500 000 ₽", 4_500_000),
        ("без цены", None),
        ("0 ₽", None),
    ],
 )
 def test_extract_price_from_element(text: str, expected: int | None) -> None:
    assert _extract_price_from_element(text) == expected
 # ── source_id from href ───────────────────────────────────────────────────────
@pytest.mark.parametrize(
    "href, expected",
    [
        ("/card/sale__flat__2075671636", "2075671636"),
        ("https://domclick.ru/card/sale__flat__123456789", "123456789"),
        ("/card/sale__flat__1", "1"),
        ("/card/something_else", None),
        ("", None),
    ],
 )
 def test_extract_source_id(href: str, expected: str | None) -> None:
    assert _extract_source_id(href) == expected
 # ── _parse_html -> list[ScrapedLot] ──────────────────────────────────────────
 # Цена в каждой карточке вынесена в отдельный <span> с символом рубля,
 # чтобы не сливаться с адресным номером дома при конкатенации card_text.
 _MOCK_HTML = """\
 <!DOCTYPE html>
 <html>
 <body>
  <!-- Карточка 1: комнатность, площадь, этаж, адрес, цена в отдельных span -->
  <a href="/card/sale__flat__2075671636">
    <span>2-комн. квартира 53,5 м² 5/15 эт.</span>
    <span>ул. Малышева, 1</span>
    <span>5 200 000 ₽</span>
  </a>
  <!-- Карточка 2: студия -->
  <a href="/card/sale__flat__1111111111">
    <span>Студия 28 м² 3/10 эт.</span>
    <span>пр-т Ленина, 50</span>
    <span>3 100 000 ₽</span>
  </a>
  <!-- Карточка 3: без цены -> должна быть отфильтрована -->
  <a href="/card/sale__flat__9999999999">
    <span>1-комн. квартира 32 м² 2/9 эт.</span>
  </a>
  <!-- Не карточка (без sale__flat__) -> должна быть отфильтрована -->
  <a href="/card/sale__apartment__123">
    <span>что-то другое</span>
  </a>
  <!-- Дубликат первой карточки -> один раз на страницу -->
  <a href="/card/sale__flat__2075671636">
    <span>2-комн. квартира 53,5 м² 5/15 эт.</span>
    <span>5 200 000 ₽</span>
  </a>
 </body>
 </html>
 """
 def test_parse_html_returns_lots() -> None:
    scraper = DomClickScraper()
    lots = scraper._parse_html(_MOCK_HTML)
    # Карточки 1 и 2 парсятся; карточка 3 (без цены) отфильтрована; дубль — дедуплицирован
    assert len(lots) == 2
 def test_parse_html_card1_fields() -> None:
    scraper = DomClickScraper()
    lots = scraper._parse_html(_MOCK_HTML)
    card1 = next(lot for lot in lots if lot.source_id == "2075671636")
    assert card1.source == "domklik"
    assert card1.source_url == "https://domclick.ru/card/sale__flat__2075671636"
    assert card1.rooms == 2
    assert card1.area_m2 == 53.5
    assert card1.floor == 5
    assert card1.total_floors == 15
    assert card1.price_rub == 5_200_000
    assert card1.lat is None
    assert card1.lon is None
    assert card1.listing_segment == "vtorichka"
 def test_parse_html_studio() -> None:
    scraper = DomClickScraper()
    lots = scraper._parse_html(_MOCK_HTML)
    studio = next(lot for lot in lots if lot.source_id == "1111111111")
    assert studio.rooms == 0  # студия
    assert studio.area_m2 == 28.0
    assert studio.price_rub == 3_100_000
 def test_parse_html_no_price_filtered() -> None:
    scraper = DomClickScraper()
    lots = scraper._parse_html(_MOCK_HTML)
    source_ids = {lot.source_id for lot in lots}
    assert "9999999999" not in source_ids
 def test_parse_html_dedup_same_href() -> None:
    """Дубликат href на одной странице должен давать одну карточку."""
    scraper = DomClickScraper()
    lots = scraper._parse_html(_MOCK_HTML)
    ids_2075 = [lot for lot in lots if lot.source_id == "2075671636"]
    assert len(ids_2075) == 1
 def test_scraper_source_name() -> None:
    scraper = DomClickScraper()
    assert scraper.name == "domklik"
    assert scraper.source == "domklik"
    assert scraper.base_url == "https://domclick.ru"
    # request_delay_sec установлен из get_scraper_delay (DB или fallback 8.0)
    assert scraper.request_delay_sec >= 0.0
 def test_fetch_around_raises() -> None:
    scraper = DomClickScraper()
    with pytest.raises(NotImplementedError, match="geo-radius"):
        asyncio.run(scraper.fetch_around(56.8, 60.6))
 def test_build_url_no_rooms() -> None:
    scraper = DomClickScraper()
    url = scraper._build_url(city_id=4, rooms=None, page=1)
    assert "city_id=4" in url
    assert "rooms" not in url
    assert "p=1" in url
 def test_build_url_with_rooms() -> None:
    scraper = DomClickScraper()
    url = scraper._build_url(city_id=4, rooms=2, page=3)
    assert "rooms=2" in url
    assert "p=3" in url
 def test_parse_failures_counter() -> None:
    """parse_failures счётчик начинается с 0."""
    scraper = DomClickScraper()
    assert scraper.parse_failures == 0
 def test_blocked_page_raises_domclick_blocked_error() -> None:
    """HTML с datadome в начале вызывает DomClickBlockedError."""
    scraper = DomClickScraper()
    blocked_html = (
        "<html><head><title>datadome protection</title></head><body>blocked</body></html>"
    )
    with pytest.raises(DomClickBlockedError):
        scraper._parse_html(blocked_html)
--- a/tradein-mvp/backend/tests/test_avito_browser_mode.py
+++ b/tradein-mvp/backend/tests/test_avito_browser_mode.py
@ -1,225 +0,0 @@
 """#901 — avito SERP fetch via BrowserFetcher behind scraper_fetch_mode flag.
 Tests:
 1. browser branch chosen: mode="browser" → _browser set, _cffi is None, HTML from mock browser.
 2. curl branch unchanged: default mode → _browser is None, _cffi session used.
 3. firewall-detect in browser-mode: firewall HTML + no rotate URL → AvitoBlockedError.
 4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks.
 Без сети, без БД, без curl_cffi.
 """
 from __future__ import annotations
 import os
 from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
 from app.services.scrapers.avito_exceptions import AvitoBlockedError
 from app.services.scrapers.base import ScrapedLot
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
 _NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
 _FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"
 def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
    """Return a mock BrowserFetcher context manager that yields html from fetch()."""
    mock = MagicMock()
    mock.__aenter__ = AsyncMock(return_value=mock)
    mock.__aexit__ = AsyncMock(return_value=None)
    mock.fetch = AsyncMock(return_value=html)
    return mock
 # ---------------------------------------------------------------------------
 # 1. browser branch chosen
 # ---------------------------------------------------------------------------
@pytest.mark.asyncio
 async def test_aenter_browser_mode_sets_browser_attribute() -> None:
    """mode=browser → __aenter__ instantiates BrowserFetcher, _cffi stays None."""
    mock_browser = _make_mock_browser()
    scraper = AvitoScraper()
    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.scraper_fetch_mode = "browser"
        mock_settings.scraper_proxy_url = None
        mock_settings.avito_proxy_rotate_url = None
        mock_settings.avito_proxy_max_rotations = 2
        with patch(
            "app.services.scrapers.avito.BrowserFetcher",
            return_value=mock_browser,
        ):
            # Patch BaseScraper.__aenter__ so we skip its real setup (DB/network)
            with patch(
                "app.services.scrapers.base.BaseScraper.__aenter__",
                new=AsyncMock(return_value=scraper),
            ):
                result = await scraper.__aenter__()
    assert result is scraper
    assert scraper._browser is mock_browser
    assert scraper._cffi is None
@pytest.mark.asyncio
 async def test_fetch_serp_html_browser_mode_returns_html() -> None:
    """_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""
    scraper = AvitoScraper()
    mock_browser = _make_mock_browser(_NORMAL_HTML)
    scraper._browser = mock_browser
    result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
    assert result == _NORMAL_HTML
    mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
    # _cffi was never set — assert remains None
    assert scraper._cffi is None
 # ---------------------------------------------------------------------------
 # 2. curl branch unchanged
 # ---------------------------------------------------------------------------
@pytest.mark.asyncio
 async def test_fetch_serp_html_curl_mode_uses_cffi() -> None:
    """Default mode (_browser is None) → _fetch_serp_html goes through curl path."""
    scraper = AvitoScraper()
    # Mock the curl session returning HTTP 200 with normal HTML
    mock_response = MagicMock()
    mock_response.status_code = 200
    mock_response.text = _NORMAL_HTML
    mock_cffi = MagicMock()
    mock_cffi.get = AsyncMock(return_value=mock_response)
    scraper._cffi = mock_cffi
    # _browser must be None (default curl path)
    assert scraper._browser is None
    result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
    assert result == _NORMAL_HTML
    mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
 # ---------------------------------------------------------------------------
 # 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL)
 # ---------------------------------------------------------------------------
@pytest.mark.asyncio
 async def test_browser_mode_firewall_raises_blocked_error() -> None:
    """Browser returns firewall page + no rotate URL configured → AvitoBlockedError."""
    scraper = AvitoScraper()
    mock_browser = _make_mock_browser(_FIREWALL_HTML)
    scraper._browser = mock_browser
    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = None  # no rotation configured
        mock_settings.avito_proxy_max_rotations = 2
        with pytest.raises(AvitoBlockedError, match="browser-mode"):
            await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
@pytest.mark.asyncio
 async def test_browser_mode_firewall_rotates_and_retries() -> None:
    """Browser returns firewall on first call, normal HTML after rotation → returns HTML."""
    scraper = AvitoScraper()
    # First call → firewall; second call → normal HTML
    mock_browser = MagicMock()
    mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML])
    scraper._browser = mock_browser
    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate"
        mock_settings.avito_proxy_max_rotations = 1
        with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)):
            result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2)
    assert result == _NORMAL_HTML
    assert mock_browser.fetch.await_count == 2
 # ---------------------------------------------------------------------------
 # 4. pagination terminates via empty-parse
 # ---------------------------------------------------------------------------
@pytest.mark.asyncio
 async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
    """fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops.
    Verifies the documented invariant: callers break on `not lots` regardless of
    fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is
    already covered by unit tests above); here we confirm the caller loop behaviour.
    """
    scraper = AvitoScraper()
    html_p1 = "<html>page1</html>"
    html_p2 = "<html>page2</html>"
    lot = ScrapedLot(
        source="avito",
        source_url="https://www.avito.ru/ekaterinburg/kvartiry/test",
        source_id="LOT_1",
        price_rub=5_000_000,
    )
    call_count = 0
    async def mock_fetch_html(url: str, page: int) -> str:
        return html_p1 if page == 1 else html_p2
    def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
        nonlocal call_count
        call_count += 1
        return [lot] if "page1" in html else []
    with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html):
        with patch.object(scraper, "_parse_html", side_effect=mock_parse):
            with patch.object(scraper, "sleep_between_requests", new=AsyncMock()):
                result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0)
    assert len(result) == 1
    assert result[0].source_id == "LOT_1"
    # page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
    assert call_count == 2
 # ---------------------------------------------------------------------------
 # 5. rotation returns False → immediate AvitoBlockedError
 # ---------------------------------------------------------------------------
@pytest.mark.asyncio
 async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
    """rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
    Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
    (e.g. rotation API timeout/failure) → we cannot recover → blocked.
    """
    scraper = AvitoScraper()
    scraper._browser = _make_mock_browser(_FIREWALL_HTML)
    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
        mock_settings.avito_proxy_max_rotations = 2
        with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
            with pytest.raises(AvitoBlockedError, match="browser-mode"):
                await scraper._fetch_serp_html("https://avito.ru/test", page=1)
--- a/tradein-mvp/backend/tests/test_pipeline_browser_routing.py
+++ b/tradein-mvp/backend/tests/test_pipeline_browser_routing.py
@ -368,26 +368,57 @@ async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
    assert result.item_id == "99887766"
-# ── 6. SERP routing: real _fetch_serp_html routes through browser ────────────
+# ── 6. SERP routing: browser-mode — _cffi never touched (refutes #916 concern) ─
@pytest.mark.asyncio
-async def test_fetch_serp_html_routes_through_browser() -> None:
+async def test_fetch_serp_html_uses_browser_when_browser_set() -> None:
-    """With _browser set and _cffi=None, the REAL _fetch_serp_html returns the
+    """SERP routing: with _browser set and _cffi=None, fetch_around routes SERP
-    browser HTML and never dereferences _cffi (no AssertionError). Proves #915
+    through _browser and never dereferences _cffi (no AssertionError).
-    SERP routing on top of #901's browser branch."""
+
-    from unittest.mock import AsyncMock
+    Refutes #916 reviewer concern that _cffi assert always fires in browser-mode.
    The pipeline sets scraper._browser before calling fetch_around; _fetch_serp_html
    is monkey-patched at method level to verify _browser is present and _cffi is
    absent at call time — proving the routing contract holds end-to-end without
    a network call.
    """
    from unittest.mock import AsyncMock, patch
    from app.services.scrapers.avito import AvitoScraper
    scraper = AvitoScraper()
-    assert scraper._cffi is None
+    assert scraper._cffi is None  # browser-mode: curl session never created
    scraper._browser = AsyncMock()
    scraper._browser.fetch = AsyncMock(return_value="<html><body>real serp listing</body></html>")
-    html = await scraper._fetch_serp_html(
+    mock_browser = AsyncMock()
-        "https://www.avito.ru/ekaterinburg/kvartiry/prodam", page=1
+    # Non-firewall SERP HTML: avoid AvitoBlockedError (_FIREWALL_MARKERS not present)
-    )
+    serp_html = "<html><body><div>ok serp result page</div></body></html>"
    mock_browser.fetch = AsyncMock(return_value=serp_html)
    scraper._browser = mock_browser
-    assert html == "<html><body>real serp listing</body></html>"
+    browser_calls: list[str] = []
-    scraper._browser.fetch.assert_awaited_once()
+
    async def _patched_fetch_serp(url: str, page: int) -> str | None:
        # Core assertion: _browser is set, _cffi is still None in browser-mode
        assert scraper._browser is not None, "_browser must be set in browser-mode"
        assert scraper._cffi is None, "_cffi must remain None in browser-mode (no AssertionError)"
        browser_calls.append(url)
        # Delegate to the mock browser (as production code does via browser_fetcher.fetch)
        return await scraper._browser.fetch(url)
    # Patch at method level — exercises fetch_around → _fetch_serp_html pathway
    scraper._fetch_serp_html = _patched_fetch_serp  # type: ignore[method-assign]
    # Patch _parse_html so 0-cards doesn't obscure the SERP-routing assertion
    with patch.object(scraper, "_parse_html", return_value=[]):
        try:
            await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
        except Exception:
            # AvitoContentBlockedError on 0-card page=1 is expected — irrelevant to routing
            pass
    # _browser.fetch was called → SERP routed through browser, not _cffi
    assert (
        len(browser_calls) >= 1
    ), "browser.fetch was never called — SERP not routed through browser"
    mock_browser.fetch.assert_awaited()
    # Reaching here without AssertionError proves _cffi=None did not crash the SERP path
--- a/tradein-mvp/browser/Dockerfile
+++ b/tradein-mvp/browser/Dockerfile
@ -1,44 +0,0 @@
 # tradein-browser — camoufox Playwright WS server (изолированный контейнер Firefox).
 # Backend подключается к этому контейнеру через playwright.firefox.connect.
 FROM python:3.12-slim
 ENV PYTHONUNBUFFERED=1 \
    PYTHONDONTWRITEBYTECODE=1
 # Firefox runtime libs + ca-certificates для TLS
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
    --mount=type=cache,target=/var/lib/apt,sharing=locked \
    apt-get update && apt-get install -y --no-install-recommends \
    libgtk-3-0 \
    libasound2 \
    libdbus-glib-1-2 \
    libx11-xcb1 \
    libxtst6 \
    ca-certificates \
    && useradd --create-home --uid 1000 app
 WORKDIR /app
 # camoufox[geoip] тянет playwright как зависимость
 RUN pip install --no-cache-dir "camoufox[geoip]" aiohttp
 COPY server.py ./server.py
 # camoufox fetch качает (а) Firefox-сборку и (б) GeoLite2 mmdb (geoip=True).
 # Firefox идёт в $HOME/.cache → ENV HOME=/home/app кладёт его в кэш app-юзера,
 # куда рантайм его ищет (фикс #899). НО mmdb пишется в site-packages пакета
 # camoufox (root-owned после pip install) — под app это PermissionError на
 # GeoLite2-City.mmdb. Поэтому fetch выполняем под ROOT (пишет и mmdb, и —
 # благодаря ENV HOME — Firefox в /home/app/.cache), затем chown'им Firefox-кэш
 # на app. mmdb остаётся root-owned в site-packages (world-readable → app читает
 # на рантайме). Verified реальной сборкой образа 2026-05-31.
 ENV HOME=/home/app
 RUN python -m camoufox fetch && chown -R app:app /home/app/.cache
 USER app
 EXPOSE 3000
 CMD ["python", "server.py"]
--- a/tradein-mvp/browser/server.py
+++ b/tradein-mvp/browser/server.py
@ -1,269 +0,0 @@
 """server.py — tradein-browser service main process.
 Этот модуль запускается как точка входа контейнера ``tradein-browser``.
 Он запускает camoufox **локально** внутри контейнера (AsyncCamoufox) и
 экспонирует простой HTTP API на базе aiohttp:
    GET  /health  → {"status": "ok"}
    POST /fetch   → {"url": "..."} → {"html": "..."}
 Такой подход выбран потому, что ``camoufox.server.launch_server`` (Playwright
 WS-сервер) несовместим с современными версиями playwright (1.45–1.60):
 ``browserServerImpl.js`` отсутствует в дистрибутиве → контейнер краш-лупится
 с MODULE_NOT_FOUND / RuntimeError при запуске через ``launch_server``.
 Локальный запуск ``AsyncCamoufox`` работает стабильно — проверено.
 Переменные окружения:
    BROWSER_PORT            — TCP-порт HTTP-сервера (default: 3000)
    BROWSER_RECYCLE_PAGES   — страниц в одном сеансе браузера до перезапуска (default: 15)
    BROWSER_NAV_TIMEOUT_MS  — таймаут page.goto в мс (default: 60000)
    BROWSER_WAIT_MS         — ожидание гидрации listings после DOMContentLoaded, мс (default: 6000)
    AVITO_PROXY_URL         — прокси ``http://user:pass@host:port`` (mobileproxy, #623);
                              SCRAPER_PROXY_URL — generic-fallback. Без него — soft-block.
 Эндпоинт /fetch доступен из Docker-сети как ``http://tradein-browser:3000/fetch``.
 """
 import asyncio
 import logging
 import os
 from urllib.parse import urlparse
 from aiohttp import web
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 logger = logging.getLogger(__name__)
 # ── конфигурация из env ────────────────────────────────────────────────────────
 BROWSER_PORT: int = int(os.environ.get("BROWSER_PORT", "3000"))
 BROWSER_RECYCLE_PAGES: int = int(os.environ.get("BROWSER_RECYCLE_PAGES", "15"))
 BROWSER_NAV_TIMEOUT_MS: int = int(os.environ.get("BROWSER_NAV_TIMEOUT_MS", "60000"))
 # 6000 (не 2500): avito гидрирует listings client-side ПОСЛЕ domcontentloaded;
 # на 2.5с в HTML генерик-шелл без объявлений (0 listings), на 5-6с — полная
 # выдача (~50 карточек, 3.2МБ). Подтверждено прод-дебагом 2026-05-31.
 BROWSER_WAIT_MS: int = int(os.environ.get("BROWSER_WAIT_MS", "6000"))
 # Прокси: основной источник — AVITO_PROXY_URL (mobileproxy, #623, лежит в
 # backend/.env.runtime, тот же что у curl_cffi-avito). SCRAPER_PROXY_URL —
 # опциональный generic-fallback на будущее (cian/yandex). Без прокси camoufox
 # фетчит с датацентр-IP → avito отдаёт soft-block заглушку.
 SCRAPER_PROXY_URL: str | None = os.environ.get("AVITO_PROXY_URL") or os.environ.get(
    "SCRAPER_PROXY_URL"
 )
 def _parse_proxy(proxy_url: str | None) -> dict[str, str] | None:
    """Парсит proxy URL → camoufox proxy dict.
    Формат входного URL: ``http://user:pass@host:port`` или ``http://host:port``.
    Возвращает None если proxy_url пуст.
    Returns:
        ``{"server": "http://host:port", "username": "...", "password": "..."}``
        или None.
    """
    if not proxy_url:
        return None
    parsed = urlparse(proxy_url)
    port_part = f":{parsed.port}" if parsed.port else ""
    server = f"{parsed.scheme}://{parsed.hostname}{port_part}"
    result: dict[str, str] = {"server": server}
    if parsed.username:
        result["username"] = parsed.username
    if parsed.password:
        result["password"] = parsed.password
    return result
 # ── состояние браузера ─────────────────────────────────────────────────────────
 _browser: object | None = None          # текущий экземпляр Browser
 _browser_cm: object | None = None       # AsyncCamoufox context manager (для __aexit__)
 _page_counter: int = 0                  # страниц с момента последнего (пере)запуска
 _fetch_lock: asyncio.Lock | None = None  # сериализация fetch-запросов
 async def _launch_browser() -> None:
    """Запускает AsyncCamoufox и сохраняет browser + CM в модульных переменных."""
    global _browser, _browser_cm, _page_counter
    from camoufox.async_api import AsyncCamoufox
    proxy = _parse_proxy(SCRAPER_PROXY_URL)
    kwargs: dict[str, object] = {
        "headless": True,
        "os": "windows",
        "locale": "ru-RU",
        "geoip": True,
        "humanize": True,
    }
    if proxy is not None:
        kwargs["proxy"] = proxy
    logger.info(
        "tradein-browser: запуск AsyncCamoufox (proxy=%s, recycle_pages=%d)",
        proxy is not None,
        BROWSER_RECYCLE_PAGES,
    )
    cm = AsyncCamoufox(**kwargs)  # type: ignore[arg-type]
    browser = await cm.__aenter__()
    _browser_cm = cm
    _browser = browser
    _page_counter = 0
    logger.info("tradein-browser: браузер запущен")
 async def _close_browser() -> None:
    """Закрывает текущий экземпляр браузера."""
    global _browser, _browser_cm, _page_counter
    cm = _browser_cm
    if cm is not None:
        try:
            await cm.__aexit__(None, None, None)  # type: ignore[attr-defined]
            logger.info("tradein-browser: браузер закрыт")
        except Exception as exc:
            logger.warning("tradein-browser: ошибка при закрытии браузера: %s", exc)
        finally:
            _browser = None
            _browser_cm = None
            _page_counter = 0
 async def _relaunch_browser() -> None:
    """Закрывает текущий браузер и запускает новый (recycle / crash-recovery)."""
    logger.info("tradein-browser: перезапуск браузера")
    await _close_browser()
    await _launch_browser()
 # ── aiohttp lifecycle hooks ────────────────────────────────────────────────────
 async def _on_startup(app: web.Application) -> None:
    global _fetch_lock
    _fetch_lock = asyncio.Lock()
    await _launch_browser()
 async def _on_cleanup(app: web.Application) -> None:
    await _close_browser()
 # ── handlers ───────────────────────────────────────────────────────────────────
 async def health_handler(request: web.Request) -> web.Response:
    return web.json_response({"status": "ok"})
 async def fetch_handler(request: web.Request) -> web.Response:
    """POST /fetch {"url": "..."} → {"html": "..."}
    Выполняет навигацию в браузере и возвращает полный HTML страницы.
    Запросы сериализованы через lock (один за раз).
    """
    global _page_counter
    try:
        body = await request.json()
    except Exception:
        return web.json_response({"error": "invalid JSON body"}, status=400)
    url: str | None = body.get("url")
    if not url:
        return web.json_response({"error": "missing 'url' field"}, status=400)
    assert _fetch_lock is not None, "_fetch_lock not initialised"
    async with _fetch_lock:
        try:
            html = await _do_fetch(url)
        except Exception as exc:
            logger.error(
                "tradein-browser: fetch error url=%r: %s: %s", url, type(exc).__name__, exc
            )
            return web.json_response({"error": f"{type(exc).__name__}: {exc}"}, status=500)
    return web.json_response({"html": html})
 async def _do_fetch(url: str) -> str:
    """Одна попытка навигации; при краше браузера — перезапускает и повторяет один раз."""
    global _page_counter
    try:
        return await _fetch_once(url)
    except Exception as exc:
        if _is_browser_crash(exc):
            logger.warning(
                "tradein-browser: краш браузера (%s), перезапуск + retry: %s",
                type(exc).__name__,
                url,
            )
            await _relaunch_browser()
            return await _fetch_once(url)
        raise
 async def _fetch_once(url: str) -> str:
    """Открывает страницу, переходит по URL, ждёт JS, возвращает HTML."""
    global _page_counter
    browser = _browser
    assert browser is not None, "browser not launched"
    page = await browser.new_page()  # type: ignore[attr-defined]
    try:
        await page.goto(url, timeout=BROWSER_NAV_TIMEOUT_MS, wait_until="domcontentloaded")  # type: ignore[attr-defined]
        if BROWSER_WAIT_MS > 0:
            await page.wait_for_timeout(BROWSER_WAIT_MS)  # type: ignore[attr-defined]
        html: str = await page.content()  # type: ignore[attr-defined]
    finally:
        await page.close()  # type: ignore[attr-defined]
    _page_counter += 1
    logger.debug("tradein-browser: fetch OK url=%r pages_since_launch=%d", url, _page_counter)
    if _page_counter >= BROWSER_RECYCLE_PAGES:
        logger.info(
            "tradein-browser: recycle threshold (%d) достигнут, перезапуск браузера",
            BROWSER_RECYCLE_PAGES,
        )
        await _relaunch_browser()
    return html
 def _is_browser_crash(exc: BaseException) -> bool:
    """Проверяет является ли исключение признаком краша / разрыва браузера."""
    cls_name = type(exc).__name__
    if cls_name in ("TargetClosedError", "BrowserClosedError", "ConnectionClosedError"):
        return True
    msg = str(exc).lower()
    return (
        "browser has been closed" in msg
        or "target closed" in msg
        or "connection closed" in msg
        or "browser disconnected" in msg
    )
 # ── entrypoint ─────────────────────────────────────────────────────────────────
 def build_app() -> web.Application:
    app = web.Application()
    app.on_startup.append(_on_startup)
    app.on_cleanup.append(_on_cleanup)
    app.router.add_get("/health", health_handler)
    app.router.add_post("/fetch", fetch_handler)
    return app
 if __name__ == "__main__":
    logger.info("tradein-browser: старт HTTP-сервера на порту %d", BROWSER_PORT)
    web.run_app(build_app(), host="0.0.0.0", port=BROWSER_PORT)
--- a/tradein-mvp/docker-compose.prod.yml
+++ b/tradein-mvp/docker-compose.prod.yml
@ -13,22 +13,6 @@
 # - Frontend строится с basePath=/trade-in (см. next.config.ts)
 services:
  browser:
    image: ghcr.io/lekss361/gendesign-tradein-browser:${IMAGE_TAG:-latest}
    container_name: tradein-browser
    mem_limit: 2.5g  # headless Firefox (camoufox) потребляет до ~1.5GB (#884/#905)
    env_file:
      - path: ./backend/.env.runtime
        required: false
    environment:
      BROWSER_PORT: "3000"
      # SCRAPER_PROXY_URL читается из .env.runtime (см. env_file выше)
    expose:
      - "3000"
    restart: unless-stopped
    networks:
      - tradein-net
  postgres:
    image: postgis/postgis:16-3.4
    container_name: tradein-postgres
@ -52,6 +36,7 @@ services:
  backend:
    image: ghcr.io/lekss361/gendesign-tradein-backend:${IMAGE_TAG:-latest}
    container_name: tradein-backend
    mem_limit: 2.5g  # headless Firefox (camoufox) потребляет до ~1.5GB (#884)
    # Prod: uvicorn БЕЗ --reload (Dockerfile CMD несёт --reload только для dev hot-reload,
    # где app/ bind-mount'ится). В prod --reload = лишний WatchFiles-наблюдатель + риск
    # reload в середине запроса. Single worker ОБЯЗАТЕЛЕН: in-app планировщик
@ -82,8 +67,6 @@ services:
      # cookies upload endpoint returns 503. Generate with `openssl rand -hex 32`.
      COOKIE_ENCRYPTION_KEY: "${COOKIE_ENCRYPTION_KEY:-}"
    depends_on:
      browser:
        condition: service_started
      postgres:
        condition: service_healthy
    volumes:
--- a/tradein-mvp/ops/systemd/README.md
+++ b/tradein-mvp/ops/systemd/README.md
@ -1,57 +0,0 @@
 # TradeIn systemd scrape trigger
 Replaces the in-app asyncio scheduler with a systemd timer that calls the admin API every 60s.
 ## Install
 Run as root on the VPS after deploy:
 ```bash
 cd /opt/gendesign/tradein-mvp/ops/systemd
 bash install.sh
 ```
 `install.sh` copies the unit files to `/etc/systemd/system/`, reloads systemd, and
 enables + starts the timer.
 ## Environment variables
 Create `/etc/tradein/trigger.env` before enabling the timer:
 ```ini
 # psycopg v3 connection string (postgresql://... or postgresql+psycopg://...)
 TRADEIN_DATABASE_URL=postgresql://tradein:<password>@localhost:5432/tradein
 # Admin username from roles.yaml with role=admin — passed as X-Authenticated-User header
 TRADEIN_ADMIN_TOKEN=admin
 # Backend base URL (no trailing slash). Default: http://localhost:8000
 TRADEIN_API_BASE=http://localhost:8000
 ```
 ## Diagnostic commands
 ```bash
 # Follow live timer logs
 journalctl -u tradein-scrape-trigger -f
 # List all active timers and next trigger times
 systemctl list-timers
 # Check timer status
 systemctl status tradein-scrape-trigger.timer
 # Check last service run
 systemctl status tradein-scrape-trigger.service
 # Manual one-shot trigger (for testing)
 systemctl start tradein-scrape-trigger.service
 ```
 ## Notes
 - The script uses `flock` on `/var/run/tradein-trigger.lock` to prevent overlapping
  runs if the previous invocation is still running.
 - Sources without a dedicated admin endpoint (e.g. `rosreestr_dkp_import`,
  `listing_source_snapshot`) are logged as warnings and skipped. Set
  `SCHEDULER_ENABLE=true` in the backend env to handle them via the in-app scheduler.
--- a/tradein-mvp/ops/systemd/install.sh
+++ b/tradein-mvp/ops/systemd/install.sh
@ -1,12 +0,0 @@
 #!/bin/bash
 # Run as root on VPS after deploy.
 # Installs (or updates) the systemd timer that triggers scrape schedules every 60s.
 set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cp "$SCRIPT_DIR/tradein-scrape-trigger.service" /etc/systemd/system/
 cp "$SCRIPT_DIR/tradein-scrape-trigger.timer" /etc/systemd/system/
 systemctl daemon-reload
 systemctl enable --now tradein-scrape-trigger.timer
 systemctl status tradein-scrape-trigger.timer
--- a/tradein-mvp/ops/systemd/tradein-scrape-trigger.service
+++ b/tradein-mvp/ops/systemd/tradein-scrape-trigger.service
@ -1,15 +0,0 @@
 [Unit]
 Description=TradeIn Scrape Schedule Trigger
 After=network-online.target docker.service
 [Service]
 Type=oneshot
 User=root
 EnvironmentFile=/etc/tradein/trigger.env
 ExecStart=/usr/bin/python3 /opt/gendesign/tradein-mvp/scripts/trigger-schedules.py
 StandardOutput=journal
 StandardError=journal
 TimeoutStartSec=55
 [Install]
 WantedBy=multi-user.target
--- a/tradein-mvp/ops/systemd/tradein-scrape-trigger.timer
+++ b/tradein-mvp/ops/systemd/tradein-scrape-trigger.timer
@ -1,11 +0,0 @@
 [Unit]
 Description=TradeIn Scrape Trigger — every 60s
 Requires=tradein-scrape-trigger.service
 [Timer]
 OnBootSec=30s
 OnUnitActiveSec=60s
 AccuracySec=5s
 [Install]
 WantedBy=timers.target
--- a/tradein-mvp/scripts/browser-smoke.sh
+++ b/tradein-mvp/scripts/browser-smoke.sh
@ -1,33 +0,0 @@
 #!/bin/sh
 # Smoke-test the tradein-browser service (camoufox HTTP wrapper, #905) from
 # inside the tradein docker network. Verifies /health and a real /fetch.
 #
 # Run from the deploy host (after `git pull`), piping the script into the
 # backend container (which shares the tradein-net with tradein-browser):
 #
 #   docker exec -i tradein-backend sh < tradein-mvp/scripts/browser-smoke.sh
 #
 # Override the target URL:
 #   URL=https://example.com docker exec -i -e URL tradein-backend sh < tradein-mvp/scripts/browser-smoke.sh
 set -eu
 B="${BROWSER_HTTP_ENDPOINT:-http://tradein-browser:3000}"
 U="${URL:-https://www.avito.ru/ekaterinburg/kvartiry/prodam}"
 printf 'health:    '
 curl -s --max-time 10 "$B/health" || echo "(no response)"
 echo
 echo "{\"url\":\"$U\"}" > /tmp/smoke_payload.json
 curl -s --max-time 120 -X POST "$B/fetch" \
  -H 'Content-Type: application/json' \
  -d @/tmp/smoke_payload.json \
  -o /tmp/smoke_result.html || true
 echo "url:       $U"
 echo "bytes:     $(wc -c < /tmp/smoke_result.html 2>/dev/null || echo 0)"
 echo "preloaded: $(grep -c preloadedState /tmp/smoke_result.html 2>/dev/null || echo 0)"
 echo "firewall:  $(grep -ci firewall-container /tmp/smoke_result.html 2>/dev/null || echo 0)"
 echo "--- first 200 bytes (error JSON shows here if /fetch failed) ---"
 head -c 200 /tmp/smoke_result.html 2>/dev/null || true
 echo
--- a/tradein-mvp/scripts/trigger-schedules.py
+++ b/tradein-mvp/scripts/trigger-schedules.py
@ -1,188 +0,0 @@
 #!/usr/bin/env python3
 """Trigger due scrape schedules via admin API. Called by systemd timer every 60s."""
 from __future__ import annotations
 import fcntl
 import logging
 import os
 import random
 import sys
 import time
 import httpx
 import psycopg
 LOCK_FILE = "/var/run/tradein-trigger.lock"
 DB_URL = os.environ["TRADEIN_DATABASE_URL"]
 API_BASE = os.environ.get("TRADEIN_API_BASE", "http://localhost:8000")
 # Username that maps to role=admin in roles.yaml — passed as X-Authenticated-User.
 ADMIN_TOKEN = os.environ["TRADEIN_ADMIN_TOKEN"]
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
 # Mapping from scrape_schedules.source → admin API path.
 # Sources handled entirely inside the scheduler loop (no separate admin endpoint)
 # are listed in _NO_ENDPOINT_SOURCES: they are skipped when the systemd trigger
 # runs, and require SCHEDULER_ENABLE=true to fire.
 _SOURCE_TO_PATH: dict[str, str] = {
    "avito_city_sweep": "/api/v1/admin/scrape/avito-city-sweep",
    "yandex_city_sweep": "/api/v1/admin/scrape/yandex-city-sweep",
    "n1_city_sweep": "/api/v1/admin/scrape/n1",
    "cian_history_backfill": "/api/v1/admin/scrape/cian-backfill-history",
    "yandex_address_backfill": "/api/v1/admin/scrape/yandex-address-backfill",
 }
 # Sources with no dedicated admin trigger endpoint.
 _NO_ENDPOINT_SOURCES = frozenset(
    {
        "rosreestr_dkp_import",
        "listing_source_snapshot",
        "asking_to_sold_ratio_refresh",
        "refresh_search_matview",
        "deactivate_stale_avito",
        "sber_index_pull",
        "rosreestr_quarter_poll",
    }
 )
 _MAX_RETRIES = 3
 _RETRY_BASE_S = 2.0
 def _normalize_db_url(url: str) -> str:
    """Strip SQLAlchemy driver prefix so psycopg v3 accepts the connection string."""
    if url.startswith("postgresql+psycopg://"):
        return "postgresql://" + url[len("postgresql+psycopg://"):]
    return url
 def get_due_schedules() -> list[dict[str, str]]:
    """Fetch scrape_schedules rows that are due for execution."""
    conn_url = _normalize_db_url(DB_URL)
    with psycopg.connect(conn_url) as conn:
        rows = conn.execute(
            """
            SELECT id, source, schedule_type
            FROM scrape_schedules
            WHERE enabled = true
              AND (next_run_at IS NULL OR next_run_at <= NOW())
            """,
        ).fetchall()
        description = conn.execute(
            "SELECT id, source, schedule_type FROM scrape_schedules LIMIT 0"
        ).description or []
    col_names = [desc.name for desc in description]
    if not col_names:
        col_names = ["id", "source", "schedule_type"]
    return [
        {col_names[i]: (str(row[i]) if row[i] is not None else "") for i in range(len(col_names))}
        for row in rows
    ]
 def trigger_source(client: httpx.Client, source: str) -> None:
    """POST to the admin trigger endpoint for a given source. Retries on connection error."""
    path = _SOURCE_TO_PATH.get(source)
    if path is None:
        if source in _NO_ENDPOINT_SOURCES:
            log.warning(
                "trigger: source=%s has no admin API endpoint — "
                "ensure SCHEDULER_ENABLE=true or add a dedicated admin endpoint",
                source,
            )
        else:
            log.warning("trigger: unknown source=%s, skipping", source)
        return
    url = f"{API_BASE}{path}"
    last_exc: Exception | None = None
    for attempt in range(1, _MAX_RETRIES + 1):
        try:
            resp = client.post(
                url,
                headers={"X-Authenticated-User": ADMIN_TOKEN},
                timeout=50.0,
            )
            log.info(
                "trigger: source=%s url=%s status=%d attempt=%d",
                source,
                url,
                resp.status_code,
                attempt,
            )
            if resp.status_code >= 500:
                log.warning(
                    "trigger: source=%s got HTTP %d — body: %s",
                    source,
                    resp.status_code,
                    resp.text[:200],
                )
            return
        except httpx.ConnectError as exc:
            last_exc = exc
            wait = _RETRY_BASE_S * (2 ** (attempt - 1))
            log.warning(
                "trigger: source=%s connection error (attempt %d/%d), "
                "retrying in %.1fs: %s",
                source,
                attempt,
                _MAX_RETRIES,
                wait,
                exc,
            )
            if attempt < _MAX_RETRIES:
                time.sleep(wait)
    log.error(
        "trigger: source=%s failed after %d attempts: %s",
        source,
        _MAX_RETRIES,
        last_exc,
    )
 def main() -> int:
    # Exclusive non-blocking flock: if a previous run is still active, exit immediately
    # with code 0 so systemd does not log a failure.
    lock_fd = open(LOCK_FILE, "w")
    try:
        fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
    except BlockingIOError:
        log.info("trigger: lock busy — previous run still active, exiting")
        lock_fd.close()
        return 0
    try:
        due = get_due_schedules()
        if not due:
            log.info("trigger: no schedules due")
            return 0
        log.info("trigger: %d schedule(s) due", len(due))
        with httpx.Client() as client:
            for i, row in enumerate(due):
                source = row.get("source", "")
                trigger_source(client, source)
                if i < len(due) - 1:
                    # Stagger requests to avoid thundering herd on the API
                    time.sleep(random.uniform(1, 4))
        return 0
    except Exception:
        log.exception("trigger: unexpected error")
        return 1
    finally:
        try:
            fcntl.flock(lock_fd.fileno(), fcntl.LOCK_UN)
        except Exception:
            pass
        lock_fd.close()
 if __name__ == "__main__":
    sys.exit(main())