Merge pull request 'refactor(tradein/scrapers): удалить legacy scrape_pipeline.py + тест-хирургия (#2397 Part E1)' (#2405) from chore/tradein-partE1-delete-scrape-pipeline into main

tradein-mvp/backend/app/api/v1/admin.py

@@ -31,14 +31,15 @@ from pydantic import BaseModel, Field, field_validator
 #
 # scraper_kit-миграция (#2397 slice A, эпик #2277): 5 debug city-sweep/full-load
 # роутов (avito-city-sweep, cian-city-sweep, cian-full-load, yandex-city-sweep,
-# yandex-full-load) переключены с app.services.scrape_pipeline (legacy) на
+# yandex-full-load) переключены с app.services.scrape_pipeline (legacy, удалён в
-# scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и app.scheduler_main
+# #2397 Part E1) на scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и
-# ._run_kit_scheduler / scraper_kit.orchestration.scheduler._job_*: config/matcher/
+# app.scheduler_main._run_kit_scheduler / scraper_kit.orchestration.scheduler
-# enrichment/proxy_provider инжектируются явно вместо module-level импортов app.*
+# ._job_*: config/matcher/enrichment/proxy_provider инжектируются явно вместо
-# внутри scrape_pipeline.py. shutdown_requested НЕ прокидывается (нет SIGTERM-drain
+# module-level импортов app.*. shutdown_requested НЕ прокидывается (нет SIGTERM-drain
 # семантики у admin BackgroundTasks — эквивалент дефолту lambda: False, поведение не
-# меняется). Боевой scheduler.py (production sweep-cron) остаётся на legacy — это
+# меняется). Боевой scheduler-путь (production sweep-cron) уже на kit (#2397 Part C
-# отдельный, ещё больший шаг (см. vault Event_Legacy_Scrapers_Partial_Deletion_Jul04).
+# убрал legacy app.services.scheduler.scheduler_loop fallback) — тот же orchestrator,
+# что и debug-роуты этого файла.
 from scraper_kit.base import save_listings
 from scraper_kit.browser_fetcher import BrowserFetcher
 from scraper_kit.orchestration.pipeline import (

tradein-mvp/backend/app/services/house_imv_backfill.py

@@ -13,9 +13,10 @@ For each house:
 
 Resumable: re-running picks up from where the previous run stopped.
 process_houses_imv_batch() IS wired into the scheduler: run_avito_city_sweep()
-(scrape_pipeline.py) calls it automatically as the final IMV-phase of every
+(scraper_kit.orchestration.pipeline, via the injected EnrichmentJobs protocol)
-avito city sweep. backfill_house_imv() (batch/pending-status entrypoint) remains
+calls it automatically as the final IMV-phase of every avito city sweep.
-manual-only, triggered via admin API.
+backfill_house_imv() (batch/pending-status entrypoint) remains manual-only,
+triggered via admin API.
 """
 
 from __future__ import annotations

tradein-mvp/backend/app/services/scrapers/avito_detail.py

@@ -388,7 +388,7 @@ async def fetch_detail(
     reconnect_on_block — если False, 403/firewall поднимает AvitoBlockedError СРАЗУ
     (без холодных reconnect'ов — ими займётся outer-loop через rebuild+rewarm прогретой
     сессии); 429 после short-retry поднимает AvitoRateLimitedError. Дефолт True
-    сохраняет старое поведение own-session / scrape_pipeline путей.
+    сохраняет старое поведение own-session путей (напр. admin endpoint).
     Raises:
         httpx.HTTPError — если status != 200 (через raise_for_status-like).
         ValueError — если item_id не извлечён из HTML.
@@ -432,8 +432,8 @@ async def fetch_detail(
     own_session = cffi_session is None
     if cffi_session is None:
         # Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed,
-        # e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy
+        # e.g. admin endpoint). Caller-provided sessions already have proxy from
-        # from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
+        # AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
         session = _build_detail_session()
     else:
         session = cffi_session
@@ -446,8 +446,8 @@ async def fetch_detail(
     # эфемерную retry_session, которую сами закрываем в finally.
     # reconnect_on_block=False (warm-batch #1551): не делаем холодных per-card
     # reconnect'ов — блок поднимает AvitoBlockedError сразу, outer-loop пересоздаёт+
-    # прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session /
+    # прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session) —
-    # scrape_pipeline) — старое поведение: backconnect-reconnect эфемерной сессией.
+    # старое поведение: backconnect-reconnect эфемерной сессией.
     backconnect = bool(settings.scraper_proxy_url) and reconnect_on_block
     req_headers = {"Referer": referer} if referer else None
     r403 = 0

tradein-mvp/backend/app/services/yandex_price_history.py

@@ -4,7 +4,9 @@
 Yandex gate-ответ уже несёт `price.previous` + `price.trend`, которые парсятся в
 `ScrapedLot.price_previous_rub` / `price_trend`. Otsenka-скрапер НЕ нужен — gate достаточно.
 
-Эта функция вызывается из scrape_pipeline СРАЗУ ПОСЛЕ save_listings(те же lots): резолвит
+Эта функция вызывается из run_yandex_city_sweep (scraper_kit.orchestration.pipeline,
+через инжектируемый EnrichmentJobs.record_yandex_price_history) СРАЗУ ПОСЛЕ
+save_listings (те же lots): резолвит
 listing_id по (source='yandex', source_id), читает последнюю по change_time точку
 offer_price_history и идемпотентно дописывает наблюдение, если цена изменилась.
 

tradein-mvp/backend/app/tasks/avito_detail_backfill.py

@@ -8,7 +8,8 @@ City sweep ENRICH_DETAIL only enriches top-N proximate listings per anchor.
 Legacy listings (older than 2h or outside radius) are never enriched.
 
 Solution: single snapshot SELECT at start (guarantees termination), same proxy
-session path as scrape_pipeline.py step 5. Block handling mirrors step 5:
+session path as the detail-phase of `run_avito_city_sweep`
+(scraper_kit.orchestration.pipeline). Block handling mirrors that phase:
 rotate IP on every block, abort after max_consecutive_blocks (mark_done not
 mark_failed -- block is temporary, retry next night via NULL detail_enriched_at).
 """
@@ -176,7 +177,7 @@ async def run_avito_detail_backfill(
             session = await build_warmed_session()
         elif not use_curl:
             # curl_cffi legacy path (scraper_fetch_mode="curl_cffi", use_curl=False):
-            # строим shared сессию через auv, как делал scrape_pipeline.
+            # строим shared сессию через auv, как делает run_avito_city_sweep (kit).
             own_session = True
             session = AsyncSession(
                 impersonate="chrome120",
@@ -280,7 +281,7 @@ async def run_avito_detail_backfill(
             source_url: str = row["source_url"]
             counters.attempted += 1
 
-            # Normalise URL -> path (mirrors scrape_pipeline.py line 311)
+            # Normalise URL -> path (mirrors run_avito_city_sweep detail-phase, kit)
             item_url = urlparse(source_url).path if source_url.startswith("http") else source_url
 
             # use_curl warm-batch (#1551): интервалы по размеру SERP-страницы (~50).

tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py

@@ -1,6 +1,6 @@
 """Test anti-bot hardening: blocked exceptions + early abort."""
 
-from unittest.mock import AsyncMock, MagicMock, call, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
@@ -170,169 +170,13 @@ async def test_fetch_around_raises_rate_limited_on_429() -> None:
             await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
 
 
-# ── Pipeline abort tests ────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_pipeline_aborts_after_3_consecutive_blocks() -> None:
-    """#1820: 3 consecutive 403 в detail-фазе прерывают enrichment, но НЕ роняют прогон.
-
-    Раньше pipeline re-raise'ил AvitoBlockedError (один блок ронял весь прогон). Теперь
-    блок ловится пер-item, и лишь N consecutive блоков прерывают enrichment-фазу —
-    PipelineResult возвращается, detail_failed считается, лишние detail-фетчи не делаются.
-    """
-    from app.services.scrape_pipeline import (
-        _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
-        PipelineResult,
-        run_avito_pipeline,
-    )
-
-    mock_db = MagicMock()
-    mock_scraper = MagicMock()
-    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
-    mock_scraper.__aexit__ = AsyncMock(return_value=None)
-    mock_scraper.fetch_around = AsyncMock(return_value=[])
-    mock_scraper._cffi = None
-
-    mock_row = MagicMock()
-    mock_row.__getitem__ = lambda self, key: (
-        "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
-    )
-    mock_mappings = MagicMock()
-    mock_mappings.all.return_value = [mock_row, mock_row, mock_row, mock_row]
-    mock_execute = MagicMock()
-    mock_execute.mappings.return_value = mock_mappings
-    mock_db.execute.return_value = mock_execute
-
-    block_error = AvitoBlockedError("HTTP 403")
-    detail_mock = AsyncMock(side_effect=block_error)
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
-        patch("app.services.scrape_pipeline.fetch_detail", detail_mock),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
-        ),
-    ):
-        # НЕ должно бросать наружу
-        result = await run_avito_pipeline(
-            mock_db,
-            56.8,
-            60.6,
-            1000,
-            enrich_houses=False,
-            enrich_detail_top_n=4,
-        )
-
-    assert isinstance(result, PipelineResult)
-    # enrichment прерван ровно после порога consecutive блоков (4-я строка не фетчилась)
-    assert detail_mock.await_count == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
-    assert result.counters.detail_attempted == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
-    assert result.counters.detail_failed == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
-    # в errors зафиксирован abort
-    assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
-
-
-@pytest.mark.asyncio
-async def test_pipeline_sleep_between_detail_requests() -> None:
-    """Pipeline должен вызывать asyncio.sleep между каждым detail request."""
-    from app.services.scrape_pipeline import run_avito_pipeline
-
-    mock_db = MagicMock()
-    mock_scraper = MagicMock()
-    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
-    mock_scraper.__aexit__ = AsyncMock(return_value=None)
-    mock_scraper.fetch_around = AsyncMock(return_value=[])
-    mock_scraper._cffi = None
-
-    mock_row = MagicMock()
-    mock_row.__getitem__ = lambda self, key: (
-        "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
-    )
-    mock_mappings = MagicMock()
-    mock_mappings.all.return_value = [mock_row, mock_row, mock_row]
-    mock_execute = MagicMock()
-    mock_execute.mappings.return_value = mock_mappings
-    mock_db.execute.return_value = mock_execute
-
-    mock_enrichment = MagicMock()
-    sleep_mock = AsyncMock()
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
-        patch(
-            "app.services.scrape_pipeline.fetch_detail",
-            AsyncMock(return_value=mock_enrichment),
-        ),
-        patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
-        patch("app.services.scrape_pipeline.asyncio.sleep", sleep_mock),
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
-        ),
-        patch("app.services.scrape_pipeline.random.uniform", return_value=1.0),
-    ):
-        await run_avito_pipeline(
-            mock_db,
-            56.8,
-            60.6,
-            1000,
-            enrich_houses=False,
-            enrich_detail_top_n=3,
-            request_delay_sec=7.0,
-        )
-
-    assert sleep_mock.call_count == 2
-    for c in sleep_mock.call_args_list:
-        assert c == call(7.0)
-
-
-@pytest.mark.asyncio
-async def test_city_sweep_marks_banned_on_block() -> None:
-    """City sweep mark'ит run как banned при AvitoBlockedError.
-
-    Migration 015 задокументировал 'banned' как 'Avito вернул 403/captcha' — именно этот сценарий.
-
-    Патчим AvitoScraper.fetch_around (не run_avito_pipeline — он больше не вызывается
-    из run_avito_city_sweep; sweep использует внутренние _avito_anchor_phases).
-    """
-    from app.services.scrape_pipeline import run_avito_city_sweep
-    from app.services.scrapers.avito import AvitoScraper
-
-    mock_db = MagicMock()
-    mock_runs = MagicMock()
-    mock_runs.is_cancelled.return_value = False
-    mock_runs.update_heartbeat = MagicMock()
-    mock_runs.mark_banned = MagicMock()
-    mock_runs.mark_done = MagicMock()
-
-    block_error = AvitoBlockedError("IP blocked at page=1")
-
-    with (
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch.object(AvitoScraper, "fetch_around", AsyncMock(side_effect=block_error)),
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(
-                __aenter__=AsyncMock(return_value=AsyncMock()),
-                __aexit__=AsyncMock(return_value=None),
-            ),
-        ),
-    ):
-        result = await run_avito_city_sweep(
-            mock_db,
-            run_id=42,
-            anchors=[(56.84, 60.60, "Центр"), (56.79, 60.53, "ЮЗ")],
-        )
-
-    assert mock_runs.mark_banned.called
-    assert not mock_runs.mark_done.called
-    assert result.anchors_done == 1
-
-
 # ── #623 firewall-детект + ротация IP ────────────────────────────
+#
+# Legacy `run_avito_pipeline` / `run_avito_city_sweep` orchestration-level tests
+# (pipeline consecutive-block abort, inter-detail sleep, city-sweep mark_banned)
+# удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная
+# regression-coverage теперь в `test_scraper_kit_pipeline_parity.py` (kit
+# `run_avito_city_sweep`: ban/rotation state-machine, detail-block propagation).
 
 _CARD_HTML = (
     "<html><body>"
@@ -437,20 +281,20 @@ async def test_fetch_around_blocked_when_rotation_exhausted(
 # ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ───
 
 
-def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_avito_proxies_helper() -> None:
-    """_avito_proxies: None без env, dict {http,https} при заданном AVITO_PROXY_URL.
+    """Kit _avito_proxies(config): None без proxy, dict {http,https} при заданном URL.
 
     Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail —
     без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup).
     """
-    from app.services import scrape_pipeline as sp
+    from types import SimpleNamespace
 
-    monkeypatch.setattr(sp.settings, "avito_proxy_url", None)
+    from scraper_kit.orchestration.pipeline import _avito_proxies
-    assert sp._avito_proxies() is None
+
+    assert _avito_proxies(SimpleNamespace(scraper_proxy_url=None)) is None
 
     url = "http://u:p@mproxy.site:16928"
-    monkeypatch.setattr(sp.settings, "avito_proxy_url", url)
+    assert _avito_proxies(SimpleNamespace(scraper_proxy_url=url)) == {"http": url, "https": url}
-    assert sp._avito_proxies() == {"http": url, "https": url}
 
 
 # ── curl_cffi 429 short-retry (backconnect conn-limit transient) ──────────────

tradein-mvp/backend/tests/test_anchor_watchdog.py

@@ -1,670 +0,0 @@
-"""Unit tests for per-anchor watchdog timeout in city sweeps (#880).
-
-Verifies that ANCHOR_TIMEOUT_SEC is applied to all three sweeps (avito / cian / yandex):
-  (a) A hung anchor (asyncio.sleep(large)) doesn't freeze the whole sweep.
-  (b) TimeoutError branch: errors_count incremented, anchor logged as timed-out.
-  (c) Sweep proceeds to the next anchor after the timeout.
-  (d) mark_done is called when the sweep finishes all anchors.
-
-All tests are network-free — they monkeypatch the inner scrape call.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import os
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-from app.services import scrape_pipeline
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.cian import CianScraper
-from app.services.scrapers.yandex_realty import YandexRealtyScraper
-
-# Two anchors: first hangs (causes timeout), second succeeds.
-TWO_ANCHORS: list[tuple[float, float, str]] = [
-    (56.8400, 60.6050, "Hang"),
-    (56.7950, 60.5300, "OK"),
-]
-
-
-# ── Shared fake scrape_runs ─────────────────────────────────────────────────
-
-
-class _FakeRuns:
-    def __init__(self) -> None:
-        self.heartbeats: list[dict[str, int]] = []
-        self.done: dict[str, int] | None = None
-        self.failed: tuple[str, dict[str, int]] | None = None
-        self.banned: tuple[str, dict[str, int]] | None = None
-
-    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
-        return False
-
-    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.heartbeats.append(dict(counters))
-
-    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.done = dict(counters)
-
-    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.failed = (error, dict(counters))
-
-    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.banned = (error, dict(counters))
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-
-
-def _fake_lot(source: str, offer_id: str) -> ScrapedLot:
-    return ScrapedLot(
-        source=source,
-        source_url=f"https://example.ru/{offer_id}/",
-        source_id=offer_id,
-        address="Екатеринбург, улица Тестовая, 1",
-        price_rub=4_000_000,
-        area_m2=42.0,
-        rooms=1,
-    )
-
-
-# ── CianScraper lifecycle stub ──────────────────────────────────────────────
-
-
-@pytest.fixture(autouse=True)
-def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
-    def _init(self: CianScraper) -> None:
-        self.request_delay_sec = 0.0
-        self._cffi = None
-
-    async def _aenter(self: CianScraper) -> CianScraper:
-        return self
-
-    async def _aexit(self: CianScraper, *_args: Any) -> None:
-        return None
-
-    monkeypatch.setattr(CianScraper, "__init__", _init)
-    monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
-    monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
-
-
-# ── YandexRealtyScraper lifecycle stub ─────────────────────────────────────
-
-
-@pytest.fixture(autouse=True)
-def _stub_yandex_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
-    def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
-        self.city = city
-        self.request_delay_sec = 0.0
-        self._cffi_session = None
-        self._cookies = {}
-
-    async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
-        return self
-
-    async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
-        return None
-
-    monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
-    monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
-    monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)
-
-
-# ── ANCHOR_TIMEOUT_SEC is exported ─────────────────────────────────────────
-
-
-def test_anchor_timeout_sec_exported() -> None:
-    from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC
-
-    assert isinstance(ANCHOR_TIMEOUT_SEC, int)
-    # Reasonable sane range: 60s–600s
-    assert (
-        60 <= ANCHOR_TIMEOUT_SEC <= 600
-    ), f"ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC} out of sane range"
-
-
-# ── Avito sweep watchdog ────────────────────────────────────────────────────
-
-
-async def test_avito_sweep_anchor_timeout_continues(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается.
-
-    run_avito_city_sweep использует внутреннюю _avito_anchor_phases (не run_avito_pipeline),
-    поэтому патчим AvitoScraper.fetch_around + save_listings.
-    """
-    from app.services.scrapers.avito import AvitoScraper
-
-    # Monkeypatch asyncio.wait_for so we can inject TimeoutError on first call only,
-    # without actually waiting in CI.
-    original_wait_for = asyncio.wait_for
-    wf_call = 0
-
-    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
-        nonlocal wf_call
-        wf_call += 1
-        if wf_call == 1:
-            # Simulate the hung anchor: cancel the coro (mimic wait_for behaviour)
-            # and raise TimeoutError.
-            try:
-                coro.close()
-            except Exception:
-                pass
-            raise TimeoutError
-        return await original_wait_for(coro, timeout=timeout)
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
-    # Stub AvitoScraper.fetch_around — second anchor returns 3 lots.
-    fetch_calls: list[str] = []
-
-    async def fake_fetch_around(
-        self: AvitoScraper,
-        lat: float,
-        lon: float,
-        radius_m: int,
-        pages: int = 1,
-        delay_override_sec: float | None = None,
-    ) -> list[ScrapedLot]:
-        fetch_calls.append(f"{lat:.4f}")
-        return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(3)]
-
-    monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
-
-    # Stub shared AsyncSession (used by avito sweep).
-    fake_session = AsyncMock()
-    fake_session.__aenter__ = AsyncMock(return_value=fake_session)
-    fake_session.__aexit__ = AsyncMock(return_value=None)
-    import curl_cffi.requests as _cffi_mod
-
-    monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
-
-    # Stub IMV phase (not relevant to this test).
-    async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
-        result = MagicMock()
-        result.checked = 0
-        result.saved = 0
-        result.errors = 0
-        return result
-
-    monkeypatch.setattr(
-        "app.services.house_imv_backfill.process_houses_imv_batch",
-        fake_imv_batch,
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_avito_city_sweep(
-        db=MagicMock(),
-        run_id=1,
-        anchors=TWO_ANCHORS,
-        enrich_houses=False,
-        detail_top_n=0,
-        enrich_imv=False,
-        request_delay_sec=0.0,
-    )
-
-    # (a) sweep completes — doesn't hang
-    # (b) first anchor timed out → errors_count >= 1
-    assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
-    # (c) second anchor reached: lots_fetched > 0
-    assert counters.lots_fetched > 0, "второй anchor должен был выполниться"
-    # (d) mark_done вызван (sweep завершился)
-    assert fake.done is not None, "mark_done должен быть вызван"
-    assert fake.failed is None, "mark_failed не должен быть вызван"
-    assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-# ── Cian sweep watchdog ─────────────────────────────────────────────────────
-
-
-async def test_cian_sweep_anchor_timeout_continues(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
-    serp_calls: list[str] = []
-    original_wait_for = asyncio.wait_for
-    wf_call = 0
-
-    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
-        nonlocal wf_call
-        wf_call += 1
-        if wf_call == 1:
-            try:
-                coro.close()
-            except Exception:
-                pass
-            raise TimeoutError
-        return await original_wait_for(coro, timeout=timeout)
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
-    async def fake_fetch_multi(
-        self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
-    ) -> list[ScrapedLot]:
-        serp_calls.append(f"{lat:.4f}")
-        return [_fake_lot("cian", f"{lat:.4f}-1")]
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        AsyncMock(return_value=None),
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=2,
-        anchors=TWO_ANCHORS,
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    # (a) sweep completes (doesn't hang at anchor 1)
-    # (b) errors_count >= 1 (timeout counted)
-    assert counters.errors_count >= 1
-    # (c) second anchor was reached (SERP called once — first anchor timed out before SERP)
-    # wf_call==2 means the second wait_for ran → second anchor's coro executed.
-    assert wf_call == 2, f"wait_for should be called once per anchor; got {wf_call}"
-    # (d) mark_done called
-    assert fake.done is not None
-    assert fake.failed is None
-    assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-# ── Yandex sweep watchdog ───────────────────────────────────────────────────
-
-
-async def test_yandex_sweep_anchor_timeout_continues(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
-    serp_calls: list[str] = []
-    original_wait_for = asyncio.wait_for
-    wf_call = 0
-
-    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
-        nonlocal wf_call
-        wf_call += 1
-        if wf_call == 1:
-            try:
-                coro.close()
-            except Exception:
-                pass
-            raise TimeoutError
-        return await original_wait_for(coro, timeout=timeout)
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1500,
-        max_pages: int = 2,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        serp_calls.append(f"{lat:.4f}")
-        return [_fake_lot("yandex", f"{lat:.4f}-1")]
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=3,
-        anchors=TWO_ANCHORS,
-        pages_per_anchor=1,
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-
-    # (a) sweep completes
-    # (b) timeout counted
-    assert counters.errors_count >= 1
-    # (c) second anchor reached
-    assert wf_call == 2
-    # (d) mark_done called
-    assert fake.done is not None
-    assert fake.failed is None
-    assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-# ── All three: timeout increments errors_count and is logged, sweep reaches mark_done ──
-
-
-async def test_cian_timeout_increments_counter_and_marks_done(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Детальная проверка: timeout anchor → errors_count+1, sweep продолжается, mark_done.
-
-    Использует fake_wait_for (зеркало первых трёх тестов): первый вызов → TimeoutError
-    (имитирует зависший anchor), второй вызов → реальный wait_for (нормальный anchor).
-    Проверяет что:
-      - errors_count увеличивается
-      - второй anchor выполнился (SERP вызван)
-      - mark_done вызван (sweep завершился)
-      - mark_banned НЕ вызван (одиночный таймаут < порога consecutive failures)
-    """
-    original_wait_for = asyncio.wait_for
-    wf_call = 0
-
-    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
-        nonlocal wf_call
-        wf_call += 1
-        if wf_call == 1:
-            try:
-                coro.close()
-            except Exception:
-                pass
-            raise TimeoutError
-        return await original_wait_for(coro, timeout=timeout)
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
-    serp_calls: list[str] = []
-
-    async def fake_fetch(
-        self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
-    ) -> list[ScrapedLot]:
-        serp_calls.append(f"{lat:.4f}")
-        return [_fake_lot("cian", f"{lat:.4f}-ok")]
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch)
-    monkeypatch.setattr(
-        scrape_pipeline,
-        "save_listings",
-        lambda _db, lots, *, run_id=None: (len(lots), 0),
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        AsyncMock(return_value=None),
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=10,
-        anchors=TWO_ANCHORS,
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    # (b) timeout branch: errors_count incremented
-    assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
-    # (c) second anchor reached (wait_for was called twice — once per anchor)
-    assert wf_call == 2, f"wait_for должен быть вызван для каждого anchor; got {wf_call}"
-    # (d) sweep reached mark_done
-    assert fake.done is not None, "mark_done должен быть вызван"
-    assert fake.banned is None, "mark_banned не должен быть вызван для одиночного таймаута"
-    assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-# ── (A) Timeout scaling — avito_anchor_timeout > ANCHOR_TIMEOUT_SEC ───────────
-
-
-def test_avito_anchor_timeout_scaling() -> None:
-    """_avito_anchor_timeout масштабируется по detail_top_n и enrich_houses.
-
-    При detail_top_n=20 и enrich_houses=True таймаут должен быть существенно выше
-    ANCHOR_TIMEOUT_SEC (240), чтобы detail-фаза не guillotine'ила SERP-фазу.
-    """
-    from app.services.scrape_pipeline import (
-        _AVITO_HOUSES_BUDGET_S,
-        _AVITO_PER_DETAIL_S,
-        ANCHOR_TIMEOUT_SEC,
-    )
-
-    detail_top_n = 20
-    enrich_houses = True
-    computed = (
-        float(ANCHOR_TIMEOUT_SEC)
-        + detail_top_n * _AVITO_PER_DETAIL_S
-        + (_AVITO_HOUSES_BUDGET_S if enrich_houses else 0.0)
-    )
-    # Должен быть значительно больше базового (минимум вдвое при n=20+houses)
-    assert (
-        computed > ANCHOR_TIMEOUT_SEC * 2
-    ), f"computed timeout {computed} должен быть > 2× base {ANCHOR_TIMEOUT_SEC}"
-    # Без detail и без houses — ровно ANCHOR_TIMEOUT_SEC
-    computed_no_detail = float(ANCHOR_TIMEOUT_SEC) + 0 * _AVITO_PER_DETAIL_S + 0.0
-    assert computed_no_detail == float(ANCHOR_TIMEOUT_SEC)
-
-    # Новые константы экспортированы / импортируются без ошибок
-    assert _AVITO_PER_DETAIL_S > 0
-    assert _AVITO_HOUSES_BUDGET_S > 0
-
-
-# ── (B) SERP counters durable across detail-phase timeout ────────────────────
-
-
-async def test_avito_serp_counters_durable_after_detail_timeout(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """SERP+save счётчики сохраняются в sweep-level counters даже когда detail-фаза
-    вызывает TimeoutError из wait_for.
-
-    Симулируем: первый anchor — SERP возвращает 5 lots, save успешен, затем wait_for
-    завершается TimeoutError (detail-фаза «зависла»). Второй anchor — полный успех.
-
-    Ожидаем:
-      - counters.lots_fetched == 5 (SERP первого anchor'а) + лоты второго
-      - counters.lots_inserted >= 1 (save первого anchor'а)
-      - counters.errors_count >= 1 (timeout первого anchor'а)
-      - mark_done вызван (sweep завершился)
-    """
-    import curl_cffi.requests as _cffi_mod
-
-    from app.services.scrapers.avito import AvitoScraper
-
-    # Отслеживаем вызовы fetch_around
-    serp_calls: list[str] = []
-    save_calls: list[int] = []
-
-    original_wait_for = asyncio.wait_for
-    wf_call = 0
-
-    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
-        nonlocal wf_call
-        wf_call += 1
-        if wf_call == 1:
-            # Запускаем корутину до конца — она дойдёт до SERP+save,
-            # а затем TimeoutError наступает как если бы detail-фаза зависла.
-            # Для упрощения: просто инжектируем TimeoutError (как во всех других тестах),
-            # но проверяем, что counter-патч произошёл ДО wait_for через отдельный
-            # механизм: monkeypatch fetch_around + save_listings записывают данные
-            # в counters до timeout-исключения.
-            try:
-                coro.close()
-            except Exception:
-                pass
-            raise TimeoutError
-        return await original_wait_for(coro, timeout=timeout)
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
-    # Stub shared AsyncSession
-    fake_session = AsyncMock()
-    fake_session.__aenter__ = AsyncMock(return_value=fake_session)
-    fake_session.__aexit__ = AsyncMock(return_value=None)
-    monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
-
-    # Stub AvitoScraper.fetch_around — возвращает lots (не AvitoBlockedError)
-    async def fake_fetch_around(
-        self: AvitoScraper,
-        lat: float,
-        lon: float,
-        radius_m: int,
-        pages: int = 1,
-        delay_override_sec: float | None = None,
-    ) -> list[ScrapedLot]:
-        serp_calls.append(f"{lat:.4f}")
-        return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(5)]
-
-    monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
-
-    # Stub save_listings — записывает lots count
-    def fake_save_listings(db: Any, lots: Any, *, run_id: int | None = None) -> tuple[int, int]:
-        save_calls.append(len(lots))
-        return (len(lots), 0)
-
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_listings)
-
-    # Stub run_avito_pipeline — используем реальный run_avito_city_sweep
-    # но с внутренним патчем, поэтому НЕ monkeypatch run_avito_pipeline.
-    # Важно: inner _avito_anchor_phases вызывает AvitoScraper и save_listings
-    # через nonlocal counters ПЕРЕД wait_for возвращает TimeoutError.
-
-    async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
-        result = MagicMock()
-        result.checked = 0
-        result.saved = 0
-        result.errors = 0
-        return result
-
-    monkeypatch.setattr(
-        "app.services.house_imv_backfill.process_houses_imv_batch",
-        fake_imv_batch,
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    # Stub fetch_house_catalog и fetch_detail чтобы не было реальных сетевых вызовов
-    # (они могут вызваться из _avito_anchor_phases если timeout не наступит раньше)
-    monkeypatch.setattr(
-        scrape_pipeline,
-        "fetch_house_catalog",
-        AsyncMock(side_effect=RuntimeError("should not be called in timeout test")),
-    )
-    monkeypatch.setattr(
-        scrape_pipeline,
-        "fetch_detail",
-        AsyncMock(side_effect=RuntimeError("should not be called in timeout test")),
-    )
-
-    counters = await scrape_pipeline.run_avito_city_sweep(
-        db=MagicMock(),
-        run_id=42,
-        anchors=TWO_ANCHORS,
-        enrich_houses=False,
-        detail_top_n=0,
-        enrich_imv=False,
-        request_delay_sec=0.0,
-    )
-
-    # SERP первого anchor'а НЕ дойдёт до счётчиков т.к. wait_for killит coro СРАЗУ
-    # (fake_wait_for делает coro.close() до любого await внутри). Это ожидаемо при
-    # coro.close() — реальный сценарий другой: SERP завершается, затем detail зависает.
-    # Тест проверяет sweep-level инварианты:
-    # - errors_count >= 1 (timeout)
-    # - sweep не падает
-    # - mark_done вызван
-    # - второй anchor дошёл (wf_call==2)
-    assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
-    assert fake.done is not None, "mark_done должен быть вызван"
-    assert fake.failed is None, "mark_failed не должен быть вызван"
-    assert wf_call == 2, f"оба anchor'а должны пройти через wait_for; got {wf_call}"
-    assert counters.anchors_done == len(TWO_ANCHORS)
-
-
-async def test_avito_serp_counters_durable_real_phases(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Проверяем что SERP+save счётчики попадают в sweep-level counters когда
-    _avito_anchor_phases выполняется нормально (без timeout): lots_fetched/inserted
-    отражают реальные данные.
-
-    Патчим wait_for через реальный asyncio.wait_for с очень большим timeout —
-    anchor_phases выполняется полностью. SERP возвращает 7 lots, save возвращает (7, 0).
-    """
-    import curl_cffi.requests as _cffi_mod
-
-    from app.services.scrapers.avito import AvitoScraper
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
-    fake_session = AsyncMock()
-    fake_session.__aenter__ = AsyncMock(return_value=fake_session)
-    fake_session.__aexit__ = AsyncMock(return_value=None)
-    monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
-
-    async def fake_fetch_around(
-        self: AvitoScraper,
-        lat: float,
-        lon: float,
-        radius_m: int,
-        pages: int = 1,
-        delay_override_sec: float | None = None,
-    ) -> list[ScrapedLot]:
-        return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(7)]
-
-    monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
-    # Не вызывать house/detail enrich
-    monkeypatch.setattr(scrape_pipeline, "fetch_house_catalog", AsyncMock(return_value=None))
-    monkeypatch.setattr(scrape_pipeline, "fetch_detail", AsyncMock(return_value=None))
-
-    async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
-        result = MagicMock()
-        result.checked = 0
-        result.saved = 0
-        result.errors = 0
-        return result
-
-    monkeypatch.setattr(
-        "app.services.house_imv_backfill.process_houses_imv_batch",
-        fake_imv_batch,
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    # Один anchor — проверяем счётчики после нормального прохода
-    counters = await scrape_pipeline.run_avito_city_sweep(
-        db=MagicMock(),
-        run_id=99,
-        anchors=[(56.8400, 60.6050, "Центр")],
-        enrich_houses=False,
-        detail_top_n=0,
-        enrich_imv=False,
-        request_delay_sec=0.0,
-    )
-
-    assert counters.lots_fetched == 7, f"lots_fetched должен быть 7; got {counters.lots_fetched}"
-    assert counters.lots_inserted == 7, f"lots_inserted должен быть 7; got {counters.lots_inserted}"
-    assert counters.anchors_done == 1
-    assert fake.done is not None

tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py

@@ -6,22 +6,24 @@
    newbuilding_id заполнен (DOM-маркер застройщика + zhk-якорь).
 3. fetch_newbuildings break-on-empty + dedup (mirror fetch_city_wide).
 4. NewbuildingSweepCounters — defaults + to_dict.
-5. run_avito_newbuilding_sweep wiring — мокаем fetch_newbuildings + save_listings,
+
-   assert save вызван и SERP-счётчики записаны (mark_done).
+Legacy `run_avito_newbuilding_sweep` orchestration wiring (fetch_newbuildings +
+save_listings + mark_done) удалён вместе с `app.services.scrape_pipeline` (#2397
+Part E1) — эквивалентная regression-coverage теперь в
+`test_scraper_kit_pipeline_parity2.py::test_avito_newbuilding_sweep` (kit
+`run_avito_newbuilding_sweep`).
 """
 
 from __future__ import annotations
 
 import os
 from pathlib import Path
-from typing import Any
+from unittest.mock import AsyncMock, patch
-from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
 
-from app.services import scrape_pipeline
 from app.services.scrapers.avito import NOVOSTROYKA_SLUG, AvitoScraper
 from app.services.scrapers.base import ScrapedLot
 
@@ -148,7 +150,7 @@ def test_fetch_city_wide_url_unchanged() -> None:
 
 
 def test_newbuilding_counters_defaults_and_to_dict() -> None:
-    from app.services.scrape_pipeline import NewbuildingSweepCounters
+    from scraper_kit.orchestration.pipeline import NewbuildingSweepCounters
 
     c = NewbuildingSweepCounters()
     assert c.lots_fetched == 0
@@ -157,89 +159,3 @@ def test_newbuilding_counters_defaults_and_to_dict() -> None:
     assert c.errors_count == 0
     d = c.to_dict()
     assert set(d.keys()) == {"lots_fetched", "lots_inserted", "lots_updated", "errors_count"}
-
-
-# ── 5. run_avito_newbuilding_sweep wiring ───────────────────────────────────
-
-
-class _FakeRuns:
-    def __init__(self) -> None:
-        self.heartbeats: list[dict[str, int]] = []
-        self.done: dict[str, int] | None = None
-        self.failed: tuple[str, dict[str, int]] | None = None
-        self.banned: tuple[str, dict[str, int]] | None = None
-
-    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
-        return False
-
-    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.heartbeats.append(dict(counters))
-
-    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.done = dict(counters)
-
-    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.failed = (error, dict(counters))
-
-    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.banned = (error, dict(counters))
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-
-
-@pytest.mark.asyncio
-async def test_run_avito_newbuilding_sweep_saves_and_counts(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    fake_runs = _FakeRuns()
-    _install_runs(monkeypatch, fake_runs)
-
-    fake_lots = [_make_lot("N1"), _make_lot("N2"), _make_lot("N3")]
-
-    async def fake_fetch_newbuildings(
-        self: AvitoScraper,
-        pages: int = 30,
-        *,
-        delay_override_sec: float | None = None,
-    ) -> list[ScrapedLot]:
-        return list(fake_lots)
-
-    monkeypatch.setattr(AvitoScraper, "fetch_newbuildings", fake_fetch_newbuildings)
-
-    save_calls: list[int] = []
-
-    def fake_save_listings(_db: Any, lots: Any, **_kw: Any) -> tuple[int, int]:
-        save_calls.append(len(lots))
-        return (len(lots), 0)
-
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_listings)
-    # Force cffi-mode (no browser) so we don't spin up BrowserFetcher.
-    monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "cffi")
-
-    # Stub shared AsyncSession (used by the sweep in cffi-mode) — no real session.
-    fake_session = AsyncMock()
-    fake_session.__aenter__ = AsyncMock(return_value=fake_session)
-    fake_session.__aexit__ = AsyncMock(return_value=None)
-    monkeypatch.setattr(scrape_pipeline, "AsyncSession", lambda **_kw: fake_session)
-
-    counters = await scrape_pipeline.run_avito_newbuilding_sweep(
-        MagicMock(),
-        run_id=42,
-        pages=2,
-        request_delay_sec=0.0,
-    )
-
-    assert save_calls == [3]
-    assert counters.lots_fetched == 3
-    assert counters.lots_inserted == 3
-    assert counters.lots_updated == 0
-    assert fake_runs.done is not None
-    assert fake_runs.done["lots_fetched"] == 3
-    assert fake_runs.failed is None
-    assert fake_runs.banned is None

tradein-mvp/backend/tests/test_ban_rotation_1790.py

@@ -1,220 +0,0 @@
-"""#1790 — ban-triggered IP rotation в enrichment-фазах scrape_pipeline.
-
-Тест 1: AvitoBlockedError при доступных ротациях → вызывается _rotate_proxy_ip
-        (changeip) + consecutive счётчик сбрасывается → enrichment продолжается.
-
-Тест 2: Ротаций больше нет (бюджет исчерпан) → abort enrichment (bounded N ротаций).
-
-Тест 3: Graceful per-item (#1820) сохранён — один блок без порога ротации НЕ вызывает.
-
-Без сети, без БД.
-"""
-
-from __future__ import annotations
-
-import os
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from app.services.scrape_pipeline import (
-    _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
-    PipelineResult,
-    run_avito_pipeline,
-)
-from app.services.scrapers.avito_exceptions import AvitoBlockedError
-
-
-def _make_lot(
-    item_id: str,
-    house_url: str | None = None,
-) -> object:
-    from app.services.scrapers.base import ScrapedLot
-
-    return ScrapedLot(
-        source="avito",
-        source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
-        source_id=item_id,
-        price_rub=5_000_000,
-        address="Test address",
-        house_source="avito" if house_url else None,
-        house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
-        house_url=house_url,
-        listing_segment="vtorichka",
-    )
-
-
-def _block_scraper(lots: list) -> MagicMock:  # type: ignore[type-arg]
-    scraper = MagicMock()
-    scraper.__aenter__ = AsyncMock(return_value=scraper)
-    scraper.__aexit__ = AsyncMock(return_value=None)
-    scraper.fetch_around = AsyncMock(return_value=lots)
-    scraper._cffi = None
-    return scraper
-
-
-# ---------------------------------------------------------------------------
-# Тест 1: ротация срабатывает при consecutive blocks == порог
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_rotation_called_on_consecutive_blocks() -> None:
-    """При N подряд-блоках (порог) вызывается _rotate_proxy_ip — IP меняется.
-
-    После успешной ротации consecutive счётчик сбрасывается и enrichment продолжается:
-    следующий (N+1)-й дом фетчится успешно.
-    """
-    n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
-    lots = [
-        _make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{100 + i}")
-        for i in range(n + 2)  # n блокируются, потом ротация, потом успех
-    ]
-    scraper = _block_scraper(lots)
-
-    # Первые n обращений — блок, (n+1)-й — успех
-    fetch_house = AsyncMock(side_effect=[AvitoBlockedError("banned")] * n + [MagicMock()])
-    rotate_mock = AsyncMock(return_value=True)  # ротация успешна
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
-        patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
-        patch(
-            "app.services.scrape_pipeline.save_house_catalog_enrichment",
-            return_value={"house_id": 99},
-        ),
-        patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
-        ),
-    ):
-        result = await run_avito_pipeline(
-            MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
-        )
-
-    assert isinstance(result, PipelineResult)
-    # _rotate_proxy_ip вызвана хотя бы один раз
-    assert rotate_mock.await_count >= 1
-    # Enrichment продолжился после ротации — (n+1)-й дом обогащён
-    assert result.counters.houses_enriched >= 1
-    # Listings сохранены
-    assert result.counters.lots_inserted == len(lots)
-
-
-# ---------------------------------------------------------------------------
-# Тест 2: бюджет ротаций исчерпан → abort
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_rotation_bounded_abort_when_budget_exhausted() -> None:
-    """После max_rotations попыток ротации enrichment abort'ится (bounded).
-
-    Используем avito_proxy_max_rotations=1 → только одна попытка ротации.
-    После неё блоки продолжаются → abort.
-    """
-    n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
-    # Достаточно домов чтобы вызвать и ротацию и повторные блоки
-    lots = [
-        _make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{200 + i}")
-        for i in range(n * 3)
-    ]
-    scraper = _block_scraper(lots)
-
-    # Все дома блокируются (не будет успешного фетча)
-    fetch_house = AsyncMock(side_effect=AvitoBlockedError("banned"))
-    # Ротация успешна, но дом после неё тоже блокируется
-    rotate_mock = AsyncMock(return_value=True)
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
-        patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
-        patch("app.services.scrape_pipeline.save_house_catalog_enrichment", return_value={}),
-        patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_pipeline.settings") as mock_settings,
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
-        ),
-    ):
-        # Лимит 1 ротация: после неё бюджет исчерпан → abort на следующей серии блоков
-        mock_settings.avito_proxy_max_rotations = 1
-        mock_settings.avito_proxy_rotate_url = "http://rotate.test/"
-        mock_settings.avito_proxy_rotate_settle_s = 0.0
-        mock_settings.scraper_fetch_mode = "curl_cffi"
-        mock_settings.scraper_proxy_url = None
-        mock_settings.scraper_skip_seen_today = False
-        # #2202: bare MagicMock settings auto-truthy on any attribute access — pin
-        # explicitly off so _rotate_avito_proxy takes the changeip path this test
-        # asserts on (pool path is covered separately in test_scrape_pipeline_proxy_pool.py).
-        mock_settings.use_proxy_pool_curl = False
-
-        result = await run_avito_pipeline(
-            MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
-        )
-
-    assert isinstance(result, PipelineResult)
-    # Ровно 1 ротация (бюджет = 1)
-    assert rotate_mock.await_count == 1
-    # Enrichment abort'нут (есть AVITO_BLOCKED в ошибках)
-    assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
-    # Listings всё равно сохранены
-    assert result.counters.lots_inserted == len(lots)
-
-
-# ---------------------------------------------------------------------------
-# Тест 3: один блок (ниже порога) — ротация НЕ вызывается (#1820 graceful)
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_single_block_below_threshold_no_rotation() -> None:
-    """Один блок (< порога) — _rotate_proxy_ip НЕ вызывается.
-
-    Graceful per-item #1820: единственный блок пишется в errors, enrichment
-    продолжается, ротация не тратится.
-    """
-    n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
-    # Только 2 дома: первый блокируется, второй успешен
-    lots = [
-        _make_lot("1", "https://www.avito.ru/catalog/houses/ekb/a/100"),
-        _make_lot("2", "https://www.avito.ru/catalog/houses/ekb/b/200"),
-    ]
-    assert n >= 2, "Порог должен быть >= 2 чтобы тест был корректен"
-    scraper = _block_scraper(lots)
-
-    fetch_house = AsyncMock(side_effect=[AvitoBlockedError("HTTP 403"), MagicMock()])
-    rotate_mock = AsyncMock(return_value=True)
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(2, 0)),
-        patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
-        patch(
-            "app.services.scrape_pipeline.save_house_catalog_enrichment",
-            return_value={"house_id": 55},
-        ),
-        patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
-        ),
-    ):
-        result = await run_avito_pipeline(
-            MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
-        )
-
-    # Ротация НЕ вызвана — один блок ниже порога
-    assert rotate_mock.await_count == 0
-    # Первый дом упал, второй обогащён
-    assert result.counters.houses_failed == 1
-    assert result.counters.houses_enriched == 1
-    assert any("BLOCKED_HOUSE" in e for e in result.counters.errors)

tradein-mvp/backend/tests/test_cian_city_sweep.py

@@ -1,16 +1,18 @@
-"""Offline unit tests for run_cian_city_sweep (#860).
+"""Offline unit tests for CianCitySweepCounters + admin sweep endpoints (#860).
 
-Pure & fast: NO live network, NO DB. Mirrors test_city_sweep.
+Pure & fast: NO live network, NO DB.
-Asserts orchestration semantics only.
+
+Legacy `run_cian_city_sweep` orchestration behavior (SERP→save→detail→houses phase
+sequencing, cooperative cancel, consecutive-SERP-failure abort, per-item error
+tolerance) удалён вместе с `app.services.scrape_pipeline` (#2397 Part E1) —
+эквивалентная regression-coverage теперь в
+`test_scraper_kit_pipeline_parity2.py::test_cian_city_sweep` (kit
+`run_cian_city_sweep`).
 
 Coverage:
-  - Все фазы (SERP → save → detail → houses) вызываются в нужном порядке
+  - CianCitySweepCounters корректно заполняются (dataclass shape)
-  - CianCitySweepCounters корректно заполняются
+  - _cian_anchor_timeout_s watchdog-формула (#2160)
-  - Cooperative cancel прерывает sweep перед anchor'ом
+  - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI (config/matcher)
-  - enrich_houses=False пропускает houses-фазу
-  - Consecutive SERP failures → mark_banned + abort
-  - Per-item detail/house error не валит весь sweep
-  - mark_done вызывается при нормальном завершении
 """
 
 from __future__ import annotations
@@ -19,117 +21,13 @@ import os
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
 
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock
-
 import pytest
 
-from app.services import scrape_pipeline
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.cian import CianScraper
-
-# Три тестовых anchor'а — не все 5, чтобы тесты были быстрее
-TEST_ANCHORS: list[tuple[float, float, str]] = [
-    (56.8400, 60.6050, "A1"),
-    (56.7950, 60.5300, "A2"),
-    (56.8970, 60.6100, "A3"),
-]
-
-
-def _fake_lot(
-    offer_id: str, *, nb: bool = False, segment: str | None = "novostroyki"
-) -> ScrapedLot:
-    """Создаём ScrapedLot с опциональным newbuilding-ссылкой.
-
-    segment по умолчанию "novostroyki" — run_cian_city_sweep теперь NB-only
-    (newbuilding_only=True), поэтому дефолтные lots должны проходить SERP-фильтр.
-    """
-    return ScrapedLot(
-        source="cian",
-        source_url=f"https://ekb.cian.ru/sale/flat/{offer_id}/",
-        source_id=offer_id,
-        address="Екатеринбург, улица Тестовая, 1",
-        price_rub=5_000_000,
-        area_m2=50.0,
-        rooms=2,
-        listing_segment=segment,
-        house_source="cian_newbuilding" if nb else "cian",
-        house_ext_id="99999" if nb else None,
-    )
-
-
-class _FakeRuns:
-    """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel."""
-
-    def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
-        self.cancel_at_anchor = cancel_at_anchor
-        self._is_cancelled_calls = 0
-        self.heartbeats: list[dict[str, int]] = []
-        self.done: dict[str, int] | None = None
-        self.failed: tuple[str, dict[str, int]] | None = None
-        self.banned: tuple[str, dict[str, int]] | None = None
-
-    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
-        self._is_cancelled_calls += 1
-        if self.cancel_at_anchor is None:
-            return False
-        return self._is_cancelled_calls >= self.cancel_at_anchor
-
-    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.heartbeats.append(dict(counters))
-
-    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.done = dict(counters)
-
-    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.failed = (error, dict(counters))
-
-    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.banned = (error, dict(counters))
-
-
-@pytest.fixture(autouse=True)
-def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
-    """asyncio.sleep → instant (no test delays)."""
-
-    async def _instant(_secs: float) -> None:
-        return None
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
-
-
-@pytest.fixture(autouse=True)
-def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Нейтрализуем CianScraper.__init__/__aenter__/__aexit__ — нет DB вызовов."""
-
-    def _init(self: CianScraper) -> None:
-        self.request_delay_sec = 0.0
-        self._cffi = None
-
-    async def _aenter(self: CianScraper) -> CianScraper:
-        return self
-
-    async def _aexit(self: CianScraper, *_args: Any) -> None:
-        return None
-
-    monkeypatch.setattr(CianScraper, "__init__", _init)
-    monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
-    monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-
-
 # ── CianCitySweepCounters ───────────────────────────────────────────────────
 
 
 def test_cian_sweep_counters_defaults() -> None:
-    from app.services.scrape_pipeline import CianCitySweepCounters
+    from scraper_kit.orchestration.pipeline import CianCitySweepCounters
 
     c = CianCitySweepCounters()
     assert c.anchors_total == 0
@@ -142,7 +40,7 @@ def test_cian_sweep_counters_defaults() -> None:
 def test_cian_sweep_counters_to_dict_all_keys() -> None:
     from dataclasses import fields
 
-    from app.services.scrape_pipeline import CianCitySweepCounters
+    from scraper_kit.orchestration.pipeline import CianCitySweepCounters
 
     c = CianCitySweepCounters()
     d = c.to_dict()
@@ -150,531 +48,6 @@ def test_cian_sweep_counters_to_dict_all_keys() -> None:
     assert set(d.keys()) == expected
 
 
-# ── Full sweep — фазы и агрегация ──────────────────────────────────────────
-
-
-async def test_sweep_iterates_all_anchors_serp_and_save(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """SERP вызывается для каждого anchor, save_listings получает накопленные lots."""
-    serp_calls: list[str] = []
-    save_calls: list[int] = []
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        serp_calls.append(f"{lat:.4f}")
-        return [_fake_lot(f"{lat}-{i}") for i in range(3)]
-
-    def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
-        save_calls.append(len(lots))
-        return len(lots), 0
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
-    # Stub detail + houses fetchers (возвращают None → graceful skip)
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        AsyncMock(return_value=None),
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=1,
-        anchors=TEST_ANCHORS,
-        pages_per_anchor=2,
-        detail_top_n=0,  # skip detail phase
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    assert len(serp_calls) == len(TEST_ANCHORS)
-    assert len(save_calls) == len(TEST_ANCHORS)
-    assert counters.anchors_done == len(TEST_ANCHORS)
-    assert counters.lots_fetched == len(TEST_ANCHORS) * 3
-    assert counters.lots_inserted == len(TEST_ANCHORS) * 3
-    assert counters.errors_count == 0
-    assert fake.done is not None
-    assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3
-    assert fake.failed is None
-    assert fake.banned is None
-
-
-async def test_newbuilding_only_drops_secondary_before_save(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """newbuilding_only=True (default): save_listings получает только novostroyki-lots.
-
-    SERP отдаёт mixed-сегмент выдачу — вторичкой авторитетно владеет run_cian_full_load,
-    поэтому sweep её отбрасывает до save и считает в lots_dropped_secondary.
-    """
-    saved_lots: list[list[ScrapedLot]] = []
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        # 1 novostroyki + 2 vtorichka на anchor
-        return [
-            _fake_lot(f"{lat}-nb", segment="novostroyki"),
-            _fake_lot(f"{lat}-v1", segment="vtorichka"),
-            _fake_lot(f"{lat}-v2", segment="vtorichka"),
-        ]
-
-    def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
-        saved_lots.append(list(lots))
-        return len(lots), 0
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=1,
-        anchors=TEST_ANCHORS,
-        pages_per_anchor=2,
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-        newbuilding_only=True,
-    )
-
-    # Каждый save-вызов содержит только novostroyki-lot
-    assert len(saved_lots) == len(TEST_ANCHORS)
-    for lots in saved_lots:
-        assert len(lots) == 1
-        assert all(lot.listing_segment == "novostroyki" for lot in lots)
-    # Counters: fetched все 3×N, сохранён 1×N, отброшено 2×N
-    assert counters.lots_fetched == len(TEST_ANCHORS) * 3
-    assert counters.lots_inserted == len(TEST_ANCHORS) * 1
-    assert counters.lots_dropped_secondary == len(TEST_ANCHORS) * 2
-
-
-async def test_newbuilding_only_false_saves_all_segments(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """newbuilding_only=False: фильтр отключён — save получает и вторичку, и новостройки."""
-    saved_lots: list[list[ScrapedLot]] = []
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        return [
-            _fake_lot(f"{lat}-nb", segment="novostroyki"),
-            _fake_lot(f"{lat}-v1", segment="vtorichka"),
-        ]
-
-    def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
-        saved_lots.append(list(lots))
-        return len(lots), 0
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=1,
-        anchors=TEST_ANCHORS,
-        pages_per_anchor=2,
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-        newbuilding_only=False,
-    )
-
-    assert len(saved_lots) == len(TEST_ANCHORS)
-    for lots in saved_lots:
-        assert len(lots) == 2
-    assert counters.lots_inserted == len(TEST_ANCHORS) * 2
-    assert counters.lots_dropped_secondary == 0
-
-
-async def test_detail_phase_runs_and_fills_counters(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """detail-фаза вызывается и счётчики заполняются."""
-    from app.services.scrapers.cian_detail import DetailEnrichment
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        return [_fake_lot(f"{lat}-{i}") for i in range(2)]
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-
-    # DB mock returning 2 rows for detail query
-    mock_db = MagicMock()
-    mock_db.execute.return_value.mappings.return_value.all.return_value = [
-        {"id": 101, "source_url": "https://ekb.cian.ru/sale/flat/101/"},
-        {"id": 102, "source_url": "https://ekb.cian.ru/sale/flat/102/"},
-    ]
-
-    fake_enrichment = DetailEnrichment(cian_id=101)
-    mock_fetch_detail = AsyncMock(return_value=fake_enrichment)
-    mock_save_detail = MagicMock()
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        mock_fetch_detail,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.save_detail_enrichment",
-        mock_save_detail,
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=mock_db,
-        run_id=2,
-        anchors=TEST_ANCHORS[:1],  # один anchor
-        detail_top_n=2,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    assert counters.detail_attempted == 2
-    assert counters.detail_enriched == 2
-    assert counters.detail_failed == 0
-    assert mock_fetch_detail.call_count == 2
-    assert mock_save_detail.call_count == 2
-    assert fake.done is not None
-
-
-async def test_enrich_houses_false_skips_houses_phase(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """enrich_houses=False → houses-фаза не выполняется."""
-    nb_fetch_calls: list[str] = []
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        return [_fake_lot("1001", nb=True)]
-
-    async def fake_fetch_nb(url: str, **_: Any) -> None:
-        nb_fetch_calls.append(url)
-        return None
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_newbuilding.fetch_newbuilding",
-        fake_fetch_nb,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        AsyncMock(return_value=None),
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=3,
-        anchors=TEST_ANCHORS[:1],
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    assert len(nb_fetch_calls) == 0
-    assert counters.houses_attempted == 0
-    assert counters.houses_enriched == 0
-    assert fake.done is not None
-
-
-async def test_cancel_midway_stops_remaining(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Cooperative cancel перед anchor #2 → anchor #1 выполняется, #2+ нет."""
-    serp_calls: list[str] = []
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        serp_calls.append(f"{lat:.4f}")
-        return [_fake_lot(f"{lat}-1")]
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        AsyncMock(return_value=None),
-    )
-
-    fake = _FakeRuns(cancel_at_anchor=2)  # cancel при is_cancelled_calls >= 2 → перед anchor #2
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=4,
-        anchors=TEST_ANCHORS,
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    assert len(serp_calls) == 1, "только anchor #1 должен выполниться"
-    assert counters.anchors_done == 1
-    assert fake.done is None
-    assert fake.banned is None
-
-
-async def test_consecutive_serp_failures_abort_with_banned(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """N подряд SERP-ошибок → mark_banned + abort sweep."""
-    from app.services.scrape_pipeline import CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
-
-    attempt_count = 0
-
-    async def fake_fetch_fail(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        nonlocal attempt_count
-        attempt_count += 1
-        raise RuntimeError("cian blocked")
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_fail)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    # Передаём больше anchor'ов чем CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
-    anchors = [*TEST_ANCHORS, (56.77, 60.55, "A4"), (56.865, 60.62, "A5")]
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=5,
-        anchors=anchors,
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    assert attempt_count == CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
-    assert counters.errors_count == CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
-    assert fake.banned is not None, "mark_banned должен быть вызван"
-    assert "consecutive" in fake.banned[0]
-    assert fake.done is None
-
-
-async def test_per_item_detail_error_does_not_abort_sweep(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Ошибка при detail-обогащении одного листинга не валит sweep."""
-    call_count = 0
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        return [_fake_lot(f"lot-{idx}") for idx in range(2)]
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-
-    mock_db = MagicMock()
-    # Возвращаем 2 listing-а для detail
-    mock_db.execute.return_value.mappings.return_value.all.return_value = [
-        {"id": 201, "source_url": "https://ekb.cian.ru/sale/flat/201/"},
-        {"id": 202, "source_url": "https://ekb.cian.ru/sale/flat/202/"},
-    ]
-
-    async def fetch_detail_sometimes_fails(url: str, **_: Any):
-        nonlocal call_count
-        call_count += 1
-        if call_count == 1:
-            raise RuntimeError("transient network error")
-        from app.services.scrapers.cian_detail import DetailEnrichment
-
-        return DetailEnrichment(cian_id=202)
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        fetch_detail_sometimes_fails,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.save_detail_enrichment",
-        MagicMock(),
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=mock_db,
-        run_id=6,
-        anchors=TEST_ANCHORS[:1],
-        detail_top_n=2,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    assert counters.detail_attempted == 2
-    assert counters.detail_failed == 1
-    assert counters.detail_enriched == 1
-    assert counters.errors_count >= 1
-    assert fake.done is not None, "sweep должен завершиться mark_done несмотря на ошибку"
-    assert fake.banned is None
-
-
-async def test_mark_done_always_called(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """mark_done вызывается даже при пустом SERP."""
-
-    async def fake_fetch_empty(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        return []
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_empty)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        AsyncMock(return_value=None),
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=7,
-        anchors=TEST_ANCHORS,
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    assert fake.done is not None
-    assert counters.lots_fetched == 0
-    assert counters.anchors_done == len(TEST_ANCHORS)
-
-
-async def test_houses_phase_uses_house_sources_join(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Houses-фаза формирует JOIN на house_sources, не h.ext_source напрямую."""
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        # Один lot с house_source="cian_newbuilding" → попадёт в houses-фазу
-        return [_fake_lot("nb-1", nb=True)]
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        AsyncMock(return_value=None),
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_newbuilding.fetch_newbuilding",
-        AsyncMock(return_value=None),
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_newbuilding.save_newbuilding_enrichment",
-        MagicMock(),
-    )
-
-    mock_db = MagicMock()
-    # DB execute → возвращает одну house-строку
-    mock_db.execute.return_value.mappings.return_value.all.return_value = [
-        {"id": 501, "cian_zhk_url": "https://ekb.cian.ru/kupit-kvartiru/zhk-test/"},
-    ]
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=mock_db,
-        run_id=10,
-        anchors=TEST_ANCHORS[:1],
-        detail_top_n=0,
-        enrich_houses=True,
-        request_delay_sec=0.0,
-    )
-
-    # Проверяем SQL — должен содержать JOIN house_sources, а не h.ext_source
-    assert mock_db.execute.called
-    executed_sql_args = [str(c.args[0]) for c in mock_db.execute.call_args_list]
-    houses_sql_calls = [s for s in executed_sql_args if "house" in s.lower()]
-    assert houses_sql_calls, "должен быть хотя бы один SQL-вызов связанный с houses"
-    houses_query = houses_sql_calls[-1]
-    assert (
-        "house_sources" in houses_query
-    ), f"запрос должен JOIN house_sources, а не обращаться к h.ext_source: {houses_query!r}"
-    assert "ext_source" not in houses_query.split("house_sources")[0] or True  # tolerate join expr
-    # Главное: нет обращения к h.ext_source (старый баг)
-    assert (
-        "h.ext_source" not in houses_query
-    ), f"колонка h.ext_source не существует в houses: {houses_query!r}"
-    assert (
-        "h.ext_id" not in houses_query
-    ), f"колонка h.ext_id не существует в houses: {houses_query!r}"
-
-    assert fake.done is not None, "mark_done должен быть вызван"
-    assert fake.failed is None, "mark_failed НЕ должен быть вызван"
-    assert counters.houses_attempted == 1
-
-
-async def test_houses_db_query_error_graceful_mark_done(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Ошибка SQL в houses-фазе (напр. UndefinedColumn) → graceful: mark_done,
-    НЕ mark_failed, houses_failed увеличивается."""
-
-    async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
-        # Два nb-лота → houses-фаза попытается запросить 2 id
-        return [_fake_lot("nb-10", nb=True), _fake_lot("nb-11", nb=True)]
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail",
-        AsyncMock(return_value=None),
-    )
-
-    mock_db = MagicMock()
-
-    def _db_execute_raises(sql, params=None):
-        sql_text = str(sql)
-        if "house" in sql_text.lower() and "cian_newbuilding" in sql_text:
-            raise RuntimeError("column h.ext_source does not exist")
-        result = MagicMock()
-        result.mappings.return_value.all.return_value = []
-        return result
-
-    mock_db.execute.side_effect = _db_execute_raises
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=mock_db,
-        run_id=11,
-        anchors=TEST_ANCHORS[:1],
-        detail_top_n=0,
-        enrich_houses=True,
-        request_delay_sec=0.0,
-    )
-
-    assert fake.done is not None, "mark_done должен быть вызван даже при ошибке houses-фазы"
-    assert fake.failed is None, "mark_failed НЕ должен быть вызван из-за ошибки houses-фазы"
-    assert counters.houses_failed > 0, "houses_failed должен вырасти при ошибке"
-    assert counters.houses_attempted == 0, "houses_attempted не растёт — до него не дошли"
-
-
 # ── Admin endpoint tests (offline) ──────────────────────────────────────────
 
 
@@ -806,19 +179,23 @@ def test_cian_anchor_timeout_defaults() -> None:
     = 900 + 500 + 180 = 1580s. Фиксированный ANCHOR_TIMEOUT_SEC=240 гильотинил бы якорь в
     SERP-фазе (#2160): каждый SERP-фетч через camoufox 13-45s.
     """
-    t = scrape_pipeline._cian_anchor_timeout_s(
+    from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s
+
+    t = _cian_anchor_timeout_s(
         pages_per_anchor=3,
         request_delay_sec=5.0,
         detail_top_n=10,
         enrich_houses=True,
     )
     assert t == pytest.approx(1580.0)
-    assert t > scrape_pipeline.ANCHOR_TIMEOUT_SEC
+    assert t > ANCHOR_TIMEOUT_SEC
 
 
 def test_cian_anchor_timeout_no_houses() -> None:
     """enrich_houses=False убирает houses-бюджет (180s) из формулы."""
-    t = scrape_pipeline._cian_anchor_timeout_s(
+    from scraper_kit.orchestration.pipeline import _cian_anchor_timeout_s
+
+    t = _cian_anchor_timeout_s(
         pages_per_anchor=3,
         request_delay_sec=5.0,
         detail_top_n=10,
@@ -829,24 +206,12 @@ def test_cian_anchor_timeout_no_houses() -> None:
 
 def test_cian_anchor_timeout_floor_at_anchor_timeout_sec() -> None:
     """Малые параметры не опускают таймаут ниже ANCHOR_TIMEOUT_SEC (max-ветка)."""
-    t = scrape_pipeline._cian_anchor_timeout_s(
+    from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s
+
+    t = _cian_anchor_timeout_s(
         pages_per_anchor=0,
         request_delay_sec=0.0,
         detail_top_n=0,
         enrich_houses=False,
     )
-    assert t == float(scrape_pipeline.ANCHOR_TIMEOUT_SEC)
+    assert t == float(ANCHOR_TIMEOUT_SEC)
-
-
-def test_cian_anchor_timeout_parity_with_scraper_kit() -> None:
-    """Strangler-инвариант: backend-хелпер ≡ scraper_kit-хелпер (обе копии в синхроне)."""
-    from scraper_kit.orchestration.pipeline import (
-        _cian_anchor_timeout_s as kit_timeout_s,
-    )
-
-    for args in (
-        dict(pages_per_anchor=3, request_delay_sec=5.0, detail_top_n=10, enrich_houses=True),
-        dict(pages_per_anchor=5, request_delay_sec=9.0, detail_top_n=20, enrich_houses=False),
-        dict(pages_per_anchor=0, request_delay_sec=0.0, detail_top_n=0, enrich_houses=False),
-    ):
-        assert scrape_pipeline._cian_anchor_timeout_s(**args) == kit_timeout_s(**args)

tradein-mvp/backend/tests/test_city_sweep.py

@@ -18,7 +18,7 @@ import pytest
 
 
 def test_ekb_anchors_count() -> None:
-    from app.services.scrape_pipeline import EKB_ANCHORS
+    from scraper_kit.orchestration.pipeline import EKB_ANCHORS
 
     assert len(EKB_ANCHORS) == 5
     for lat, lon, name in EKB_ANCHORS:
@@ -31,7 +31,7 @@ def test_ekb_anchors_count() -> None:
 
 
 def test_city_sweep_counters_defaults() -> None:
-    from app.services.scrape_pipeline import CitySweepCounters
+    from scraper_kit.orchestration.pipeline import CitySweepCounters
 
     c = CitySweepCounters()
     assert c.anchors_total == 0
@@ -40,7 +40,7 @@ def test_city_sweep_counters_defaults() -> None:
 
 
 def test_city_sweep_counters_to_dict() -> None:
-    from app.services.scrape_pipeline import CitySweepCounters
+    from scraper_kit.orchestration.pipeline import CitySweepCounters
 
     c = CitySweepCounters(anchors_total=5, anchors_done=2, lots_fetched=150)
     d = c.to_dict()
@@ -55,7 +55,7 @@ def test_city_sweep_counters_to_dict() -> None:
 def test_city_sweep_counters_to_dict_all_keys() -> None:
     from dataclasses import fields
 
-    from app.services.scrape_pipeline import CitySweepCounters
+    from scraper_kit.orchestration.pipeline import CitySweepCounters
 
     c = CitySweepCounters()
     d = c.to_dict()
@@ -207,7 +207,8 @@ def test_update_heartbeat_persists_total_seen_new_count_columns() -> None:
 
 def test_mark_done_yandex_counters_populate_columns() -> None:
     """End-to-end shape: YandexCitySweepCounters.to_dict() → mark_done → columns set."""
-    from app.services.scrape_pipeline import YandexCitySweepCounters
+    from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
+
     from app.services.scrape_runs import mark_done
 
     counters = YandexCitySweepCounters(
@@ -230,7 +231,8 @@ def test_mark_done_avito_cian_counters_populate_columns() -> None:
     counters.to_dict() → mark_done, что и Yandex — фикс #1926 должен заполнять
     total_seen/new_count и для них (issue: «то же касается CitySweepCounters»).
     """
-    from app.services.scrape_pipeline import CitySweepCounters
+    from scraper_kit.orchestration.pipeline import CitySweepCounters
+
     from app.services.scrape_runs import mark_done
 
     counters = CitySweepCounters(

tradein-mvp/backend/tests/test_domclick_sweep.py

@@ -1,14 +1,15 @@
-"""Offline unit tests for run_domclick_city_sweep + DomClick parser.
+"""Offline unit tests for DomClickCitySweepCounters + DomClick parser.
 
-Pure & fast: NO live network, NO DB. Mirrors test_cian_city_sweep orchestration
+Legacy `run_domclick_city_sweep` orchestration behavior (sweep phases, cooperative
-semantics, but DomClick is CITYWIDE (no anchor-loop) и SERP-only (no detail/houses).
+cancel, honest mark_done/mark_failed statuses) удалён вместе с
+`app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная regression-coverage
+теперь в `test_scraper_kit_pipeline_parity2.py::test_domclick_city_sweep_{done,
+blocked_failed}` (kit `run_domclick_city_sweep`).
 
-Coverage:
+Остались:
-  - fetch_city вызывается, save_listings получает накопленные lots с source='domklik'
+  - DomClickCitySweepCounters (dataclass, из scraper_kit) — defaults + to_dict
-  - DomClickCitySweepCounters корректно заполняется (lots_fetched/inserted/updated)
+  - DomClickScraper._map_item — BFF offer-item → ScrapedLot маппинг (не зависит
-  - Cooperative cancel прерывает sweep перед SERP
+    от оркестрации, `app.services.scrapers.domclick` живёт до Part E2)
-  - fetch_city exception не валит sweep (graceful, errors_count++, mark_done)
-  - _parse_html: минимальная DomClick карточка → ScrapedLot с rooms/area/floor/price
 """
 
 from __future__ import annotations
@@ -17,94 +18,15 @@ import os
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
 
-from typing import Any
-from unittest.mock import MagicMock
-
 import pytest
 
-from app.services import scrape_pipeline
-from app.services.scrapers.base import ScrapedLot
 from app.services.scrapers.domclick import DomClickScraper
 
-
-def _fake_lot(offer_id: str) -> ScrapedLot:
-    return ScrapedLot(
-        source="domklik",
-        source_url=f"https://domclick.ru/card/sale__flat__{offer_id}",
-        source_id=offer_id,
-        address="Екатеринбург, улица Тестовая, 1",
-        price_rub=5_000_000,
-        area_m2=50.0,
-        rooms=2,
-        lat=None,
-        lon=None,
-    )
-
-
-class _FakeRuns:
-    """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel."""
-
-    def __init__(self, *, cancel: bool = False) -> None:
-        self.cancel = cancel
-        self.heartbeats: list[dict[str, int]] = []
-        self.done: dict[str, int] | None = None
-        self.failed: tuple[str, dict[str, int]] | None = None
-
-    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
-        return self.cancel
-
-    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.heartbeats.append(dict(counters))
-
-    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.done = dict(counters)
-
-    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.failed = (error, dict(counters))
-
-
-@pytest.fixture(autouse=True)
-def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
-    async def _instant(_secs: float) -> None:
-        return None
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
-
-
-@pytest.fixture(autouse=True)
-def _stub_domclick_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Нейтрализуем DomClickScraper.__init__/__aenter__/__aexit__ — нет DB/browser."""
-
-    def _init(self: DomClickScraper) -> None:
-        self.request_delay_sec = 0.0
-        self.parse_failures = 0
-        self.geo_filtered = 0
-        self.blocked = False
-        self.fetch_errors = 0
-
-    async def _aenter(self: DomClickScraper) -> DomClickScraper:
-        return self
-
-    async def _aexit(self: DomClickScraper, *_args: Any) -> None:
-        return None
-
-    monkeypatch.setattr(DomClickScraper, "__init__", _init)
-    monkeypatch.setattr(DomClickScraper, "__aenter__", _aenter)
-    monkeypatch.setattr(DomClickScraper, "__aexit__", _aexit)
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
-
-
 # ── DomClickCitySweepCounters ───────────────────────────────────────────────
 
 
 def test_domclick_sweep_counters_defaults() -> None:
-    from app.services.scrape_pipeline import DomClickCitySweepCounters
+    from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters
 
     c = DomClickCitySweepCounters()
     assert c.lots_fetched == 0
@@ -119,7 +41,7 @@ def test_domclick_sweep_counters_defaults() -> None:
 def test_domclick_sweep_counters_to_dict_all_keys() -> None:
     from dataclasses import fields
 
-    from app.services.scrape_pipeline import DomClickCitySweepCounters
+    from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters
 
     c = DomClickCitySweepCounters()
     d = c.to_dict()
@@ -127,221 +49,12 @@ def test_domclick_sweep_counters_to_dict_all_keys() -> None:
     assert set(d.keys()) == expected
 
 
-# ── Full sweep — фазы и агрегация ──────────────────────────────────────────
+# ── _map_item ─────────────────────────────────────────────────────────────────
-
-
-async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPatch) -> None:
-    """fetch_city → save_listings получает 2 lots с source='domklik', counters заполнены."""
-    fetch_calls: list[tuple[int, int]] = []
-    save_calls: list[list[ScrapedLot]] = []
-
-    async def fake_fetch_city(
-        self: DomClickScraper,
-        city_id: int,
-        rooms: list[int] | None = None,
-        pages: int = 20,
-    ) -> list[ScrapedLot]:
-        fetch_calls.append((city_id, pages))
-        return [_fake_lot("100"), _fake_lot("101")]
-
-    def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
-        save_calls.append(list(lots))
-        return len(lots), 0
-
-    monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_city)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_domclick_city_sweep(
-        db=MagicMock(),
-        run_id=1,
-        city_id=4,
-        rooms=[1, 2],
-        pages=3,
-        request_delay_sec=0.0,
-    )
-
-    assert len(fetch_calls) == 1
-    assert fetch_calls[0] == (4, 3)
-    assert len(save_calls) == 1
-    assert len(save_calls[0]) == 2
-    assert all(lot.source == "domklik" for lot in save_calls[0])
-    assert counters.lots_fetched == 2
-    assert counters.lots_inserted == 2
-    assert counters.lots_updated == 0
-    assert counters.errors_count == 0
-    assert fake.done is not None
-    assert fake.done["lots_fetched"] == 2
-    assert fake.failed is None
-
-
-async def test_sweep_empty_serp_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Пустой SERP → save не вызывается, но mark_done всё равно вызывается."""
-    save_called = False
-
-    async def fake_fetch_empty(
-        self: DomClickScraper,
-        city_id: int,
-        rooms: list[int] | None = None,
-        pages: int = 20,
-    ) -> list[ScrapedLot]:
-        return []
-
-    def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
-        nonlocal save_called
-        save_called = True
-        return 0, 0
-
-    monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_empty)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_domclick_city_sweep(
-        db=MagicMock(), run_id=2, request_delay_sec=0.0
-    )
-
-    assert save_called is False
-    assert counters.lots_fetched == 0
-    assert fake.done is not None
-
-
-async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Cooperative cancel перед SERP → fetch_city НЕ вызывается, mark_done НЕ вызывается."""
-    fetch_called = False
-
-    async def fake_fetch(
-        self: DomClickScraper,
-        city_id: int,
-        rooms: list[int] | None = None,
-        pages: int = 20,
-    ) -> list[ScrapedLot]:
-        nonlocal fetch_called
-        fetch_called = True
-        return [_fake_lot("1")]
-
-    monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
-
-    fake = _FakeRuns(cancel=True)
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_domclick_city_sweep(
-        db=MagicMock(), run_id=3, request_delay_sec=0.0
-    )
-
-    assert fetch_called is False
-    assert counters.lots_fetched == 0
-    assert fake.done is None
-
-
-async def test_fetch_city_exception_no_reraise(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Исключение в fetch_city → errors_count++, sweep НЕ ре-райзит (graceful).
-
-    Honest-status (#1968): полный крах SERP-фазы (errors_count=1, 0 lots) → mark_failed,
-    НЕ mark_done. Run без единого листинга по внешней причине — это реальный failure,
-    а не успешный пустой прогон. Re-raise при этом НЕ происходит (task завершается чисто).
-    """
-
-    async def fake_fetch_fail(
-        self: DomClickScraper,
-        city_id: int,
-        rooms: list[int] | None = None,
-        pages: int = 20,
-    ) -> list[ScrapedLot]:
-        raise RuntimeError("browser crashed")
-
-    monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_fail)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_domclick_city_sweep(
-        db=MagicMock(), run_id=4, request_delay_sec=0.0
-    )
-
-    assert counters.errors_count == 1
-    assert counters.lots_fetched == 0
-    # 0 lots + error → honest mark_failed (не маскируем как done)
-    assert fake.failed is not None
-    assert fake.done is None
-
-
-async def test_fetch_errors_and_no_lots_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Scraper fetch_errors>0 + 0 лотов (без block) → mark_failed (FIX 3 / closes #1968).
-
-    Нераспознанный block-вариант приходит как fetch_error (не blocked=True), но
-    honest-status всё равно обязан пометить run failed при пустом результате.
-    """
-
-    async def fake_fetch_errs(
-        self: DomClickScraper,
-        city_id: int,
-        rooms: list[int] | None = None,
-        pages: int = 100,
-    ) -> list[ScrapedLot]:
-        # Имитируем не-block ошибки извлечения JSON: blocked остаётся False
-        self.fetch_errors = 3
-        return []
-
-    monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_errs)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_domclick_city_sweep(
-        db=MagicMock(), run_id=6, request_delay_sec=0.0
-    )
-
-    assert counters.lots_fetched == 0
-    assert counters.blocked == 0
-    assert counters.errors_count >= 3  # fetch_errors свёрнуты в errors_count
-    assert fake.failed is not None
-    assert fake.done is None
-
-
-async def test_blocked_and_no_lots_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
-    """QRATOR-блок + 0 лотов → mark_failed (честный статус, closes #1968)."""
-
-    async def fake_fetch_blocked(
-        self: DomClickScraper,
-        city_id: int,
-        rooms: list[int] | None = None,
-        pages: int = 100,
-    ) -> list[ScrapedLot]:
-        # Имитируем QRATOR-блок: установим флаг на scraper и вернём пустой список
-        self.blocked = True
-        return []
-
-    monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_blocked)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_domclick_city_sweep(
-        db=MagicMock(), run_id=5, request_delay_sec=0.0
-    )
-
-    assert counters.lots_fetched == 0
-    assert counters.blocked == 1
-    # Честный статус: mark_failed должен быть вызван, mark_done — нет
-    assert fake.failed is not None
-    assert "QRATOR" in fake.failed[0]
-    assert fake.done is None
-
-
-# ── _map_item via sweep — quick integration smoke ────────────────────────────
 
 
 def test_map_item_basic_mapping() -> None:
     """DomClickScraper._map_item маппит BFF offer-item → ScrapedLot корректно."""
-    scraper = DomClickScraper()  # __init__ stubbed
+    scraper = DomClickScraper()
     item = {
         "id": 111222333,
         "path": "https://domclick.ru/card/sale__flat__111222333",

tradein-mvp/backend/tests/test_pipeline_browser_routing.py

@@ -6,9 +6,12 @@
   1. fetch_detail(browser_fetcher=mock) → DetailEnrichment (browser branch taken)
   2. fetch_detail(browser_fetcher=mock) → AvitoBlockedError при firewall HTML
   3. fetch_house_catalog(browser_fetcher=mock) → browser branch taken
-  4. run_avito_pipeline в browser-mode → scraper._browser установлен,
+  4. Curl mode (default) — browser_fetcher=None, curl path используется
-     fetch_detail/fetch_house_catalog вызваны с browser_fetcher
+
-  5. Curl mode (default) — browser_fetcher=None, curl path используется
+Legacy `run_avito_pipeline` browser/curl-mode integration tests (было ранее секцией 4)
+удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — browser-routing
+покрытие на уровне `fetch_detail`/`fetch_house_catalog`/`AvitoScraper._fetch_serp_html`
+ниже остаётся нетронутым.
 """
 
 from __future__ import annotations
@@ -17,7 +20,7 @@ import json
 import os
 import urllib.parse
 from pathlib import Path
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
@@ -212,135 +215,7 @@ async def test_fetch_house_catalog_browser_firewall_raises() -> None:
         )
 
 
-# ── 4. run_avito_pipeline browser-mode routing ───────────────────────────────
+# ── 4. Curl mode (default) — browser_fetcher stays None ──────────────────────
-
-
-@pytest.mark.asyncio
-async def test_run_avito_pipeline_browser_mode_sets_scraper_browser(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """В browser-mode scraper._browser установлен, fetch_detail/fetch_house_catalog
-    вызываются с browser_fetcher (не с cffi_session).
-    """
-    from app.services import scrape_pipeline
-
-    monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
-
-    mock_bf = make_mock_fetcher(DETAIL_HTML)
-    captured_scraper: list = []
-
-    # Mock AvitoScraper.fetch_around → возвращает пустой список (нет лотов → нет DB)
-    async def mock_fetch_around(
-        self: object, lat: float, lon: float, radius_m: int, **kwargs: object
-    ) -> list:
-        captured_scraper.append(self)
-        return []
-
-    mock_db = MagicMock()
-    mock_db.execute.return_value.mappings.return_value.all.return_value = []
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
-    ):
-        from app.services.scrape_pipeline import run_avito_pipeline
-
-        await run_avito_pipeline(
-            mock_db,
-            lat=56.84,
-            lon=60.60,
-            radius_m=1500,
-            enrich_houses=False,
-            enrich_detail_top_n=0,
-            pages=1,
-            shared_browser=mock_bf,
-        )
-
-    assert len(captured_scraper) == 1
-    scraper_instance = captured_scraper[0]
-    assert scraper_instance._browser is mock_bf
-    # curl session должен быть None — в browser-mode pipeline не создаёт _cffi сессию
-    assert scraper_instance._cffi is None
-
-
-@pytest.mark.asyncio
-async def test_run_avito_pipeline_browser_mode_own_browser_created(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Если shared_browser не передан в browser-mode, BrowserFetcher создаётся
-    и __aenter__/__aexit__ вызываются."""
-    from app.services import scrape_pipeline
-
-    monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
-
-    mock_bf_instance = make_mock_fetcher(DETAIL_HTML)
-    mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance)
-    mock_bf_instance.__aexit__ = AsyncMock(return_value=None)
-
-    mock_db = MagicMock()
-    mock_db.execute.return_value.mappings.return_value.all.return_value = []
-
-    async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
-        return []
-
-    with (
-        patch("app.services.scrape_pipeline.BrowserFetcher", return_value=mock_bf_instance),
-        patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
-    ):
-        await scrape_pipeline.run_avito_pipeline(
-            mock_db,
-            lat=56.84,
-            lon=60.60,
-            enrich_houses=False,
-            enrich_detail_top_n=0,
-            pages=1,
-        )
-
-    mock_bf_instance.__aenter__.assert_awaited_once()
-    mock_bf_instance.__aexit__.assert_awaited_once()
-
-
-# ── 5. Curl mode (default) — browser_fetcher stays None ──────────────────────
-
-
-@pytest.mark.asyncio
-async def test_run_avito_pipeline_curl_mode_no_browser_fetcher(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """В curl-mode (default) browser_fetcher=None, scraper._cffi установлен."""
-    from app.services import scrape_pipeline
-
-    monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "curl_cffi", raising=False)
-
-    captured_scraper: list = []
-
-    async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
-        captured_scraper.append(self)
-        return []
-
-    mock_db = MagicMock()
-    mock_db.execute.return_value.mappings.return_value.all.return_value = []
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
-    ):
-        await scrape_pipeline.run_avito_pipeline(
-            mock_db,
-            lat=56.84,
-            lon=60.60,
-            enrich_houses=False,
-            enrich_detail_top_n=0,
-            pages=1,
-        )
-
-    assert len(captured_scraper) == 1
-    scraper_instance = captured_scraper[0]
-    # curl mode: _cffi должен быть установлен
-    assert scraper_instance._cffi is not None
-    # browser не установлен в curl mode
-    assert getattr(scraper_instance, "_browser", None) is None
 
 
 @pytest.mark.asyncio
@@ -368,7 +243,7 @@ async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
     assert result.item_id == "99887766"
 
 
-# ── 6. SERP routing: real _fetch_serp_html routes through browser ────────────
+# ── 5. SERP routing: real _fetch_serp_html routes through browser ────────────
 
 
 @pytest.mark.asyncio

tradein-mvp/backend/tests/test_provider_rotation_1848.py

@@ -1,245 +0,0 @@
-"""Тесты Fix 2: provider-aware _rotate_proxy_ip (#1848).
-
-Тест 1: source='cian' → cian_proxy_rotate_url используется.
-Тест 2: source='yandex' → yandex_proxy_rotate_url используется.
-Тест 3: source='avito' (default) → avito_proxy_rotate_url (backward compat).
-Тест 4: cian без cian url → fallback на avito_proxy_rotate_url.
-Тест 5: rotate_url=None → returns False (no HTTP call).
-Тест 6: network error → returns False (no crash).
-Тест 7: yandex fallback на avito.
-"""
-
-from __future__ import annotations
-
-import os
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from app.services.scrape_pipeline import _rotate_proxy_ip
-
-# ── Helpers ──────────────────────────────────────────────────────────────────
-
-
-def _async_session_ctx(resp_json: dict | None = None):
-    """Возвращает (async_cm_class, session_mock) для curl_cffi AsyncSession."""
-    session = AsyncMock()
-    if resp_json is not None:
-        mock_resp = MagicMock()
-        mock_resp.json.return_value = resp_json
-        session.get = AsyncMock(return_value=mock_resp)
-    else:
-        session.get = AsyncMock(side_effect=RuntimeError("network error"))
-
-    # curl_cffi AsyncSession используется как `async with AsyncSession(...) as rot:`
-    # Нам нужен callable класс-замена, вызов которого возвращает async context manager.
-    class _MockAsyncSession:
-        def __init__(self, timeout=30):
-            pass
-
-        async def __aenter__(self):
-            return session
-
-        async def __aexit__(self, *_):
-            pass
-
-    return _MockAsyncSession, session
-
-
-# ── Тест 1: cian_proxy_rotate_url ────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_cian_uses_cian_url() -> None:
-    """source='cian' → cian_proxy_rotate_url (не avito_proxy_rotate_url)."""
-    mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "1.2.3.4"})
-
-    with (
-        patch("app.services.scrape_pipeline.settings") as s,
-        patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-    ):
-        s.cian_proxy_rotate_url = "http://cian-rotate.test/"
-        s.avito_proxy_rotate_url = "http://avito-rotate.test/"
-        s.yandex_proxy_rotate_url = None
-        s.cian_proxy_max_rotations = 4
-        s.avito_proxy_max_rotations = 4
-        s.yandex_proxy_max_rotations = 4
-        s.avito_proxy_rotate_settle_s = 0.0
-
-        result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
-
-    assert result is True
-    called_url: str = session.get.call_args[0][0]
-    assert "cian-rotate.test" in called_url
-    assert "avito-rotate.test" not in called_url
-
-
-# ── Тест 2: yandex_proxy_rotate_url ─────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_yandex_uses_yandex_url() -> None:
-    """source='yandex' → yandex_proxy_rotate_url."""
-    mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "5.6.7.8"})
-
-    with (
-        patch("app.services.scrape_pipeline.settings") as s,
-        patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-    ):
-        s.yandex_proxy_rotate_url = "http://yandex-rotate.test/"
-        s.avito_proxy_rotate_url = "http://avito-rotate.test/"
-        s.cian_proxy_rotate_url = None
-        s.yandex_proxy_max_rotations = 4
-        s.avito_proxy_max_rotations = 4
-        s.cian_proxy_max_rotations = 4
-        s.avito_proxy_rotate_settle_s = 0.0
-
-        result = await _rotate_proxy_ip(reason="timeout", rotations_done=0, source="yandex")
-
-    assert result is True
-    called_url: str = session.get.call_args[0][0]
-    assert "yandex-rotate.test" in called_url
-    assert "avito-rotate.test" not in called_url
-
-
-# ── Тест 3: avito backward compat ────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_avito_backward_compat() -> None:
-    """source не передан (дефолт 'avito') → avito_proxy_rotate_url."""
-    mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "9.9.9.9"})
-
-    with (
-        patch("app.services.scrape_pipeline.settings") as s,
-        patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-    ):
-        s.avito_proxy_rotate_url = "http://avito-only.test/"
-        s.cian_proxy_rotate_url = None
-        s.yandex_proxy_rotate_url = None
-        s.avito_proxy_max_rotations = 4
-        s.cian_proxy_max_rotations = 4
-        s.yandex_proxy_max_rotations = 4
-        s.avito_proxy_rotate_settle_s = 0.0
-
-        # Вызов без source= — должен работать как раньше
-        result = await _rotate_proxy_ip(reason="block", rotations_done=0)
-
-    assert result is True
-    called_url: str = session.get.call_args[0][0]
-    assert "avito-only.test" in called_url
-
-
-# ── Тест 4: cian fallback на avito когда нет cian url ─────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_cian_fallback_to_avito() -> None:
-    """cian_proxy_rotate_url=None → fallback на avito_proxy_rotate_url."""
-    mock_session_cls, session = _async_session_ctx(resp_json={})
-
-    with (
-        patch("app.services.scrape_pipeline.settings") as s,
-        patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-    ):
-        s.cian_proxy_rotate_url = None
-        s.avito_proxy_rotate_url = "http://avito-fallback.test/"
-        s.yandex_proxy_rotate_url = None
-        s.cian_proxy_max_rotations = 4
-        s.avito_proxy_max_rotations = 4
-        s.yandex_proxy_max_rotations = 4
-        s.avito_proxy_rotate_settle_s = 0.0
-
-        result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
-
-    assert result is True
-    called_url: str = session.get.call_args[0][0]
-    assert "avito-fallback.test" in called_url
-
-
-# ── Тест 5: rotate_url=None → returns False ──────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_returns_false_when_no_url() -> None:
-    """Никакой URL не задан → False, HTTP-запрос не делается."""
-    mock_session_cls, session = _async_session_ctx(resp_json={})
-
-    with (
-        patch("app.services.scrape_pipeline.settings") as s,
-        patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
-    ):
-        s.cian_proxy_rotate_url = None
-        s.avito_proxy_rotate_url = None
-        s.yandex_proxy_rotate_url = None
-        s.cian_proxy_max_rotations = 4
-        s.avito_proxy_max_rotations = 4
-        s.yandex_proxy_max_rotations = 4
-
-        result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
-
-    assert result is False
-    # HTTP-запрос не делался
-    session.get.assert_not_called()
-
-
-# ── Тест 6: network error → returns False ────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_returns_false_on_network_error() -> None:
-    """Ошибка сети → returns False (не крашит caller)."""
-    mock_session_cls, _session = _async_session_ctx(resp_json=None)  # raises RuntimeError
-
-    with (
-        patch("app.services.scrape_pipeline.settings") as s,
-        patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-    ):
-        s.cian_proxy_rotate_url = "http://cian-rotate.test/"
-        s.avito_proxy_rotate_url = None
-        s.yandex_proxy_rotate_url = None
-        s.cian_proxy_max_rotations = 4
-        s.avito_proxy_max_rotations = 4
-        s.yandex_proxy_max_rotations = 4
-        s.avito_proxy_rotate_settle_s = 0.0
-        s.proxy_rotate_attempts = 1  # одна попытка, без retry-sleep (#1950)
-        s.proxy_rotate_attempt_timeout_s = 8.0
-
-        result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
-
-    assert result is False
-
-
-# ── Тест 7: yandex fallback на avito ─────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_proxy_ip_yandex_fallback_to_avito() -> None:
-    """yandex_proxy_rotate_url=None → fallback на avito_proxy_rotate_url."""
-    mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "11.22.33.44"})
-
-    with (
-        patch("app.services.scrape_pipeline.settings") as s,
-        patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-    ):
-        s.yandex_proxy_rotate_url = None
-        s.avito_proxy_rotate_url = "http://shared-proxy.test/"
-        s.cian_proxy_rotate_url = None
-        s.yandex_proxy_max_rotations = 4
-        s.avito_proxy_max_rotations = 4
-        s.cian_proxy_max_rotations = 4
-        s.avito_proxy_rotate_settle_s = 0.0
-
-        result = await _rotate_proxy_ip(reason="503", rotations_done=0, source="yandex")
-
-    assert result is True
-    called_url: str = session.get.call_args[0][0]
-    assert "shared-proxy.test" in called_url

tradein-mvp/backend/tests/test_scrape_pipeline.py

@@ -1,270 +0,0 @@
-"""Offline smoke for scrape_pipeline. Mocks scrapers + DB session."""
-
-from datetime import date, timedelta
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-from app.services.scrape_pipeline import (
-    _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
-    PipelineCounters,
-    PipelineResult,
-    run_avito_full_load,
-    run_avito_pipeline,
-)
-from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
-from app.services.scrapers.base import ScrapedLot
-
-
-def _make_lot(
-    item_id: str,
-    house_url: str | None = None,
-    price: int = 5_000_000,
-) -> ScrapedLot:
-    return ScrapedLot(
-        source="avito",
-        source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
-        source_id=item_id,
-        price_rub=price,
-        address="Test address",
-        house_source="avito" if house_url else None,
-        house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
-        house_url=house_url,
-        listing_segment="vtorichka",
-    )
-
-
-def test_counters_dataclass_default() -> None:
-    c = PipelineCounters()
-    assert c.lots_fetched == 0
-    assert c.unique_houses == 0
-    assert c.errors == []
-
-
-@pytest.mark.asyncio
-async def test_pipeline_search_failure_graceful() -> None:
-    """Если AvitoScraper.fetch_around raises — pipeline возвращает PipelineResult
-    с errors, не падает наружу."""
-    mock_db = MagicMock()
-    mock_scraper = MagicMock()
-    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
-    mock_scraper.__aexit__ = AsyncMock(return_value=None)
-    mock_scraper.fetch_around = AsyncMock(side_effect=RuntimeError("test net error"))
-
-    with patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper):
-        result = await run_avito_pipeline(
-            mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
-        )
-
-    assert isinstance(result, PipelineResult)
-    assert result.counters.lots_fetched == 0
-    assert len(result.counters.errors) == 1
-    assert "search:" in result.counters.errors[0]
-
-
-@pytest.mark.asyncio
-async def test_pipeline_group_by_house_unique_set() -> None:
-    """3 lots с house_url'ами (2 одинаковых, 1 уникальный) → unique_houses=2."""
-    mock_db = MagicMock()
-    lots = [
-        _make_lot("1", "https://www.avito.ru/catalog/houses/ekb/test/100"),
-        _make_lot("2", "https://www.avito.ru/catalog/houses/ekb/test/100"),  # дубль
-        _make_lot("3", "https://www.avito.ru/catalog/houses/ekb/other/200"),
-        _make_lot("4", None),  # без дома
-    ]
-    mock_scraper = MagicMock()
-    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
-    mock_scraper.__aexit__ = AsyncMock(return_value=None)
-    mock_scraper.fetch_around = AsyncMock(return_value=lots)
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(4, 0)),
-    ):
-        result = await run_avito_pipeline(
-            mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
-        )
-
-    assert result.counters.lots_fetched == 4
-    assert result.counters.lots_inserted == 4
-    assert result.counters.unique_houses == 2
-
-
-# ── run_avito_full_load: incremental_days → since plumbing (#avito-split) ─────
-
-
-def _full_load_scraper() -> MagicMock:
-    """AvitoScraper mock с async-CM + AsyncMock fetch_all_secondary."""
-    scraper = MagicMock()
-    scraper.__aenter__ = AsyncMock(return_value=scraper)
-    scraper.__aexit__ = AsyncMock(return_value=None)
-    scraper.fetch_all_secondary = AsyncMock(return_value=[])
-    return scraper
-
-
-@pytest.mark.asyncio
-async def test_full_load_incremental_days_passes_since() -> None:
-    """incremental_days=2 → fetch_all_secondary(since=date.today()-2d)."""
-    mock_db = MagicMock()
-    scraper = _full_load_scraper()
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
-    ):
-        await run_avito_full_load(mock_db, run_id=1, incremental_days=2)
-
-    kwargs = scraper.fetch_all_secondary.call_args.kwargs
-    assert kwargs["since"] == date.today() - timedelta(days=2)
-
-
-@pytest.mark.asyncio
-async def test_full_load_no_incremental_days_since_none() -> None:
-    """incremental_days=None (default) → fetch_all_secondary(since=None) — exhaustive."""
-    mock_db = MagicMock()
-    scraper = _full_load_scraper()
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
-    ):
-        await run_avito_full_load(mock_db, run_id=2)
-
-    kwargs = scraper.fetch_all_secondary.call_args.kwargs
-    assert kwargs["since"] is None
-
-
-# ── #1820: AvitoBlockedError per-item не роняет весь pipeline ─────────────────
-
-
-def _block_test_scraper(lots: list[ScrapedLot]) -> MagicMock:
-    scraper = MagicMock()
-    scraper.__aenter__ = AsyncMock(return_value=scraper)
-    scraper.__aexit__ = AsyncMock(return_value=None)
-    scraper.fetch_around = AsyncMock(return_value=lots)
-    scraper._cffi = None
-    return scraper
-
-
-@pytest.mark.asyncio
-async def test_pipeline_single_house_block_does_not_crash() -> None:
-    """#1820: один заблокированный house-фетч НЕ роняет прогон.
-
-    Listings из SEARCH+SAVE сохранены, блок зафиксирован в counters (houses_failed/errors),
-    остальные дома обогащаются дальше (enrichment не прерван < порога).
-    """
-    mock_db = MagicMock()
-    lots = [
-        _make_lot("1", "https://www.avito.ru/catalog/houses/ekb/a/100"),
-        _make_lot("2", "https://www.avito.ru/catalog/houses/ekb/b/200"),
-    ]
-    scraper = _block_test_scraper(lots)
-
-    # 1-й house блокируется, 2-й — успех.
-    fetch_house = AsyncMock(side_effect=[AvitoBlockedError("HTTP 403"), MagicMock()])
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(2, 0)),
-        patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
-        patch(
-            "app.services.scrape_pipeline.save_house_catalog_enrichment",
-            return_value={"house_id": 42},
-        ),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
-        ),
-    ):
-        result = await run_avito_pipeline(
-            mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
-        )
-
-    assert isinstance(result, PipelineResult)
-    # listings сохранены всегда
-    assert result.counters.lots_inserted == 2
-    # один дом упал по блоку, второй обогащён
-    assert result.counters.houses_failed == 1
-    assert result.counters.houses_enriched == 1
-    assert fetch_house.await_count == 2  # enrichment НЕ прерван — оба дома попробованы
-    assert any("BLOCKED_HOUSE" in e for e in result.counters.errors)
-    assert 42 in result.touched_house_ids
-
-
-@pytest.mark.asyncio
-async def test_pipeline_consecutive_house_blocks_abort_but_return_result() -> None:
-    """#1820: N consecutive house-блоков прерывают enrichment, но PipelineResult возвращён."""
-    mock_db = MagicMock()
-    n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
-    # Создаём n+2 дома — после n блоков подряд enrichment прерывается.
-    lots = [
-        _make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{100 + i}")
-        for i in range(n + 2)
-    ]
-    scraper = _block_test_scraper(lots)
-
-    fetch_house = AsyncMock(side_effect=AvitoRateLimitedError("HTTP 429"))
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
-        patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
-        patch("app.services.scrape_pipeline.save_house_catalog_enrichment", return_value={}),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
-        ),
-    ):
-        result = await run_avito_pipeline(
-            mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
-        )
-
-    assert isinstance(result, PipelineResult)
-    assert result.counters.lots_inserted == len(lots)  # listings сохранены
-    # enrichment прерван ровно после порога — лишние дома НЕ фетчились
-    assert fetch_house.await_count == n
-    assert result.counters.houses_failed == n
-    assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
-
-
-@pytest.mark.asyncio
-async def test_pipeline_detail_block_aborts_skips_step5b() -> None:
-    """#1820: consecutive detail-блоки прерывают detail-фазу И Step 5b (houses из detail)."""
-    mock_db = MagicMock()
-    n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
-    scraper = _block_test_scraper([])  # SERP пуст → сразу detail-фаза по priority_rows
-
-    # priority_rows: n+1 строк, все блокируются в detail.
-    row = MagicMock()
-    row.__getitem__ = lambda self, key: (
-        "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
-    )
-    mappings = MagicMock()
-    mappings.all.return_value = [row] * (n + 1)
-    execute = MagicMock()
-    execute.mappings.return_value = mappings
-    mock_db.execute.return_value = execute
-
-    fetch_detail = AsyncMock(side_effect=AvitoBlockedError("HTTP 403"))
-    fetch_house = AsyncMock()  # должен НЕ вызываться (Step 5b пропущен)
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.fetch_detail", fetch_detail),
-        patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch(
-            "curl_cffi.requests.AsyncSession",
-            return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
-        ),
-    ):
-        result = await run_avito_pipeline(
-            mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=n + 1
-        )
-
-    assert isinstance(result, PipelineResult)
-    assert fetch_detail.await_count == n  # detail прерван на пороге
-    assert result.counters.detail_failed == n
-    assert fetch_house.await_count == 0  # Step 5b пропущен (enrichment_aborted)

tradein-mvp/backend/tests/test_scrape_pipeline_proxy_pool.py

@@ -1,408 +0,0 @@
-"""Tests for scrape_pipeline.py proxy-pool wiring (#2202).
-
-Issue #2202: боевые скрейперы (scrape_pipeline.py Avito orchestrator) читали только
-статический settings.scraper_proxy_url — один mobile-IP на все прогоны, SPOF. Этот
-файл покрывает новые pool-aware helpers:
-
-  - _pool_proxy_dict: acquire() health-filtered lease за флагом USE_PROXY_POOL_CURL,
-    fallback на статический env при флаге off / пустом пуле / ошибке.
-  - _rotate_pool_proxy: переключение на ДРУГОЙ здоровый прокси из пула при бане
-    (mark_health(False)+release текущего, acquire() следующего, мутация session.proxies).
-  - _rotate_avito_proxy: единая точка ротации — пул first, changeip fallback.
-
-Acceptance-тест (issue AC4): пул с 1 «забаненным» прокси — ротация выбирает
-СЛЕДУЮЩИЙ здоровый, не зависший на забаненном и не переиспользующий только что
-провалившийся lease раньше более свежего здорового кандидата.
-
-Offline: FakeSession эмулирует scrape_proxies-таблицу (тот же паттерн, что
-tests/services/test_proxy_pool.py), без live БД/сети.
-"""
-
-from __future__ import annotations
-
-import os
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-
-from datetime import UTC, datetime
-from types import SimpleNamespace
-from typing import Any
-from unittest.mock import AsyncMock, patch
-
-import pytest
-
-from app.services import proxy_pool
-from app.services.proxy_pool import MAX_CONSECUTIVE_FAILS
-from app.services.scrape_pipeline import (
-    _pool_proxy_dict,
-    _release_pool_lease,
-    _rotate_avito_proxy,
-    _rotate_pool_proxy,
-)
-
-# ── stateful fake session (mirrors tests/services/test_proxy_pool.py) ─────────
-
-
-class _FakeResult:
-    def __init__(self, rows: list[dict[str, Any]]):
-        self._rows = rows
-
-    def mappings(self) -> _FakeResult:
-        return self
-
-    def fetchone(self) -> dict[str, Any] | None:
-        return self._rows[0] if self._rows else None
-
-    def all(self) -> list[dict[str, Any]]:
-        return list(self._rows)
-
-
-class FakeSession:
-    """Эмуляция Session поверх in-memory списка scrape_proxies-строк."""
-
-    def __init__(self, rows: list[dict[str, Any]]):
-        self.rows = rows
-
-    def _by_id(self, pid: int) -> dict[str, Any] | None:
-        return next((r for r in self.rows if r["id"] == pid), None)
-
-    def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
-        sql = str(stmt)
-        p = params or {}
-
-        if "FOR UPDATE SKIP LOCKED" in sql:  # acquire SELECT
-            provider = p["provider"]
-            max_fails = p["max_fails"]
-            cands = [
-                r
-                for r in self.rows
-                if r["enabled"]
-                and r["consecutive_fails"] < max_fails
-                and r["provider_affinity"] in (provider, "any")
-                and r["leased_by"] is None
-            ]
-            cands.sort(
-                key=lambda r: (
-                    r["last_ok_at"] is None,
-                    r["last_ok_at"] or datetime.min.replace(tzinfo=UTC),
-                    r["id"],
-                )
-            )
-            return _FakeResult(cands[:1])
-
-        if "SET leased_by = CAST(:run_id" in sql:  # acquire lease UPDATE
-            row = self._by_id(p["id"])
-            if row is not None:
-                row["leased_by"] = p["run_id"]
-                row["leased_at"] = datetime.now(UTC)
-            return _FakeResult([])
-
-        if "SET leased_by = NULL" in sql:  # release
-            row = self._by_id(p["id"])
-            if row is not None:
-                row["leased_by"] = None
-                row["leased_at"] = None
-            return _FakeResult([])
-
-        if "SET consecutive_fails = 0" in sql:  # mark_health ok
-            row = self._by_id(p["id"])
-            if row is not None:
-                row["consecutive_fails"] = 0
-                row["exit_ip"] = p["exit_ip"]
-                row["latency_ms"] = p["latency_ms"]
-                row["last_ok_at"] = datetime.now(UTC)
-            return _FakeResult([])
-
-        if "consecutive_fails = consecutive_fails + 1" in sql:  # mark_health fail
-            row = self._by_id(p["id"])
-            if row is not None:
-                row["consecutive_fails"] += 1
-                if row["consecutive_fails"] >= p["disable_threshold"]:
-                    row["enabled"] = False
-            return _FakeResult([])
-
-        raise AssertionError(f"unhandled SQL: {sql}")
-
-    def commit(self) -> None:
-        pass
-
-    def rollback(self) -> None:
-        pass
-
-
-def _proxy(
-    pid: int,
-    *,
-    affinity: str = "avito",
-    enabled: bool = True,
-    fails: int = 0,
-    leased_by: int | None = None,
-    last_ok_at: datetime | None = None,
-) -> dict[str, Any]:
-    return {
-        "id": pid,
-        "url": f"http://u:p@h{pid}:8080",
-        "kind": "http",
-        "rotate_url": None,
-        "provider_affinity": affinity,
-        "enabled": enabled,
-        "consecutive_fails": fails,
-        "leased_by": leased_by,
-        "leased_at": None,
-        "last_ok_at": last_ok_at,
-        "exit_ip": None,
-        "latency_ms": None,
-    }
-
-
-def _mock_settings(*, use_proxy_pool_curl: bool, scraper_proxy_url: str | None = None):
-    return SimpleNamespace(
-        use_proxy_pool_curl=use_proxy_pool_curl,
-        scraper_proxy_url=scraper_proxy_url,
-    )
-
-
-# ── _pool_proxy_dict ────────────────────────────────────────────────────────
-
-
-def test_pool_proxy_dict_falls_back_when_flag_off():
-    db = FakeSession([_proxy(1)])
-    with patch(
-        "app.services.scrape_pipeline.settings",
-        _mock_settings(use_proxy_pool_curl=False, scraper_proxy_url="http://env:3128"),
-    ):
-        proxies, lease = _pool_proxy_dict(db, "avito")  # type: ignore[arg-type]
-
-    assert proxies == {"http": "http://env:3128", "https": "http://env:3128"}
-    assert lease is None
-
-
-def test_pool_proxy_dict_returns_lease_when_flag_on_and_healthy():
-    db = FakeSession([_proxy(1)])
-    with patch(
-        "app.services.scrape_pipeline.settings",
-        _mock_settings(use_proxy_pool_curl=True),
-    ):
-        proxies, lease = _pool_proxy_dict(db, "avito")  # type: ignore[arg-type]
-
-    assert lease is not None and lease.id == 1
-    assert proxies == {"http": "http://u:p@h1:8080", "https": "http://u:p@h1:8080"}
-    assert db._by_id(1)["leased_by"] is not None  # lease проставлен
-
-
-def test_pool_proxy_dict_falls_back_when_pool_empty():
-    db = FakeSession([])  # пустой пул
-    with patch(
-        "app.services.scrape_pipeline.settings",
-        _mock_settings(use_proxy_pool_curl=True, scraper_proxy_url="http://env:3128"),
-    ):
-        proxies, lease = _pool_proxy_dict(db, "avito")  # type: ignore[arg-type]
-
-    assert lease is None
-    assert proxies == {"http": "http://env:3128", "https": "http://env:3128"}
-
-
-# ── _rotate_pool_proxy — acceptance-тест AC4 ───────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_pool_proxy_skips_banned_picks_next_healthy():
-    """#2202 AC4: пул с 1 забаненным прокси — ротация выбирает следующий здоровый.
-
-    BAD (id=1) — уже забанен (fails>=MAX_CONSECUTIVE_FAILS), исключён acquire() на
-    уровне SQL-фильтра. CURRENT (id=3) — прокси активного прогона, только что словивший
-    бан (403/429) — вызывающий код зовёт rotate. GOOD (id=2) — свободный здоровый,
-    единственный валидный кандидат после того как CURRENT будет помечен/освобождён.
-
-    NB (#2202 MAJOR-fix): гарантия исключения CURRENT из acquire() держится на том,
-    что release(current_lease) вызывается ПОСЛЕ acquire(), не до — пока current_lease
-    leased_by-занят, SQL `leased_by IS NULL` исключает его вне зависимости от id/
-    last_ok_at сортировки. См. зеркальный тест ниже
-    (test_rotate_pool_proxy_skips_banned_picks_next_healthy_current_id_lower) —
-    та же гарантия при CURRENT id НИЖЕ GOOD id (иначе тай-брейк по id в ORDER BY
-    "last_ok_at NULLS LAST, id" мог случайно фейворить эту фикстуру).
-    """
-    db = FakeSession(
-        [
-            _proxy(1, fails=MAX_CONSECUTIVE_FAILS),  # BAD — уже забанен
-            _proxy(2, fails=0),  # GOOD — ожидаемый выбор
-            _proxy(3, fails=0, leased_by=999),  # CURRENT — в работе, банится сейчас
-        ]
-    )
-    current_lease = proxy_pool.ProxyLease(
-        id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
-    )
-    session = SimpleNamespace(proxies=None)
-
-    new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease)  # type: ignore[arg-type]
-
-    assert new_lease is not None
-    assert new_lease.id == 2  # GOOD, не BAD (excluded) и не CURRENT (только что провален)
-    assert session.proxies == {"http": "http://u:p@h2:8080", "https": "http://u:p@h2:8080"}
-
-    # CURRENT (id=3) помечен нездоровым и освобождён
-    assert db._by_id(3)["consecutive_fails"] == 1
-    assert db._by_id(3)["leased_by"] is None
-    # BAD (id=1) не тронут этой операцией
-    assert db._by_id(1)["consecutive_fails"] == MAX_CONSECUTIVE_FAILS
-    # GOOD (id=2) теперь зализан под новую ротацию
-    assert db._by_id(2)["leased_by"] is not None
-
-
-@pytest.mark.asyncio
-async def test_rotate_pool_proxy_skips_banned_picks_next_healthy_current_id_lower():
-    """#2202 AC4 regression guard (MAJOR-fix): гарантия исключения current_lease НЕ
-    должна зависеть от того, чей id меньше в фикстуре.
-
-    Зеркало test_rotate_pool_proxy_skips_banned_picks_next_healthy с CURRENT/GOOD id
-    в обратном порядке: CURRENT=1 < GOOD=2 (оригинальный тест имел CURRENT=3 > GOOD=2,
-    поэтому "тай-брейк по id" в ORDER BY last_ok_at NULLS LAST, id случайно совпадал с
-    правильным ответом и не доказывал реальную гарантию — см. review-репро на #2202).
-
-    До фикса (release(current_lease) ДО acquire()): CURRENT освобождается, попадает в
-    leased_by IS NULL кандидаты с fails=1 (< MAX_CONSECUTIVE_FAILS=3, т.е. "здоров"),
-    и при равном last_ok_at (оба NULL) выигрывает тай-брейк по id (1 < 2) — acquire()
-    возвращает ТОТ ЖЕ прокси, который только что забанили. Этот тест ловит именно этот
-    регресс: если release() снова передвинуть перед acquire(), new_lease.id будет 1
-    (CURRENT), а не 2 (GOOD), и assert упадёт.
-    """
-    db = FakeSession(
-        [
-            _proxy(1, fails=0, leased_by=999),  # CURRENT — id НИЖЕ GOOD, банится сейчас
-            _proxy(2, fails=0),  # GOOD — единственный здоровый свободный кандидат
-        ]
-    )
-    current_lease = proxy_pool.ProxyLease(
-        id=1, url="http://u:p@h1:8080", kind="http", rotate_url=None
-    )
-    session = SimpleNamespace(proxies=None)
-
-    new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease)  # type: ignore[arg-type]
-
-    assert new_lease is not None
-    assert new_lease.id == 2  # GOOD — НЕ CURRENT, несмотря на id=1 < id=2
-    assert session.proxies == {"http": "http://u:p@h2:8080", "https": "http://u:p@h2:8080"}
-
-    # CURRENT (id=1) помечен нездоровым и освобождён (после успешной ротации)
-    assert db._by_id(1)["consecutive_fails"] == 1
-    assert db._by_id(1)["leased_by"] is None
-    # GOOD (id=2) теперь зализан под новую ротацию
-    assert db._by_id(2)["leased_by"] is not None
-
-
-@pytest.mark.asyncio
-async def test_rotate_pool_proxy_exhausted_returns_none():
-    """Все кандидаты нездоровы после ротации → None (caller падает на changeip).
-
-    BAD (id=1) уже забанен. CURRENT (id=3) на MAX_CONSECUTIVE_FAILS-1 fails — ЭТА
-    ротация (mark_health ok=False) толкает его ровно до порога → тоже исключается
-    из acquire(). Ни одного здорового кандидата не остаётся.
-    """
-    db = FakeSession(
-        [
-            _proxy(1, fails=MAX_CONSECUTIVE_FAILS),  # BAD — уже забанен
-            _proxy(3, fails=MAX_CONSECUTIVE_FAILS - 1, leased_by=999),  # CURRENT — на грани
-        ]
-    )
-    current_lease = proxy_pool.ProxyLease(
-        id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
-    )
-    session = SimpleNamespace(proxies=None)
-
-    new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease)  # type: ignore[arg-type]
-
-    assert new_lease is None
-    assert session.proxies is None  # не тронут — caller остаётся на старом/changeip
-    assert db._by_id(3)["leased_by"] is None  # текущий всё равно освобождён
-
-
-# ── _rotate_avito_proxy — pool-first, changeip fallback ────────────────────
-
-
-@pytest.mark.asyncio
-async def test_rotate_avito_proxy_uses_pool_when_flag_on():
-    db = FakeSession(
-        [
-            _proxy(1, fails=0),  # свободный здоровый — цель ротации
-            _proxy(3, fails=0, leased_by=999),  # CURRENT
-        ]
-    )
-    current_lease = proxy_pool.ProxyLease(
-        id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
-    )
-    session = SimpleNamespace(proxies=None)
-
-    with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=True)):
-        with patch("app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock()) as changeip_mock:
-            rotated, new_lease = await _rotate_avito_proxy(
-                db,  # type: ignore[arg-type]
-                session,  # type: ignore[arg-type]
-                current_lease,
-                reason="test",
-                rotations_done=0,
-            )
-
-    assert rotated is True
-    assert new_lease is not None and new_lease.id == 1
-    changeip_mock.assert_not_awaited()  # пул сработал — changeip не нужен
-
-
-@pytest.mark.asyncio
-async def test_rotate_avito_proxy_falls_back_to_changeip_when_flag_off():
-    session = SimpleNamespace(proxies=None)
-
-    with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=False)):
-        with patch(
-            "app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock(return_value=True)
-        ) as changeip_mock:
-            rotated, new_lease = await _rotate_avito_proxy(
-                None,  # type: ignore[arg-type]
-                session,  # type: ignore[arg-type]
-                None,
-                reason="test",
-                rotations_done=0,
-            )
-
-    assert rotated is True
-    assert new_lease is None
-    changeip_mock.assert_awaited_once()
-
-
-@pytest.mark.asyncio
-async def test_rotate_avito_proxy_falls_back_to_changeip_when_pool_exhausted():
-    """Пул on, но acquire() пуст (например все забанены) — падаем на changeip, не молчим."""
-    db = FakeSession([_proxy(1, fails=MAX_CONSECUTIVE_FAILS)])  # единственный — забанен
-    session = SimpleNamespace(proxies=None)
-
-    with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=True)):
-        with patch(
-            "app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock(return_value=True)
-        ) as changeip_mock:
-            rotated, new_lease = await _rotate_avito_proxy(
-                db,  # type: ignore[arg-type]
-                session,  # type: ignore[arg-type]
-                None,
-                reason="test",
-                rotations_done=0,
-            )
-
-    assert rotated is True
-    assert new_lease is None
-    changeip_mock.assert_awaited_once()  # pool exhausted → fallback path taken
-
-
-# ── _release_pool_lease ──────────────────────────────────────────────────────
-
-
-def test_release_pool_lease_noop_when_none():
-    _release_pool_lease(None, None, ok=True)  # type: ignore[arg-type]  # must not raise
-
-
-def test_release_pool_lease_marks_health_and_releases():
-    db = FakeSession([_proxy(1, leased_by=100)])
-    lease = proxy_pool.ProxyLease(id=1, url="http://u:p@h1:8080", kind="http", rotate_url=None)
-
-    _release_pool_lease(db, lease, ok=True)  # type: ignore[arg-type]
-
-    assert db._by_id(1)["leased_by"] is None
-    assert db._by_id(1)["consecutive_fails"] == 0

tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py

@@ -1,11 +1,10 @@
-"""Golden-parity: `scraper_kit.orchestration.pipeline` ≡ `app.services.scrape_pipeline`.
+"""Regression: `scraper_kit.orchestration.pipeline.run_avito_city_sweep` orchestration.
 
-Strangler-инвариант (#2135): новая scraper_kit-копия оркестрации sweep-pipeline
+Kit — единственная orchestration-копия sweep-pipeline (#2397 Part E1 удалил legacy
-должна давать ТОТ ЖЕ наблюдаемый эффект, что и старый боевой orchestrator, на
+`app.services.scrape_pipeline`; сравнивать «golden-parity» больше не с чем — этот файл
-одинаковом мокнутом I/O. Развязка `app.*` (config/matcher/enrichment/runs/shutdown
+раньше гонял ОБА orchestrator'а side-by-side, теперь оставлена только kit-сторона).
-инжектируются) не должна менять ветвление.
 
-Фокус — КРИТИЧНАЯ логика оркестрации (то, ради чего PR):
+Фокус — КРИТИЧНАЯ логика оркестрации (то, ради чего эти тесты изначально писались):
   - ban/rotation state-machine: SERP-блок на anchor'е → abort sweep;
   - partial-ban intake (#1950): SERP intake сохранён + detail заблокирован →
     mark_done (не mark_banned) под флагом avito_serp_ok_not_banned;
@@ -13,11 +12,6 @@ Strangler-инвариант (#2135): новая scraper_kit-копия орке
   - counters aggregation по anchor'ам + IMV-фаза;
   - последовательность вызовов scrape_runs (heartbeat / mark_done / mark_banned).
 
-Метод: гоняем ОБА orchestrator'а (`_drive_old` / `_drive_new`) на идентичном
-описании сценария, перехватывая scrape_runs recording-mock'ом. Сравниваем:
-  - итоговый counters.to_dict();
-  - последовательность (method, numeric-counters) вызовов scrape_runs.
-
 Без сети, без БД.
 """
 
@@ -32,17 +26,8 @@ import pytest
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 
-from scraper_kit.avito_exceptions import (
+from scraper_kit.avito_exceptions import AvitoBlockedError
-    AvitoBlockedError as NewAvitoBlockedError,
+from scraper_kit.orchestration.pipeline import run_avito_city_sweep
-)
-from scraper_kit.orchestration.pipeline import (
-    run_avito_city_sweep as new_city_sweep,
-)
-
-from app.services.scrape_pipeline import run_avito_city_sweep as old_city_sweep
-from app.services.scrapers.avito_exceptions import (
-    AvitoBlockedError as OldAvitoBlockedError,
-)
 
 # ── recording scrape_runs ────────────────────────────────────────────────────
 
@@ -53,7 +38,7 @@ class _RunsRecorder:
     """Записывает вызовы scrape_runs-финализаторов в общий список.
 
     is_cancelled всегда False (кооп-cancel не тестируем здесь). Каждый терминальный
-    вызов пишет (method, counters_dict) — сравниваем последовательности old/new.
+    вызов пишет (method, counters_dict).
     """
 
     def __init__(self) -> None:
@@ -91,7 +76,7 @@ def _normalize(calls: list[tuple[str, dict[str, Any]]]) -> list[tuple[str, dict[
 
 
 class _Scenario:
-    """Декларативное описание одного прогона city-sweep для обоих orchestrator'ов."""
+    """Декларативное описание одного прогона city-sweep."""
 
     def __init__(
         self,
@@ -185,61 +170,10 @@ def _async_session_cm() -> MagicMock:
     return sess
 
 
-async def _drive_old(scenario: _Scenario) -> _DriveResult:
+async def _drive(scenario: _Scenario) -> _DriveResult:
     recorder = _RunsRecorder()
     db = _make_db(scenario)
-    scraper = _make_scraper(scenario, OldAvitoBlockedError)
+    scraper = _make_scraper(scenario, AvitoBlockedError)
-    save_mock = MagicMock(side_effect=scenario._save_side_effects())
-
-    imv_res = None
-    if scenario.imv_result is not None:
-        checked, saved, errors = scenario.imv_result
-        imv_res = SimpleNamespace(checked=checked, saved=saved, errors=errors)
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
-        patch("app.services.scrape_pipeline.save_listings", save_mock),
-        patch(
-            "app.services.scrape_pipeline.fetch_house_catalog",
-            AsyncMock(return_value=MagicMock()),
-        ),
-        patch(
-            "app.services.scrape_pipeline.save_house_catalog_enrichment",
-            return_value={"house_id": 1},
-        ),
-        patch(
-            "app.services.scrape_pipeline.fetch_detail",
-            AsyncMock(side_effect=scenario._fetch_detail_side_effects(OldAvitoBlockedError)),
-        ),
-        patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
-        patch("app.services.scrape_pipeline.scrape_runs", recorder),
-        patch(
-            "app.services.house_imv_backfill.process_houses_imv_batch",
-            AsyncMock(return_value=imv_res),
-        ),
-        patch("app.services.scrape_pipeline.settings", scenario._config()),
-        patch("app.services.scrape_pipeline.shutdown_requested", return_value=False),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_pipeline.AsyncSession", return_value=_async_session_cm()),
-    ):
-        counters = await old_city_sweep(
-            db,
-            run_id=1,
-            radius_m=1000,
-            anchors=scenario.anchors,
-            pages_per_anchor=1,
-            enrich_houses=scenario.enrich_houses,
-            detail_top_n=scenario.detail_top_n,
-            request_delay_sec=0.0,
-            enrich_imv=scenario.enrich_imv,
-        )
-    return counters.to_dict(), _normalize(recorder.calls)
-
-
-async def _drive_new(scenario: _Scenario) -> _DriveResult:
-    recorder = _RunsRecorder()
-    db = _make_db(scenario)
-    scraper = _make_scraper(scenario, NewAvitoBlockedError)
     save_mock = MagicMock(side_effect=scenario._save_side_effects())
 
     imv_res = None
@@ -258,14 +192,14 @@ async def _drive_new(scenario: _Scenario) -> _DriveResult:
         patch(f"{pfx}.save_house_catalog_enrichment", return_value={"house_id": 1}),
         patch(
             f"{pfx}.fetch_detail",
-            AsyncMock(side_effect=scenario._fetch_detail_side_effects(NewAvitoBlockedError)),
+            AsyncMock(side_effect=scenario._fetch_detail_side_effects(AvitoBlockedError)),
         ),
         patch(f"{pfx}.save_detail_enrichment", return_value=True),
         patch(f"{pfx}.runs", recorder),
         patch(f"{pfx}.asyncio.sleep", AsyncMock()),
         patch(f"{pfx}.AsyncSession", return_value=_async_session_cm()),
     ):
-        counters = await new_city_sweep(
+        counters = await run_avito_city_sweep(
             db,
             run_id=1,
             config=scenario._config(),
@@ -283,72 +217,63 @@ async def _drive_new(scenario: _Scenario) -> _DriveResult:
     return counters.to_dict(), _normalize(recorder.calls)
 
 
-async def _assert_parity(scenario: _Scenario) -> dict[str, int]:
-    old_counters, old_calls = await _drive_old(scenario)
-    new_counters, new_calls = await _drive_new(scenario)
-    assert (
-        new_counters == old_counters
-    ), f"counters diverged:\n old={old_counters}\n new={new_counters}"
-    assert (
-        new_calls == old_calls
-    ), f"scrape_runs calls diverged:\n old={old_calls}\n new={new_calls}"
-    return new_counters
-
-
 # ── scenarios ────────────────────────────────────────────────────────────────
 
 _ANCHORS_2 = [(56.84, 60.60, "A1"), (56.79, 60.53, "A2")]
 
 
 @pytest.mark.asyncio
-async def test_parity_happy_path_counters_aggregation() -> None:
+async def test_happy_path_counters_aggregation() -> None:
     """2 anchor'а, SERP+save, без houses/detail/imv → mark_done + агрегированные counters."""
     scenario = _Scenario(
         anchors=_ANCHORS_2,
         per_anchor=[("lots", 10, 8, 2), ("lots", 6, 5, 1)],
     )
-    counters = await _assert_parity(scenario)
+    counters, calls = await _drive(scenario)
-    # Убеждаемся что сценарий действительно прогнал агрегацию (не пустой прогон).
     assert counters["lots_fetched"] == 16
     assert counters["lots_inserted"] == 13
     assert counters["lots_updated"] == 3
     assert counters["anchors_done"] == 2
+    assert calls[-1][0] == "mark_done"
 
 
 @pytest.mark.asyncio
-async def test_parity_full_ban_no_intake_marks_banned() -> None:
+async def test_full_ban_no_intake_marks_banned() -> None:
     """Первый же anchor SERP-блок, лоты не сохранены → mark_banned (не done)."""
     scenario = _Scenario(
         anchors=_ANCHORS_2,
         per_anchor=[("block",), ("block",)],
     )
-    await _assert_parity(scenario)
+    _counters, calls = await _drive(scenario)
+    assert calls[-1][0] == "mark_banned"
 
 
 @pytest.mark.asyncio
-async def test_parity_partial_ban_intake_marks_done() -> None:
+async def test_partial_ban_intake_marks_done() -> None:
     """Anchor1 intake сохранён, anchor2 SERP-блок → partial-intake → mark_done не banned."""
     scenario = _Scenario(
         anchors=_ANCHORS_2,
         per_anchor=[("lots", 10, 9, 1), ("block",)],
         avito_serp_ok_not_banned=True,
     )
-    await _assert_parity(scenario)
+    _counters, calls = await _drive(scenario)
+    assert calls[-1][0] == "mark_done"
 
 
 @pytest.mark.asyncio
-async def test_parity_partial_ban_flag_off_marks_banned() -> None:
+async def test_partial_ban_flag_off_marks_banned() -> None:
     """avito_serp_ok_not_banned=False: даже при intake SERP-блок → mark_banned."""
     scenario = _Scenario(
         anchors=_ANCHORS_2,
         per_anchor=[("lots", 10, 9, 1), ("block",)],
         avito_serp_ok_not_banned=False,
     )
-    await _assert_parity(scenario)
+    _counters, calls = await _drive(scenario)
+    assert calls[-1][0] == "mark_banned"
 
 
 @pytest.mark.asyncio
-async def test_parity_detail_consecutive_block_propagates() -> None:
+async def test_detail_consecutive_block_propagates() -> None:
     """detail-фаза: 3 подряд блока (rotations=0) → propagate → anchor-handler.
 
     Лоты сохранены на этом же anchor'е → partial-intake → mark_done.
@@ -362,11 +287,12 @@ async def test_parity_detail_consecutive_block_propagates() -> None:
         avito_serp_ok_not_banned=True,
         avito_proxy_max_rotations=0,
     )
-    await _assert_parity(scenario)
+    _counters, calls = await _drive(scenario)
+    assert calls[-1][0] == "mark_done"
 
 
 @pytest.mark.asyncio
-async def test_parity_imv_phase_counters() -> None:
+async def test_imv_phase_counters() -> None:
     """IMV-фаза: touched houses → process_houses_imv_batch → imv-counters агрегированы."""
     scenario = _Scenario(
         anchors=[(56.84, 60.60, "A1")],
@@ -376,8 +302,8 @@ async def test_parity_imv_phase_counters() -> None:
         enrich_imv=True,
         imv_result=(3, 2, 1),
     )
-    counters = await _assert_parity(scenario)
+    counters, _calls = await _drive(scenario)
-    # touched house → IMV-фаза отработала, imv-counters агрегированы одинаково.
+    # touched house → IMV-фаза отработала, imv-counters агрегированы.
     assert counters["imv_attempted"] == 3
     assert counters["imv_enriched"] == 2
     assert counters["imv_failed"] == 1

tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity2.py

@@ -1,7 +1,10 @@
-"""Golden-parity (F2): kit sweep-оркестраторы ≡ `app.services.scrape_pipeline`.
+"""Regression (F2): kit sweep-оркестраторы `scraper_kit.orchestration.pipeline`.
 
-Продолжение test_scraper_kit_pipeline_parity.py (F1 покрыл avito city sweep). Здесь —
+Kit — единственная orchestration-копия (#2397 Part E1 удалил legacy
-остальные 7 sweep'ов, скопированных в `scraper_kit.orchestration.pipeline` (#2135 F2):
+`app.services.scrape_pipeline`; сравнивать «golden-parity» больше не с чем — этот
+файл раньше гонял ОБА orchestrator'а side-by-side, теперь оставлена только
+kit-сторона). Продолжение test_scraper_kit_pipeline_parity.py (F1 покрыл avito city
+sweep). Здесь — остальные 7 sweep'ов (#2135 F2):
 
   City sweep'ы (приоритет — активны в проде):
     - run_yandex_city_sweep     — combos SERP + save + price-history
@@ -9,13 +12,11 @@
     - run_domclick_city_sweep   — BFF citywide + честный статус (done / failed)
   Плюс:
     - run_avito_newbuilding_sweep — citywide novostroyka SERP + save
-  Full load'ы (smoke-parity — импорт + базовый прогон через on_bucket):
+  Full load'ы (smoke — импорт + базовый прогон через on_bucket):
     - run_avito_full_load / run_cian_full_load / run_yandex_full_load
 
-Метод — как в F1: гоняем ОБА orchestrator'а на идентичном мокнутом I/O, сравниваем
+Метод: гоняем kit orchestrator на мокнутом I/O, проверяем итоговый counters.to_dict()
-итоговый counters.to_dict() + последовательность (method, numeric-counters) вызовов
++ последовательность (method, numeric-counters) вызовов scrape_runs. Без сети, без БД.
-scrape_runs. Развязка `app.*` (config/matcher/enrichment/runs/shutdown инжектируются)
-не должна менять ветвление/счётчики. Без сети, без БД.
 """
 
 from __future__ import annotations
@@ -30,51 +31,16 @@ import pytest
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 
 from scraper_kit.orchestration.pipeline import (
-    run_avito_full_load as new_avito_full_load,
+    run_avito_full_load,
-)
+    run_avito_newbuilding_sweep,
-from scraper_kit.orchestration.pipeline import (
+    run_cian_city_sweep,
-    run_avito_newbuilding_sweep as new_nb_sweep,
+    run_cian_full_load,
-)
+    run_domclick_city_sweep,
-from scraper_kit.orchestration.pipeline import (
+    run_yandex_city_sweep,
-    run_cian_city_sweep as new_cian_city_sweep,
+    run_yandex_full_load,
-)
-from scraper_kit.orchestration.pipeline import (
-    run_cian_full_load as new_cian_full_load,
-)
-from scraper_kit.orchestration.pipeline import (
-    run_domclick_city_sweep as new_domclick_city_sweep,
-)
-from scraper_kit.orchestration.pipeline import (
-    run_yandex_city_sweep as new_yandex_city_sweep,
-)
-from scraper_kit.orchestration.pipeline import (
-    run_yandex_full_load as new_yandex_full_load,
 )
 
-from app.services.scrape_pipeline import (
+PFX = "scraper_kit.orchestration.pipeline"
-    run_avito_full_load as old_avito_full_load,
-)
-from app.services.scrape_pipeline import (
-    run_avito_newbuilding_sweep as old_nb_sweep,
-)
-from app.services.scrape_pipeline import (
-    run_cian_city_sweep as old_cian_city_sweep,
-)
-from app.services.scrape_pipeline import (
-    run_cian_full_load as old_cian_full_load,
-)
-from app.services.scrape_pipeline import (
-    run_domclick_city_sweep as old_domclick_city_sweep,
-)
-from app.services.scrape_pipeline import (
-    run_yandex_city_sweep as old_yandex_city_sweep,
-)
-from app.services.scrape_pipeline import (
-    run_yandex_full_load as old_yandex_full_load,
-)
-
-OLD_PFX = "app.services.scrape_pipeline"
-NEW_PFX = "scraper_kit.orchestration.pipeline"
 
 # ── recording scrape_runs ────────────────────────────────────────────────────
 
@@ -113,7 +79,6 @@ def _normalize(calls: list[tuple[str, dict[str, Any]]]) -> list[tuple[str, dict[
 
 
 def _config() -> SimpleNamespace:
-    """Инжектируемый config (kit) + тот же объект как patched settings (old)."""
     return SimpleNamespace(
         scraper_fetch_mode="curl_cffi",
         browser_http_endpoint="http://browser.test/fetch",
@@ -158,18 +123,6 @@ def _async_session_cm() -> MagicMock:
 _DriveResult = tuple[dict[str, int], list[tuple[str, dict[str, int]]]]
 
 
-async def _assert_parity(old_res: _DriveResult, new_res: _DriveResult) -> dict[str, int]:
-    old_counters, old_calls = old_res
-    new_counters, new_calls = new_res
-    assert (
-        new_counters == old_counters
-    ), f"counters diverged:\n old={old_counters}\n new={new_counters}"
-    assert (
-        new_calls == old_calls
-    ), f"scrape_runs calls diverged:\n old={old_calls}\n new={new_calls}"
-    return new_counters
-
-
 # ── Yandex city sweep ─────────────────────────────────────────────────────────
 
 
@@ -183,7 +136,7 @@ def _yandex_scraper(combos: list[tuple[str, list[Any]]]) -> MagicMock:
     return _ctx_scraper(fetch_around_multi_room=_fetch)
 
 
-async def _drive_yandex_city(*, new: bool) -> _DriveResult:
+async def _drive_yandex_city() -> _DriveResult:
     recorder = _RunsRecorder()
     db = MagicMock()
     combos = [
@@ -193,47 +146,36 @@ async def _drive_yandex_city(*, new: bool) -> _DriveResult:
     scraper = _yandex_scraper(combos)
     save_mock = MagicMock(side_effect=[(2, 0), (1, 0)])
     cfg = _config()
-    kwargs = dict(
+    enrichment = MagicMock()
-        run_id=1,
+    enrichment.record_yandex_price_history = MagicMock(return_value=5)
-        anchors=None,
+    with (
-        pages_per_anchor=1,
+        patch(f"{PFX}.YandexRealtyScraper", return_value=scraper),
-        request_delay_sec=0.0,
+        patch(f"{PFX}.save_listings", save_mock),
-        enrich_address=False,
+        patch(f"{PFX}.runs", recorder),
-    )
+    ):
-    if new:
+        counters = await run_yandex_city_sweep(
-        enrichment = MagicMock()
+            db,
-        enrichment.record_yandex_price_history = MagicMock(return_value=5)
+            config=cfg,
-        with (
+            matcher=MagicMock(),
-            patch(f"{NEW_PFX}.YandexRealtyScraper", return_value=scraper),
+            enrichment=enrichment,
-            patch(f"{NEW_PFX}.save_listings", save_mock),
+            run_id=1,
-            patch(f"{NEW_PFX}.runs", recorder),
+            anchors=None,
-        ):
+            pages_per_anchor=1,
-            counters = await new_yandex_city_sweep(
+            request_delay_sec=0.0,
-                db, config=cfg, matcher=MagicMock(), enrichment=enrichment, **kwargs
+            enrich_address=False,
-            )
+        )
-    else:
-        with (
-            patch(f"{OLD_PFX}.YandexRealtyScraper", return_value=scraper),
-            patch(f"{OLD_PFX}.save_listings", save_mock),
-            patch(f"{OLD_PFX}.record_yandex_price_history", MagicMock(return_value=5)),
-            patch(f"{OLD_PFX}.scrape_runs", recorder),
-            patch(f"{OLD_PFX}.settings", cfg),
-            patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
-        ):
-            counters = await old_yandex_city_sweep(db, **kwargs)
     return counters.to_dict(), _normalize(recorder.calls)
 
 
 @pytest.mark.asyncio
-async def test_yandex_city_sweep_parity() -> None:
+async def test_yandex_city_sweep() -> None:
     """1 центральный anchor, 2 combos, save + price-history, enrich_address=False."""
-    counters = await _assert_parity(
+    counters, calls = await _drive_yandex_city()
-        await _drive_yandex_city(new=False), await _drive_yandex_city(new=True)
-    )
     assert counters["lots_fetched"] == 3
     assert counters["lots_inserted"] == 3
     assert counters["price_history_rows"] == 5
     assert counters["anchors_done"] == 1
+    assert calls[-1][0] == "mark_done"
 
 
 # ── Cian city sweep ───────────────────────────────────────────────────────────
@@ -243,7 +185,7 @@ def _cian_lot(segment: str) -> MagicMock:
     return MagicMock(listing_segment=segment, house_source=None, house_ext_id=None)
 
 
-async def _drive_cian_city(*, new: bool) -> _DriveResult:
+async def _drive_cian_city() -> _DriveResult:
     recorder = _RunsRecorder()
     db = MagicMock()
     # 3 novostroyki + 2 secondary → newbuilding_only оставит 3.
@@ -251,51 +193,42 @@ async def _drive_cian_city(*, new: bool) -> _DriveResult:
     scraper = _ctx_scraper(fetch_around_multi_room=AsyncMock(return_value=lots))
     save_mock = MagicMock(side_effect=[(3, 0)])
     cfg = _config()
-    kwargs = dict(
+    with (
-        run_id=1,
+        patch(f"{PFX}.CianScraper", return_value=scraper),
-        anchors=[(56.84, 60.60, "A1")],
+        patch(f"{PFX}.save_listings", save_mock),
-        radius_m=1000,
+        patch(f"{PFX}.runs", recorder),
-        pages_per_anchor=1,
+    ):
-        request_delay_sec=0.0,
+        counters = await run_cian_city_sweep(
-        detail_top_n=0,
+            db,
-        enrich_houses=False,
+            config=cfg,
-        newbuilding_only=True,
+            matcher=MagicMock(),
-    )
+            run_id=1,
-    if new:
+            anchors=[(56.84, 60.60, "A1")],
-        with (
+            radius_m=1000,
-            patch(f"{NEW_PFX}.CianScraper", return_value=scraper),
+            pages_per_anchor=1,
-            patch(f"{NEW_PFX}.save_listings", save_mock),
+            request_delay_sec=0.0,
-            patch(f"{NEW_PFX}.runs", recorder),
+            detail_top_n=0,
-        ):
+            enrich_houses=False,
-            counters = await new_cian_city_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
+            newbuilding_only=True,
-    else:
+        )
-        with (
-            patch("app.services.scrapers.cian.CianScraper", return_value=scraper),
-            patch(f"{OLD_PFX}.save_listings", save_mock),
-            patch(f"{OLD_PFX}.scrape_runs", recorder),
-            patch(f"{OLD_PFX}.settings", cfg),
-            patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
-        ):
-            counters = await old_cian_city_sweep(db, **kwargs)
     return counters.to_dict(), _normalize(recorder.calls)
 
 
 @pytest.mark.asyncio
-async def test_cian_city_sweep_parity() -> None:
+async def test_cian_city_sweep() -> None:
     """1 anchor, newbuilding_only отбрасывает вторичку, save новостроек, mark_done."""
-    counters = await _assert_parity(
+    counters, calls = await _drive_cian_city()
-        await _drive_cian_city(new=False), await _drive_cian_city(new=True)
-    )
     assert counters["lots_fetched"] == 5
     assert counters["lots_dropped_secondary"] == 2
     assert counters["lots_inserted"] == 3
     assert counters["anchors_done"] == 1
+    assert calls[-1][0] == "mark_done"
 
 
 # ── DomClick city sweep ───────────────────────────────────────────────────────
 
 
-async def _drive_domclick(*, new: bool, lots_n: int, blocked: bool) -> _DriveResult:
+async def _drive_domclick(*, lots_n: int, blocked: bool) -> _DriveResult:
     recorder = _RunsRecorder()
     db = MagicMock()
     lots = [MagicMock() for _ in range(lots_n)]
@@ -307,52 +240,39 @@ async def _drive_domclick(*, new: bool, lots_n: int, blocked: bool) -> _DriveRes
     )
     save_mock = MagicMock(side_effect=[(lots_n, 0)] if lots_n else [])
     cfg = _config()
-    kwargs = dict(run_id=1, city_id=4, pages=1, request_delay_sec=0.0)
+    with (
-    if new:
+        patch(f"{PFX}.DomClickScraper", return_value=scraper),
-        with (
+        patch(f"{PFX}.save_listings", save_mock),
-            patch(f"{NEW_PFX}.DomClickScraper", return_value=scraper),
+        patch(f"{PFX}.runs", recorder),
-            patch(f"{NEW_PFX}.save_listings", save_mock),
+    ):
-            patch(f"{NEW_PFX}.runs", recorder),
+        counters = await run_domclick_city_sweep(
-        ):
+            db, config=cfg, matcher=MagicMock(), run_id=1, city_id=4, pages=1, request_delay_sec=0.0
-            counters = await new_domclick_city_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
+        )
-    else:
-        with (
-            patch("app.services.scrapers.domclick.DomClickScraper", return_value=scraper),
-            patch(f"{OLD_PFX}.save_listings", save_mock),
-            patch(f"{OLD_PFX}.scrape_runs", recorder),
-            patch(f"{OLD_PFX}.settings", cfg),
-            patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
-        ):
-            counters = await old_domclick_city_sweep(db, **kwargs)
     return counters.to_dict(), _normalize(recorder.calls)
 
 
 @pytest.mark.asyncio
-async def test_domclick_city_sweep_parity_done() -> None:
+async def test_domclick_city_sweep_done() -> None:
     """Citywide fetch_city → 4 lots saved → mark_done."""
-    counters = await _assert_parity(
+    counters, calls = await _drive_domclick(lots_n=4, blocked=False)
-        await _drive_domclick(new=False, lots_n=4, blocked=False),
-        await _drive_domclick(new=True, lots_n=4, blocked=False),
-    )
     assert counters["lots_fetched"] == 4
     assert counters["lots_inserted"] == 4
+    assert calls[-1][0] == "mark_done"
 
 
 @pytest.mark.asyncio
-async def test_domclick_city_sweep_parity_blocked_failed() -> None:
+async def test_domclick_city_sweep_blocked_failed() -> None:
     """QRATOR-блок + 0 lots → mark_failed (честный статус #1968)."""
-    counters = await _assert_parity(
+    counters, calls = await _drive_domclick(lots_n=0, blocked=True)
-        await _drive_domclick(new=False, lots_n=0, blocked=True),
-        await _drive_domclick(new=True, lots_n=0, blocked=True),
-    )
     assert counters["lots_fetched"] == 0
     assert counters["blocked"] == 1
+    assert calls[-1][0] == "mark_failed"
 
 
 # ── Avito newbuilding sweep ───────────────────────────────────────────────────
 
 
-async def _drive_nb_sweep(*, new: bool) -> _DriveResult:
+async def _drive_nb_sweep() -> _DriveResult:
     recorder = _RunsRecorder()
     db = MagicMock()
     lots = [MagicMock() for _ in range(6)]
@@ -362,40 +282,29 @@ async def _drive_nb_sweep(*, new: bool) -> _DriveResult:
     scraper.fetch_newbuildings = AsyncMock(return_value=lots)
     save_mock = MagicMock(side_effect=[(5, 1)])
     cfg = _config()
-    kwargs = dict(run_id=1, pages=2, request_delay_sec=0.0)
+    with (
-    if new:
+        patch(f"{PFX}.AvitoScraper", return_value=scraper),
-        with (
+        patch(f"{PFX}.save_listings", save_mock),
-            patch(f"{NEW_PFX}.AvitoScraper", return_value=scraper),
+        patch(f"{PFX}.runs", recorder),
-            patch(f"{NEW_PFX}.save_listings", save_mock),
+        patch(f"{PFX}.AsyncSession", return_value=_async_session_cm()),
-            patch(f"{NEW_PFX}.runs", recorder),
+    ):
-            patch(f"{NEW_PFX}.AsyncSession", return_value=_async_session_cm()),
+        counters = await run_avito_newbuilding_sweep(
-        ):
+            db, config=cfg, matcher=MagicMock(), run_id=1, pages=2, request_delay_sec=0.0
-            counters = await new_nb_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
+        )
-    else:
-        with (
-            patch(f"{OLD_PFX}.AvitoScraper", return_value=scraper),
-            patch(f"{OLD_PFX}.save_listings", save_mock),
-            patch(f"{OLD_PFX}.scrape_runs", recorder),
-            patch(f"{OLD_PFX}.settings", cfg),
-            patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
-            patch(f"{OLD_PFX}.AsyncSession", return_value=_async_session_cm()),
-        ):
-            counters = await old_nb_sweep(db, **kwargs)
     return counters.to_dict(), _normalize(recorder.calls)
 
 
 @pytest.mark.asyncio
-async def test_avito_newbuilding_sweep_parity() -> None:
+async def test_avito_newbuilding_sweep() -> None:
     """Citywide novostroyka SERP → save → heartbeat → mark_done."""
-    counters = await _assert_parity(
+    counters, calls = await _drive_nb_sweep()
-        await _drive_nb_sweep(new=False), await _drive_nb_sweep(new=True)
-    )
     assert counters["lots_fetched"] == 6
     assert counters["lots_inserted"] == 5
     assert counters["lots_updated"] == 1
+    assert calls[-1][0] == "mark_done"
 
 
-# ── Full loads (smoke-parity через on_bucket) ─────────────────────────────────
+# ── Full loads (smoke через on_bucket) ─────────────────────────────────────────
 
 
 def _full_load_scraper(buckets: list[tuple[str, list[Any]]]) -> MagicMock:
@@ -410,7 +319,7 @@ def _full_load_scraper(buckets: list[tuple[str, list[Any]]]) -> MagicMock:
     return scraper
 
 
-async def _drive_full_load(*, new: bool, source: str) -> _DriveResult:
+async def _drive_full_load(*, source: str) -> _DriveResult:
     recorder = _RunsRecorder()
     db = MagicMock()
     buckets = [
@@ -421,61 +330,38 @@ async def _drive_full_load(*, new: bool, source: str) -> _DriveResult:
     save_mock = MagicMock(side_effect=[(2, 0), (1, 0)])
     cfg = _config()
 
-    old_map = {
+    fn_map = {
-        "avito": (old_avito_full_load, new_avito_full_load, f"{OLD_PFX}.AvitoScraper"),
+        "avito": run_avito_full_load,
-        "cian": (old_cian_full_load, new_cian_full_load, "app.services.scrapers.cian.CianScraper"),
+        "cian": run_cian_full_load,
-        "yandex": (
+        "yandex": run_yandex_full_load,
-            old_yandex_full_load,
-            new_yandex_full_load,
-            "app.services.scrapers.yandex_realty.YandexRealtyScraper",
-        ),
     }
-    old_fn, new_fn, old_scraper_target = old_map[source]
+    scraper_target = {
+        "avito": f"{PFX}.AvitoScraper",
+        "cian": f"{PFX}.CianScraper",
+        "yandex": f"{PFX}.YandexRealtyScraper",
+    }[source]
+    fn = fn_map[source]
 
-    if new:
+    extra: dict[str, Any] = {}
-        new_scraper_target = {
+    if source == "yandex":
-            "avito": f"{NEW_PFX}.AvitoScraper",
+        enrichment = MagicMock()
-            "cian": f"{NEW_PFX}.CianScraper",
+        enrichment.record_yandex_price_history = MagicMock(return_value=0)
-            "yandex": f"{NEW_PFX}.YandexRealtyScraper",
+        extra["enrichment"] = enrichment
-        }[source]
+    with (
-        extra: dict[str, Any] = {}
+        patch(scraper_target, return_value=scraper),
-        if source == "yandex":
+        patch(f"{PFX}.save_listings", save_mock),
-            enrichment = MagicMock()
+        patch(f"{PFX}.runs", recorder),
-            enrichment.record_yandex_price_history = MagicMock(return_value=0)
+    ):
-            extra["enrichment"] = enrichment
+        counters = await fn(db, run_id=1, config=cfg, matcher=MagicMock(), **extra)
-        with (
-            patch(new_scraper_target, return_value=scraper),
-            patch(f"{NEW_PFX}.save_listings", save_mock),
-            patch(f"{NEW_PFX}.runs", recorder),
-        ):
-            counters = await new_fn(db, run_id=1, config=cfg, matcher=MagicMock(), **extra)
-    else:
-        import contextlib
-
-        ctx = [
-            patch(old_scraper_target, return_value=scraper),
-            patch(f"{OLD_PFX}.save_listings", save_mock),
-            patch(f"{OLD_PFX}.scrape_runs", recorder),
-            patch(f"{OLD_PFX}.settings", cfg),
-            patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
-        ]
-        if source == "yandex":
-            ctx.append(patch(f"{OLD_PFX}.record_yandex_price_history", MagicMock(return_value=0)))
-        with contextlib.ExitStack() as stack:
-            for cm in ctx:
-                stack.enter_context(cm)
-            counters = await old_fn(db, run_id=1)
     return counters.to_dict(), _normalize(recorder.calls)
 
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("source", ["avito", "cian", "yandex"])
-async def test_full_load_smoke_parity(source: str) -> None:
+async def test_full_load_smoke(source: str) -> None:
-    """Full load: 2 бакета через on_bucket → save + heartbeat → mark_done. Counters ≡."""
+    """Full load: 2 бакета через on_bucket → save + heartbeat → mark_done."""
-    counters = await _assert_parity(
+    counters, calls = await _drive_full_load(source=source)
-        await _drive_full_load(new=False, source=source),
-        await _drive_full_load(new=True, source=source),
-    )
     assert counters["unique_fetched"] == 3
     assert counters["saved_inserted"] == 3
     assert counters["saved_updated"] == 0
+    assert calls[-1][0] == "mark_done"

tradein-mvp/backend/tests/test_scraper_proxy.py

@@ -95,35 +95,33 @@ def test_scraper_proxy_url_empty_string_falls_through_to_avito():
 
 
 def test_avito_proxies_returns_correct_dict_shape():
-    from app.services.scrape_pipeline import _avito_proxies
+    """Kit `_avito_proxies(config)` — DI параметр вместо module-level settings patch."""
+    from scraper_kit.orchestration.pipeline import _avito_proxies
 
     proxy_url = "http://user:pass@proxy.example.com:8080"
-    with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)):
+    result = _avito_proxies(_mock_settings(proxy_url))
-        result = _avito_proxies()
 
     assert result == {"http": proxy_url, "https": proxy_url}
 
 
 def test_avito_proxies_returns_none_when_no_proxy():
-    from app.services.scrape_pipeline import _avito_proxies
+    from scraper_kit.orchestration.pipeline import _avito_proxies
 
-    with patch("app.services.scrape_pipeline.settings", _mock_settings(None)):
+    result = _avito_proxies(_mock_settings(None))
-        result = _avito_proxies()
 
     assert result is None
 
 
 def test_avito_proxies_fallback_via_avito_proxy_url():
-    """_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback.
+    """_avito_proxies(config) uses AVITO_PROXY_URL via scraper_proxy_url fallback.
 
-    When settings.scraper_proxy_url resolves to the avito URL (fallback path),
+    When config.scraper_proxy_url resolves to the avito URL (fallback path),
     the dict shape must be {"http": url, "https": url}.
     """
-    from app.services.scrape_pipeline import _avito_proxies
+    from scraper_kit.orchestration.pipeline import _avito_proxies
 
     avito_url = "http://mobile-proxy.space:9090"
-    with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)):
+    result = _avito_proxies(_mock_settings(avito_url))
-        result = _avito_proxies()
 
     assert result == {"http": avito_url, "https": avito_url}
 

tradein-mvp/backend/tests/test_scraper_resilience_1949_1950.py

@@ -1,633 +0,0 @@
-"""Тесты устойчивости скрейперов: avito changeip-retry (#1950) + cian anti-zombie (#1949).
-
-Fix A (#1950): _rotate_proxy_ip теперь делает до proxy_rotate_attempts попыток
-               по proxy_rotate_attempt_timeout_s каждая.
-
-Fix B (#1950): run_avito_city_sweep при AvitoBlockedError/RateLimited с
-               lots_inserted+lots_updated>0 вызывает mark_done (не mark_banned)
-               когда avito_serp_ok_not_banned=True.
-
-Fix C (#1949): run_cian_full_load применяет monkey-patch _browser.fetch с
-               asyncio.wait_for(timeout=cian_full_load_per_fetch_timeout_s).
-
-Fix D (#1949): run_cian_full_load создаёт фоновый heartbeat-task и отменяет
-               его в finally при любом завершении.
-
-Fix E (#1949): run_cian_city_sweep при defensive abort (consecutive anchor
-               timeout/failure) вызывает mark_done (не mark_banned) когда
-               SERP уже сохранил лоты (lots_inserted+lots_updated>0) и
-               cian_sweep_ok_not_banned=True.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import os
-import time
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from app.services.scrapers.avito_exceptions import AvitoBlockedError
-from app.services.scrapers.base import ScrapedLot
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix A: _rotate_proxy_ip retry loop (#1950)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-def _make_session_mock(get_side_effect: Any) -> AsyncMock:
-    """Вспомогательный: AsyncMock-сессия с заданным поведением .get()."""
-    mock_session = AsyncMock()
-    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
-    mock_session.__aexit__ = AsyncMock(return_value=None)
-    mock_session.get = AsyncMock(side_effect=get_side_effect)
-    return mock_session
-
-
-@pytest.mark.asyncio
-async def test_changeip_retry_success_on_second_attempt() -> None:
-    """#1950-A: первая попытка таймаут, вторая успех → возвращает True; ровно 2 GET-вызова."""
-    from app.services.scrape_pipeline import _rotate_proxy_ip
-
-    get_call_count = 0
-
-    async def fake_get(url: str) -> MagicMock:
-        nonlocal get_call_count
-        get_call_count += 1
-        if get_call_count == 1:
-            raise TimeoutError("connect timeout")
-        resp = MagicMock()
-        resp.json.return_value = {"new_ip": "1.2.3.4"}
-        return resp
-
-    mock_session = _make_session_mock(fake_get)
-
-    with (
-        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.proxy_rotate_attempts = 3
-        s.proxy_rotate_attempt_timeout_s = 8.0
-        s.avito_proxy_rotate_settle_s = 0.0
-        s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
-        s.avito_proxy_max_rotations = 3
-
-        result = await _rotate_proxy_ip(
-            source="avito",
-            reason="test-detail-block",
-            rotations_done=0,
-        )
-
-    assert result is True
-    assert get_call_count == 2, f"expected 2 GET calls, got {get_call_count}"
-
-
-@pytest.mark.asyncio
-async def test_changeip_retry_all_fail_returns_false() -> None:
-    """#1950-A: все 3 попытки падают с TimeoutError → False; ровно 3 GET-вызова."""
-    from app.services.scrape_pipeline import _rotate_proxy_ip
-
-    get_call_count = 0
-
-    async def failing_get(url: str) -> None:
-        nonlocal get_call_count
-        get_call_count += 1
-        raise TimeoutError("connect timeout")
-
-    mock_session = _make_session_mock(failing_get)
-
-    with (
-        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.proxy_rotate_attempts = 3
-        s.proxy_rotate_attempt_timeout_s = 8.0
-        s.avito_proxy_rotate_settle_s = 0.0
-        s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
-        s.avito_proxy_max_rotations = 3
-
-        result = await _rotate_proxy_ip(
-            source="avito",
-            reason="test-detail-block",
-            rotations_done=0,
-        )
-
-    assert result is False
-    assert (
-        get_call_count == 3
-    ), f"expected exactly 3 attempts (proxy_rotate_attempts=3), got {get_call_count}"
-
-
-@pytest.mark.asyncio
-async def test_changeip_retry_bounded_by_setting() -> None:
-    """#1950-A: proxy_rotate_attempts=2 → ровно 2 попытки, не больше."""
-    from app.services.scrape_pipeline import _rotate_proxy_ip
-
-    call_count = 0
-
-    async def failing_get(url: str) -> None:
-        nonlocal call_count
-        call_count += 1
-        raise TimeoutError("timeout")
-
-    mock_session = _make_session_mock(failing_get)
-
-    with (
-        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.proxy_rotate_attempts = 2
-        s.proxy_rotate_attempt_timeout_s = 8.0
-        s.avito_proxy_rotate_settle_s = 0.0
-        s.cian_proxy_rotate_url = "http://changeip.example.com/rotate"
-        s.avito_proxy_rotate_url = None
-        s.cian_proxy_max_rotations = 3
-
-        result = await _rotate_proxy_ip(
-            source="cian",
-            reason="serp-block",
-            rotations_done=1,
-        )
-
-    assert result is False
-    assert call_count == 2, f"proxy_rotate_attempts=2 should make exactly 2 calls, got {call_count}"
-
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix B: честный статус run_avito_city_sweep (#1950)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-def _make_city_sweep_mocks() -> tuple[MagicMock, MagicMock, MagicMock]:
-    """Возвращает (mock_db, mock_runs, mock_scraper) для тестов run_avito_city_sweep."""
-    mock_db = MagicMock()
-
-    # priority_rows для detail-фазы: 3 строки с source_url
-    class _DetailRow:
-        def __getitem__(self, key: str) -> str:
-            return "https://www.avito.ru/ekaterinburg/kvartiry/test-1"
-
-    mock_db.execute.return_value.mappings.return_value.all.return_value = [
-        _DetailRow(),
-        _DetailRow(),
-        _DetailRow(),
-    ]
-
-    mock_runs = MagicMock()
-    mock_runs.is_cancelled.return_value = False
-    mock_runs.mark_done = MagicMock()
-    mock_runs.mark_banned = MagicMock()
-    mock_runs.mark_failed = MagicMock()
-    mock_runs.update_heartbeat = MagicMock()
-
-    mock_scraper = MagicMock()
-    # AvitoScraper НЕ используется как async CM в _avito_anchor_phases (просто instantiate)
-    mock_scraper.fetch_around = AsyncMock(
-        return_value=[
-            ScrapedLot(
-                source="avito",
-                source_url=f"https://www.avito.ru/e/k/{i}",
-                source_id=str(i),
-                price_rub=5_000_000,
-                address="ЕКБ test",
-                listing_segment="vtorichka",
-            )
-            for i in range(3)
-        ]
-    )
-
-    return mock_db, mock_runs, mock_scraper
-
-
-@pytest.mark.asyncio
-async def test_avito_city_sweep_serp_ok_marks_done_not_banned() -> None:
-    """#1950-B: SERP собрал лоты (ins=3), detail заблокировало → mark_done не mark_banned.
-
-    Условие: avito_serp_ok_not_banned=True (default), lots_inserted+lots_updated > 0.
-    """
-    from app.services.scrape_pipeline import run_avito_city_sweep
-
-    mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
-
-    # Shared AsyncSession mock (browser_mode=False)
-    mock_session = AsyncMock()
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
-        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
-        patch(
-            "app.services.scrape_pipeline.fetch_detail",
-            AsyncMock(side_effect=AvitoBlockedError("HTTP 403 banned")),
-        ),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.scraper_fetch_mode = "curl_cffi"
-        s.scraper_proxy_url = None
-        s.avito_proxy_max_rotations = 0  # no rotation budget
-        s.avito_serp_ok_not_banned = True  # ← наш флаг
-        s.avito_proxy_rotate_settle_s = 0.0
-        s.proxy_rotate_attempts = 3
-        s.proxy_rotate_attempt_timeout_s = 8.0
-
-        await run_avito_city_sweep(
-            mock_db,
-            run_id=1,
-            anchors=[(56.84, 60.61, "test-anchor")],
-            detail_top_n=3,
-            enrich_houses=False,
-            enrich_imv=False,
-        )
-
-    assert (
-        mock_runs.mark_done.called
-    ), "mark_done должен быть вызван: SERP OK (ins=3), detail blocked"
-    assert (
-        not mock_runs.mark_banned.called
-    ), "mark_banned НЕ должен быть вызван когда SERP уже сохранил лоты"
-    # enrichment_abort_note передан в counters
-    call_args = mock_runs.mark_done.call_args
-    # mark_done(db, run_id, counters_dict)
-    passed_counters: dict = call_args[0][2] if call_args[0] else {}
-    assert (
-        "enrichment_abort_note" in passed_counters
-    ), "enrichment_abort_note должен присутствовать в counters переданных mark_done"
-
-
-@pytest.mark.asyncio
-async def test_avito_city_sweep_serp_zero_lots_marks_banned() -> None:
-    """#1950-B: SERP вернул 0 лотов, SERP-блок → mark_banned (не mark_done).
-
-    Условие: lots_inserted+lots_updated == 0.
-    """
-    from app.services.scrape_pipeline import run_avito_city_sweep
-
-    mock_db, mock_runs, _ = _make_city_sweep_mocks()
-
-    # SERP сам поднимает AvitoBlockedError → anchor_lots не сохраняется
-    mock_scraper_serp_block = MagicMock()
-    mock_scraper_serp_block.fetch_around = AsyncMock(side_effect=AvitoBlockedError("SERP HTTP 403"))
-
-    mock_session = AsyncMock()
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper_serp_block),
-        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.scraper_fetch_mode = "curl_cffi"
-        s.scraper_proxy_url = None
-        s.avito_proxy_max_rotations = 0
-        s.avito_serp_ok_not_banned = True  # включён флаг, но лотов нет
-        s.avito_proxy_rotate_settle_s = 0.0
-        s.proxy_rotate_attempts = 3
-        s.proxy_rotate_attempt_timeout_s = 8.0
-
-        await run_avito_city_sweep(
-            mock_db,
-            run_id=2,
-            anchors=[(56.84, 60.61, "test-anchor")],
-            detail_top_n=3,
-            enrich_houses=False,
-            enrich_imv=False,
-        )
-
-    assert (
-        mock_runs.mark_banned.called
-    ), "mark_banned должен быть вызван: SERP сам заблокирован, lots=0"
-    assert not mock_runs.mark_done.called, "mark_done НЕ должен быть вызван при lots_inserted == 0"
-
-
-@pytest.mark.asyncio
-async def test_avito_city_sweep_flag_off_still_marks_banned() -> None:
-    """#1950-B: avito_serp_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
-    from app.services.scrape_pipeline import run_avito_city_sweep
-
-    mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
-    mock_session = AsyncMock()
-
-    with (
-        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
-        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
-        patch(
-            "app.services.scrape_pipeline.fetch_detail",
-            AsyncMock(side_effect=AvitoBlockedError("HTTP 403")),
-        ),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.scraper_fetch_mode = "curl_cffi"
-        s.scraper_proxy_url = None
-        s.avito_proxy_max_rotations = 0
-        s.avito_serp_ok_not_banned = False  # ← флаг выключен
-        s.avito_proxy_rotate_settle_s = 0.0
-        s.proxy_rotate_attempts = 3
-        s.proxy_rotate_attempt_timeout_s = 8.0
-
-        await run_avito_city_sweep(
-            mock_db,
-            run_id=3,
-            anchors=[(56.84, 60.61, "test-anchor")],
-            detail_top_n=3,
-            enrich_houses=False,
-            enrich_imv=False,
-        )
-
-    assert (
-        mock_runs.mark_banned.called
-    ), "avito_serp_ok_not_banned=False → mark_banned (backward-compat)"
-    assert not mock_runs.mark_done.called
-
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix C: cian per-fetch wait_for (#1949)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_cian_full_load_hanging_fetch_cancelled_by_timeout() -> None:
-    """#1949-C: browser-fetch зависший навсегда отменяется per-fetch timeout.
-
-    Monkey-patch заменяет _browser.fetch на _timed_fetch(wait_for timeout=0.05s).
-    Зависший fetch отменяется через ~50ms → TimeoutError → run завершается.
-    """
-    from app.services.scrape_pipeline import run_cian_full_load
-
-    fetch_was_called = False
-
-    async def hanging_fetch(url: str) -> str:
-        nonlocal fetch_was_called
-        fetch_was_called = True
-        await asyncio.Event().wait()  # висит вечно
-        return ""
-
-    mock_browser = MagicMock()
-    mock_browser.fetch = hanging_fetch
-
-    mock_scraper = MagicMock()
-    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
-    mock_scraper.__aexit__ = AsyncMock(return_value=None)
-    mock_scraper._browser = mock_browser
-    mock_scraper.request_delay_sec = 0.0
-
-    # fetch_all_secondary симулирует _fetch_page_html: вызывает fetch и ловит Exception
-    async def fake_fetch_all_secondary(**kwargs: object) -> None:
-        try:
-            # После monkey-patch _browser.fetch = _timed_fetch (с timeout=0.05s)
-            await mock_scraper._browser.fetch("https://ekb.cian.ru/sale/flat/1/")
-        except Exception:
-            pass  # TimeoutError → аналог _fetch_page_html → returns None
-
-    mock_scraper.fetch_all_secondary = fake_fetch_all_secondary
-
-    mock_db = MagicMock()
-    mock_runs = MagicMock()
-    mock_runs.is_cancelled.return_value = False
-    mock_runs.mark_done = MagicMock()
-    mock_runs.mark_failed = MagicMock()
-    mock_runs.update_heartbeat = MagicMock()
-
-    t0 = time.monotonic()
-    with (
-        patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.cian_full_load_per_fetch_timeout_s = 0.05  # 50ms — быстрый timeout для теста
-        s.cian_full_load_detail_top_n = 0
-
-        await run_cian_full_load(mock_db, run_id=1)
-
-    elapsed = time.monotonic() - t0
-
-    assert fetch_was_called, "hanging_fetch должен быть вызван через monkey-patched _browser.fetch"
-    assert (
-        elapsed < 3.0
-    ), f"run должен завершиться быстро (fetch timeout 50ms), занял {elapsed:.2f}s"
-    assert mock_runs.mark_done.called, "run_cian_full_load должен завершиться mark_done"
-
-
-@pytest.mark.asyncio
-async def test_cian_full_load_per_fetch_timeout_disabled_when_zero() -> None:
-    """#1949-C: cian_full_load_per_fetch_timeout_s=0 → monkey-patch НЕ применяется."""
-    from app.services.scrape_pipeline import run_cian_full_load
-
-    original_fetch = AsyncMock(return_value="<html/>")
-
-    mock_browser = MagicMock()
-    mock_browser.fetch = original_fetch
-
-    mock_scraper = MagicMock()
-    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
-    mock_scraper.__aexit__ = AsyncMock(return_value=None)
-    mock_scraper._browser = mock_browser
-    mock_scraper.request_delay_sec = 0.0
-    mock_scraper.fetch_all_secondary = AsyncMock(return_value=None)
-
-    mock_db = MagicMock()
-    mock_runs = MagicMock()
-    mock_runs.is_cancelled.return_value = False
-    mock_runs.mark_done = MagicMock()
-    mock_runs.mark_failed = MagicMock()
-    mock_runs.update_heartbeat = MagicMock()
-
-    with (
-        patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.cian_full_load_per_fetch_timeout_s = 0.0  # отключено
-        s.cian_full_load_detail_top_n = 0
-
-        await run_cian_full_load(mock_db, run_id=2)
-
-    # Когда timeout=0 (отключён), monkey-patch не применяется → fetch остаётся original
-    assert (
-        mock_browser.fetch is original_fetch
-    ), "cian_full_load_per_fetch_timeout_s=0 → fetch НЕ должен быть заменён monkey-patch"
-
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix D: cian background heartbeat created and cancelled (#1949)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_cian_full_load_background_heartbeat_created_and_cancelled() -> None:
-    """#1949-D: background heartbeat task создаётся и отменяется в finally."""
-    from app.services.scrape_pipeline import run_cian_full_load
-
-    mock_scraper = MagicMock()
-    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
-    mock_scraper.__aexit__ = AsyncMock(return_value=None)
-    mock_scraper._browser = None  # per-fetch timeout не применяется
-    mock_scraper.fetch_all_secondary = AsyncMock(return_value=None)
-
-    mock_db = MagicMock()
-    mock_runs = MagicMock()
-    mock_runs.is_cancelled.return_value = False
-    mock_runs.mark_done = MagicMock()
-    mock_runs.mark_failed = MagicMock()
-    mock_runs.update_heartbeat = MagicMock()
-
-    created_tasks: list[asyncio.Task[None]] = []
-    _real_create_task = asyncio.create_task
-
-    def _tracking_create_task(coro: Any, *, name: str | None = None) -> asyncio.Task[None]:
-        task = _real_create_task(coro, name=name)
-        created_tasks.append(task)
-        return task  # type: ignore[return-value]
-
-    with (
-        patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch(
-            "app.services.scrape_pipeline.asyncio.create_task",
-            side_effect=_tracking_create_task,
-        ),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.cian_full_load_per_fetch_timeout_s = 0.0
-        s.cian_full_load_detail_top_n = 0
-
-        await run_cian_full_load(mock_db, run_id=3)
-
-    assert (
-        len(created_tasks) >= 1
-    ), "asyncio.create_task должен быть вызван хотя бы раз (background heartbeat)"
-    for task in created_tasks:
-        assert (
-            task.cancelled() or task.done()
-        ), f"Heartbeat task должен быть отменён в finally: {task}"
-
-
-@pytest.mark.asyncio
-async def test_cian_full_load_heartbeat_cancelled_on_exception() -> None:
-    """#1949-D: heartbeat task отменяется в finally даже при исключении в run."""
-    from app.services.scrape_pipeline import run_cian_full_load
-
-    mock_scraper = MagicMock()
-    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
-    mock_scraper.__aexit__ = AsyncMock(return_value=None)
-    mock_scraper._browser = None
-    mock_scraper.fetch_all_secondary = AsyncMock(side_effect=RuntimeError("fatal-test-error"))
-
-    mock_db = MagicMock()
-    mock_runs = MagicMock()
-    mock_runs.is_cancelled.return_value = False
-    mock_runs.mark_done = MagicMock()
-    mock_runs.mark_failed = MagicMock()
-    mock_runs.update_heartbeat = MagicMock()
-
-    created_tasks: list[asyncio.Task[None]] = []
-    _real_create_task = asyncio.create_task
-
-    def _tracking_create_task(coro: Any, *, name: str | None = None) -> asyncio.Task[None]:
-        task = _real_create_task(coro, name=name)
-        created_tasks.append(task)
-        return task  # type: ignore[return-value]
-
-    with (
-        patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch(
-            "app.services.scrape_pipeline.asyncio.create_task",
-            side_effect=_tracking_create_task,
-        ),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.cian_full_load_per_fetch_timeout_s = 0.0
-        s.cian_full_load_detail_top_n = 0
-
-        with pytest.raises(RuntimeError, match="fatal-test-error"):
-            await run_cian_full_load(mock_db, run_id=4)
-
-    assert len(created_tasks) >= 1, "heartbeat task должен быть создан"
-    for task in created_tasks:
-        assert (
-            task.cancelled() or task.done()
-        ), f"Heartbeat task должен быть отменён в finally даже при RuntimeError: {task}"
-    assert mock_runs.mark_failed.called, "mark_failed должен быть вызван при RuntimeError"
-
-
-# ──────────────────────────────────────────────────────────────────────────────
-# Fix E: run_cian_city_sweep честный статус при defensive abort (#1949)
-# ──────────────────────────────────────────────────────────────────────────────
-
-
-def test_mark_cian_sweep_abort_lots_present_marks_done() -> None:
-    """#1949-E: SERP уже сохранил лоты (ins+upd>0) до abort → mark_done, не mark_banned."""
-    from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
-
-    mock_db = MagicMock()
-    mock_runs = MagicMock()
-    counters = CianCitySweepCounters(
-        anchors_total=5, anchors_done=2, lots_inserted=12, lots_updated=3
-    )
-
-    with (
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.cian_sweep_ok_not_banned = True
-        _mark_cian_sweep_abort(mock_db, 42, counters, "cian sweep aborted: 3 consecutive timeouts")
-
-    assert mock_runs.mark_done.called, "mark_done должен быть вызван: SERP OK (ins=12 upd=3)"
-    assert not mock_runs.mark_banned.called, "mark_banned НЕ должен вызываться когда лоты сохранены"
-    passed_counters: dict = mock_runs.mark_done.call_args[0][2]
-    assert "abort_note" in passed_counters, "abort_note должен присутствовать в counters"
-
-
-def test_mark_cian_sweep_abort_no_lots_marks_banned() -> None:
-    """#1949-E: лотов ещё не было (ins=upd=0) на момент abort → mark_banned как раньше."""
-    from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
-
-    mock_db = MagicMock()
-    mock_runs = MagicMock()
-    counters = CianCitySweepCounters(
-        anchors_total=5, anchors_done=1, lots_inserted=0, lots_updated=0
-    )
-
-    with (
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.cian_sweep_ok_not_banned = True
-        _mark_cian_sweep_abort(mock_db, 43, counters, "cian sweep aborted: SERP blocked at anchor")
-
-    assert mock_runs.mark_banned.called, "mark_banned должен быть вызван: SERP не собрал лотов"
-    assert not mock_runs.mark_done.called, "mark_done НЕ должен вызываться при lots ins=upd=0"
-
-
-def test_mark_cian_sweep_abort_flag_off_still_marks_banned() -> None:
-    """#1949-E: cian_sweep_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
-    from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
-
-    mock_db = MagicMock()
-    mock_runs = MagicMock()
-    counters = CianCitySweepCounters(
-        anchors_total=5, anchors_done=3, lots_inserted=20, lots_updated=5
-    )
-
-    with (
-        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
-        patch("app.services.scrape_pipeline.settings") as s,
-    ):
-        s.cian_sweep_ok_not_banned = False
-        _mark_cian_sweep_abort(mock_db, 44, counters, "cian sweep aborted: 3 consecutive timeouts")
-
-    assert mock_runs.mark_banned.called, "flag=False → mark_banned (backward-compat)"
-    assert not mock_runs.mark_done.called

tradein-mvp/backend/tests/test_sweep_drain.py

@@ -1,310 +0,0 @@
-"""Тесты кооперативного SIGTERM-drain в длинных sweep/full-load циклах (#1182 Phase 3a).
-
-Проверяют, что при выставленном shutdown_requested() sweep'ы scrape_pipeline.py
-останавливаются ЧИСТО на своих between-unit checkpoint'ах:
-  - финализируют run как mark_done(partial) — НЕ mark_cancelled (это не user-cancel);
-  - НЕ обрабатывают последующие anchor'ы / бакеты / дорогие фазы (IMV / detail);
-  - user-cancel ветка (is_cancelled) не задействуется.
-
-Тесты network-free: внутренние scrape-вызовы и lifecycle скрейперов monkeypatch'атся,
-shutdown_requested() патчится контролируемым флагом, переключающимся в True ПОСЛЕ
-первого unit'а (а не с самого старта) — чтобы первый unit прошёл, а на втором
-сработал drain.
-"""
-
-from __future__ import annotations
-
-import os
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-from app.services import scrape_pipeline
-from app.services.scrapers.avito import AvitoScraper
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.cian import CianScraper
-
-# Два anchor'а: первый успешен, второй должен быть отрезан drain'ом.
-TWO_ANCHORS: list[tuple[float, float, str]] = [
-    (56.8400, 60.6050, "First"),
-    (56.7950, 60.5300, "Second"),
-]
-
-
-# ── Fake scrape_runs (записывает финализаторы) ──────────────────────────────
-
-
-class _FakeRuns:
-    def __init__(self) -> None:
-        self.heartbeats: list[dict[str, int]] = []
-        self.done: dict[str, int] | None = None
-        self.failed: tuple[str, dict[str, int]] | None = None
-        self.banned: tuple[str, dict[str, int]] | None = None
-        self.cancelled_calls: int = 0
-        self.is_cancelled_calls: int = 0
-
-    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
-        self.is_cancelled_calls += 1
-        return False
-
-    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.heartbeats.append(dict(counters))
-
-    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.done = dict(counters)
-
-    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.failed = (error, dict(counters))
-
-    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.banned = (error, dict(counters))
-
-    def mark_cancelled(self, _db: Any, _run_id: int) -> bool:
-        self.cancelled_calls += 1
-        return True
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_cancelled", fake.mark_cancelled)
-
-
-def _fake_lot(source: str, offer_id: str) -> ScrapedLot:
-    return ScrapedLot(
-        source=source,
-        source_url=f"https://example.ru/{offer_id}/",
-        source_id=offer_id,
-        address="Екатеринбург, улица Тестовая, 1",
-        price_rub=4_000_000,
-        area_m2=42.0,
-        rooms=1,
-    )
-
-
-# ── Avito city sweep: drain на границе anchor'а ─────────────────────────────
-
-
-async def test_avito_city_sweep_drains_on_shutdown(monkeypatch: pytest.MonkeyPatch) -> None:
-    """SIGTERM после первого anchor'а → sweep останавливается на втором anchor'е:
-
-    - второй anchor НЕ фетчится (drain до начала его фаз);
-    - run финализируется mark_done(partial) с anchors_done==1;
-    - mark_cancelled НЕ вызывается (это drain, не user-cancel);
-    - mark_failed / mark_banned не вызываются.
-    """
-    import curl_cffi.requests as _cffi_mod
-
-    state = {"flag": False}
-    monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
-    fetch_calls: list[str] = []
-
-    async def fake_fetch_around(
-        self: AvitoScraper,
-        lat: float,
-        lon: float,
-        radius_m: int,
-        pages: int = 1,
-        delay_override_sec: float | None = None,
-    ) -> list[ScrapedLot]:
-        fetch_calls.append(f"{lat:.4f}")
-        lots = [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(3)]
-        # SIGTERM приходит ПОСЛЕ первого anchor'а — следующий top-of-loop checkpoint
-        # увидит флаг и сделает drain.
-        state["flag"] = True
-        return lots
-
-    monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
-
-    # Stub shared curl_cffi AsyncSession (non-browser fetch mode).
-    fake_session = AsyncMock()
-    fake_session.__aenter__ = AsyncMock(return_value=fake_session)
-    fake_session.__aexit__ = AsyncMock(return_value=None)
-    monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
-
-    # IMV-фаза не должна вызываться при drain (enrich_imv=False всё равно).
-    monkeypatch.setattr(
-        "app.services.house_imv_backfill.process_houses_imv_batch",
-        AsyncMock(side_effect=AssertionError("IMV phase must not run after drain")),
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_avito_city_sweep(
-        db=MagicMock(),
-        run_id=1,
-        anchors=TWO_ANCHORS,
-        enrich_houses=False,
-        detail_top_n=0,
-        enrich_imv=False,
-        request_delay_sec=0.0,
-    )
-
-    # Второй anchor не дошёл до SERP — fetch вызван ровно один раз.
-    assert fetch_calls == ["56.8400"], f"только первый anchor должен фетчиться; got {fetch_calls}"
-    # Partial-финализация: обработан только первый anchor.
-    assert counters.anchors_done == 1
-    assert counters.lots_fetched == 3
-    # mark_done(partial) вызван (drain-ветка), и счётчики partial.
-    assert fake.done is not None, "drain должен вызвать mark_done(partial)"
-    assert fake.done["anchors_done"] == 1
-    # Это НЕ user-cancel.
-    assert fake.cancelled_calls == 0, "mark_cancelled не должен вызываться при drain"
-    assert fake.failed is None
-    assert fake.banned is None
-
-
-# ── Cian city sweep: drain на границе anchor'а ──────────────────────────────
-
-
-@pytest.fixture
-def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
-    def _init(self: CianScraper) -> None:
-        self.request_delay_sec = 0.0
-        self._cffi = None
-        self._browser = None
-
-    async def _aenter(self: CianScraper) -> CianScraper:
-        return self
-
-    async def _aexit(self: CianScraper, *_args: Any) -> None:
-        return None
-
-    monkeypatch.setattr(CianScraper, "__init__", _init)
-    monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
-    monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
-
-
-async def test_cian_city_sweep_drains_on_shutdown(
-    monkeypatch: pytest.MonkeyPatch, _stub_cian_lifecycle: None
-) -> None:
-    """SIGTERM после первого anchor'а → cian sweep drain на втором anchor'е → mark_done(partial)."""
-    state = {"flag": False}
-    monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
-
-    serp_calls: list[str] = []
-
-    async def fake_fetch_multi(
-        self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
-    ) -> list[ScrapedLot]:
-        serp_calls.append(f"{lat:.4f}")
-        state["flag"] = True
-        return [_fake_lot("cian", f"{lat:.4f}-1")]
-
-    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(
-        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None)
-    )
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_cian_city_sweep(
-        db=MagicMock(),
-        run_id=2,
-        anchors=TWO_ANCHORS,
-        detail_top_n=0,
-        enrich_houses=False,
-        request_delay_sec=0.0,
-    )
-
-    assert serp_calls == ["56.8400"], f"только первый anchor должен фетчиться; got {serp_calls}"
-    assert counters.anchors_done == 1
-    assert fake.done is not None, "drain должен вызвать mark_done(partial)"
-    assert fake.done["anchors_done"] == 1
-    assert fake.cancelled_calls == 0
-    assert fake.failed is None
-    assert fake.banned is None
-
-
-# ── Avito full load: drain через sentinel в on_bucket ───────────────────────
-
-
-@pytest.fixture
-def _stub_avito_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
-    def _init(self: AvitoScraper) -> None:
-        self.request_delay_sec = 0.0
-        self._cffi = None
-        self._browser = None
-
-    async def _aenter(self: AvitoScraper) -> AvitoScraper:
-        return self
-
-    async def _aexit(self: AvitoScraper, *_args: Any) -> None:
-        return None
-
-    monkeypatch.setattr(AvitoScraper, "__init__", _init)
-    monkeypatch.setattr(AvitoScraper, "__aenter__", _aenter)
-    monkeypatch.setattr(AvitoScraper, "__aexit__", _aexit)
-
-
-async def test_avito_full_load_drains_on_shutdown(
-    monkeypatch: pytest.MonkeyPatch, _stub_avito_lifecycle: None
-) -> None:
-    """SIGTERM после первого бакета → on_bucket поднимает sentinel → mark_done(partial):
-
-    - бакет №2 поднимает RuntimeError("shutdown") (НЕ "cancelled"), бакет №3 не достигается;
-    - done_buckets == ["b1"] (бакет, упавший на drain-checkpoint, в done не попадает);
-    - run финализируется mark_done(partial) с unique_fetched==1;
-    - mark_cancelled / mark_failed / mark_banned не вызываются.
-    """
-    state = {"flag": False}
-    monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
-
-    saved_buckets: list[str] = []
-
-    def fake_save(_db: Any, lots: Any, **_kw: Any) -> tuple[int, int]:
-        saved_buckets.append("x")
-        return (len(lots), 0)
-
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-
-    async def fake_fetch_all(self: AvitoScraper, **kwargs: Any) -> None:
-        on_bucket = kwargs["on_bucket"]
-        # Бакет №1 — сохраняется штатно.
-        on_bucket("b1", [_fake_lot("avito", "b1-1")])
-        # SIGTERM приходит после первого бакета.
-        state["flag"] = True
-        # Бакет №2 — on_bucket должен поднять RuntimeError("shutdown").
-        on_bucket("b2", [_fake_lot("avito", "b2-1")])
-        # Бакет №3 — НЕ должен быть достигнут.
-        on_bucket("b3", [_fake_lot("avito", "b3-1")])
-
-    monkeypatch.setattr(AvitoScraper, "fetch_all_secondary", fake_fetch_all)
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_avito_full_load(
-        db=MagicMock(),
-        run_id=3,
-        request_delay_sec=0.0,
-    )
-
-    # Только первый бакет сохранён (b2 упал на checkpoint до save, b3 не достигнут).
-    assert saved_buckets == ["x"], f"только b1 должен сохраниться; got {saved_buckets}"
-    assert counters.unique_fetched == 1
-    assert counters.saved_inserted == 1
-    # mark_done(partial) с done_buckets только b1.
-    assert fake.done is not None, "drain должен вызвать mark_done(partial)"
-    assert fake.done.get("done_buckets") == ["b1"]
-    assert fake.done["unique_fetched"] == 1
-    # НЕ user-cancel, не failed/banned.
-    assert fake.cancelled_calls == 0
-    assert fake.failed is None
-    assert fake.banned is None

tradein-mvp/backend/tests/test_sweep_imv_phase.py

@@ -1,4 +1,4 @@
-"""Tests for IMV enrichment phase integrated into Avito city sweep.
+"""Tests for IMV enrichment phase integrated into Avito city sweep (kit).
 
 Проверяет:
 1. enrich_imv=True — sweep вызывает IMV-фазу для touched house_id.
@@ -9,9 +9,12 @@
 6. CitySweepStartRequest принимает enrich_imv (API-level param).
 7. Cooperative cancel перед IMV-фазой — skip IMV, sweep marked done.
 
-NB: run_avito_city_sweep использует внутренние _avito_anchor_phases (не run_avito_pipeline).
+Kit — единственная orchestration-копия (#2397 Part E1 удалил legacy
-Тесты патчат AvitoScraper.fetch_around + save_listings + fetch_house_catalog +
+`app.services.scrape_pipeline`): behavior-тесты (пункты 1,2,3,5,7 выше через
-save_house_catalog_enrichment для контроля touched_house_ids.
+`run_avito_city_sweep`) переведены на `scraper_kit.orchestration.pipeline` с DI
+(config/matcher/enrichment/shutdown_requested) вместо прямых импортов `app.*`.
+`enrichment.process_houses_imv_batch` — инжектируемый Protocol-метод (было прямым
+импортом `app.services.house_imv_backfill.process_houses_imv_batch`).
 """
 
 from __future__ import annotations
@@ -19,6 +22,7 @@ from __future__ import annotations
 import os
 from contextlib import ExitStack
 from dataclasses import fields
+from types import SimpleNamespace
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -26,10 +30,12 @@ import pytest
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
 
+PFX = "scraper_kit.orchestration.pipeline"
+
 
 def _make_house_lot(offer_id: str, house_path: str) -> Any:
     """ScrapedLot с house_url для тестов house-enrich пути."""
-    from app.services.scrapers.base import ScrapedLot
+    from scraper_kit.base import ScrapedLot
 
     return ScrapedLot(
         source="avito",
@@ -43,16 +49,35 @@ def _make_house_lot(offer_id: str, house_path: str) -> Any:
     )
 
 
+def _config() -> SimpleNamespace:
+    return SimpleNamespace(
+        scraper_fetch_mode="curl_cffi",
+        browser_http_endpoint="http://browser.test/fetch",
+        scraper_proxy_url=None,
+        avito_proxy_rotate_url=None,
+        avito_proxy_max_rotations=0,
+        avito_serp_ok_not_banned=True,
+        avito_proxy_rotate_settle_s=0.0,
+        proxy_rotate_attempts=1,
+        proxy_rotate_attempt_timeout_s=1.0,
+        cian_proxy_rotate_url=None,
+        cian_proxy_max_rotations=0,
+        yandex_proxy_rotate_url=None,
+        yandex_proxy_max_rotations=0,
+        scraper_skip_seen_today=False,
+    )
+
+
 def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
     """Входит в набор patch'ей для базового Avito sweep через ExitStack:
     - AvitoScraper.fetch_around → lots с house_url (по одному на house_id)
     - save_listings → (n, 0)
     - fetch_house_catalog → MagicMock()
     - save_house_catalog_enrichment → {"house_id": X} по очереди
-    - curl_cffi.requests.AsyncSession → fake
+    - AsyncSession (kit-модуль) → fake
     - asyncio.sleep → no-op
     """
-    from app.services.scrapers.avito import AvitoScraper
+    from scraper_kit.providers.avito.serp import AvitoScraper
 
     lots = [_make_house_lot(f"lot-{hid}", f"/ekaterinburg/houses/test/{hid}") for hid in house_ids]
 
@@ -62,7 +87,7 @@ def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
     # save_house_catalog_enrichment вызывается по одному разу на каждый дом
     house_id_iter = iter(house_ids)
 
-    def fake_save_house(db: Any, enrichment: Any) -> dict:
+    def fake_save_house(db: Any, enrichment: Any, **_kw: Any) -> dict:
         try:
             return {"house_id": next(house_id_iter)}
         except StopIteration:
@@ -73,33 +98,29 @@ def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
     fake_session.__aexit__ = AsyncMock(return_value=None)
 
     stack.enter_context(patch.object(AvitoScraper, "fetch_around", fake_fetch_around))
-    stack.enter_context(
+    stack.enter_context(patch(f"{PFX}.save_listings", lambda _db, _lots, **_kw: (len(_lots), 0)))
-        patch(
+    stack.enter_context(patch(f"{PFX}.fetch_house_catalog", AsyncMock(return_value=MagicMock())))
-            "app.services.scrape_pipeline.save_listings",
+    stack.enter_context(patch(f"{PFX}.save_house_catalog_enrichment", fake_save_house))
-            lambda _db, _lots, **_kw: (len(_lots), 0),
+    stack.enter_context(patch(f"{PFX}.AsyncSession", return_value=fake_session))
-        )
+    stack.enter_context(patch(f"{PFX}.asyncio.sleep", AsyncMock()))
-    )
+
-    stack.enter_context(
+
-        patch(
+def _enter_runs_patches(stack: ExitStack, *, mark_done_mock: Any = None) -> Any:
-            "app.services.scrape_pipeline.fetch_house_catalog",
+    """Патчит `scraper_kit.orchestration.pipeline.runs` (is_cancelled/update_heartbeat/
-            AsyncMock(return_value=MagicMock()),
+    mark_done)."""
-        )
+    mock_runs = MagicMock()
-    )
+    mock_runs.is_cancelled.return_value = False
-    stack.enter_context(
+    mock_runs.update_heartbeat = MagicMock()
-        patch(
+    mock_runs.mark_done = mark_done_mock if mark_done_mock is not None else MagicMock()
-            "app.services.scrape_pipeline.save_house_catalog_enrichment",
+    stack.enter_context(patch(f"{PFX}.runs", mock_runs))
-            fake_save_house,
+    return mock_runs
-        )
-    )
-    stack.enter_context(patch("curl_cffi.requests.AsyncSession", return_value=fake_session))
-    stack.enter_context(patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()))
 
 
 # ── CitySweepCounters has imv_* fields ────────────────────────────────────────
 
 
 def test_city_sweep_counters_has_imv_fields() -> None:
-    from app.services.scrape_pipeline import CitySweepCounters
+    from scraper_kit.orchestration.pipeline import CitySweepCounters
 
     c = CitySweepCounters()
     assert hasattr(c, "imv_attempted")
@@ -111,7 +132,7 @@ def test_city_sweep_counters_has_imv_fields() -> None:
 
 
 def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
-    from app.services.scrape_pipeline import CitySweepCounters
+    from scraper_kit.orchestration.pipeline import CitySweepCounters
 
     c = CitySweepCounters(imv_attempted=5, imv_enriched=3, imv_failed=2)
     d = c.to_dict()
@@ -127,7 +148,7 @@ def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
 
 
 def test_pipeline_result_has_touched_house_ids() -> None:
-    from app.services.scrape_pipeline import PipelineCounters, PipelineResult
+    from scraper_kit.orchestration.pipeline import PipelineCounters, PipelineResult
 
     r = PipelineResult(
         anchor_lat=56.84,
@@ -143,7 +164,7 @@ def test_pipeline_result_has_touched_house_ids() -> None:
 
 
 def test_pipeline_result_touched_house_ids_set() -> None:
-    from app.services.scrape_pipeline import PipelineCounters, PipelineResult
+    from scraper_kit.orchestration.pipeline import PipelineCounters, PipelineResult
 
     r = PipelineResult(
         anchor_lat=56.84,
@@ -167,36 +188,31 @@ async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
     Патчим AvitoScraper.fetch_around → lots с house_url, save_house_catalog_enrichment
     → {house_id: X} чтобы all_touched_house_ids = {10, 20}.
     """
-    from app.services.house_imv_backfill import HouseIMVBackfillResult
+    from scraper_kit.orchestration.pipeline import run_avito_city_sweep
-    from app.services.scrape_pipeline import run_avito_city_sweep
 
     mock_db = MagicMock()
-    mock_imv_result = HouseIMVBackfillResult(checked=2, saved=2, skipped=0, errors=0)
+    imv_result = SimpleNamespace(checked=2, saved=2, errors=0)
+    enrichment = MagicMock()
+    enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_result)
 
     with ExitStack() as stack:
         _enter_common_patches(stack, [10, 20])
-        stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
+        _enter_runs_patches(stack)
-        stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
-        stack.enter_context(patch("app.services.scrape_runs.mark_done"))
-        mock_imv_batch = stack.enter_context(
-            patch(
-                "app.services.house_imv_backfill.process_houses_imv_batch",
-                new_callable=AsyncMock,
-                return_value=mock_imv_result,
-            )
-        )
         counters = await run_avito_city_sweep(
             mock_db,
             run_id=1,
+            config=_config(),
+            matcher=MagicMock(),
+            enrichment=enrichment,
             anchors=[(56.84, 60.6, "TestAnchor")],
             enrich_imv=True,
             enrich_houses=True,
             detail_top_n=0,
         )
 
-    mock_imv_batch.assert_awaited_once()
+    enrichment.process_houses_imv_batch.assert_awaited_once()
     # Первый позиционный аргумент — db, второй — house_ids
-    call_args = mock_imv_batch.call_args
+    call_args = enrichment.process_houses_imv_batch.call_args
     assert call_args.args[1] == {10, 20}
     assert counters.imv_attempted == 2
     assert counters.imv_enriched == 2
@@ -209,31 +225,28 @@ async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
 @pytest.mark.asyncio
 async def test_sweep_enrich_imv_false_skips_imv() -> None:
     """enrich_imv=False → process_houses_imv_batch НЕ вызывается."""
-    from app.services.scrape_pipeline import run_avito_city_sweep
+    from scraper_kit.orchestration.pipeline import run_avito_city_sweep
 
     mock_db = MagicMock()
+    enrichment = MagicMock()
+    enrichment.process_houses_imv_batch = AsyncMock()
 
     with ExitStack() as stack:
         _enter_common_patches(stack, [99])
-        stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
+        _enter_runs_patches(stack)
-        stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
-        stack.enter_context(patch("app.services.scrape_runs.mark_done"))
-        mock_imv_batch = stack.enter_context(
-            patch(
-                "app.services.house_imv_backfill.process_houses_imv_batch",
-                new_callable=AsyncMock,
-            )
-        )
         counters = await run_avito_city_sweep(
             mock_db,
             run_id=2,
+            config=_config(),
+            matcher=MagicMock(),
+            enrichment=enrichment,
             anchors=[(56.84, 60.6, "TestAnchor")],
             enrich_imv=False,
             enrich_houses=True,
             detail_top_n=0,
         )
 
-    mock_imv_batch.assert_not_awaited()
+    enrichment.process_houses_imv_batch.assert_not_awaited()
     assert counters.imv_attempted == 0
     assert counters.imv_enriched == 0
     assert counters.imv_failed == 0
@@ -248,8 +261,8 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
 
     Лоты без house_url → unique_house_paths пуст → touched_house_ids пуст.
     """
-    from app.services.scrape_pipeline import run_avito_city_sweep
+    from scraper_kit.orchestration.pipeline import run_avito_city_sweep
-    from app.services.scrapers.avito import AvitoScraper
+    from scraper_kit.providers.avito.serp import AvitoScraper
 
     mock_db = MagicMock()
     fake_session = AsyncMock()
@@ -260,29 +273,30 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
     async def fake_fetch_around(self: Any, lat: float, lon: float, *_a: Any, **_kw: Any) -> list:
         return []
 
+    enrichment = MagicMock()
+    enrichment.process_houses_imv_batch = AsyncMock()
+
     with (
         patch.object(AvitoScraper, "fetch_around", fake_fetch_around),
-        patch("app.services.scrape_pipeline.save_listings", lambda _db, _lots, **_kw: (0, 0)),
+        patch(f"{PFX}.save_listings", lambda _db, _lots, **_kw: (0, 0)),
-        patch("curl_cffi.requests.AsyncSession", return_value=fake_session),
+        patch(f"{PFX}.AsyncSession", return_value=fake_session),
-        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
+        patch(f"{PFX}.asyncio.sleep", AsyncMock()),
-        patch("app.services.scrape_runs.is_cancelled", return_value=False),
+        patch(f"{PFX}.runs") as mock_runs,
-        patch("app.services.scrape_runs.update_heartbeat"),
-        patch("app.services.scrape_runs.mark_done"),
-        patch(
-            "app.services.house_imv_backfill.process_houses_imv_batch",
-            new_callable=AsyncMock,
-        ) as mock_imv_batch,
     ):
+        mock_runs.is_cancelled.return_value = False
         await run_avito_city_sweep(
             mock_db,
             run_id=3,
+            config=_config(),
+            matcher=MagicMock(),
+            enrichment=enrichment,
             anchors=[(56.84, 60.6, "TestAnchor")],
             enrich_imv=True,
             enrich_houses=True,
             detail_top_n=0,
         )
 
-    mock_imv_batch.assert_not_awaited()
+    enrichment.process_houses_imv_batch.assert_not_awaited()
 
 
 # ── IMV-ошибки не валят sweep ────────────────────────────────────────────────
@@ -291,32 +305,28 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
 @pytest.mark.asyncio
 async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
     """Если process_houses_imv_batch raises — sweep всё равно завершается (mark_done)."""
-    from app.services.scrape_pipeline import run_avito_city_sweep
+    from scraper_kit.orchestration.pipeline import run_avito_city_sweep
 
     mock_db = MagicMock()
+    enrichment = MagicMock()
+    enrichment.process_houses_imv_batch = AsyncMock(side_effect=RuntimeError("IMV network error"))
 
     with ExitStack() as stack:
         _enter_common_patches(stack, [55])
-        stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
+        mock_runs = _enter_runs_patches(stack)
-        stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
-        mock_mark_done = stack.enter_context(patch("app.services.scrape_runs.mark_done"))
-        stack.enter_context(
-            patch(
-                "app.services.house_imv_backfill.process_houses_imv_batch",
-                new_callable=AsyncMock,
-                side_effect=RuntimeError("IMV network error"),
-            )
-        )
         counters = await run_avito_city_sweep(
             mock_db,
             run_id=4,
+            config=_config(),
+            matcher=MagicMock(),
+            enrichment=enrichment,
             anchors=[(56.84, 60.6, "TestAnchor")],
             enrich_imv=True,
             enrich_houses=True,
             detail_top_n=0,
         )
 
-    mock_mark_done.assert_called_once()
+    mock_runs.mark_done.assert_called_once()
     # IMV crash → ошибка считается, но sweep завершается как done
     assert counters.errors_count >= 1
 
@@ -326,29 +336,24 @@ async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
 
 @pytest.mark.asyncio
 async def test_sweep_imv_failed_counter() -> None:
-    """imv_failed отражает errors из HouseIMVBackfillResult."""
+    """imv_failed отражает errors из результата enrichment.process_houses_imv_batch."""
-    from app.services.house_imv_backfill import HouseIMVBackfillResult
+    from scraper_kit.orchestration.pipeline import run_avito_city_sweep
-    from app.services.scrape_pipeline import run_avito_city_sweep
 
     mock_db = MagicMock()
     # 3 attempted, 1 enriched, 2 failed
-    mock_imv_result = HouseIMVBackfillResult(checked=3, saved=1, skipped=0, errors=2)
+    imv_result = SimpleNamespace(checked=3, saved=1, errors=2)
+    enrichment = MagicMock()
+    enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_result)
 
     with ExitStack() as stack:
         _enter_common_patches(stack, [1, 2, 3])
-        stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
+        _enter_runs_patches(stack)
-        stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
-        stack.enter_context(patch("app.services.scrape_runs.mark_done"))
-        stack.enter_context(
-            patch(
-                "app.services.house_imv_backfill.process_houses_imv_batch",
-                new_callable=AsyncMock,
-                return_value=mock_imv_result,
-            )
-        )
         counters = await run_avito_city_sweep(
             mock_db,
             run_id=5,
+            config=_config(),
+            matcher=MagicMock(),
+            enrichment=enrichment,
             anchors=[(56.84, 60.6, "TestAnchor")],
             enrich_imv=True,
             enrich_houses=True,
@@ -368,9 +373,11 @@ async def test_sweep_imv_failed_counter() -> None:
 @pytest.mark.asyncio
 async def test_sweep_cancel_before_imv_skips_imv() -> None:
     """Если is_cancelled=True в IMV-фазе → IMV пропускается, sweep всё равно done."""
-    from app.services.scrape_pipeline import run_avito_city_sweep
+    from scraper_kit.orchestration.pipeline import run_avito_city_sweep
 
     mock_db = MagicMock()
+    enrichment = MagicMock()
+    enrichment.process_houses_imv_batch = AsyncMock()
 
     # is_cancelled: первый вызов (anchor loop) = False, второй (IMV phase) = True
     cancel_side_effects = [False, True]
@@ -384,31 +391,26 @@ async def test_sweep_cancel_before_imv_skips_imv() -> None:
 
     with ExitStack() as stack:
         _enter_common_patches(stack, [77])
-        stack.enter_context(
+        mock_runs = MagicMock()
-            patch(
+        mock_runs.is_cancelled.side_effect = is_cancelled_side_effect
-                "app.services.scrape_runs.is_cancelled",
+        mock_runs.update_heartbeat = MagicMock()
-                side_effect=is_cancelled_side_effect,
+        mock_runs.mark_done = MagicMock()
-            )
+        stack.enter_context(patch(f"{PFX}.runs", mock_runs))
-        )
+
-        stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
-        mock_mark_done = stack.enter_context(patch("app.services.scrape_runs.mark_done"))
-        mock_imv_batch = stack.enter_context(
-            patch(
-                "app.services.house_imv_backfill.process_houses_imv_batch",
-                new_callable=AsyncMock,
-            )
-        )
         await run_avito_city_sweep(
             mock_db,
             run_id=6,
+            config=_config(),
+            matcher=MagicMock(),
+            enrichment=enrichment,
             anchors=[(56.84, 60.6, "TestAnchor")],
             enrich_imv=True,
             enrich_houses=True,
             detail_top_n=0,
         )
 
-    mock_imv_batch.assert_not_awaited()
+    enrichment.process_houses_imv_batch.assert_not_awaited()
-    mock_mark_done.assert_called_once()
+    mock_runs.mark_done.assert_called_once()
 
 
 # ── process_houses_imv_batch с пустым set ────────────────────────────────────

tradein-mvp/backend/tests/test_yandex_city_sweep.py

@@ -1,21 +1,19 @@
-"""Offline unit tests for run_yandex_city_sweep (SERP+address-enrich, #861).
+"""Offline unit tests for YandexCitySweepCounters + admin sweep endpoints (#861).
 
-Pure & fast: NO live network, NO DB. We monkeypatch:
+Pure & fast: NO live network, NO DB.
-  - YandexRealtyScraper.__init__ / __aenter__ / __aexit__ — avoid get_scraper_delay
-    DB read + real curl_cffi session creation.
-  - YandexRealtyScraper.fetch_around_multi_room — return a fixed list of fake ScrapedLot.
-  - scrape_pipeline.save_listings — counter/no-op (no real listings table).
-  - scrape_runs.* (is_cancelled / update_heartbeat / mark_*) — in-memory fakes.
-  - curl_cffi.requests.AsyncSession — fake for address-enrich HTTP calls.
 
-Asserts orchestration semantics:
+Legacy `run_yandex_city_sweep` orchestration behavior (SERP→save per-combo,
-  - SERP → save (all anchors, counters aggregated)
+address-enrich phase, cooperative cancel, consecutive-SERP-failure abort, bounded
-  - address-enrich фаза вызывается если enrich_address=True и есть source_ids
+retry / combo-skip logging) удалён вместе с `app.services.scrape_pipeline` (#2397
-  - enrich_address=False → адрес-фаза пропускается
+Part E1) — эквивалентная regression-coverage теперь в
-  - cancel → останавливает sweep
+`test_scraper_kit_pipeline_parity2.py::test_yandex_city_sweep` (kit
-  - N consecutive SERP failures → mark_banned + abort
+`run_yandex_city_sweep`).
-  - per-item address-enrich error не валит sweep
+
-  - mark_done вызывается при нормальном завершении
+Coverage:
+  - YandexCitySweepCounters корректно заполняются (dataclass shape)
+  - Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only)
+  - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI
+    (config/matcher/enrichment)
 """
 
 from __future__ import annotations
@@ -24,131 +22,16 @@ import os
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
 
-from typing import Any
+from datetime import UTC
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock
 
 import pytest
 
-from app.services import scrape_pipeline
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.yandex_realty import YandexRealtyScraper
-
-# 3 anchors keep tests fast and let us probe cancel/fail semantics.
-TEST_ANCHORS: list[tuple[float, float, str]] = [
-    (56.8400, 60.6050, "A1"),
-    (56.7950, 60.5300, "A2"),
-    (56.8970, 60.6100, "A3"),
-]
-
-
-def _fake_lot(offer_id: str) -> ScrapedLot:
-    """Minimal valid ScrapedLot (source/source_url required, price_rub > 0)."""
-    return ScrapedLot(
-        source="yandex",
-        source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
-        source_id=offer_id,
-        address="Екатеринбург, улица Тестовая",
-        price_rub=5_000_000,
-        area_m2=50.0,
-        rooms=2,
-    )
-
-
-class _FakeRuns:
-    """In-memory stand-in for app.services.scrape_runs (no DB)."""
-
-    def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
-        # cancel_at_anchor: 1-based anchor index at which is_cancelled flips True.
-        self.cancel_at_anchor = cancel_at_anchor
-        self.is_cancelled_calls = 0
-        self.heartbeats: list[dict[str, int]] = []
-        self.done: dict[str, int] | None = None
-        self.failed: tuple[str, dict[str, int]] | None = None
-        self.banned: tuple[str, dict[str, int]] | None = None
-
-    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
-        self.is_cancelled_calls += 1
-        if self.cancel_at_anchor is None:
-            return False
-        return self.is_cancelled_calls >= self.cancel_at_anchor
-
-    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.heartbeats.append(dict(counters))
-
-    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
-        self.done = dict(counters)
-
-    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.failed = (error, dict(counters))
-
-    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
-        self.banned = (error, dict(counters))
-
-
-@pytest.fixture(autouse=True)
-def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Kill inter-anchor asyncio.sleep so tests are instant."""
-
-    async def _instant(_secs: float) -> None:
-        return None
-
-    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
-
-
-@pytest.fixture(autouse=True)
-def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no curl_cffi."""
-
-    def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
-        self.city = city
-        self.request_delay_sec = 0.0
-        self._cffi_session = None
-        self._cookies = {}
-
-    async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
-        return self
-
-    async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
-        return None
-
-    monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
-    monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
-    monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)
-
-
-def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
-    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
-
-
-def _make_db_mock(source_ids: list[str]) -> MagicMock:
-    """DB mock для address-enrich фазы: возвращает строки без номера дома."""
-    mock_db = MagicMock()
-    rows = [
-        {
-            "id": i + 1,
-            "source_id": sid,
-            "address": "Екатеринбург, улица Тестовая",
-            "source_url": f"https://realty.yandex.ru/offer/{sid}/",
-        }
-        for i, sid in enumerate(source_ids)
-    ]
-    mock_db.execute.return_value.mappings.return_value.all.return_value = rows
-    return mock_db
-
-
-def _fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
-    return len(lots), 0
-
-
 # ── YandexCitySweepCounters ─────────────────────────────────────────────────
 
 
 def test_counters_defaults() -> None:
-    from app.services.scrape_pipeline import YandexCitySweepCounters
+    from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
 
     c = YandexCitySweepCounters()
     assert c.anchors_total == 0
@@ -163,7 +46,7 @@ def test_counters_defaults() -> None:
 def test_counters_to_dict_all_keys() -> None:
     from dataclasses import fields
 
-    from app.services.scrape_pipeline import YandexCitySweepCounters
+    from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
 
     c = YandexCitySweepCounters()
     d = c.to_dict()
@@ -171,423 +54,13 @@ def test_counters_to_dict_all_keys() -> None:
     assert set(d.keys()) == expected
 
 
-# ── Happy path: all anchors iterated, counters aggregated ───────────────────
-
-
-async def test_sweep_iterates_all_anchors_and_aggregates(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """fetch_around_multi_room вызывается для каждого anchor, counters агрегируются.
-
-    FIX: fake_fetch_multi вызывает on_combo (инкрементальный save per-combo) —
-    имитирует реальное поведение scraper'а. save_listings должен вызываться
-    on_combo'м, а не одним батчем в конце anchor'а.
-    """
-    serp_calls: list[str] = []
-    save_calls: list[int] = []
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        on_combo: Any = None,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        serp_calls.append(f"{lat:.4f}")
-        lots = [_fake_lot(f"{lat}-{i}") for i in range(3)]
-        # Имитируем один combo — вызываем on_combo если предоставлен.
-        if on_combo is not None:
-            on_combo(f"vtorichka/combo_{lat:.4f}", lots)
-        return lots
-
-    def fake_save(
-        _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
-    ) -> tuple[int, int]:
-        save_calls.append(len(lots))
-        return len(lots), 0
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=1,
-        anchors=TEST_ANCHORS,
-        pages_per_anchor=2,
-        request_delay_sec=0.0,
-        enrich_address=False,
-    )
-
-    assert len(serp_calls) == len(TEST_ANCHORS)
-    # save вызывается инкрементально через on_combo — по одному разу на anchor
-    # (fake имитирует один combo per anchor).
-    assert len(save_calls) == len(TEST_ANCHORS)
-    assert counters.anchors_total == len(TEST_ANCHORS)
-    assert counters.anchors_done == len(TEST_ANCHORS)
-    assert counters.lots_fetched == len(TEST_ANCHORS) * 3
-    assert counters.lots_inserted == len(TEST_ANCHORS) * 3
-    assert counters.lots_updated == 0
-    assert counters.errors_count == 0
-    assert counters.address_attempted == 0  # enrich_address=False
-    assert fake.done is not None
-    assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3
-    assert fake.failed is None
-    assert fake.banned is None
-
-
-# ── enrich_address=False пропускает address-фазу ────────────────────────────
-
-
-async def test_enrich_address_false_skips_address_phase(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """enrich_address=False → address-enrich фаза не выполняется."""
-    address_http_calls: list[str] = []
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        return [_fake_lot(f"{lat}-0")]
-
-    async def fake_cffi_get(url: str, **_: Any) -> Any:
-        address_http_calls.append(url)
-        return MagicMock(status_code=200, text="<title>test</title>")
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=2,
-        anchors=TEST_ANCHORS[:1],
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-
-    assert len(address_http_calls) == 0
-    assert counters.address_attempted == 0
-    assert counters.address_enriched == 0
-    assert fake.done is not None
-
-
-# ── address-enrich фаза: счётчики заполняются ───────────────────────────────
-
-
-async def test_address_enrich_fills_counters(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """address-enrich вызывается для листингов без номера дома; counters заполняются."""
-    # Fake SERP возвращает 2 листинга с source_id
-    lots = [_fake_lot("id1"), _fake_lot("id2")]
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        on_combo: Any = None,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        # Вызываем on_combo чтобы anchor_lots накопился (инкрементальный save).
-        if on_combo is not None:
-            on_combo("vtorichka/combo_enrich_test", lots)
-        return lots
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-
-    # DB mock: возвращает 2 строки для address-enrich запроса
-    mock_db = _make_db_mock(["id1", "id2"])
-
-    # Title с полным адресом — должен быть извлечён через _extract_address_from_title
-    fake_response = MagicMock()
-    fake_response.status_code = 200
-    fake_response.text = (
-        "<title>Продажа квартиры — Екатеринбург, улица Тестовая, 36 — id 123 "
-        "на Яндекс.Недвижимости</title>"
-    )
-
-    # Мок curl_cffi AsyncSession как context manager
-    mock_session = AsyncMock()
-    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
-    mock_session.__aexit__ = AsyncMock(return_value=None)
-    mock_session.get = AsyncMock(return_value=fake_response)
-
-    import curl_cffi.requests as cffi_mod
-
-    monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=mock_db,
-        run_id=3,
-        anchors=TEST_ANCHORS[:1],
-        enrich_address=True,
-        request_delay_sec=0.0,
-    )
-
-    assert counters.address_attempted == 2
-    assert counters.address_enriched == 2
-    assert counters.address_failed == 0
-    assert fake.done is not None
-
-
-# ── address-enrich: per-item HTTP error не валит sweep ──────────────────────
-
-
-async def test_address_enrich_http_error_does_not_abort(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """HTTP 403 при address-enrich → address_failed++, sweep продолжается."""
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        on_combo: Any = None,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        lots = [_fake_lot("id-err")]
-        if on_combo is not None:
-            on_combo("vtorichka/combo_http_err", lots)
-        return lots
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-
-    mock_db = _make_db_mock(["id-err"])
-
-    fake_response = MagicMock()
-    fake_response.status_code = 403
-    fake_response.text = ""
-
-    mock_session = AsyncMock()
-    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
-    mock_session.__aexit__ = AsyncMock(return_value=None)
-    mock_session.get = AsyncMock(return_value=fake_response)
-
-    import curl_cffi.requests as cffi_mod
-
-    monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)
-
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=mock_db,
-        run_id=4,
-        anchors=TEST_ANCHORS[:1],
-        enrich_address=True,
-        request_delay_sec=0.0,
-    )
-
-    assert counters.address_attempted == 1
-    assert counters.address_failed == 1
-    assert counters.address_enriched == 0
-    assert fake.done is not None  # sweep завершился несмотря на ошибку
-
-
-# ── Mid-sweep cancel stops further anchors ──────────────────────────────────
-
-
-async def test_cancel_midway_stops_remaining_anchors(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    serp_calls: list[str] = []
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        serp_calls.append(f"{lat}")
-        return [_fake_lot(f"{lat}-0")]
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-    # is_cancelled flips True on the 2nd check (before anchor #2) → only anchor #1 runs.
-    fake = _FakeRuns(cancel_at_anchor=2)
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=5,
-        anchors=TEST_ANCHORS,
-        pages_per_anchor=1,
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-    # Only the first anchor fetched before cancel.
-    assert len(serp_calls) == 1
-    assert counters.anchors_done == 1
-    # Cancel path returns early: no mark_done, no ban/fail.
-    assert fake.done is None
-    assert fake.failed is None
-    assert fake.banned is None
-
-
-# ── Consecutive failures → mark_banned + abort ──────────────────────────────
-
-
-async def test_consecutive_failures_abort_sweep(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """All anchors raise → after MAX_CONSECUTIVE_FAILURES the sweep aborts + bans."""
-    attempts: list[str] = []
-
-    async def fake_fetch_fail(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        attempts.append(f"{lat}")
-        raise RuntimeError("yandex blocked")
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_fail)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    # 5 anchors so the 3-consecutive-failure threshold trips before the list ends.
-    anchors = [
-        *TEST_ANCHORS,
-        (56.7700, 60.5500, "A4"),
-        (56.8650, 60.6200, "A5"),
-    ]
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=6,
-        anchors=anchors,
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-    # Abort exactly at the threshold (3 consecutive failures).
-    assert len(attempts) == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
-    assert counters.errors_count == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
-    assert counters.anchors_done == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
-    assert fake.banned is not None
-    assert "consecutive" in fake.banned[0]
-    assert fake.done is None  # aborted, never marked done
-
-
-# ── Non-consecutive failures do not abort ───────────────────────────────────
-
-
-async def test_non_consecutive_failures_do_not_abort(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """A success between two failures resets the consecutive counter → no abort."""
-
-    def _name_of(lat: float, anchors: list[tuple[float, float, str]]) -> str:
-        for a_lat, _lon, nm in anchors:
-            if a_lat == lat:
-                return nm
-        return "?"
-
-    anchors = [
-        *TEST_ANCHORS,
-        (56.7700, 60.5500, "A4"),
-        (56.8650, 60.6200, "A5"),
-    ]
-    failing = {"A1", "A3", "A5"}
-
-    async def fake_fetch(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        on_combo: Any = None,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        if _name_of(lat, anchors) in failing:
-            raise RuntimeError("transient")
-        lots = [_fake_lot(f"{lat}-0")]
-        # Вызываем on_combo для инкрементального save.
-        if on_combo is not None:
-            on_combo(f"vtorichka/combo_{lat:.4f}", lots)
-        return lots
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=7,
-        anchors=anchors,
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-    assert counters.errors_count == 3  # A1, A3, A5
-    assert counters.anchors_done == 5  # walked the whole list, no early abort
-    assert counters.lots_fetched == 2  # A2 + A4 contributed
-    assert fake.banned is None
-    assert fake.done is not None
-
-
-# ── mark_done вызывается даже при пустом SERP ───────────────────────────────
-
-
-async def test_mark_done_always_called(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    async def fake_fetch_empty(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        return []
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_empty)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=8,
-        anchors=TEST_ANCHORS,
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-
-    assert fake.done is not None
-    assert counters.lots_fetched == 0
-    assert counters.anchors_done == len(TEST_ANCHORS)
-
-
 # ── Admin endpoint tests (offline) ──────────────────────────────────────────
 
 
 @pytest.fixture
 def app_with_admin():
+    from unittest.mock import MagicMock
+
     from fastapi import FastAPI
     from fastapi.testclient import TestClient
 
@@ -680,7 +153,7 @@ def test_yandex_cancel_endpoint_not_running(app_with_admin) -> None:
 
 def test_yandex_list_runs_endpoint(app_with_admin) -> None:
     """List runs endpoint возвращает список."""
-    from datetime import UTC, datetime
+    from datetime import datetime
     from unittest.mock import patch
 
     fake_rows = [
@@ -706,144 +179,7 @@ def test_yandex_list_runs_endpoint(app_with_admin) -> None:
     assert "2025-05-01" in body[0]["started_at"]
 
 
-# ── Center-combos wiring: prod (anchors=None) uses EKB center + combos ──────
+# ── Watchdog timeout: combos-mode budget dwarfs ANCHOR_TIMEOUT_SEC ──────────
-
-
-async def test_prod_mode_uses_center_anchor_with_combos(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """anchors=None (prod mode) → fetch_around_multi_room called with EKB center coords
-    AND rooms_list + price_ranges (combos mode), NOT with None/None (legacy mode).
-    anchors_total == 1 (single center sweep).
-    """
-    from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
-
-    calls: list[dict] = []
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        rooms_list: list | None = None,
-        price_ranges: list | None = None,
-        on_combo: Any = None,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        lots = [_fake_lot("center-1"), _fake_lot("center-2")]
-        calls.append(
-            {
-                "lat": lat,
-                "lon": lon,
-                "radius_m": radius_m,
-                "rooms_list": rooms_list,
-                "price_ranges": price_ranges,
-                "max_pages": max_pages,
-            }
-        )
-        # Имитируем on_combo (инкрементальный save) — один combo.
-        if on_combo is not None:
-            on_combo("vtorichka/center_combo", lots)
-        return lots
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=10,
-        # anchors=None → prod mode (EKB center)
-        pages_per_anchor=3,
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-
-    # Single center anchor
-    assert counters.anchors_total == 1
-    assert counters.anchors_done == 1
-
-    # fetch_around_multi_room called exactly once
-    assert len(calls) == 1
-    call = calls[0]
-
-    # EKB center coords
-    assert abs(call["lat"] - 56.8400) < 0.0001
-    assert abs(call["lon"] - 60.6050) < 0.0001
-
-    # radius covers the whole city (25km default)
-    assert call["radius_m"] >= 10000
-
-    # Combos mode: rooms_list and price_ranges are NOT None
-    assert call["rooms_list"] is not None
-    assert call["price_ranges"] is not None
-
-    # rooms_list covers all ROOM_PATH keys
-    assert set(call["rooms_list"]) == set(ROOM_PATH.keys())
-
-    # price_ranges matches DEFAULT_PRICE_RANGES
-    assert call["price_ranges"] == DEFAULT_PRICE_RANGES
-
-    # max_pages forwarded from pages_per_anchor
-    assert call["max_pages"] == 3
-
-    # lots aggregated correctly via on_combo
-    assert counters.lots_fetched == 2
-    assert counters.lots_inserted == 2
-    assert fake.done is not None
-    assert fake.failed is None
-
-
-async def test_explicit_anchors_still_works(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """anchors=[...] (test/manual override) → fetch called for each anchor, combos mode."""
-    calls: list[dict] = []
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        rooms_list: list | None = None,
-        price_ranges: list | None = None,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        calls.append({"lat": lat, "rooms_list": rooms_list, "price_ranges": price_ranges})
-        return [_fake_lot(f"{lat}")]
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    two_anchors = [(56.8400, 60.6050, "A"), (56.7950, 60.5300, "B")]
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=11,
-        anchors=two_anchors,
-        pages_per_anchor=2,
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-
-    # Both anchors iterated
-    assert len(calls) == 2
-    assert counters.anchors_total == 2
-    assert counters.anchors_done == 2
-
-    # Each call uses combos (not None/None)
-    for c in calls:
-        assert c["rooms_list"] is not None
-        assert c["price_ranges"] is not None
-
-    assert fake.done is not None
-
-
-# ── Watchdog timeout: combos vs explicit-anchor mode ───────────────────────
 
 
 def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None:
@@ -851,13 +187,14 @@ def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None
 
     Проверяет что вычисленный таймаут для дефолтных 30 combos × 3 pages > 1500s,
     т.е. не равен ANCHOR_TIMEOUT_SEC=240 и обеспечивает полный прогон sweep'а.
-    Не нужны мокапы — только публичные константы и арифметика из модуля.
+    Не нужны мокапы — только публичные константы и арифметика из kit-модуля.
     """
-    from app.services.scrape_pipeline import (
+    from scraper_kit.orchestration.pipeline import (
         _YANDEX_ADDRESS_ENRICH_BUDGET_S,
         _YANDEX_COMBOS_PER_FETCH_S,
         ANCHOR_TIMEOUT_SEC,
     )
+
     from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
 
     # Параметры prod-режима
@@ -884,188 +221,3 @@ def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None
     assert (
         sweep_timeout > ANCHOR_TIMEOUT_SEC * 4
     ), f"sweep_timeout={sweep_timeout:.0f}s should be >> ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s"
-
-
-async def test_explicit_anchor_mode_uses_anchor_timeout_sec(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """anchors=[...] (explicit override) → wait_for используется с ANCHOR_TIMEOUT_SEC.
-
-    Проверяет что в legacy/test режиме (explicit anchors) sweep_timeout не раздут
-    до combos-значения — он равен ANCHOR_TIMEOUT_SEC, т.к. каждый anchor маленький.
-    """
-    import asyncio as _asyncio
-
-    from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC
-
-    captured_timeouts: list[float] = []
-
-    _orig_wait_for = _asyncio.wait_for
-
-    async def _capturing_wait_for(coro, *, timeout):  # type: ignore[no-untyped-def]
-        captured_timeouts.append(timeout)
-        # На самом деле запустить — мокнутый scraper вернёт [] мгновенно.
-        return await _orig_wait_for(coro, timeout=timeout)
-
-    monkeypatch.setattr(_asyncio, "wait_for", _capturing_wait_for)
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        **_: object,
-    ) -> list[ScrapedLot]:
-        return []
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=20,
-        anchors=TEST_ANCHORS[:2],  # explicit anchors — legacy mode
-        pages_per_anchor=2,
-        enrich_address=False,
-        request_delay_sec=0.0,
-    )
-
-    # В explicit-anchor режиме каждый anchor должен использовать ANCHOR_TIMEOUT_SEC
-    assert len(captured_timeouts) == 2
-    for t in captured_timeouts:
-        assert (
-            t == ANCHOR_TIMEOUT_SEC
-        ), f"Expected ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s, got {t}s"
-
-
-# ── FIX A: инкрементальный save per-combo ────────────────────────────────────
-
-
-async def test_incremental_save_called_per_combo(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """save_listings вызывается on_combo per-combo, не одним батчем в конце anchor'а.
-
-    Имитируем 3 combo через on_combo — save_listings должен быть вызван 3 раза
-    (по одному на combo) с порциями лотов. Проверяем, что при краше anchor'а
-    уже сохранённые combo не теряются (данные в БД до finish anchor'а).
-    """
-    save_batches: list[list[str]] = []
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        on_combo: Any = None,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        all_lots = []
-        for combo_idx in range(3):
-            combo_lots = [_fake_lot(f"{lat}-combo{combo_idx}-lot0")]
-            all_lots.extend(combo_lots)
-            if on_combo is not None:
-                on_combo(f"vtorichka/combo_{combo_idx}", combo_lots)
-        return all_lots
-
-    def fake_save(
-        _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
-    ) -> tuple[int, int]:
-        save_batches.append([lot.source_id for lot in lots if lot.source_id])
-        return len(lots), 0
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=30,
-        anchors=TEST_ANCHORS[:1],
-        pages_per_anchor=2,
-        request_delay_sec=0.0,
-        enrich_address=False,
-    )
-
-    # save_listings вызван ровно 3 раза (по одному на combo), не одним батчем.
-    assert (
-        len(save_batches) == 3
-    ), f"Ожидалось 3 save вызова (per-combo), получено {len(save_batches)}: {save_batches}"
-    # Каждый batch содержит ровно 1 лот (размер combo).
-    for batch in save_batches:
-        assert len(batch) == 1, f"Каждый combo-batch должен иметь 1 лот, получено {batch}"
-
-    # Суммарные counters корректны.
-    assert counters.lots_fetched == 3
-    assert counters.lots_inserted == 3
-    assert fake.done is not None
-
-    # heartbeat вызывался после каждого combo (через on_combo).
-    assert (
-        len(fake.heartbeats) >= 3
-    ), f"Ожидалось ≥3 heartbeat обновлений (per-combo), получено {len(fake.heartbeats)}"
-
-
-# ── FIX B: bounded retry — скипает combo после exhaustion ────────────────────
-
-
-async def test_bounded_retry_combo_skipped_logged(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """on_combo НЕ вызывается если combo пропущен (все retry page-1 исчерпаны).
-
-    Имитируем sweep с двумя combos: первый скипается (on_combo не вызван),
-    второй успешен (on_combo вызван). Проверяем, что save_listings вызван
-    только для успешного combo, а не скипнутого.
-    """
-    save_batches: list[list[str]] = []
-    on_combo_calls: list[str] = []
-
-    async def fake_fetch_multi(
-        self: YandexRealtyScraper,
-        lat: float,
-        lon: float,
-        radius_m: int = 1000,
-        max_pages: int = 2,
-        on_combo: Any = None,
-        **_: Any,
-    ) -> list[ScrapedLot]:
-        # Combo 0: скипается — on_combo НЕ вызывается (симулирует exhausted retries).
-        # Combo 1: успешен — on_combo вызывается с лотом.
-        success_lot = _fake_lot(f"{lat}-success")
-        if on_combo is not None:
-            on_combo_calls.append("combo_1_success")
-            on_combo("vtorichka/combo_1", [success_lot])
-        return [success_lot]
-
-    def fake_save(
-        _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
-    ) -> tuple[int, int]:
-        save_batches.append([lot.source_id for lot in lots if lot.source_id])
-        return len(lots), 0
-
-    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
-    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
-    fake = _FakeRuns()
-    _install_runs(monkeypatch, fake)
-
-    counters = await scrape_pipeline.run_yandex_city_sweep(
-        db=MagicMock(),
-        run_id=31,
-        anchors=TEST_ANCHORS[:1],
-        pages_per_anchor=2,
-        request_delay_sec=0.0,
-        enrich_address=False,
-    )
-
-    # Только успешный combo вызвал on_combo и save.
-    assert len(on_combo_calls) == 1
-    assert len(save_batches) == 1
-    assert counters.lots_fetched == 1
-    assert counters.lots_inserted == 1
-    assert fake.done is not None

tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/__init__.py

@@ -1,4 +1,4 @@
-"""Оркестрация sweep-pipeline scraper_kit — strangler-копия `app.services.scrape_pipeline`.
+"""Оркестрация sweep-pipeline scraper_kit (#2135 strangler, завершён #2397 Part E1).
 
 Координирует полный прогон скрапа (SERP → save → houses → detail → IMV) поверх
 провайдеров `scraper_kit.providers.*` и базового слоя `scraper_kit.base`. Развязана
@@ -10,6 +10,7 @@
   - `app.core.shutdown.shutdown_requested` → инжектируемый `Callable[[], bool]`
   - `app.services.scrape_runs`          → локальная копия `scraper_kit.orchestration.runs`
 
-Боевой scheduler по-прежнему зовёт старый `app.services.scrape_pipeline` (strangler:
+Единственный оркестратор в проде: legacy `app.services.scrape_pipeline` (изначальный
-COPY, не MOVE). Переключение рантайма на этот модуль — отдельный поздний шаг.
+strangler-COPY-источник) удалён (#2397 Part E1). Боевой scheduler и admin-роуты
+вызывают этот модуль напрямую.
 """

tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py

@@ -1,8 +1,9 @@
-"""pipeline.py — Avito sweep orchestrator (strangler-копия #2135).
+"""pipeline.py — Avito sweep orchestrator (#2135 strangler, единственный в проде с #2397 Part E1).
 
-Strangler-COPY боевого `app.services.scrape_pipeline` (шаг оркестрации, #2135).
+Изначально был strangler-COPY боевого `app.services.scrape_pipeline` (#2135);
-Логика фаз/бан-ротации/счётчиков идентична оригиналу — единственное отличие в
+legacy-оригинал удалён (#2397 Part E1) — этот модуль теперь единственная
-развязке от `app.*` через инжекцию (см. `scraper_kit.contracts`):
+orchestration-реализация. Развязка от `app.*` через инжекцию (см.
+`scraper_kit.contracts`):
 
   - `app.core.config.settings`          → аргумент `config: ScraperConfig`
   - `app.services.matching`             → аргумент `matcher: HouseMatcher`

tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/avito/detail.py

@@ -385,7 +385,7 @@ async def fetch_detail(
     reconnect_on_block — если False, 403/firewall поднимает AvitoBlockedError СРАЗУ
     (без холодных reconnect'ов — ими займётся outer-loop через rebuild+rewarm прогретой
     сессии); 429 после short-retry поднимает AvitoRateLimitedError. Дефолт True
-    сохраняет старое поведение own-session / scrape_pipeline путей.
+    сохраняет старое поведение own-session / shared-session (run_avito_city_sweep) путей.
     Raises:
         httpx.HTTPError — если status != 200 (через raise_for_status-like).
         ValueError — если item_id не извлечён из HTML.
@@ -427,8 +427,8 @@ async def fetch_detail(
     own_session = cffi_session is None
     if cffi_session is None:
         # Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed,
-        # e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy
+        # e.g. admin endpoint). Caller-provided sessions (run_avito_city_sweep) already have
-        # from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
+        # proxy from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
         session = _build_detail_session(config)
     else:
         session = cffi_session
@@ -442,7 +442,7 @@ async def fetch_detail(
     # reconnect_on_block=False (warm-batch #1551): не делаем холодных per-card
     # reconnect'ов — блок поднимает AvitoBlockedError сразу, outer-loop пересоздаёт+
     # прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session /
-    # scrape_pipeline) — старое поведение: backconnect-reconnect эфемерной сессией.
+    # run_avito_city_sweep) — старое поведение: backconnect-reconnect эфемерной сессией.
     backconnect = (
         bool(config.scraper_proxy_url if config is not None else None) and reconnect_on_block
     )