lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request 'refactor(tradein/scrapers): удалить legacy scrape_pipeline.py + тест-хирургия (#2397 Part E1)' (#2405) from chore/tradein-partE1-delete-scrape-pipeline into main
All checks were successful
Deploy Trade-In / changes (push) Successful in 11s
Details
Deploy Trade-In / build-frontend (push) Has been skipped
Details
Deploy Trade-In / build-browser (push) Has been skipped
Details
Deploy Trade-In / test (push) Successful in 1m37s
Details
Deploy Trade-In / build-backend (push) Successful in 1m20s
Details
Deploy Trade-In / deploy (push) Successful in 1m15s
Details

Browse source
This commit is contained in:
lekss361 2026-07-04 11:09:39 +00:00
commit 84716e61de
27 changed files with 412 additions and 9216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
tradein-mvp/backend/app/api/v1/admin.py
View file

@ -31,14 +31,15 @@ from pydantic import BaseModel, Field, field_validator
#
# scraper_kit-миграция (#2397 slice A, эпик #2277): 5 debug city-sweep/full-load
# роутов (avito-city-sweep, cian-city-sweep, cian-full-load, yandex-city-sweep,
# yandex-full-load) переключены с app.services.scrape_pipeline (legacy) на
# scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и app.scheduler_main
# ._run_kit_scheduler / scraper_kit.orchestration.scheduler._job_*: config/matcher/
# enrichment/proxy_provider инжектируются явно вместо module-level импортов app.*
# внутри scrape_pipeline.py. shutdown_requested НЕ прокидывается (нет SIGTERM-drain
# yandex-full-load) переключены с app.services.scrape_pipeline (legacy, удалён в
# #2397 Part E1) на scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и
# app.scheduler_main._run_kit_scheduler / scraper_kit.orchestration.scheduler
# ._job_*: config/matcher/enrichment/proxy_provider инжектируются явно вместо
# module-level импортов app.*. shutdown_requested НЕ прокидывается (нет SIGTERM-drain
# семантики у admin BackgroundTasks — эквивалент дефолту lambda: False, поведение не
# меняется). Боевой scheduler.py (production sweep-cron) остаётся на legacy — это
# отдельный, ещё больший шаг (см. vault Event_Legacy_Scrapers_Partial_Deletion_Jul04).
# меняется). Боевой scheduler-путь (production sweep-cron) уже на kit (#2397 Part C
# убрал legacy app.services.scheduler.scheduler_loop fallback) — тот же orchestrator,
# что и debug-роуты этого файла.
from scraper_kit.base import save_listings
from scraper_kit.browser_fetcher import BrowserFetcher
from scraper_kit.orchestration.pipeline import (

7
tradein-mvp/backend/app/services/house_imv_backfill.py
View file

@ -13,9 +13,10 @@ For each house:
Resumable: re-running picks up from where the previous run stopped.
process_houses_imv_batch() IS wired into the scheduler: run_avito_city_sweep()
(scrape_pipeline.py) calls it automatically as the final IMV-phase of every
avito city sweep. backfill_house_imv() (batch/pending-status entrypoint) remains
manual-only, triggered via admin API.
(scraper_kit.orchestration.pipeline, via the injected EnrichmentJobs protocol)
calls it automatically as the final IMV-phase of every avito city sweep.
backfill_house_imv() (batch/pending-status entrypoint) remains manual-only,
triggered via admin API.
"""
from __future__ import annotations

3734
tradein-mvp/backend/app/services/scrape_pipeline.py
View file

File diff suppressed because it is too large Load diff

10
tradein-mvp/backend/app/services/scrapers/avito_detail.py
View file

@ -388,7 +388,7 @@ async def fetch_detail(
reconnect_on_block — если False, 403/firewall поднимает AvitoBlockedError СРАЗУ
(без холодных reconnect'ов — ими займётся outer-loop через rebuild+rewarm прогретой
сессии); 429 после short-retry поднимает AvitoRateLimitedError. Дефолт True
сохраняет старое поведение own-session / scrape_pipeline путей.
сохраняет старое поведение own-session путей (напр. admin endpoint).
Raises:
httpx.HTTPError — если status != 200 (через raise_for_status-like).
ValueError — если item_id не извлечён из HTML.
@ -432,8 +432,8 @@ async def fetch_detail(
own_session = cffi_session is None
if cffi_session is None:
# Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed,
# e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy
# from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
# e.g. admin endpoint). Caller-provided sessions already have proxy from
# AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
session = _build_detail_session()
else:
session = cffi_session
@ -446,8 +446,8 @@ async def fetch_detail(
# эфемерную retry_session, которую сами закрываем в finally.
# reconnect_on_block=False (warm-batch #1551): не делаем холодных per-card
# reconnect'ов — блок поднимает AvitoBlockedError сразу, outer-loop пересоздаёт+
# прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session /
# scrape_pipeline) — старое поведение: backconnect-reconnect эфемерной сессией.
# прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session) —
# старое поведение: backconnect-reconnect эфемерной сессией.
backconnect = bool(settings.scraper_proxy_url) and reconnect_on_block
req_headers = {"Referer": referer} if referer else None
r403 = 0

4
tradein-mvp/backend/app/services/yandex_price_history.py
View file

@ -4,7 +4,9 @@
Yandex gate-ответ уже несёт `price.previous` + `price.trend`, которые парсятся в
`ScrapedLot.price_previous_rub` / `price_trend`. Otsenka-скрапер НЕ нужен — gate достаточно.
Эта функция вызывается из scrape_pipeline СРАЗУ ПОСЛЕ save_listings(те же lots): резолвит
Эта функция вызывается из run_yandex_city_sweep (scraper_kit.orchestration.pipeline,
через инжектируемый EnrichmentJobs.record_yandex_price_history) СРАЗУ ПОСЛЕ
save_listings (те же lots): резолвит
listing_id по (source='yandex', source_id), читает последнюю по change_time точку
offer_price_history и идемпотентно дописывает наблюдение, если цена изменилась.

7
tradein-mvp/backend/app/tasks/avito_detail_backfill.py
View file

@ -8,7 +8,8 @@ City sweep ENRICH_DETAIL only enriches top-N proximate listings per anchor.
Legacy listings (older than 2h or outside radius) are never enriched.
Solution: single snapshot SELECT at start (guarantees termination), same proxy
session path as scrape_pipeline.py step 5. Block handling mirrors step 5:
session path as the detail-phase of `run_avito_city_sweep`
(scraper_kit.orchestration.pipeline). Block handling mirrors that phase:
rotate IP on every block, abort after max_consecutive_blocks (mark_done not
mark_failed -- block is temporary, retry next night via NULL detail_enriched_at).
"""
@ -176,7 +177,7 @@ async def run_avito_detail_backfill(
session = await build_warmed_session()
elif not use_curl:
# curl_cffi legacy path (scraper_fetch_mode="curl_cffi", use_curl=False):
# строим shared сессию через auv, как делал scrape_pipeline.
# строим shared сессию через auv, как делает run_avito_city_sweep (kit).
own_session = True
session = AsyncSession(
impersonate="chrome120",
@ -280,7 +281,7 @@ async def run_avito_detail_backfill(
source_url: str = row["source_url"]
counters.attempted += 1
# Normalise URL -> path (mirrors scrape_pipeline.py line 311)
# Normalise URL -> path (mirrors run_avito_city_sweep detail-phase, kit)
item_url = urlparse(source_url).path if source_url.startswith("http") else source_url
# use_curl warm-batch (#1551): интервалы по размеру SERP-страницы (~50).

184
tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py
View file

@ -1,6 +1,6 @@
"""Test anti-bot hardening: blocked exceptions + early abort."""
from unittest.mock import AsyncMock, MagicMock, call, patch
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -170,169 +170,13 @@ async def test_fetch_around_raises_rate_limited_on_429() -> None:
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
# ── Pipeline abort tests ────────────────────────────────────────
@pytest.mark.asyncio
async def test_pipeline_aborts_after_3_consecutive_blocks() -> None:
"""#1820: 3 consecutive 403 в detail-фазе прерывают enrichment, но НЕ роняют прогон.
Раньше pipeline re-raise'ил AvitoBlockedError (один блок ронял весь прогон). Теперь
блок ловится пер-item, и лишь N consecutive блоков прерывают enrichment-фазу —
PipelineResult возвращается, detail_failed считается, лишние detail-фетчи не делаются.
"""
from app.services.scrape_pipeline import (
_AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
PipelineResult,
run_avito_pipeline,
)
mock_db = MagicMock()
mock_scraper = MagicMock()
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
mock_scraper.__aexit__ = AsyncMock(return_value=None)
mock_scraper.fetch_around = AsyncMock(return_value=[])
mock_scraper._cffi = None
mock_row = MagicMock()
mock_row.__getitem__ = lambda self, key: (
"https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
)
mock_mappings = MagicMock()
mock_mappings.all.return_value = [mock_row, mock_row, mock_row, mock_row]
mock_execute = MagicMock()
mock_execute.mappings.return_value = mock_mappings
mock_db.execute.return_value = mock_execute
block_error = AvitoBlockedError("HTTP 403")
detail_mock = AsyncMock(side_effect=block_error)
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.fetch_detail", detail_mock),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
),
):
# НЕ должно бросать наружу
result = await run_avito_pipeline(
mock_db,
56.8,
60.6,
1000,
enrich_houses=False,
enrich_detail_top_n=4,
)
assert isinstance(result, PipelineResult)
# enrichment прерван ровно после порога consecutive блоков (4-я строка не фетчилась)
assert detail_mock.await_count == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
assert result.counters.detail_attempted == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
assert result.counters.detail_failed == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
# в errors зафиксирован abort
assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
@pytest.mark.asyncio
async def test_pipeline_sleep_between_detail_requests() -> None:
"""Pipeline должен вызывать asyncio.sleep между каждым detail request."""
from app.services.scrape_pipeline import run_avito_pipeline
mock_db = MagicMock()
mock_scraper = MagicMock()
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
mock_scraper.__aexit__ = AsyncMock(return_value=None)
mock_scraper.fetch_around = AsyncMock(return_value=[])
mock_scraper._cffi = None
mock_row = MagicMock()
mock_row.__getitem__ = lambda self, key: (
"https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
)
mock_mappings = MagicMock()
mock_mappings.all.return_value = [mock_row, mock_row, mock_row]
mock_execute = MagicMock()
mock_execute.mappings.return_value = mock_mappings
mock_db.execute.return_value = mock_execute
mock_enrichment = MagicMock()
sleep_mock = AsyncMock()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
patch(
"app.services.scrape_pipeline.fetch_detail",
AsyncMock(return_value=mock_enrichment),
),
patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
patch("app.services.scrape_pipeline.asyncio.sleep", sleep_mock),
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
),
patch("app.services.scrape_pipeline.random.uniform", return_value=1.0),
):
await run_avito_pipeline(
mock_db,
56.8,
60.6,
1000,
enrich_houses=False,
enrich_detail_top_n=3,
request_delay_sec=7.0,
)
assert sleep_mock.call_count == 2
for c in sleep_mock.call_args_list:
assert c == call(7.0)
@pytest.mark.asyncio
async def test_city_sweep_marks_banned_on_block() -> None:
"""City sweep mark'ит run как banned при AvitoBlockedError.
Migration 015 задокументировал 'banned' как 'Avito вернул 403/captcha' — именно этот сценарий.
Патчим AvitoScraper.fetch_around (не run_avito_pipeline — он больше не вызывается
из run_avito_city_sweep; sweep использует внутренние _avito_anchor_phases).
"""
from app.services.scrape_pipeline import run_avito_city_sweep
from app.services.scrapers.avito import AvitoScraper
mock_db = MagicMock()
mock_runs = MagicMock()
mock_runs.is_cancelled.return_value = False
mock_runs.update_heartbeat = MagicMock()
mock_runs.mark_banned = MagicMock()
mock_runs.mark_done = MagicMock()
block_error = AvitoBlockedError("IP blocked at page=1")
with (
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch.object(AvitoScraper, "fetch_around", AsyncMock(side_effect=block_error)),
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(
__aenter__=AsyncMock(return_value=AsyncMock()),
__aexit__=AsyncMock(return_value=None),
),
),
):
result = await run_avito_city_sweep(
mock_db,
run_id=42,
anchors=[(56.84, 60.60, "Центр"), (56.79, 60.53, "ЮЗ")],
)
assert mock_runs.mark_banned.called
assert not mock_runs.mark_done.called
assert result.anchors_done == 1
# ── #623 firewall-детект + ротация IP ────────────────────────────
#
# Legacy `run_avito_pipeline` / `run_avito_city_sweep` orchestration-level tests
# (pipeline consecutive-block abort, inter-detail sleep, city-sweep mark_banned)
# удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная
# regression-coverage теперь в `test_scraper_kit_pipeline_parity.py` (kit
# `run_avito_city_sweep`: ban/rotation state-machine, detail-block propagation).
_CARD_HTML = (
"<html><body>"
@ -437,20 +281,20 @@ async def test_fetch_around_blocked_when_rotation_exhausted(
# ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ───
def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None:
"""_avito_proxies: None без env, dict {http,https} при заданном AVITO_PROXY_URL.
def test_avito_proxies_helper() -> None:
"""Kit _avito_proxies(config): None без proxy, dict {http,https} при заданном URL.
Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail —
без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup).
"""
from app.services import scrape_pipeline as sp
from types import SimpleNamespace
monkeypatch.setattr(sp.settings, "avito_proxy_url", None)
assert sp._avito_proxies() is None
from scraper_kit.orchestration.pipeline import _avito_proxies
assert _avito_proxies(SimpleNamespace(scraper_proxy_url=None)) is None
url = "http://u:p@mproxy.site:16928"
monkeypatch.setattr(sp.settings, "avito_proxy_url", url)
assert sp._avito_proxies() == {"http": url, "https": url}
assert _avito_proxies(SimpleNamespace(scraper_proxy_url=url)) == {"http": url, "https": url}
# ── curl_cffi 429 short-retry (backconnect conn-limit transient) ──────────────

670
tradein-mvp/backend/tests/test_anchor_watchdog.py
View file

@ -1,670 +0,0 @@
"""Unit tests for per-anchor watchdog timeout in city sweeps (#880).
Verifies that ANCHOR_TIMEOUT_SEC is applied to all three sweeps (avito / cian / yandex):
(a) A hung anchor (asyncio.sleep(large)) doesn't freeze the whole sweep.
(b) TimeoutError branch: errors_count incremented, anchor logged as timed-out.
(c) Sweep proceeds to the next anchor after the timeout.
(d) mark_done is called when the sweep finishes all anchors.
All tests are network-free — they monkeypatch the inner scrape call.
"""
from __future__ import annotations
import asyncio
import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.cian import CianScraper
from app.services.scrapers.yandex_realty import YandexRealtyScraper
# Two anchors: first hangs (causes timeout), second succeeds.
TWO_ANCHORS: list[tuple[float, float, str]] = [
(56.8400, 60.6050, "Hang"),
(56.7950, 60.5300, "OK"),
]
# ── Shared fake scrape_runs ─────────────────────────────────────────────────
class _FakeRuns:
def __init__(self) -> None:
self.heartbeats: list[dict[str, int]] = []
self.done: dict[str, int] | None = None
self.failed: tuple[str, dict[str, int]] | None = None
self.banned: tuple[str, dict[str, int]] | None = None
def is_cancelled(self, _db: Any, _run_id: int) -> bool:
return False
def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.heartbeats.append(dict(counters))
def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.done = dict(counters)
def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.failed = (error, dict(counters))
def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.banned = (error, dict(counters))
def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
def _fake_lot(source: str, offer_id: str) -> ScrapedLot:
return ScrapedLot(
source=source,
source_url=f"https://example.ru/{offer_id}/",
source_id=offer_id,
address="Екатеринбург, улица Тестовая, 1",
price_rub=4_000_000,
area_m2=42.0,
rooms=1,
)
# ── CianScraper lifecycle stub ──────────────────────────────────────────────
@pytest.fixture(autouse=True)
def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
def _init(self: CianScraper) -> None:
self.request_delay_sec = 0.0
self._cffi = None
async def _aenter(self: CianScraper) -> CianScraper:
return self
async def _aexit(self: CianScraper, *_args: Any) -> None:
return None
monkeypatch.setattr(CianScraper, "__init__", _init)
monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
# ── YandexRealtyScraper lifecycle stub ─────────────────────────────────────
@pytest.fixture(autouse=True)
def _stub_yandex_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
self.city = city
self.request_delay_sec = 0.0
self._cffi_session = None
self._cookies = {}
async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
return self
async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
return None
monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)
# ── ANCHOR_TIMEOUT_SEC is exported ─────────────────────────────────────────
def test_anchor_timeout_sec_exported() -> None:
from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC
assert isinstance(ANCHOR_TIMEOUT_SEC, int)
# Reasonable sane range: 60s–600s
assert (
60 <= ANCHOR_TIMEOUT_SEC <= 600
), f"ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC} out of sane range"
# ── Avito sweep watchdog ────────────────────────────────────────────────────
async def test_avito_sweep_anchor_timeout_continues(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается.
run_avito_city_sweep использует внутреннюю _avito_anchor_phases (не run_avito_pipeline),
поэтому патчим AvitoScraper.fetch_around + save_listings.
"""
from app.services.scrapers.avito import AvitoScraper
# Monkeypatch asyncio.wait_for so we can inject TimeoutError on first call only,
# without actually waiting in CI.
original_wait_for = asyncio.wait_for
wf_call = 0
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
nonlocal wf_call
wf_call += 1
if wf_call == 1:
# Simulate the hung anchor: cancel the coro (mimic wait_for behaviour)
# and raise TimeoutError.
try:
coro.close()
except Exception:
pass
raise TimeoutError
return await original_wait_for(coro, timeout=timeout)
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
# Stub AvitoScraper.fetch_around — second anchor returns 3 lots.
fetch_calls: list[str] = []
async def fake_fetch_around(
self: AvitoScraper,
lat: float,
lon: float,
radius_m: int,
pages: int = 1,
delay_override_sec: float | None = None,
) -> list[ScrapedLot]:
fetch_calls.append(f"{lat:.4f}")
return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(3)]
monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
# Stub shared AsyncSession (used by avito sweep).
fake_session = AsyncMock()
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
fake_session.__aexit__ = AsyncMock(return_value=None)
import curl_cffi.requests as _cffi_mod
monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
# Stub IMV phase (not relevant to this test).
async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
result = MagicMock()
result.checked = 0
result.saved = 0
result.errors = 0
return result
monkeypatch.setattr(
"app.services.house_imv_backfill.process_houses_imv_batch",
fake_imv_batch,
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_avito_city_sweep(
db=MagicMock(),
run_id=1,
anchors=TWO_ANCHORS,
enrich_houses=False,
detail_top_n=0,
enrich_imv=False,
request_delay_sec=0.0,
)
# (a) sweep completes — doesn't hang
# (b) first anchor timed out → errors_count >= 1
assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
# (c) second anchor reached: lots_fetched > 0
assert counters.lots_fetched > 0, "второй anchor должен был выполниться"
# (d) mark_done вызван (sweep завершился)
assert fake.done is not None, "mark_done должен быть вызван"
assert fake.failed is None, "mark_failed не должен быть вызван"
assert counters.anchors_done == len(TWO_ANCHORS)
# ── Cian sweep watchdog ─────────────────────────────────────────────────────
async def test_cian_sweep_anchor_timeout_continues(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
serp_calls: list[str] = []
original_wait_for = asyncio.wait_for
wf_call = 0
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
nonlocal wf_call
wf_call += 1
if wf_call == 1:
try:
coro.close()
except Exception:
pass
raise TimeoutError
return await original_wait_for(coro, timeout=timeout)
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
async def fake_fetch_multi(
self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
) -> list[ScrapedLot]:
serp_calls.append(f"{lat:.4f}")
return [_fake_lot("cian", f"{lat:.4f}-1")]
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
AsyncMock(return_value=None),
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=2,
anchors=TWO_ANCHORS,
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
)
# (a) sweep completes (doesn't hang at anchor 1)
# (b) errors_count >= 1 (timeout counted)
assert counters.errors_count >= 1
# (c) second anchor was reached (SERP called once — first anchor timed out before SERP)
# wf_call==2 means the second wait_for ran → second anchor's coro executed.
assert wf_call == 2, f"wait_for should be called once per anchor; got {wf_call}"
# (d) mark_done called
assert fake.done is not None
assert fake.failed is None
assert counters.anchors_done == len(TWO_ANCHORS)
# ── Yandex sweep watchdog ───────────────────────────────────────────────────
async def test_yandex_sweep_anchor_timeout_continues(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
serp_calls: list[str] = []
original_wait_for = asyncio.wait_for
wf_call = 0
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
nonlocal wf_call
wf_call += 1
if wf_call == 1:
try:
coro.close()
except Exception:
pass
raise TimeoutError
return await original_wait_for(coro, timeout=timeout)
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1500,
max_pages: int = 2,
**_: Any,
) -> list[ScrapedLot]:
serp_calls.append(f"{lat:.4f}")
return [_fake_lot("yandex", f"{lat:.4f}-1")]
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=3,
anchors=TWO_ANCHORS,
pages_per_anchor=1,
enrich_address=False,
request_delay_sec=0.0,
)
# (a) sweep completes
# (b) timeout counted
assert counters.errors_count >= 1
# (c) second anchor reached
assert wf_call == 2
# (d) mark_done called
assert fake.done is not None
assert fake.failed is None
assert counters.anchors_done == len(TWO_ANCHORS)
# ── All three: timeout increments errors_count and is logged, sweep reaches mark_done ──
async def test_cian_timeout_increments_counter_and_marks_done(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Детальная проверка: timeout anchor → errors_count+1, sweep продолжается, mark_done.
Использует fake_wait_for (зеркало первых трёх тестов): первый вызов → TimeoutError
(имитирует зависший anchor), второй вызов → реальный wait_for (нормальный anchor).
Проверяет что:
- errors_count увеличивается
- второй anchor выполнился (SERP вызван)
- mark_done вызван (sweep завершился)
- mark_banned НЕ вызван (одиночный таймаут < порога consecutive failures)
"""
original_wait_for = asyncio.wait_for
wf_call = 0
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
nonlocal wf_call
wf_call += 1
if wf_call == 1:
try:
coro.close()
except Exception:
pass
raise TimeoutError
return await original_wait_for(coro, timeout=timeout)
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
serp_calls: list[str] = []
async def fake_fetch(
self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
) -> list[ScrapedLot]:
serp_calls.append(f"{lat:.4f}")
return [_fake_lot("cian", f"{lat:.4f}-ok")]
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch)
monkeypatch.setattr(
scrape_pipeline,
"save_listings",
lambda _db, lots, *, run_id=None: (len(lots), 0),
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
AsyncMock(return_value=None),
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=10,
anchors=TWO_ANCHORS,
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
)
# (b) timeout branch: errors_count incremented
assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
# (c) second anchor reached (wait_for was called twice — once per anchor)
assert wf_call == 2, f"wait_for должен быть вызван для каждого anchor; got {wf_call}"
# (d) sweep reached mark_done
assert fake.done is not None, "mark_done должен быть вызван"
assert fake.banned is None, "mark_banned не должен быть вызван для одиночного таймаута"
assert counters.anchors_done == len(TWO_ANCHORS)
# ── (A) Timeout scaling — avito_anchor_timeout > ANCHOR_TIMEOUT_SEC ───────────
def test_avito_anchor_timeout_scaling() -> None:
"""_avito_anchor_timeout масштабируется по detail_top_n и enrich_houses.
При detail_top_n=20 и enrich_houses=True таймаут должен быть существенно выше
ANCHOR_TIMEOUT_SEC (240), чтобы detail-фаза не guillotine'ила SERP-фазу.
"""
from app.services.scrape_pipeline import (
_AVITO_HOUSES_BUDGET_S,
_AVITO_PER_DETAIL_S,
ANCHOR_TIMEOUT_SEC,
)
detail_top_n = 20
enrich_houses = True
computed = (
float(ANCHOR_TIMEOUT_SEC)
+ detail_top_n * _AVITO_PER_DETAIL_S
+ (_AVITO_HOUSES_BUDGET_S if enrich_houses else 0.0)
)
# Должен быть значительно больше базового (минимум вдвое при n=20+houses)
assert (
computed > ANCHOR_TIMEOUT_SEC * 2
), f"computed timeout {computed} должен быть > 2× base {ANCHOR_TIMEOUT_SEC}"
# Без detail и без houses — ровно ANCHOR_TIMEOUT_SEC
computed_no_detail = float(ANCHOR_TIMEOUT_SEC) + 0 * _AVITO_PER_DETAIL_S + 0.0
assert computed_no_detail == float(ANCHOR_TIMEOUT_SEC)
# Новые константы экспортированы / импортируются без ошибок
assert _AVITO_PER_DETAIL_S > 0
assert _AVITO_HOUSES_BUDGET_S > 0
# ── (B) SERP counters durable across detail-phase timeout ────────────────────
async def test_avito_serp_counters_durable_after_detail_timeout(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""SERP+save счётчики сохраняются в sweep-level counters даже когда detail-фаза
вызывает TimeoutError из wait_for.
Симулируем: первый anchor — SERP возвращает 5 lots, save успешен, затем wait_for
завершается TimeoutError (detail-фаза «зависла»). Второй anchor — полный успех.
Ожидаем:
- counters.lots_fetched == 5 (SERP первого anchor'а) + лоты второго
- counters.lots_inserted >= 1 (save первого anchor'а)
- counters.errors_count >= 1 (timeout первого anchor'а)
- mark_done вызван (sweep завершился)
"""
import curl_cffi.requests as _cffi_mod
from app.services.scrapers.avito import AvitoScraper
# Отслеживаем вызовы fetch_around
serp_calls: list[str] = []
save_calls: list[int] = []
original_wait_for = asyncio.wait_for
wf_call = 0
async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
nonlocal wf_call
wf_call += 1
if wf_call == 1:
# Запускаем корутину до конца — она дойдёт до SERP+save,
# а затем TimeoutError наступает как если бы detail-фаза зависла.
# Для упрощения: просто инжектируем TimeoutError (как во всех других тестах),
# но проверяем, что counter-патч произошёл ДО wait_for через отдельный
# механизм: monkeypatch fetch_around + save_listings записывают данные
# в counters до timeout-исключения.
try:
coro.close()
except Exception:
pass
raise TimeoutError
return await original_wait_for(coro, timeout=timeout)
monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
# Stub shared AsyncSession
fake_session = AsyncMock()
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
fake_session.__aexit__ = AsyncMock(return_value=None)
monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
# Stub AvitoScraper.fetch_around — возвращает lots (не AvitoBlockedError)
async def fake_fetch_around(
self: AvitoScraper,
lat: float,
lon: float,
radius_m: int,
pages: int = 1,
delay_override_sec: float | None = None,
) -> list[ScrapedLot]:
serp_calls.append(f"{lat:.4f}")
return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(5)]
monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
# Stub save_listings — записывает lots count
def fake_save_listings(db: Any, lots: Any, *, run_id: int | None = None) -> tuple[int, int]:
save_calls.append(len(lots))
return (len(lots), 0)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_listings)
# Stub run_avito_pipeline — используем реальный run_avito_city_sweep
# но с внутренним патчем, поэтому НЕ monkeypatch run_avito_pipeline.
# Важно: inner _avito_anchor_phases вызывает AvitoScraper и save_listings
# через nonlocal counters ПЕРЕД wait_for возвращает TimeoutError.
async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
result = MagicMock()
result.checked = 0
result.saved = 0
result.errors = 0
return result
monkeypatch.setattr(
"app.services.house_imv_backfill.process_houses_imv_batch",
fake_imv_batch,
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
# Stub fetch_house_catalog и fetch_detail чтобы не было реальных сетевых вызовов
# (они могут вызваться из _avito_anchor_phases если timeout не наступит раньше)
monkeypatch.setattr(
scrape_pipeline,
"fetch_house_catalog",
AsyncMock(side_effect=RuntimeError("should not be called in timeout test")),
)
monkeypatch.setattr(
scrape_pipeline,
"fetch_detail",
AsyncMock(side_effect=RuntimeError("should not be called in timeout test")),
)
counters = await scrape_pipeline.run_avito_city_sweep(
db=MagicMock(),
run_id=42,
anchors=TWO_ANCHORS,
enrich_houses=False,
detail_top_n=0,
enrich_imv=False,
request_delay_sec=0.0,
)
# SERP первого anchor'а НЕ дойдёт до счётчиков т.к. wait_for killит coro СРАЗУ
# (fake_wait_for делает coro.close() до любого await внутри). Это ожидаемо при
# coro.close() — реальный сценарий другой: SERP завершается, затем detail зависает.
# Тест проверяет sweep-level инварианты:
# - errors_count >= 1 (timeout)
# - sweep не падает
# - mark_done вызван
# - второй anchor дошёл (wf_call==2)
assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
assert fake.done is not None, "mark_done должен быть вызван"
assert fake.failed is None, "mark_failed не должен быть вызван"
assert wf_call == 2, f"оба anchor'а должны пройти через wait_for; got {wf_call}"
assert counters.anchors_done == len(TWO_ANCHORS)
async def test_avito_serp_counters_durable_real_phases(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Проверяем что SERP+save счётчики попадают в sweep-level counters когда
_avito_anchor_phases выполняется нормально (без timeout): lots_fetched/inserted
отражают реальные данные.
Патчим wait_for через реальный asyncio.wait_for с очень большим timeout —
anchor_phases выполняется полностью. SERP возвращает 7 lots, save возвращает (7, 0).
"""
import curl_cffi.requests as _cffi_mod
from app.services.scrapers.avito import AvitoScraper
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
fake_session = AsyncMock()
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
fake_session.__aexit__ = AsyncMock(return_value=None)
monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
async def fake_fetch_around(
self: AvitoScraper,
lat: float,
lon: float,
radius_m: int,
pages: int = 1,
delay_override_sec: float | None = None,
) -> list[ScrapedLot]:
return [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(7)]
monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
# Не вызывать house/detail enrich
monkeypatch.setattr(scrape_pipeline, "fetch_house_catalog", AsyncMock(return_value=None))
monkeypatch.setattr(scrape_pipeline, "fetch_detail", AsyncMock(return_value=None))
async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
result = MagicMock()
result.checked = 0
result.saved = 0
result.errors = 0
return result
monkeypatch.setattr(
"app.services.house_imv_backfill.process_houses_imv_batch",
fake_imv_batch,
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
# Один anchor — проверяем счётчики после нормального прохода
counters = await scrape_pipeline.run_avito_city_sweep(
db=MagicMock(),
run_id=99,
anchors=[(56.8400, 60.6050, "Центр")],
enrich_houses=False,
detail_top_n=0,
enrich_imv=False,
request_delay_sec=0.0,
)
assert counters.lots_fetched == 7, f"lots_fetched должен быть 7; got {counters.lots_fetched}"
assert counters.lots_inserted == 7, f"lots_inserted должен быть 7; got {counters.lots_inserted}"
assert counters.anchors_done == 1
assert fake.done is not None

100
tradein-mvp/backend/tests/test_avito_newbuilding_sweep.py
View file

@ -6,22 +6,24 @@
newbuilding_id заполнен (DOM-маркер застройщика + zhk-якорь).
3. fetch_newbuildings break-on-empty + dedup (mirror fetch_city_wide).
4. NewbuildingSweepCounters — defaults + to_dict.
5. run_avito_newbuilding_sweep wiring — мокаем fetch_newbuildings + save_listings,
assert save вызван и SERP-счётчики записаны (mark_done).
Legacy `run_avito_newbuilding_sweep` orchestration wiring (fetch_newbuildings +
save_listings + mark_done) удалён вместе с `app.services.scrape_pipeline` (#2397
Part E1) — эквивалентная regression-coverage теперь в
`test_scraper_kit_pipeline_parity2.py::test_avito_newbuilding_sweep` (kit
`run_avito_newbuilding_sweep`).
"""
from __future__ import annotations
import os
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from app.services import scrape_pipeline
from app.services.scrapers.avito import NOVOSTROYKA_SLUG, AvitoScraper
from app.services.scrapers.base import ScrapedLot
@ -148,7 +150,7 @@ def test_fetch_city_wide_url_unchanged() -> None:
def test_newbuilding_counters_defaults_and_to_dict() -> None:
from app.services.scrape_pipeline import NewbuildingSweepCounters
from scraper_kit.orchestration.pipeline import NewbuildingSweepCounters
c = NewbuildingSweepCounters()
assert c.lots_fetched == 0
@ -157,89 +159,3 @@ def test_newbuilding_counters_defaults_and_to_dict() -> None:
assert c.errors_count == 0
d = c.to_dict()
assert set(d.keys()) == {"lots_fetched", "lots_inserted", "lots_updated", "errors_count"}
# ── 5. run_avito_newbuilding_sweep wiring ───────────────────────────────────
class _FakeRuns:
def __init__(self) -> None:
self.heartbeats: list[dict[str, int]] = []
self.done: dict[str, int] | None = None
self.failed: tuple[str, dict[str, int]] | None = None
self.banned: tuple[str, dict[str, int]] | None = None
def is_cancelled(self, _db: Any, _run_id: int) -> bool:
return False
def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.heartbeats.append(dict(counters))
def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.done = dict(counters)
def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.failed = (error, dict(counters))
def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.banned = (error, dict(counters))
def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
@pytest.mark.asyncio
async def test_run_avito_newbuilding_sweep_saves_and_counts(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_runs = _FakeRuns()
_install_runs(monkeypatch, fake_runs)
fake_lots = [_make_lot("N1"), _make_lot("N2"), _make_lot("N3")]
async def fake_fetch_newbuildings(
self: AvitoScraper,
pages: int = 30,
*,
delay_override_sec: float | None = None,
) -> list[ScrapedLot]:
return list(fake_lots)
monkeypatch.setattr(AvitoScraper, "fetch_newbuildings", fake_fetch_newbuildings)
save_calls: list[int] = []
def fake_save_listings(_db: Any, lots: Any, **_kw: Any) -> tuple[int, int]:
save_calls.append(len(lots))
return (len(lots), 0)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_listings)
# Force cffi-mode (no browser) so we don't spin up BrowserFetcher.
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "cffi")
# Stub shared AsyncSession (used by the sweep in cffi-mode) — no real session.
fake_session = AsyncMock()
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
fake_session.__aexit__ = AsyncMock(return_value=None)
monkeypatch.setattr(scrape_pipeline, "AsyncSession", lambda **_kw: fake_session)
counters = await scrape_pipeline.run_avito_newbuilding_sweep(
MagicMock(),
run_id=42,
pages=2,
request_delay_sec=0.0,
)
assert save_calls == [3]
assert counters.lots_fetched == 3
assert counters.lots_inserted == 3
assert counters.lots_updated == 0
assert fake_runs.done is not None
assert fake_runs.done["lots_fetched"] == 3
assert fake_runs.failed is None
assert fake_runs.banned is None

220
tradein-mvp/backend/tests/test_ban_rotation_1790.py
View file

@ -1,220 +0,0 @@
"""#1790 — ban-triggered IP rotation в enrichment-фазах scrape_pipeline.
Тест 1: AvitoBlockedError при доступных ротациях → вызывается _rotate_proxy_ip
(changeip) + consecutive счётчик сбрасывается → enrichment продолжается.
Тест 2: Ротаций больше нет (бюджет исчерпан) → abort enrichment (bounded N ротаций).
Тест 3: Graceful per-item (#1820) сохранён — один блок без порога ротации НЕ вызывает.
Без сети, без БД.
"""
from __future__ import annotations
import os
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrape_pipeline import (
_AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
PipelineResult,
run_avito_pipeline,
)
from app.services.scrapers.avito_exceptions import AvitoBlockedError
def _make_lot(
item_id: str,
house_url: str | None = None,
) -> object:
from app.services.scrapers.base import ScrapedLot
return ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
source_id=item_id,
price_rub=5_000_000,
address="Test address",
house_source="avito" if house_url else None,
house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
house_url=house_url,
listing_segment="vtorichka",
)
def _block_scraper(lots: list) -> MagicMock: # type: ignore[type-arg]
scraper = MagicMock()
scraper.__aenter__ = AsyncMock(return_value=scraper)
scraper.__aexit__ = AsyncMock(return_value=None)
scraper.fetch_around = AsyncMock(return_value=lots)
scraper._cffi = None
return scraper
# ---------------------------------------------------------------------------
# Тест 1: ротация срабатывает при consecutive blocks == порог
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_rotation_called_on_consecutive_blocks() -> None:
"""При N подряд-блоках (порог) вызывается _rotate_proxy_ip — IP меняется.
После успешной ротации consecutive счётчик сбрасывается и enrichment продолжается:
следующий (N+1)-й дом фетчится успешно.
"""
n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
lots = [
_make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{100 + i}")
for i in range(n + 2) # n блокируются, потом ротация, потом успех
]
scraper = _block_scraper(lots)
# Первые n обращений — блок, (n+1)-й — успех
fetch_house = AsyncMock(side_effect=[AvitoBlockedError("banned")] * n + [MagicMock()])
rotate_mock = AsyncMock(return_value=True) # ротация успешна
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
patch(
"app.services.scrape_pipeline.save_house_catalog_enrichment",
return_value={"house_id": 99},
),
patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
),
):
result = await run_avito_pipeline(
MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
)
assert isinstance(result, PipelineResult)
# _rotate_proxy_ip вызвана хотя бы один раз
assert rotate_mock.await_count >= 1
# Enrichment продолжился после ротации — (n+1)-й дом обогащён
assert result.counters.houses_enriched >= 1
# Listings сохранены
assert result.counters.lots_inserted == len(lots)
# ---------------------------------------------------------------------------
# Тест 2: бюджет ротаций исчерпан → abort
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_rotation_bounded_abort_when_budget_exhausted() -> None:
"""После max_rotations попыток ротации enrichment abort'ится (bounded).
Используем avito_proxy_max_rotations=1 → только одна попытка ротации.
После неё блоки продолжаются → abort.
"""
n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
# Достаточно домов чтобы вызвать и ротацию и повторные блоки
lots = [
_make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{200 + i}")
for i in range(n * 3)
]
scraper = _block_scraper(lots)
# Все дома блокируются (не будет успешного фетча)
fetch_house = AsyncMock(side_effect=AvitoBlockedError("banned"))
# Ротация успешна, но дом после неё тоже блокируется
rotate_mock = AsyncMock(return_value=True)
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
patch("app.services.scrape_pipeline.save_house_catalog_enrichment", return_value={}),
patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as mock_settings,
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
),
):
# Лимит 1 ротация: после неё бюджет исчерпан → abort на следующей серии блоков
mock_settings.avito_proxy_max_rotations = 1
mock_settings.avito_proxy_rotate_url = "http://rotate.test/"
mock_settings.avito_proxy_rotate_settle_s = 0.0
mock_settings.scraper_fetch_mode = "curl_cffi"
mock_settings.scraper_proxy_url = None
mock_settings.scraper_skip_seen_today = False
# #2202: bare MagicMock settings auto-truthy on any attribute access — pin
# explicitly off so _rotate_avito_proxy takes the changeip path this test
# asserts on (pool path is covered separately in test_scrape_pipeline_proxy_pool.py).
mock_settings.use_proxy_pool_curl = False
result = await run_avito_pipeline(
MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
)
assert isinstance(result, PipelineResult)
# Ровно 1 ротация (бюджет = 1)
assert rotate_mock.await_count == 1
# Enrichment abort'нут (есть AVITO_BLOCKED в ошибках)
assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
# Listings всё равно сохранены
assert result.counters.lots_inserted == len(lots)
# ---------------------------------------------------------------------------
# Тест 3: один блок (ниже порога) — ротация НЕ вызывается (#1820 graceful)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_single_block_below_threshold_no_rotation() -> None:
"""Один блок (< порога) — _rotate_proxy_ip НЕ вызывается.
Graceful per-item #1820: единственный блок пишется в errors, enrichment
продолжается, ротация не тратится.
"""
n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
# Только 2 дома: первый блокируется, второй успешен
lots = [
_make_lot("1", "https://www.avito.ru/catalog/houses/ekb/a/100"),
_make_lot("2", "https://www.avito.ru/catalog/houses/ekb/b/200"),
]
assert n >= 2, "Порог должен быть >= 2 чтобы тест был корректен"
scraper = _block_scraper(lots)
fetch_house = AsyncMock(side_effect=[AvitoBlockedError("HTTP 403"), MagicMock()])
rotate_mock = AsyncMock(return_value=True)
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.save_listings", return_value=(2, 0)),
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
patch(
"app.services.scrape_pipeline.save_house_catalog_enrichment",
return_value={"house_id": 55},
),
patch("app.services.scrape_pipeline._rotate_proxy_ip", rotate_mock),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
),
):
result = await run_avito_pipeline(
MagicMock(), 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
)
# Ротация НЕ вызвана — один блок ниже порога
assert rotate_mock.await_count == 0
# Первый дом упал, второй обогащён
assert result.counters.houses_failed == 1
assert result.counters.houses_enriched == 1
assert any("BLOCKED_HOUSE" in e for e in result.counters.errors)

685
tradein-mvp/backend/tests/test_cian_city_sweep.py
View file

@ -1,16 +1,18 @@
"""Offline unit tests for run_cian_city_sweep (#860).
"""Offline unit tests for CianCitySweepCounters + admin sweep endpoints (#860).
Pure & fast: NO live network, NO DB. Mirrors test_city_sweep.
Asserts orchestration semantics only.
Pure & fast: NO live network, NO DB.
Legacy `run_cian_city_sweep` orchestration behavior (SERP→save→detail→houses phase
sequencing, cooperative cancel, consecutive-SERP-failure abort, per-item error
tolerance) удалён вместе с `app.services.scrape_pipeline` (#2397 Part E1) —
эквивалентная regression-coverage теперь в
`test_scraper_kit_pipeline_parity2.py::test_cian_city_sweep` (kit
`run_cian_city_sweep`).
Coverage:
- Все фазы (SERP → save → detail → houses) вызываются в нужном порядке
- CianCitySweepCounters корректно заполняются
- Cooperative cancel прерывает sweep перед anchor'ом
- enrich_houses=False пропускает houses-фазу
- Consecutive SERP failures → mark_banned + abort
- Per-item detail/house error не валит весь sweep
- mark_done вызывается при нормальном завершении
- CianCitySweepCounters корректно заполняются (dataclass shape)
- _cian_anchor_timeout_s watchdog-формула (#2160)
- Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI (config/matcher)
"""
from __future__ import annotations
@ -19,117 +21,13 @@ import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.cian import CianScraper
# Три тестовых anchor'а — не все 5, чтобы тесты были быстрее
TEST_ANCHORS: list[tuple[float, float, str]] = [
(56.8400, 60.6050, "A1"),
(56.7950, 60.5300, "A2"),
(56.8970, 60.6100, "A3"),
]
def _fake_lot(
offer_id: str, *, nb: bool = False, segment: str | None = "novostroyki"
) -> ScrapedLot:
"""Создаём ScrapedLot с опциональным newbuilding-ссылкой.
segment по умолчанию "novostroyki" — run_cian_city_sweep теперь NB-only
(newbuilding_only=True), поэтому дефолтные lots должны проходить SERP-фильтр.
"""
return ScrapedLot(
source="cian",
source_url=f"https://ekb.cian.ru/sale/flat/{offer_id}/",
source_id=offer_id,
address="Екатеринбург, улица Тестовая, 1",
price_rub=5_000_000,
area_m2=50.0,
rooms=2,
listing_segment=segment,
house_source="cian_newbuilding" if nb else "cian",
house_ext_id="99999" if nb else None,
)
class _FakeRuns:
"""Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel."""
def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
self.cancel_at_anchor = cancel_at_anchor
self._is_cancelled_calls = 0
self.heartbeats: list[dict[str, int]] = []
self.done: dict[str, int] | None = None
self.failed: tuple[str, dict[str, int]] | None = None
self.banned: tuple[str, dict[str, int]] | None = None
def is_cancelled(self, _db: Any, _run_id: int) -> bool:
self._is_cancelled_calls += 1
if self.cancel_at_anchor is None:
return False
return self._is_cancelled_calls >= self.cancel_at_anchor
def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.heartbeats.append(dict(counters))
def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.done = dict(counters)
def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.failed = (error, dict(counters))
def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.banned = (error, dict(counters))
@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
"""asyncio.sleep → instant (no test delays)."""
async def _instant(_secs: float) -> None:
return None
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
@pytest.fixture(autouse=True)
def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
"""Нейтрализуем CianScraper.__init__/__aenter__/__aexit__ — нет DB вызовов."""
def _init(self: CianScraper) -> None:
self.request_delay_sec = 0.0
self._cffi = None
async def _aenter(self: CianScraper) -> CianScraper:
return self
async def _aexit(self: CianScraper, *_args: Any) -> None:
return None
monkeypatch.setattr(CianScraper, "__init__", _init)
monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
# ── CianCitySweepCounters ───────────────────────────────────────────────────
def test_cian_sweep_counters_defaults() -> None:
from app.services.scrape_pipeline import CianCitySweepCounters
from scraper_kit.orchestration.pipeline import CianCitySweepCounters
c = CianCitySweepCounters()
assert c.anchors_total == 0
@ -142,7 +40,7 @@ def test_cian_sweep_counters_defaults() -> None:
def test_cian_sweep_counters_to_dict_all_keys() -> None:
from dataclasses import fields
from app.services.scrape_pipeline import CianCitySweepCounters
from scraper_kit.orchestration.pipeline import CianCitySweepCounters
c = CianCitySweepCounters()
d = c.to_dict()
@ -150,531 +48,6 @@ def test_cian_sweep_counters_to_dict_all_keys() -> None:
assert set(d.keys()) == expected
# ── Full sweep — фазы и агрегация ──────────────────────────────────────────
async def test_sweep_iterates_all_anchors_serp_and_save(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""SERP вызывается для каждого anchor, save_listings получает накопленные lots."""
serp_calls: list[str] = []
save_calls: list[int] = []
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
serp_calls.append(f"{lat:.4f}")
return [_fake_lot(f"{lat}-{i}") for i in range(3)]
def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
save_calls.append(len(lots))
return len(lots), 0
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
# Stub detail + houses fetchers (возвращают None → graceful skip)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
AsyncMock(return_value=None),
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=1,
anchors=TEST_ANCHORS,
pages_per_anchor=2,
detail_top_n=0, # skip detail phase
enrich_houses=False,
request_delay_sec=0.0,
)
assert len(serp_calls) == len(TEST_ANCHORS)
assert len(save_calls) == len(TEST_ANCHORS)
assert counters.anchors_done == len(TEST_ANCHORS)
assert counters.lots_fetched == len(TEST_ANCHORS) * 3
assert counters.lots_inserted == len(TEST_ANCHORS) * 3
assert counters.errors_count == 0
assert fake.done is not None
assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3
assert fake.failed is None
assert fake.banned is None
async def test_newbuilding_only_drops_secondary_before_save(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""newbuilding_only=True (default): save_listings получает только novostroyki-lots.
SERP отдаёт mixed-сегмент выдачу — вторичкой авторитетно владеет run_cian_full_load,
поэтому sweep её отбрасывает до save и считает в lots_dropped_secondary.
"""
saved_lots: list[list[ScrapedLot]] = []
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
# 1 novostroyki + 2 vtorichka на anchor
return [
_fake_lot(f"{lat}-nb", segment="novostroyki"),
_fake_lot(f"{lat}-v1", segment="vtorichka"),
_fake_lot(f"{lat}-v2", segment="vtorichka"),
]
def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
saved_lots.append(list(lots))
return len(lots), 0
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=1,
anchors=TEST_ANCHORS,
pages_per_anchor=2,
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
newbuilding_only=True,
)
# Каждый save-вызов содержит только novostroyki-lot
assert len(saved_lots) == len(TEST_ANCHORS)
for lots in saved_lots:
assert len(lots) == 1
assert all(lot.listing_segment == "novostroyki" for lot in lots)
# Counters: fetched все 3×N, сохранён 1×N, отброшено 2×N
assert counters.lots_fetched == len(TEST_ANCHORS) * 3
assert counters.lots_inserted == len(TEST_ANCHORS) * 1
assert counters.lots_dropped_secondary == len(TEST_ANCHORS) * 2
async def test_newbuilding_only_false_saves_all_segments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""newbuilding_only=False: фильтр отключён — save получает и вторичку, и новостройки."""
saved_lots: list[list[ScrapedLot]] = []
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
return [
_fake_lot(f"{lat}-nb", segment="novostroyki"),
_fake_lot(f"{lat}-v1", segment="vtorichka"),
]
def fake_save(_db: Any, lots: list, *, run_id: int | None = None):
saved_lots.append(list(lots))
return len(lots), 0
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=1,
anchors=TEST_ANCHORS,
pages_per_anchor=2,
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
newbuilding_only=False,
)
assert len(saved_lots) == len(TEST_ANCHORS)
for lots in saved_lots:
assert len(lots) == 2
assert counters.lots_inserted == len(TEST_ANCHORS) * 2
assert counters.lots_dropped_secondary == 0
async def test_detail_phase_runs_and_fills_counters(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""detail-фаза вызывается и счётчики заполняются."""
from app.services.scrapers.cian_detail import DetailEnrichment
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
return [_fake_lot(f"{lat}-{i}") for i in range(2)]
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
# DB mock returning 2 rows for detail query
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = [
{"id": 101, "source_url": "https://ekb.cian.ru/sale/flat/101/"},
{"id": 102, "source_url": "https://ekb.cian.ru/sale/flat/102/"},
]
fake_enrichment = DetailEnrichment(cian_id=101)
mock_fetch_detail = AsyncMock(return_value=fake_enrichment)
mock_save_detail = MagicMock()
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
mock_fetch_detail,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.save_detail_enrichment",
mock_save_detail,
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=mock_db,
run_id=2,
anchors=TEST_ANCHORS[:1], # один anchor
detail_top_n=2,
enrich_houses=False,
request_delay_sec=0.0,
)
assert counters.detail_attempted == 2
assert counters.detail_enriched == 2
assert counters.detail_failed == 0
assert mock_fetch_detail.call_count == 2
assert mock_save_detail.call_count == 2
assert fake.done is not None
async def test_enrich_houses_false_skips_houses_phase(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""enrich_houses=False → houses-фаза не выполняется."""
nb_fetch_calls: list[str] = []
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
return [_fake_lot("1001", nb=True)]
async def fake_fetch_nb(url: str, **_: Any) -> None:
nb_fetch_calls.append(url)
return None
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.fetch_newbuilding",
fake_fetch_nb,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
AsyncMock(return_value=None),
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=3,
anchors=TEST_ANCHORS[:1],
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
)
assert len(nb_fetch_calls) == 0
assert counters.houses_attempted == 0
assert counters.houses_enriched == 0
assert fake.done is not None
async def test_cancel_midway_stops_remaining(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Cooperative cancel перед anchor #2 → anchor #1 выполняется, #2+ нет."""
serp_calls: list[str] = []
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
serp_calls.append(f"{lat:.4f}")
return [_fake_lot(f"{lat}-1")]
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
AsyncMock(return_value=None),
)
fake = _FakeRuns(cancel_at_anchor=2) # cancel при is_cancelled_calls >= 2 → перед anchor #2
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=4,
anchors=TEST_ANCHORS,
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
)
assert len(serp_calls) == 1, "только anchor #1 должен выполниться"
assert counters.anchors_done == 1
assert fake.done is None
assert fake.banned is None
async def test_consecutive_serp_failures_abort_with_banned(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""N подряд SERP-ошибок → mark_banned + abort sweep."""
from app.services.scrape_pipeline import CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
attempt_count = 0
async def fake_fetch_fail(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
nonlocal attempt_count
attempt_count += 1
raise RuntimeError("cian blocked")
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_fail)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
# Передаём больше anchor'ов чем CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
anchors = [*TEST_ANCHORS, (56.77, 60.55, "A4"), (56.865, 60.62, "A5")]
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=5,
anchors=anchors,
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
)
assert attempt_count == CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
assert counters.errors_count == CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES
assert fake.banned is not None, "mark_banned должен быть вызван"
assert "consecutive" in fake.banned[0]
assert fake.done is None
async def test_per_item_detail_error_does_not_abort_sweep(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Ошибка при detail-обогащении одного листинга не валит sweep."""
call_count = 0
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
return [_fake_lot(f"lot-{idx}") for idx in range(2)]
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
mock_db = MagicMock()
# Возвращаем 2 listing-а для detail
mock_db.execute.return_value.mappings.return_value.all.return_value = [
{"id": 201, "source_url": "https://ekb.cian.ru/sale/flat/201/"},
{"id": 202, "source_url": "https://ekb.cian.ru/sale/flat/202/"},
]
async def fetch_detail_sometimes_fails(url: str, **_: Any):
nonlocal call_count
call_count += 1
if call_count == 1:
raise RuntimeError("transient network error")
from app.services.scrapers.cian_detail import DetailEnrichment
return DetailEnrichment(cian_id=202)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
fetch_detail_sometimes_fails,
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.save_detail_enrichment",
MagicMock(),
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=mock_db,
run_id=6,
anchors=TEST_ANCHORS[:1],
detail_top_n=2,
enrich_houses=False,
request_delay_sec=0.0,
)
assert counters.detail_attempted == 2
assert counters.detail_failed == 1
assert counters.detail_enriched == 1
assert counters.errors_count >= 1
assert fake.done is not None, "sweep должен завершиться mark_done несмотря на ошибку"
assert fake.banned is None
async def test_mark_done_always_called(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""mark_done вызывается даже при пустом SERP."""
async def fake_fetch_empty(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
return []
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_empty)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
AsyncMock(return_value=None),
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=7,
anchors=TEST_ANCHORS,
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
)
assert fake.done is not None
assert counters.lots_fetched == 0
assert counters.anchors_done == len(TEST_ANCHORS)
async def test_houses_phase_uses_house_sources_join(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Houses-фаза формирует JOIN на house_sources, не h.ext_source напрямую."""
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
# Один lot с house_source="cian_newbuilding" → попадёт в houses-фазу
return [_fake_lot("nb-1", nb=True)]
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
AsyncMock(return_value=None),
)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.fetch_newbuilding",
AsyncMock(return_value=None),
)
monkeypatch.setattr(
"app.services.scrapers.cian_newbuilding.save_newbuilding_enrichment",
MagicMock(),
)
mock_db = MagicMock()
# DB execute → возвращает одну house-строку
mock_db.execute.return_value.mappings.return_value.all.return_value = [
{"id": 501, "cian_zhk_url": "https://ekb.cian.ru/kupit-kvartiru/zhk-test/"},
]
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=mock_db,
run_id=10,
anchors=TEST_ANCHORS[:1],
detail_top_n=0,
enrich_houses=True,
request_delay_sec=0.0,
)
# Проверяем SQL — должен содержать JOIN house_sources, а не h.ext_source
assert mock_db.execute.called
executed_sql_args = [str(c.args[0]) for c in mock_db.execute.call_args_list]
houses_sql_calls = [s for s in executed_sql_args if "house" in s.lower()]
assert houses_sql_calls, "должен быть хотя бы один SQL-вызов связанный с houses"
houses_query = houses_sql_calls[-1]
assert (
"house_sources" in houses_query
), f"запрос должен JOIN house_sources, а не обращаться к h.ext_source: {houses_query!r}"
assert "ext_source" not in houses_query.split("house_sources")[0] or True # tolerate join expr
# Главное: нет обращения к h.ext_source (старый баг)
assert (
"h.ext_source" not in houses_query
), f"колонка h.ext_source не существует в houses: {houses_query!r}"
assert (
"h.ext_id" not in houses_query
), f"колонка h.ext_id не существует в houses: {houses_query!r}"
assert fake.done is not None, "mark_done должен быть вызван"
assert fake.failed is None, "mark_failed НЕ должен быть вызван"
assert counters.houses_attempted == 1
async def test_houses_db_query_error_graceful_mark_done(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Ошибка SQL в houses-фазе (напр. UndefinedColumn) → graceful: mark_done,
НЕ mark_failed, houses_failed увеличивается."""
async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8):
# Два nb-лота → houses-фаза попытается запросить 2 id
return [_fake_lot("nb-10", nb=True), _fake_lot("nb-11", nb=True)]
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail",
AsyncMock(return_value=None),
)
mock_db = MagicMock()
def _db_execute_raises(sql, params=None):
sql_text = str(sql)
if "house" in sql_text.lower() and "cian_newbuilding" in sql_text:
raise RuntimeError("column h.ext_source does not exist")
result = MagicMock()
result.mappings.return_value.all.return_value = []
return result
mock_db.execute.side_effect = _db_execute_raises
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=mock_db,
run_id=11,
anchors=TEST_ANCHORS[:1],
detail_top_n=0,
enrich_houses=True,
request_delay_sec=0.0,
)
assert fake.done is not None, "mark_done должен быть вызван даже при ошибке houses-фазы"
assert fake.failed is None, "mark_failed НЕ должен быть вызван из-за ошибки houses-фазы"
assert counters.houses_failed > 0, "houses_failed должен вырасти при ошибке"
assert counters.houses_attempted == 0, "houses_attempted не растёт — до него не дошли"
# ── Admin endpoint tests (offline) ──────────────────────────────────────────
@ -806,19 +179,23 @@ def test_cian_anchor_timeout_defaults() -> None:
= 900 + 500 + 180 = 1580s. Фиксированный ANCHOR_TIMEOUT_SEC=240 гильотинил бы якорь в
SERP-фазе (#2160): каждый SERP-фетч через camoufox 13-45s.
"""
t = scrape_pipeline._cian_anchor_timeout_s(
from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s
t = _cian_anchor_timeout_s(
pages_per_anchor=3,
request_delay_sec=5.0,
detail_top_n=10,
enrich_houses=True,
)
assert t == pytest.approx(1580.0)
assert t > scrape_pipeline.ANCHOR_TIMEOUT_SEC
assert t > ANCHOR_TIMEOUT_SEC
def test_cian_anchor_timeout_no_houses() -> None:
"""enrich_houses=False убирает houses-бюджет (180s) из формулы."""
t = scrape_pipeline._cian_anchor_timeout_s(
from scraper_kit.orchestration.pipeline import _cian_anchor_timeout_s
t = _cian_anchor_timeout_s(
pages_per_anchor=3,
request_delay_sec=5.0,
detail_top_n=10,
@ -829,24 +206,12 @@ def test_cian_anchor_timeout_no_houses() -> None:
def test_cian_anchor_timeout_floor_at_anchor_timeout_sec() -> None:
"""Малые параметры не опускают таймаут ниже ANCHOR_TIMEOUT_SEC (max-ветка)."""
t = scrape_pipeline._cian_anchor_timeout_s(
from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s
t = _cian_anchor_timeout_s(
pages_per_anchor=0,
request_delay_sec=0.0,
detail_top_n=0,
enrich_houses=False,
)
assert t == float(scrape_pipeline.ANCHOR_TIMEOUT_SEC)
def test_cian_anchor_timeout_parity_with_scraper_kit() -> None:
"""Strangler-инвариант: backend-хелпер ≡ scraper_kit-хелпер (обе копии в синхроне)."""
from scraper_kit.orchestration.pipeline import (
_cian_anchor_timeout_s as kit_timeout_s,
)
for args in (
dict(pages_per_anchor=3, request_delay_sec=5.0, detail_top_n=10, enrich_houses=True),
dict(pages_per_anchor=5, request_delay_sec=9.0, detail_top_n=20, enrich_houses=False),
dict(pages_per_anchor=0, request_delay_sec=0.0, detail_top_n=0, enrich_houses=False),
):
assert scrape_pipeline._cian_anchor_timeout_s(**args) == kit_timeout_s(**args)
assert t == float(ANCHOR_TIMEOUT_SEC)

14
tradein-mvp/backend/tests/test_city_sweep.py
View file

@ -18,7 +18,7 @@ import pytest
def test_ekb_anchors_count() -> None:
from app.services.scrape_pipeline import EKB_ANCHORS
from scraper_kit.orchestration.pipeline import EKB_ANCHORS
assert len(EKB_ANCHORS) == 5
for lat, lon, name in EKB_ANCHORS:
@ -31,7 +31,7 @@ def test_ekb_anchors_count() -> None:
def test_city_sweep_counters_defaults() -> None:
from app.services.scrape_pipeline import CitySweepCounters
from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters()
assert c.anchors_total == 0
@ -40,7 +40,7 @@ def test_city_sweep_counters_defaults() -> None:
def test_city_sweep_counters_to_dict() -> None:
from app.services.scrape_pipeline import CitySweepCounters
from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters(anchors_total=5, anchors_done=2, lots_fetched=150)
d = c.to_dict()
@ -55,7 +55,7 @@ def test_city_sweep_counters_to_dict() -> None:
def test_city_sweep_counters_to_dict_all_keys() -> None:
from dataclasses import fields
from app.services.scrape_pipeline import CitySweepCounters
from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters()
d = c.to_dict()
@ -207,7 +207,8 @@ def test_update_heartbeat_persists_total_seen_new_count_columns() -> None:
def test_mark_done_yandex_counters_populate_columns() -> None:
"""End-to-end shape: YandexCitySweepCounters.to_dict() → mark_done → columns set."""
from app.services.scrape_pipeline import YandexCitySweepCounters
from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
from app.services.scrape_runs import mark_done
counters = YandexCitySweepCounters(
@ -230,7 +231,8 @@ def test_mark_done_avito_cian_counters_populate_columns() -> None:
counters.to_dict() → mark_done, что и Yandex — фикс #1926 должен заполнять
total_seen/new_count и для них (issue: «то же касается CitySweepCounters»).
"""
from app.services.scrape_pipeline import CitySweepCounters
from scraper_kit.orchestration.pipeline import CitySweepCounters
from app.services.scrape_runs import mark_done
counters = CitySweepCounters(

315
tradein-mvp/backend/tests/test_domclick_sweep.py
View file

@ -1,14 +1,15 @@
"""Offline unit tests for run_domclick_city_sweep + DomClick parser.
"""Offline unit tests for DomClickCitySweepCounters + DomClick parser.
Pure & fast: NO live network, NO DB. Mirrors test_cian_city_sweep orchestration
semantics, but DomClick is CITYWIDE (no anchor-loop) и SERP-only (no detail/houses).
Legacy `run_domclick_city_sweep` orchestration behavior (sweep phases, cooperative
cancel, honest mark_done/mark_failed statuses) удалён вместе с
`app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная regression-coverage
теперь в `test_scraper_kit_pipeline_parity2.py::test_domclick_city_sweep_{done,
blocked_failed}` (kit `run_domclick_city_sweep`).
Coverage:
- fetch_city вызывается, save_listings получает накопленные lots с source='domklik'
- DomClickCitySweepCounters корректно заполняется (lots_fetched/inserted/updated)
- Cooperative cancel прерывает sweep перед SERP
- fetch_city exception не валит sweep (graceful, errors_count++, mark_done)
- _parse_html: минимальная DomClick карточка → ScrapedLot с rooms/area/floor/price
Остались:
- DomClickCitySweepCounters (dataclass, из scraper_kit) — defaults + to_dict
- DomClickScraper._map_item — BFF offer-item → ScrapedLot маппинг (не зависит
от оркестрации, `app.services.scrapers.domclick` живёт до Part E2)
"""
from __future__ import annotations
@ -17,94 +18,15 @@ import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from typing import Any
from unittest.mock import MagicMock
import pytest
from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.domclick import DomClickScraper
def _fake_lot(offer_id: str) -> ScrapedLot:
return ScrapedLot(
source="domklik",
source_url=f"https://domclick.ru/card/sale__flat__{offer_id}",
source_id=offer_id,
address="Екатеринбург, улица Тестовая, 1",
price_rub=5_000_000,
area_m2=50.0,
rooms=2,
lat=None,
lon=None,
)
class _FakeRuns:
"""Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel."""
def __init__(self, *, cancel: bool = False) -> None:
self.cancel = cancel
self.heartbeats: list[dict[str, int]] = []
self.done: dict[str, int] | None = None
self.failed: tuple[str, dict[str, int]] | None = None
def is_cancelled(self, _db: Any, _run_id: int) -> bool:
return self.cancel
def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.heartbeats.append(dict(counters))
def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.done = dict(counters)
def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.failed = (error, dict(counters))
@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
async def _instant(_secs: float) -> None:
return None
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
@pytest.fixture(autouse=True)
def _stub_domclick_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
"""Нейтрализуем DomClickScraper.__init__/__aenter__/__aexit__ — нет DB/browser."""
def _init(self: DomClickScraper) -> None:
self.request_delay_sec = 0.0
self.parse_failures = 0
self.geo_filtered = 0
self.blocked = False
self.fetch_errors = 0
async def _aenter(self: DomClickScraper) -> DomClickScraper:
return self
async def _aexit(self: DomClickScraper, *_args: Any) -> None:
return None
monkeypatch.setattr(DomClickScraper, "__init__", _init)
monkeypatch.setattr(DomClickScraper, "__aenter__", _aenter)
monkeypatch.setattr(DomClickScraper, "__aexit__", _aexit)
def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
# ── DomClickCitySweepCounters ───────────────────────────────────────────────
def test_domclick_sweep_counters_defaults() -> None:
from app.services.scrape_pipeline import DomClickCitySweepCounters
from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters
c = DomClickCitySweepCounters()
assert c.lots_fetched == 0
@ -119,7 +41,7 @@ def test_domclick_sweep_counters_defaults() -> None:
def test_domclick_sweep_counters_to_dict_all_keys() -> None:
from dataclasses import fields
from app.services.scrape_pipeline import DomClickCitySweepCounters
from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters
c = DomClickCitySweepCounters()
d = c.to_dict()
@ -127,221 +49,12 @@ def test_domclick_sweep_counters_to_dict_all_keys() -> None:
assert set(d.keys()) == expected
# ── Full sweep — фазы и агрегация ──────────────────────────────────────────
async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPatch) -> None:
"""fetch_city → save_listings получает 2 lots с source='domklik', counters заполнены."""
fetch_calls: list[tuple[int, int]] = []
save_calls: list[list[ScrapedLot]] = []
async def fake_fetch_city(
self: DomClickScraper,
city_id: int,
rooms: list[int] | None = None,
pages: int = 20,
) -> list[ScrapedLot]:
fetch_calls.append((city_id, pages))
return [_fake_lot("100"), _fake_lot("101")]
def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
save_calls.append(list(lots))
return len(lots), 0
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_city)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_domclick_city_sweep(
db=MagicMock(),
run_id=1,
city_id=4,
rooms=[1, 2],
pages=3,
request_delay_sec=0.0,
)
assert len(fetch_calls) == 1
assert fetch_calls[0] == (4, 3)
assert len(save_calls) == 1
assert len(save_calls[0]) == 2
assert all(lot.source == "domklik" for lot in save_calls[0])
assert counters.lots_fetched == 2
assert counters.lots_inserted == 2
assert counters.lots_updated == 0
assert counters.errors_count == 0
assert fake.done is not None
assert fake.done["lots_fetched"] == 2
assert fake.failed is None
async def test_sweep_empty_serp_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
"""Пустой SERP → save не вызывается, но mark_done всё равно вызывается."""
save_called = False
async def fake_fetch_empty(
self: DomClickScraper,
city_id: int,
rooms: list[int] | None = None,
pages: int = 20,
) -> list[ScrapedLot]:
return []
def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
nonlocal save_called
save_called = True
return 0, 0
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_empty)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_domclick_city_sweep(
db=MagicMock(), run_id=2, request_delay_sec=0.0
)
assert save_called is False
assert counters.lots_fetched == 0
assert fake.done is not None
async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -> None:
"""Cooperative cancel перед SERP → fetch_city НЕ вызывается, mark_done НЕ вызывается."""
fetch_called = False
async def fake_fetch(
self: DomClickScraper,
city_id: int,
rooms: list[int] | None = None,
pages: int = 20,
) -> list[ScrapedLot]:
nonlocal fetch_called
fetch_called = True
return [_fake_lot("1")]
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch)
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
fake = _FakeRuns(cancel=True)
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_domclick_city_sweep(
db=MagicMock(), run_id=3, request_delay_sec=0.0
)
assert fetch_called is False
assert counters.lots_fetched == 0
assert fake.done is None
async def test_fetch_city_exception_no_reraise(monkeypatch: pytest.MonkeyPatch) -> None:
"""Исключение в fetch_city → errors_count++, sweep НЕ ре-райзит (graceful).
Honest-status (#1968): полный крах SERP-фазы (errors_count=1, 0 lots) → mark_failed,
НЕ mark_done. Run без единого листинга по внешней причине — это реальный failure,
а не успешный пустой прогон. Re-raise при этом НЕ происходит (task завершается чисто).
"""
async def fake_fetch_fail(
self: DomClickScraper,
city_id: int,
rooms: list[int] | None = None,
pages: int = 20,
) -> list[ScrapedLot]:
raise RuntimeError("browser crashed")
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_fail)
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_domclick_city_sweep(
db=MagicMock(), run_id=4, request_delay_sec=0.0
)
assert counters.errors_count == 1
assert counters.lots_fetched == 0
# 0 lots + error → honest mark_failed (не маскируем как done)
assert fake.failed is not None
assert fake.done is None
async def test_fetch_errors_and_no_lots_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
"""Scraper fetch_errors>0 + 0 лотов (без block) → mark_failed (FIX 3 / closes #1968).
Нераспознанный block-вариант приходит как fetch_error (не blocked=True), но
honest-status всё равно обязан пометить run failed при пустом результате.
"""
async def fake_fetch_errs(
self: DomClickScraper,
city_id: int,
rooms: list[int] | None = None,
pages: int = 100,
) -> list[ScrapedLot]:
# Имитируем не-block ошибки извлечения JSON: blocked остаётся False
self.fetch_errors = 3
return []
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_errs)
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_domclick_city_sweep(
db=MagicMock(), run_id=6, request_delay_sec=0.0
)
assert counters.lots_fetched == 0
assert counters.blocked == 0
assert counters.errors_count >= 3 # fetch_errors свёрнуты в errors_count
assert fake.failed is not None
assert fake.done is None
async def test_blocked_and_no_lots_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
"""QRATOR-блок + 0 лотов → mark_failed (честный статус, closes #1968)."""
async def fake_fetch_blocked(
self: DomClickScraper,
city_id: int,
rooms: list[int] | None = None,
pages: int = 100,
) -> list[ScrapedLot]:
# Имитируем QRATOR-блок: установим флаг на scraper и вернём пустой список
self.blocked = True
return []
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_blocked)
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_domclick_city_sweep(
db=MagicMock(), run_id=5, request_delay_sec=0.0
)
assert counters.lots_fetched == 0
assert counters.blocked == 1
# Честный статус: mark_failed должен быть вызван, mark_done — нет
assert fake.failed is not None
assert "QRATOR" in fake.failed[0]
assert fake.done is None
# ── _map_item via sweep — quick integration smoke ────────────────────────────
# ── _map_item ─────────────────────────────────────────────────────────────────
def test_map_item_basic_mapping() -> None:
"""DomClickScraper._map_item маппит BFF offer-item → ScrapedLot корректно."""
scraper = DomClickScraper() # __init__ stubbed
scraper = DomClickScraper()
item = {
"id": 111222333,
"path": "https://domclick.ru/card/sale__flat__111222333",

143
tradein-mvp/backend/tests/test_pipeline_browser_routing.py
View file

@ -6,9 +6,12 @@
1. fetch_detail(browser_fetcher=mock) → DetailEnrichment (browser branch taken)
2. fetch_detail(browser_fetcher=mock) → AvitoBlockedError при firewall HTML
3. fetch_house_catalog(browser_fetcher=mock) → browser branch taken
4. run_avito_pipeline в browser-mode → scraper._browser установлен,
fetch_detail/fetch_house_catalog вызваны с browser_fetcher
5. Curl mode (default) — browser_fetcher=None, curl path используется
4. Curl mode (default) — browser_fetcher=None, curl path используется
Legacy `run_avito_pipeline` browser/curl-mode integration tests (было ранее секцией 4)
удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — browser-routing
покрытие на уровне `fetch_detail`/`fetch_house_catalog`/`AvitoScraper._fetch_serp_html`
ниже остаётся нетронутым.
"""
from __future__ import annotations
@ -17,7 +20,7 @@ import json
import os
import urllib.parse
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -212,135 +215,7 @@ async def test_fetch_house_catalog_browser_firewall_raises() -> None:
)
# ── 4. run_avito_pipeline browser-mode routing ───────────────────────────────
@pytest.mark.asyncio
async def test_run_avito_pipeline_browser_mode_sets_scraper_browser(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""В browser-mode scraper._browser установлен, fetch_detail/fetch_house_catalog
вызываются с browser_fetcher (не с cffi_session).
"""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
mock_bf = make_mock_fetcher(DETAIL_HTML)
captured_scraper: list = []
# Mock AvitoScraper.fetch_around → возвращает пустой список (нет лотов → нет DB)
async def mock_fetch_around(
self: object, lat: float, lon: float, radius_m: int, **kwargs: object
) -> list:
captured_scraper.append(self)
return []
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
with (
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
from app.services.scrape_pipeline import run_avito_pipeline
await run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
radius_m=1500,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
shared_browser=mock_bf,
)
assert len(captured_scraper) == 1
scraper_instance = captured_scraper[0]
assert scraper_instance._browser is mock_bf
# curl session должен быть None — в browser-mode pipeline не создаёт _cffi сессию
assert scraper_instance._cffi is None
@pytest.mark.asyncio
async def test_run_avito_pipeline_browser_mode_own_browser_created(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Если shared_browser не передан в browser-mode, BrowserFetcher создаётся
и __aenter__/__aexit__ вызываются."""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
mock_bf_instance = make_mock_fetcher(DETAIL_HTML)
mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance)
mock_bf_instance.__aexit__ = AsyncMock(return_value=None)
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
return []
with (
patch("app.services.scrape_pipeline.BrowserFetcher", return_value=mock_bf_instance),
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
await scrape_pipeline.run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
)
mock_bf_instance.__aenter__.assert_awaited_once()
mock_bf_instance.__aexit__.assert_awaited_once()
# ── 5. Curl mode (default) — browser_fetcher stays None ──────────────────────
@pytest.mark.asyncio
async def test_run_avito_pipeline_curl_mode_no_browser_fetcher(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""В curl-mode (default) browser_fetcher=None, scraper._cffi установлен."""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "curl_cffi", raising=False)
captured_scraper: list = []
async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
captured_scraper.append(self)
return []
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
with (
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
await scrape_pipeline.run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
)
assert len(captured_scraper) == 1
scraper_instance = captured_scraper[0]
# curl mode: _cffi должен быть установлен
assert scraper_instance._cffi is not None
# browser не установлен в curl mode
assert getattr(scraper_instance, "_browser", None) is None
# ── 4. Curl mode (default) — browser_fetcher stays None ──────────────────────
@pytest.mark.asyncio
@ -368,7 +243,7 @@ async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
assert result.item_id == "99887766"
# ── 6. SERP routing: real _fetch_serp_html routes through browser ────────────
# ── 5. SERP routing: real _fetch_serp_html routes through browser ────────────
@pytest.mark.asyncio

245
tradein-mvp/backend/tests/test_provider_rotation_1848.py
View file

@ -1,245 +0,0 @@
"""Тесты Fix 2: provider-aware _rotate_proxy_ip (#1848).
Тест 1: source='cian' → cian_proxy_rotate_url используется.
Тест 2: source='yandex' → yandex_proxy_rotate_url используется.
Тест 3: source='avito' (default) → avito_proxy_rotate_url (backward compat).
Тест 4: cian без cian url → fallback на avito_proxy_rotate_url.
Тест 5: rotate_url=None → returns False (no HTTP call).
Тест 6: network error → returns False (no crash).
Тест 7: yandex fallback на avito.
"""
from __future__ import annotations
import os
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrape_pipeline import _rotate_proxy_ip
# ── Helpers ──────────────────────────────────────────────────────────────────
def _async_session_ctx(resp_json: dict | None = None):
"""Возвращает (async_cm_class, session_mock) для curl_cffi AsyncSession."""
session = AsyncMock()
if resp_json is not None:
mock_resp = MagicMock()
mock_resp.json.return_value = resp_json
session.get = AsyncMock(return_value=mock_resp)
else:
session.get = AsyncMock(side_effect=RuntimeError("network error"))
# curl_cffi AsyncSession используется как `async with AsyncSession(...) as rot:`
# Нам нужен callable класс-замена, вызов которого возвращает async context manager.
class _MockAsyncSession:
def __init__(self, timeout=30):
pass
async def __aenter__(self):
return session
async def __aexit__(self, *_):
pass
return _MockAsyncSession, session
# ── Тест 1: cian_proxy_rotate_url ────────────────────────────────────────────
@pytest.mark.asyncio
async def test_rotate_proxy_ip_cian_uses_cian_url() -> None:
"""source='cian' → cian_proxy_rotate_url (не avito_proxy_rotate_url)."""
mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "1.2.3.4"})
with (
patch("app.services.scrape_pipeline.settings") as s,
patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
):
s.cian_proxy_rotate_url = "http://cian-rotate.test/"
s.avito_proxy_rotate_url = "http://avito-rotate.test/"
s.yandex_proxy_rotate_url = None
s.cian_proxy_max_rotations = 4
s.avito_proxy_max_rotations = 4
s.yandex_proxy_max_rotations = 4
s.avito_proxy_rotate_settle_s = 0.0
result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
assert result is True
called_url: str = session.get.call_args[0][0]
assert "cian-rotate.test" in called_url
assert "avito-rotate.test" not in called_url
# ── Тест 2: yandex_proxy_rotate_url ─────────────────────────────────────────
@pytest.mark.asyncio
async def test_rotate_proxy_ip_yandex_uses_yandex_url() -> None:
"""source='yandex' → yandex_proxy_rotate_url."""
mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "5.6.7.8"})
with (
patch("app.services.scrape_pipeline.settings") as s,
patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
):
s.yandex_proxy_rotate_url = "http://yandex-rotate.test/"
s.avito_proxy_rotate_url = "http://avito-rotate.test/"
s.cian_proxy_rotate_url = None
s.yandex_proxy_max_rotations = 4
s.avito_proxy_max_rotations = 4
s.cian_proxy_max_rotations = 4
s.avito_proxy_rotate_settle_s = 0.0
result = await _rotate_proxy_ip(reason="timeout", rotations_done=0, source="yandex")
assert result is True
called_url: str = session.get.call_args[0][0]
assert "yandex-rotate.test" in called_url
assert "avito-rotate.test" not in called_url
# ── Тест 3: avito backward compat ────────────────────────────────────────────
@pytest.mark.asyncio
async def test_rotate_proxy_ip_avito_backward_compat() -> None:
"""source не передан (дефолт 'avito') → avito_proxy_rotate_url."""
mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "9.9.9.9"})
with (
patch("app.services.scrape_pipeline.settings") as s,
patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
):
s.avito_proxy_rotate_url = "http://avito-only.test/"
s.cian_proxy_rotate_url = None
s.yandex_proxy_rotate_url = None
s.avito_proxy_max_rotations = 4
s.cian_proxy_max_rotations = 4
s.yandex_proxy_max_rotations = 4
s.avito_proxy_rotate_settle_s = 0.0
# Вызов без source= — должен работать как раньше
result = await _rotate_proxy_ip(reason="block", rotations_done=0)
assert result is True
called_url: str = session.get.call_args[0][0]
assert "avito-only.test" in called_url
# ── Тест 4: cian fallback на avito когда нет cian url ─────────────────────
@pytest.mark.asyncio
async def test_rotate_proxy_ip_cian_fallback_to_avito() -> None:
"""cian_proxy_rotate_url=None → fallback на avito_proxy_rotate_url."""
mock_session_cls, session = _async_session_ctx(resp_json={})
with (
patch("app.services.scrape_pipeline.settings") as s,
patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
):
s.cian_proxy_rotate_url = None
s.avito_proxy_rotate_url = "http://avito-fallback.test/"
s.yandex_proxy_rotate_url = None
s.cian_proxy_max_rotations = 4
s.avito_proxy_max_rotations = 4
s.yandex_proxy_max_rotations = 4
s.avito_proxy_rotate_settle_s = 0.0
result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
assert result is True
called_url: str = session.get.call_args[0][0]
assert "avito-fallback.test" in called_url
# ── Тест 5: rotate_url=None → returns False ──────────────────────────────────
@pytest.mark.asyncio
async def test_rotate_proxy_ip_returns_false_when_no_url() -> None:
"""Никакой URL не задан → False, HTTP-запрос не делается."""
mock_session_cls, session = _async_session_ctx(resp_json={})
with (
patch("app.services.scrape_pipeline.settings") as s,
patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
):
s.cian_proxy_rotate_url = None
s.avito_proxy_rotate_url = None
s.yandex_proxy_rotate_url = None
s.cian_proxy_max_rotations = 4
s.avito_proxy_max_rotations = 4
s.yandex_proxy_max_rotations = 4
result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
assert result is False
# HTTP-запрос не делался
session.get.assert_not_called()
# ── Тест 6: network error → returns False ────────────────────────────────────
@pytest.mark.asyncio
async def test_rotate_proxy_ip_returns_false_on_network_error() -> None:
"""Ошибка сети → returns False (не крашит caller)."""
mock_session_cls, _session = _async_session_ctx(resp_json=None) # raises RuntimeError
with (
patch("app.services.scrape_pipeline.settings") as s,
patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
):
s.cian_proxy_rotate_url = "http://cian-rotate.test/"
s.avito_proxy_rotate_url = None
s.yandex_proxy_rotate_url = None
s.cian_proxy_max_rotations = 4
s.avito_proxy_max_rotations = 4
s.yandex_proxy_max_rotations = 4
s.avito_proxy_rotate_settle_s = 0.0
s.proxy_rotate_attempts = 1 # одна попытка, без retry-sleep (#1950)
s.proxy_rotate_attempt_timeout_s = 8.0
result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")
assert result is False
# ── Тест 7: yandex fallback на avito ─────────────────────────────────────────
@pytest.mark.asyncio
async def test_rotate_proxy_ip_yandex_fallback_to_avito() -> None:
"""yandex_proxy_rotate_url=None → fallback на avito_proxy_rotate_url."""
mock_session_cls, session = _async_session_ctx(resp_json={"new_ip": "11.22.33.44"})
with (
patch("app.services.scrape_pipeline.settings") as s,
patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
):
s.yandex_proxy_rotate_url = None
s.avito_proxy_rotate_url = "http://shared-proxy.test/"
s.cian_proxy_rotate_url = None
s.yandex_proxy_max_rotations = 4
s.avito_proxy_max_rotations = 4
s.cian_proxy_max_rotations = 4
s.avito_proxy_rotate_settle_s = 0.0
result = await _rotate_proxy_ip(reason="503", rotations_done=0, source="yandex")
assert result is True
called_url: str = session.get.call_args[0][0]
assert "shared-proxy.test" in called_url

270
tradein-mvp/backend/tests/test_scrape_pipeline.py
View file

@ -1,270 +0,0 @@
"""Offline smoke for scrape_pipeline. Mocks scrapers + DB session."""
from datetime import date, timedelta
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrape_pipeline import (
_AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
PipelineCounters,
PipelineResult,
run_avito_full_load,
run_avito_pipeline,
)
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
from app.services.scrapers.base import ScrapedLot
def _make_lot(
item_id: str,
house_url: str | None = None,
price: int = 5_000_000,
) -> ScrapedLot:
return ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
source_id=item_id,
price_rub=price,
address="Test address",
house_source="avito" if house_url else None,
house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
house_url=house_url,
listing_segment="vtorichka",
)
def test_counters_dataclass_default() -> None:
c = PipelineCounters()
assert c.lots_fetched == 0
assert c.unique_houses == 0
assert c.errors == []
@pytest.mark.asyncio
async def test_pipeline_search_failure_graceful() -> None:
"""Если AvitoScraper.fetch_around raises — pipeline возвращает PipelineResult
с errors, не падает наружу."""
mock_db = MagicMock()
mock_scraper = MagicMock()
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
mock_scraper.__aexit__ = AsyncMock(return_value=None)
mock_scraper.fetch_around = AsyncMock(side_effect=RuntimeError("test net error"))
with patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper):
result = await run_avito_pipeline(
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
)
assert isinstance(result, PipelineResult)
assert result.counters.lots_fetched == 0
assert len(result.counters.errors) == 1
assert "search:" in result.counters.errors[0]
@pytest.mark.asyncio
async def test_pipeline_group_by_house_unique_set() -> None:
"""3 lots с house_url'ами (2 одинаковых, 1 уникальный) → unique_houses=2."""
mock_db = MagicMock()
lots = [
_make_lot("1", "https://www.avito.ru/catalog/houses/ekb/test/100"),
_make_lot("2", "https://www.avito.ru/catalog/houses/ekb/test/100"), # дубль
_make_lot("3", "https://www.avito.ru/catalog/houses/ekb/other/200"),
_make_lot("4", None), # без дома
]
mock_scraper = MagicMock()
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
mock_scraper.__aexit__ = AsyncMock(return_value=None)
mock_scraper.fetch_around = AsyncMock(return_value=lots)
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.save_listings", return_value=(4, 0)),
):
result = await run_avito_pipeline(
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
)
assert result.counters.lots_fetched == 4
assert result.counters.lots_inserted == 4
assert result.counters.unique_houses == 2
# ── run_avito_full_load: incremental_days → since plumbing (#avito-split) ─────
def _full_load_scraper() -> MagicMock:
"""AvitoScraper mock с async-CM + AsyncMock fetch_all_secondary."""
scraper = MagicMock()
scraper.__aenter__ = AsyncMock(return_value=scraper)
scraper.__aexit__ = AsyncMock(return_value=None)
scraper.fetch_all_secondary = AsyncMock(return_value=[])
return scraper
@pytest.mark.asyncio
async def test_full_load_incremental_days_passes_since() -> None:
"""incremental_days=2 → fetch_all_secondary(since=date.today()-2d)."""
mock_db = MagicMock()
scraper = _full_load_scraper()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
):
await run_avito_full_load(mock_db, run_id=1, incremental_days=2)
kwargs = scraper.fetch_all_secondary.call_args.kwargs
assert kwargs["since"] == date.today() - timedelta(days=2)
@pytest.mark.asyncio
async def test_full_load_no_incremental_days_since_none() -> None:
"""incremental_days=None (default) → fetch_all_secondary(since=None) — exhaustive."""
mock_db = MagicMock()
scraper = _full_load_scraper()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
):
await run_avito_full_load(mock_db, run_id=2)
kwargs = scraper.fetch_all_secondary.call_args.kwargs
assert kwargs["since"] is None
# ── #1820: AvitoBlockedError per-item не роняет весь pipeline ─────────────────
def _block_test_scraper(lots: list[ScrapedLot]) -> MagicMock:
scraper = MagicMock()
scraper.__aenter__ = AsyncMock(return_value=scraper)
scraper.__aexit__ = AsyncMock(return_value=None)
scraper.fetch_around = AsyncMock(return_value=lots)
scraper._cffi = None
return scraper
@pytest.mark.asyncio
async def test_pipeline_single_house_block_does_not_crash() -> None:
"""#1820: один заблокированный house-фетч НЕ роняет прогон.
Listings из SEARCH+SAVE сохранены, блок зафиксирован в counters (houses_failed/errors),
остальные дома обогащаются дальше (enrichment не прерван < порога).
"""
mock_db = MagicMock()
lots = [
_make_lot("1", "https://www.avito.ru/catalog/houses/ekb/a/100"),
_make_lot("2", "https://www.avito.ru/catalog/houses/ekb/b/200"),
]
scraper = _block_test_scraper(lots)
# 1-й house блокируется, 2-й — успех.
fetch_house = AsyncMock(side_effect=[AvitoBlockedError("HTTP 403"), MagicMock()])
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.save_listings", return_value=(2, 0)),
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
patch(
"app.services.scrape_pipeline.save_house_catalog_enrichment",
return_value={"house_id": 42},
),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
),
):
result = await run_avito_pipeline(
mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
)
assert isinstance(result, PipelineResult)
# listings сохранены всегда
assert result.counters.lots_inserted == 2
# один дом упал по блоку, второй обогащён
assert result.counters.houses_failed == 1
assert result.counters.houses_enriched == 1
assert fetch_house.await_count == 2 # enrichment НЕ прерван — оба дома попробованы
assert any("BLOCKED_HOUSE" in e for e in result.counters.errors)
assert 42 in result.touched_house_ids
@pytest.mark.asyncio
async def test_pipeline_consecutive_house_blocks_abort_but_return_result() -> None:
"""#1820: N consecutive house-блоков прерывают enrichment, но PipelineResult возвращён."""
mock_db = MagicMock()
n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
# Создаём n+2 дома — после n блоков подряд enrichment прерывается.
lots = [
_make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{100 + i}")
for i in range(n + 2)
]
scraper = _block_test_scraper(lots)
fetch_house = AsyncMock(side_effect=AvitoRateLimitedError("HTTP 429"))
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
patch("app.services.scrape_pipeline.save_house_catalog_enrichment", return_value={}),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
),
):
result = await run_avito_pipeline(
mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
)
assert isinstance(result, PipelineResult)
assert result.counters.lots_inserted == len(lots) # listings сохранены
# enrichment прерван ровно после порога — лишние дома НЕ фетчились
assert fetch_house.await_count == n
assert result.counters.houses_failed == n
assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
@pytest.mark.asyncio
async def test_pipeline_detail_block_aborts_skips_step5b() -> None:
"""#1820: consecutive detail-блоки прерывают detail-фазу И Step 5b (houses из detail)."""
mock_db = MagicMock()
n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
scraper = _block_test_scraper([]) # SERP пуст → сразу detail-фаза по priority_rows
# priority_rows: n+1 строк, все блокируются в detail.
row = MagicMock()
row.__getitem__ = lambda self, key: (
"https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
)
mappings = MagicMock()
mappings.all.return_value = [row] * (n + 1)
execute = MagicMock()
execute.mappings.return_value = mappings
mock_db.execute.return_value = execute
fetch_detail = AsyncMock(side_effect=AvitoBlockedError("HTTP 403"))
fetch_house = AsyncMock() # должен НЕ вызываться (Step 5b пропущен)
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.fetch_detail", fetch_detail),
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch(
"curl_cffi.requests.AsyncSession",
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
),
):
result = await run_avito_pipeline(
mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=n + 1
)
assert isinstance(result, PipelineResult)
assert fetch_detail.await_count == n # detail прерван на пороге
assert result.counters.detail_failed == n
assert fetch_house.await_count == 0 # Step 5b пропущен (enrichment_aborted)

408
tradein-mvp/backend/tests/test_scrape_pipeline_proxy_pool.py
View file

@ -1,408 +0,0 @@
"""Tests for scrape_pipeline.py proxy-pool wiring (#2202).
Issue #2202: боевые скрейперы (scrape_pipeline.py Avito orchestrator) читали только
статический settings.scraper_proxy_url — один mobile-IP на все прогоны, SPOF. Этот
файл покрывает новые pool-aware helpers:
- _pool_proxy_dict: acquire() health-filtered lease за флагом USE_PROXY_POOL_CURL,
fallback на статический env при флаге off / пустом пуле / ошибке.
- _rotate_pool_proxy: переключение на ДРУГОЙ здоровый прокси из пула при бане
(mark_health(False)+release текущего, acquire() следующего, мутация session.proxies).
- _rotate_avito_proxy: единая точка ротации — пул first, changeip fallback.
Acceptance-тест (issue AC4): пул с 1 «забаненным» прокси — ротация выбирает
СЛЕДУЮЩИЙ здоровый, не зависший на забаненном и не переиспользующий только что
провалившийся lease раньше более свежего здорового кандидата.
Offline: FakeSession эмулирует scrape_proxies-таблицу (тот же паттерн, что
tests/services/test_proxy_pool.py), без live БД/сети.
"""
from __future__ import annotations
import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from datetime import UTC, datetime
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, patch
import pytest
from app.services import proxy_pool
from app.services.proxy_pool import MAX_CONSECUTIVE_FAILS
from app.services.scrape_pipeline import (
_pool_proxy_dict,
_release_pool_lease,
_rotate_avito_proxy,
_rotate_pool_proxy,
)
# ── stateful fake session (mirrors tests/services/test_proxy_pool.py) ─────────
class _FakeResult:
def __init__(self, rows: list[dict[str, Any]]):
self._rows = rows
def mappings(self) -> _FakeResult:
return self
def fetchone(self) -> dict[str, Any] | None:
return self._rows[0] if self._rows else None
def all(self) -> list[dict[str, Any]]:
return list(self._rows)
class FakeSession:
"""Эмуляция Session поверх in-memory списка scrape_proxies-строк."""
def __init__(self, rows: list[dict[str, Any]]):
self.rows = rows
def _by_id(self, pid: int) -> dict[str, Any] | None:
return next((r for r in self.rows if r["id"] == pid), None)
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
sql = str(stmt)
p = params or {}
if "FOR UPDATE SKIP LOCKED" in sql: # acquire SELECT
provider = p["provider"]
max_fails = p["max_fails"]
cands = [
r
for r in self.rows
if r["enabled"]
and r["consecutive_fails"] < max_fails
and r["provider_affinity"] in (provider, "any")
and r["leased_by"] is None
]
cands.sort(
key=lambda r: (
r["last_ok_at"] is None,
r["last_ok_at"] or datetime.min.replace(tzinfo=UTC),
r["id"],
)
)
return _FakeResult(cands[:1])
if "SET leased_by = CAST(:run_id" in sql: # acquire lease UPDATE
row = self._by_id(p["id"])
if row is not None:
row["leased_by"] = p["run_id"]
row["leased_at"] = datetime.now(UTC)
return _FakeResult([])
if "SET leased_by = NULL" in sql: # release
row = self._by_id(p["id"])
if row is not None:
row["leased_by"] = None
row["leased_at"] = None
return _FakeResult([])
if "SET consecutive_fails = 0" in sql: # mark_health ok
row = self._by_id(p["id"])
if row is not None:
row["consecutive_fails"] = 0
row["exit_ip"] = p["exit_ip"]
row["latency_ms"] = p["latency_ms"]
row["last_ok_at"] = datetime.now(UTC)
return _FakeResult([])
if "consecutive_fails = consecutive_fails + 1" in sql: # mark_health fail
row = self._by_id(p["id"])
if row is not None:
row["consecutive_fails"] += 1
if row["consecutive_fails"] >= p["disable_threshold"]:
row["enabled"] = False
return _FakeResult([])
raise AssertionError(f"unhandled SQL: {sql}")
def commit(self) -> None:
pass
def rollback(self) -> None:
pass
def _proxy(
pid: int,
*,
affinity: str = "avito",
enabled: bool = True,
fails: int = 0,
leased_by: int | None = None,
last_ok_at: datetime | None = None,
) -> dict[str, Any]:
return {
"id": pid,
"url": f"http://u:p@h{pid}:8080",
"kind": "http",
"rotate_url": None,
"provider_affinity": affinity,
"enabled": enabled,
"consecutive_fails": fails,
"leased_by": leased_by,
"leased_at": None,
"last_ok_at": last_ok_at,
"exit_ip": None,
"latency_ms": None,
}
def _mock_settings(*, use_proxy_pool_curl: bool, scraper_proxy_url: str | None = None):
return SimpleNamespace(
use_proxy_pool_curl=use_proxy_pool_curl,
scraper_proxy_url=scraper_proxy_url,
)
# ── _pool_proxy_dict ────────────────────────────────────────────────────────
def test_pool_proxy_dict_falls_back_when_flag_off():
db = FakeSession([_proxy(1)])
with patch(
"app.services.scrape_pipeline.settings",
_mock_settings(use_proxy_pool_curl=False, scraper_proxy_url="http://env:3128"),
):
proxies, lease = _pool_proxy_dict(db, "avito") # type: ignore[arg-type]
assert proxies == {"http": "http://env:3128", "https": "http://env:3128"}
assert lease is None
def test_pool_proxy_dict_returns_lease_when_flag_on_and_healthy():
db = FakeSession([_proxy(1)])
with patch(
"app.services.scrape_pipeline.settings",
_mock_settings(use_proxy_pool_curl=True),
):
proxies, lease = _pool_proxy_dict(db, "avito") # type: ignore[arg-type]
assert lease is not None and lease.id == 1
assert proxies == {"http": "http://u:p@h1:8080", "https": "http://u:p@h1:8080"}
assert db._by_id(1)["leased_by"] is not None # lease проставлен
def test_pool_proxy_dict_falls_back_when_pool_empty():
db = FakeSession([]) # пустой пул
with patch(
"app.services.scrape_pipeline.settings",
_mock_settings(use_proxy_pool_curl=True, scraper_proxy_url="http://env:3128"),
):
proxies, lease = _pool_proxy_dict(db, "avito") # type: ignore[arg-type]
assert lease is None
assert proxies == {"http": "http://env:3128", "https": "http://env:3128"}
# ── _rotate_pool_proxy — acceptance-тест AC4 ───────────────────────────────
@pytest.mark.asyncio
async def test_rotate_pool_proxy_skips_banned_picks_next_healthy():
"""#2202 AC4: пул с 1 забаненным прокси — ротация выбирает следующий здоровый.
BAD (id=1) — уже забанен (fails>=MAX_CONSECUTIVE_FAILS), исключён acquire() на
уровне SQL-фильтра. CURRENT (id=3) — прокси активного прогона, только что словивший
бан (403/429) — вызывающий код зовёт rotate. GOOD (id=2) — свободный здоровый,
единственный валидный кандидат после того как CURRENT будет помечен/освобождён.
NB (#2202 MAJOR-fix): гарантия исключения CURRENT из acquire() держится на том,
что release(current_lease) вызывается ПОСЛЕ acquire(), не до — пока current_lease
leased_by-занят, SQL `leased_by IS NULL` исключает его вне зависимости от id/
last_ok_at сортировки. См. зеркальный тест ниже
(test_rotate_pool_proxy_skips_banned_picks_next_healthy_current_id_lower) —
та же гарантия при CURRENT id НИЖЕ GOOD id (иначе тай-брейк по id в ORDER BY
"last_ok_at NULLS LAST, id" мог случайно фейворить эту фикстуру).
"""
db = FakeSession(
[
_proxy(1, fails=MAX_CONSECUTIVE_FAILS), # BAD — уже забанен
_proxy(2, fails=0), # GOOD — ожидаемый выбор
_proxy(3, fails=0, leased_by=999), # CURRENT — в работе, банится сейчас
]
)
current_lease = proxy_pool.ProxyLease(
id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
)
session = SimpleNamespace(proxies=None)
new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease) # type: ignore[arg-type]
assert new_lease is not None
assert new_lease.id == 2 # GOOD, не BAD (excluded) и не CURRENT (только что провален)
assert session.proxies == {"http": "http://u:p@h2:8080", "https": "http://u:p@h2:8080"}
# CURRENT (id=3) помечен нездоровым и освобождён
assert db._by_id(3)["consecutive_fails"] == 1
assert db._by_id(3)["leased_by"] is None
# BAD (id=1) не тронут этой операцией
assert db._by_id(1)["consecutive_fails"] == MAX_CONSECUTIVE_FAILS
# GOOD (id=2) теперь зализан под новую ротацию
assert db._by_id(2)["leased_by"] is not None
@pytest.mark.asyncio
async def test_rotate_pool_proxy_skips_banned_picks_next_healthy_current_id_lower():
"""#2202 AC4 regression guard (MAJOR-fix): гарантия исключения current_lease НЕ
должна зависеть от того, чей id меньше в фикстуре.
Зеркало test_rotate_pool_proxy_skips_banned_picks_next_healthy с CURRENT/GOOD id
в обратном порядке: CURRENT=1 < GOOD=2 (оригинальный тест имел CURRENT=3 > GOOD=2,
поэтому "тай-брейк по id" в ORDER BY last_ok_at NULLS LAST, id случайно совпадал с
правильным ответом и не доказывал реальную гарантию — см. review-репро на #2202).
До фикса (release(current_lease) ДО acquire()): CURRENT освобождается, попадает в
leased_by IS NULL кандидаты с fails=1 (< MAX_CONSECUTIVE_FAILS=3, т.е. "здоров"),
и при равном last_ok_at (оба NULL) выигрывает тай-брейк по id (1 < 2) — acquire()
возвращает ТОТ ЖЕ прокси, который только что забанили. Этот тест ловит именно этот
регресс: если release() снова передвинуть перед acquire(), new_lease.id будет 1
(CURRENT), а не 2 (GOOD), и assert упадёт.
"""
db = FakeSession(
[
_proxy(1, fails=0, leased_by=999), # CURRENT — id НИЖЕ GOOD, банится сейчас
_proxy(2, fails=0), # GOOD — единственный здоровый свободный кандидат
]
)
current_lease = proxy_pool.ProxyLease(
id=1, url="http://u:p@h1:8080", kind="http", rotate_url=None
)
session = SimpleNamespace(proxies=None)
new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease) # type: ignore[arg-type]
assert new_lease is not None
assert new_lease.id == 2 # GOOD — НЕ CURRENT, несмотря на id=1 < id=2
assert session.proxies == {"http": "http://u:p@h2:8080", "https": "http://u:p@h2:8080"}
# CURRENT (id=1) помечен нездоровым и освобождён (после успешной ротации)
assert db._by_id(1)["consecutive_fails"] == 1
assert db._by_id(1)["leased_by"] is None
# GOOD (id=2) теперь зализан под новую ротацию
assert db._by_id(2)["leased_by"] is not None
@pytest.mark.asyncio
async def test_rotate_pool_proxy_exhausted_returns_none():
"""Все кандидаты нездоровы после ротации → None (caller падает на changeip).
BAD (id=1) уже забанен. CURRENT (id=3) на MAX_CONSECUTIVE_FAILS-1 fails — ЭТА
ротация (mark_health ok=False) толкает его ровно до порога → тоже исключается
из acquire(). Ни одного здорового кандидата не остаётся.
"""
db = FakeSession(
[
_proxy(1, fails=MAX_CONSECUTIVE_FAILS), # BAD — уже забанен
_proxy(3, fails=MAX_CONSECUTIVE_FAILS - 1, leased_by=999), # CURRENT — на грани
]
)
current_lease = proxy_pool.ProxyLease(
id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
)
session = SimpleNamespace(proxies=None)
new_lease = await _rotate_pool_proxy(db, "avito", session, current_lease) # type: ignore[arg-type]
assert new_lease is None
assert session.proxies is None # не тронут — caller остаётся на старом/changeip
assert db._by_id(3)["leased_by"] is None # текущий всё равно освобождён
# ── _rotate_avito_proxy — pool-first, changeip fallback ────────────────────
@pytest.mark.asyncio
async def test_rotate_avito_proxy_uses_pool_when_flag_on():
db = FakeSession(
[
_proxy(1, fails=0), # свободный здоровый — цель ротации
_proxy(3, fails=0, leased_by=999), # CURRENT
]
)
current_lease = proxy_pool.ProxyLease(
id=3, url="http://u:p@h3:8080", kind="http", rotate_url=None
)
session = SimpleNamespace(proxies=None)
with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=True)):
with patch("app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock()) as changeip_mock:
rotated, new_lease = await _rotate_avito_proxy(
db, # type: ignore[arg-type]
session, # type: ignore[arg-type]
current_lease,
reason="test",
rotations_done=0,
)
assert rotated is True
assert new_lease is not None and new_lease.id == 1
changeip_mock.assert_not_awaited() # пул сработал — changeip не нужен
@pytest.mark.asyncio
async def test_rotate_avito_proxy_falls_back_to_changeip_when_flag_off():
session = SimpleNamespace(proxies=None)
with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=False)):
with patch(
"app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock(return_value=True)
) as changeip_mock:
rotated, new_lease = await _rotate_avito_proxy(
None, # type: ignore[arg-type]
session, # type: ignore[arg-type]
None,
reason="test",
rotations_done=0,
)
assert rotated is True
assert new_lease is None
changeip_mock.assert_awaited_once()
@pytest.mark.asyncio
async def test_rotate_avito_proxy_falls_back_to_changeip_when_pool_exhausted():
"""Пул on, но acquire() пуст (например все забанены) — падаем на changeip, не молчим."""
db = FakeSession([_proxy(1, fails=MAX_CONSECUTIVE_FAILS)]) # единственный — забанен
session = SimpleNamespace(proxies=None)
with patch("app.services.scrape_pipeline.settings", _mock_settings(use_proxy_pool_curl=True)):
with patch(
"app.services.scrape_pipeline._rotate_proxy_ip", AsyncMock(return_value=True)
) as changeip_mock:
rotated, new_lease = await _rotate_avito_proxy(
db, # type: ignore[arg-type]
session, # type: ignore[arg-type]
None,
reason="test",
rotations_done=0,
)
assert rotated is True
assert new_lease is None
changeip_mock.assert_awaited_once() # pool exhausted → fallback path taken
# ── _release_pool_lease ──────────────────────────────────────────────────────
def test_release_pool_lease_noop_when_none():
_release_pool_lease(None, None, ok=True) # type: ignore[arg-type] # must not raise
def test_release_pool_lease_marks_health_and_releases():
db = FakeSession([_proxy(1, leased_by=100)])
lease = proxy_pool.ProxyLease(id=1, url="http://u:p@h1:8080", kind="http", rotate_url=None)
_release_pool_lease(db, lease, ok=True) # type: ignore[arg-type]
assert db._by_id(1)["leased_by"] is None
assert db._by_id(1)["consecutive_fails"] == 0

136
tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py
View file

@ -1,11 +1,10 @@
"""Golden-parity: `scraper_kit.orchestration.pipeline` ≡ `app.services.scrape_pipeline`.
"""Regression: `scraper_kit.orchestration.pipeline.run_avito_city_sweep` orchestration.
Strangler-инвариант (#2135): новая scraper_kit-копия оркестрации sweep-pipeline
должна давать ТОТ ЖЕ наблюдаемый эффект, что и старый боевой orchestrator, на
одинаковом мокнутом I/O. Развязка `app.*` (config/matcher/enrichment/runs/shutdown
инжектируются) не должна менять ветвление.
Kit — единственная orchestration-копия sweep-pipeline (#2397 Part E1 удалил legacy
`app.services.scrape_pipeline`; сравнивать «golden-parity» больше не с чем — этот файл
раньше гонял ОБА orchestrator'а side-by-side, теперь оставлена только kit-сторона).
Фокус — КРИТИЧНАЯ логика оркестрации (то, ради чего PR):
Фокус — КРИТИЧНАЯ логика оркестрации (то, ради чего эти тесты изначально писались):
- ban/rotation state-machine: SERP-блок на anchor'е → abort sweep;
- partial-ban intake (#1950): SERP intake сохранён + detail заблокирован →
mark_done (не mark_banned) под флагом avito_serp_ok_not_banned;
@ -13,11 +12,6 @@ Strangler-инвариант (#2135): новая scraper_kit-копия орке
- counters aggregation по anchor'ам + IMV-фаза;
- последовательность вызовов scrape_runs (heartbeat / mark_done / mark_banned).
Метод: гоняем ОБА orchestrator'а (`_drive_old` / `_drive_new`) на идентичном
описании сценария, перехватывая scrape_runs recording-mock'ом. Сравниваем:
- итоговый counters.to_dict();
- последовательность (method, numeric-counters) вызовов scrape_runs.
Без сети, без БД.
"""
@ -32,17 +26,8 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.avito_exceptions import (
AvitoBlockedError as NewAvitoBlockedError,
)
from scraper_kit.orchestration.pipeline import (
run_avito_city_sweep as new_city_sweep,
)
from app.services.scrape_pipeline import run_avito_city_sweep as old_city_sweep
from app.services.scrapers.avito_exceptions import (
AvitoBlockedError as OldAvitoBlockedError,
)
from scraper_kit.avito_exceptions import AvitoBlockedError
from scraper_kit.orchestration.pipeline import run_avito_city_sweep
# ── recording scrape_runs ────────────────────────────────────────────────────
@ -53,7 +38,7 @@ class _RunsRecorder:
"""Записывает вызовы scrape_runs-финализаторов в общий список.
is_cancelled всегда False (кооп-cancel не тестируем здесь). Каждый терминальный
вызов пишет (method, counters_dict) — сравниваем последовательности old/new.
вызов пишет (method, counters_dict).
"""
def __init__(self) -> None:
@ -91,7 +76,7 @@ def _normalize(calls: list[tuple[str, dict[str, Any]]]) -> list[tuple[str, dict[
class _Scenario:
"""Декларативное описание одного прогона city-sweep для обоих orchestrator'ов."""
"""Декларативное описание одного прогона city-sweep."""
def __init__(
self,
@ -185,61 +170,10 @@ def _async_session_cm() -> MagicMock:
return sess
async def _drive_old(scenario: _Scenario) -> _DriveResult:
async def _drive(scenario: _Scenario) -> _DriveResult:
recorder = _RunsRecorder()
db = _make_db(scenario)
scraper = _make_scraper(scenario, OldAvitoBlockedError)
save_mock = MagicMock(side_effect=scenario._save_side_effects())
imv_res = None
if scenario.imv_result is not None:
checked, saved, errors = scenario.imv_result
imv_res = SimpleNamespace(checked=checked, saved=saved, errors=errors)
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
patch("app.services.scrape_pipeline.save_listings", save_mock),
patch(
"app.services.scrape_pipeline.fetch_house_catalog",
AsyncMock(return_value=MagicMock()),
),
patch(
"app.services.scrape_pipeline.save_house_catalog_enrichment",
return_value={"house_id": 1},
),
patch(
"app.services.scrape_pipeline.fetch_detail",
AsyncMock(side_effect=scenario._fetch_detail_side_effects(OldAvitoBlockedError)),
),
patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
patch("app.services.scrape_pipeline.scrape_runs", recorder),
patch(
"app.services.house_imv_backfill.process_houses_imv_batch",
AsyncMock(return_value=imv_res),
),
patch("app.services.scrape_pipeline.settings", scenario._config()),
patch("app.services.scrape_pipeline.shutdown_requested", return_value=False),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.AsyncSession", return_value=_async_session_cm()),
):
counters = await old_city_sweep(
db,
run_id=1,
radius_m=1000,
anchors=scenario.anchors,
pages_per_anchor=1,
enrich_houses=scenario.enrich_houses,
detail_top_n=scenario.detail_top_n,
request_delay_sec=0.0,
enrich_imv=scenario.enrich_imv,
)
return counters.to_dict(), _normalize(recorder.calls)
async def _drive_new(scenario: _Scenario) -> _DriveResult:
recorder = _RunsRecorder()
db = _make_db(scenario)
scraper = _make_scraper(scenario, NewAvitoBlockedError)
scraper = _make_scraper(scenario, AvitoBlockedError)
save_mock = MagicMock(side_effect=scenario._save_side_effects())
imv_res = None
@ -258,14 +192,14 @@ async def _drive_new(scenario: _Scenario) -> _DriveResult:
patch(f"{pfx}.save_house_catalog_enrichment", return_value={"house_id": 1}),
patch(
f"{pfx}.fetch_detail",
AsyncMock(side_effect=scenario._fetch_detail_side_effects(NewAvitoBlockedError)),
AsyncMock(side_effect=scenario._fetch_detail_side_effects(AvitoBlockedError)),
),
patch(f"{pfx}.save_detail_enrichment", return_value=True),
patch(f"{pfx}.runs", recorder),
patch(f"{pfx}.asyncio.sleep", AsyncMock()),
patch(f"{pfx}.AsyncSession", return_value=_async_session_cm()),
):
counters = await new_city_sweep(
counters = await run_avito_city_sweep(
db,
run_id=1,
config=scenario._config(),
@ -283,72 +217,63 @@ async def _drive_new(scenario: _Scenario) -> _DriveResult:
return counters.to_dict(), _normalize(recorder.calls)
async def _assert_parity(scenario: _Scenario) -> dict[str, int]:
old_counters, old_calls = await _drive_old(scenario)
new_counters, new_calls = await _drive_new(scenario)
assert (
new_counters == old_counters
), f"counters diverged:\n old={old_counters}\n new={new_counters}"
assert (
new_calls == old_calls
), f"scrape_runs calls diverged:\n old={old_calls}\n new={new_calls}"
return new_counters
# ── scenarios ────────────────────────────────────────────────────────────────
_ANCHORS_2 = [(56.84, 60.60, "A1"), (56.79, 60.53, "A2")]
@pytest.mark.asyncio
async def test_parity_happy_path_counters_aggregation() -> None:
async def test_happy_path_counters_aggregation() -> None:
"""2 anchor'а, SERP+save, без houses/detail/imv → mark_done + агрегированные counters."""
scenario = _Scenario(
anchors=_ANCHORS_2,
per_anchor=[("lots", 10, 8, 2), ("lots", 6, 5, 1)],
)
counters = await _assert_parity(scenario)
# Убеждаемся что сценарий действительно прогнал агрегацию (не пустой прогон).
counters, calls = await _drive(scenario)
assert counters["lots_fetched"] == 16
assert counters["lots_inserted"] == 13
assert counters["lots_updated"] == 3
assert counters["anchors_done"] == 2
assert calls[-1][0] == "mark_done"
@pytest.mark.asyncio
async def test_parity_full_ban_no_intake_marks_banned() -> None:
async def test_full_ban_no_intake_marks_banned() -> None:
"""Первый же anchor SERP-блок, лоты не сохранены → mark_banned (не done)."""
scenario = _Scenario(
anchors=_ANCHORS_2,
per_anchor=[("block",), ("block",)],
)
await _assert_parity(scenario)
_counters, calls = await _drive(scenario)
assert calls[-1][0] == "mark_banned"
@pytest.mark.asyncio
async def test_parity_partial_ban_intake_marks_done() -> None:
async def test_partial_ban_intake_marks_done() -> None:
"""Anchor1 intake сохранён, anchor2 SERP-блок → partial-intake → mark_done не banned."""
scenario = _Scenario(
anchors=_ANCHORS_2,
per_anchor=[("lots", 10, 9, 1), ("block",)],
avito_serp_ok_not_banned=True,
)
await _assert_parity(scenario)
_counters, calls = await _drive(scenario)
assert calls[-1][0] == "mark_done"
@pytest.mark.asyncio
async def test_parity_partial_ban_flag_off_marks_banned() -> None:
async def test_partial_ban_flag_off_marks_banned() -> None:
"""avito_serp_ok_not_banned=False: даже при intake SERP-блок → mark_banned."""
scenario = _Scenario(
anchors=_ANCHORS_2,
per_anchor=[("lots", 10, 9, 1), ("block",)],
avito_serp_ok_not_banned=False,
)
await _assert_parity(scenario)
_counters, calls = await _drive(scenario)
assert calls[-1][0] == "mark_banned"
@pytest.mark.asyncio
async def test_parity_detail_consecutive_block_propagates() -> None:
async def test_detail_consecutive_block_propagates() -> None:
"""detail-фаза: 3 подряд блока (rotations=0) → propagate → anchor-handler.
Лоты сохранены на этом же anchor'е → partial-intake → mark_done.
@ -362,11 +287,12 @@ async def test_parity_detail_consecutive_block_propagates() -> None:
avito_serp_ok_not_banned=True,
avito_proxy_max_rotations=0,
)
await _assert_parity(scenario)
_counters, calls = await _drive(scenario)
assert calls[-1][0] == "mark_done"
@pytest.mark.asyncio
async def test_parity_imv_phase_counters() -> None:
async def test_imv_phase_counters() -> None:
"""IMV-фаза: touched houses → process_houses_imv_batch → imv-counters агрегированы."""
scenario = _Scenario(
anchors=[(56.84, 60.60, "A1")],
@ -376,8 +302,8 @@ async def test_parity_imv_phase_counters() -> None:
enrich_imv=True,
imv_result=(3, 2, 1),
)
counters = await _assert_parity(scenario)
# touched house → IMV-фаза отработала, imv-counters агрегированы одинаково.
counters, _calls = await _drive(scenario)
# touched house → IMV-фаза отработала, imv-counters агрегированы.
assert counters["imv_attempted"] == 3
assert counters["imv_enriched"] == 2
assert counters["imv_failed"] == 1

294
tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity2.py
View file

@ -1,7 +1,10 @@
"""Golden-parity (F2): kit sweep-оркестраторы ≡ `app.services.scrape_pipeline`.
"""Regression (F2): kit sweep-оркестраторы `scraper_kit.orchestration.pipeline`.
Продолжение test_scraper_kit_pipeline_parity.py (F1 покрыл avito city sweep). Здесь —
остальные 7 sweep'ов, скопированных в `scraper_kit.orchestration.pipeline` (#2135 F2):
Kit — единственная orchestration-копия (#2397 Part E1 удалил legacy
`app.services.scrape_pipeline`; сравнивать «golden-parity» больше не с чем — этот
файл раньше гонял ОБА orchestrator'а side-by-side, теперь оставлена только
kit-сторона). Продолжение test_scraper_kit_pipeline_parity.py (F1 покрыл avito city
sweep). Здесь — остальные 7 sweep'ов (#2135 F2):
City sweep'ы (приоритет — активны в проде):
- run_yandex_city_sweep — combos SERP + save + price-history
@ -9,13 +12,11 @@
- run_domclick_city_sweep — BFF citywide + честный статус (done / failed)
Плюс:
- run_avito_newbuilding_sweep — citywide novostroyka SERP + save
Full load'ы (smoke-parity — импорт + базовый прогон через on_bucket):
Full load'ы (smoke — импорт + базовый прогон через on_bucket):
- run_avito_full_load / run_cian_full_load / run_yandex_full_load
Метод — как в F1: гоняем ОБА orchestrator'а на идентичном мокнутом I/O, сравниваем
итоговый counters.to_dict() + последовательность (method, numeric-counters) вызовов
scrape_runs. Развязка `app.*` (config/matcher/enrichment/runs/shutdown инжектируются)
не должна менять ветвление/счётчики. Без сети, без БД.
Метод: гоняем kit orchestrator на мокнутом I/O, проверяем итоговый counters.to_dict()
+ последовательность (method, numeric-counters) вызовов scrape_runs. Без сети, без БД.
"""
from __future__ import annotations
@ -30,51 +31,16 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from scraper_kit.orchestration.pipeline import (
run_avito_full_load as new_avito_full_load,
)
from scraper_kit.orchestration.pipeline import (
run_avito_newbuilding_sweep as new_nb_sweep,
)
from scraper_kit.orchestration.pipeline import (
run_cian_city_sweep as new_cian_city_sweep,
)
from scraper_kit.orchestration.pipeline import (
run_cian_full_load as new_cian_full_load,
)
from scraper_kit.orchestration.pipeline import (
run_domclick_city_sweep as new_domclick_city_sweep,
)
from scraper_kit.orchestration.pipeline import (
run_yandex_city_sweep as new_yandex_city_sweep,
)
from scraper_kit.orchestration.pipeline import (
run_yandex_full_load as new_yandex_full_load,
run_avito_full_load,
run_avito_newbuilding_sweep,
run_cian_city_sweep,
run_cian_full_load,
run_domclick_city_sweep,
run_yandex_city_sweep,
run_yandex_full_load,
)
from app.services.scrape_pipeline import (
run_avito_full_load as old_avito_full_load,
)
from app.services.scrape_pipeline import (
run_avito_newbuilding_sweep as old_nb_sweep,
)
from app.services.scrape_pipeline import (
run_cian_city_sweep as old_cian_city_sweep,
)
from app.services.scrape_pipeline import (
run_cian_full_load as old_cian_full_load,
)
from app.services.scrape_pipeline import (
run_domclick_city_sweep as old_domclick_city_sweep,
)
from app.services.scrape_pipeline import (
run_yandex_city_sweep as old_yandex_city_sweep,
)
from app.services.scrape_pipeline import (
run_yandex_full_load as old_yandex_full_load,
)
OLD_PFX = "app.services.scrape_pipeline"
NEW_PFX = "scraper_kit.orchestration.pipeline"
PFX = "scraper_kit.orchestration.pipeline"
# ── recording scrape_runs ────────────────────────────────────────────────────
@ -113,7 +79,6 @@ def _normalize(calls: list[tuple[str, dict[str, Any]]]) -> list[tuple[str, dict[
def _config() -> SimpleNamespace:
"""Инжектируемый config (kit) + тот же объект как patched settings (old)."""
return SimpleNamespace(
scraper_fetch_mode="curl_cffi",
browser_http_endpoint="http://browser.test/fetch",
@ -158,18 +123,6 @@ def _async_session_cm() -> MagicMock:
_DriveResult = tuple[dict[str, int], list[tuple[str, dict[str, int]]]]
async def _assert_parity(old_res: _DriveResult, new_res: _DriveResult) -> dict[str, int]:
old_counters, old_calls = old_res
new_counters, new_calls = new_res
assert (
new_counters == old_counters
), f"counters diverged:\n old={old_counters}\n new={new_counters}"
assert (
new_calls == old_calls
), f"scrape_runs calls diverged:\n old={old_calls}\n new={new_calls}"
return new_counters
# ── Yandex city sweep ─────────────────────────────────────────────────────────
@ -183,7 +136,7 @@ def _yandex_scraper(combos: list[tuple[str, list[Any]]]) -> MagicMock:
return _ctx_scraper(fetch_around_multi_room=_fetch)
async def _drive_yandex_city(*, new: bool) -> _DriveResult:
async def _drive_yandex_city() -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
combos = [
@ -193,47 +146,36 @@ async def _drive_yandex_city(*, new: bool) -> _DriveResult:
scraper = _yandex_scraper(combos)
save_mock = MagicMock(side_effect=[(2, 0), (1, 0)])
cfg = _config()
kwargs = dict(
enrichment = MagicMock()
enrichment.record_yandex_price_history = MagicMock(return_value=5)
with (
patch(f"{PFX}.YandexRealtyScraper", return_value=scraper),
patch(f"{PFX}.save_listings", save_mock),
patch(f"{PFX}.runs", recorder),
):
counters = await run_yandex_city_sweep(
db,
config=cfg,
matcher=MagicMock(),
enrichment=enrichment,
run_id=1,
anchors=None,
pages_per_anchor=1,
request_delay_sec=0.0,
enrich_address=False,
)
if new:
enrichment = MagicMock()
enrichment.record_yandex_price_history = MagicMock(return_value=5)
with (
patch(f"{NEW_PFX}.YandexRealtyScraper", return_value=scraper),
patch(f"{NEW_PFX}.save_listings", save_mock),
patch(f"{NEW_PFX}.runs", recorder),
):
counters = await new_yandex_city_sweep(
db, config=cfg, matcher=MagicMock(), enrichment=enrichment, **kwargs
)
else:
with (
patch(f"{OLD_PFX}.YandexRealtyScraper", return_value=scraper),
patch(f"{OLD_PFX}.save_listings", save_mock),
patch(f"{OLD_PFX}.record_yandex_price_history", MagicMock(return_value=5)),
patch(f"{OLD_PFX}.scrape_runs", recorder),
patch(f"{OLD_PFX}.settings", cfg),
patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
):
counters = await old_yandex_city_sweep(db, **kwargs)
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
async def test_yandex_city_sweep_parity() -> None:
async def test_yandex_city_sweep() -> None:
"""1 центральный anchor, 2 combos, save + price-history, enrich_address=False."""
counters = await _assert_parity(
await _drive_yandex_city(new=False), await _drive_yandex_city(new=True)
)
counters, calls = await _drive_yandex_city()
assert counters["lots_fetched"] == 3
assert counters["lots_inserted"] == 3
assert counters["price_history_rows"] == 5
assert counters["anchors_done"] == 1
assert calls[-1][0] == "mark_done"
# ── Cian city sweep ───────────────────────────────────────────────────────────
@ -243,7 +185,7 @@ def _cian_lot(segment: str) -> MagicMock:
return MagicMock(listing_segment=segment, house_source=None, house_ext_id=None)
async def _drive_cian_city(*, new: bool) -> _DriveResult:
async def _drive_cian_city() -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
# 3 novostroyki + 2 secondary → newbuilding_only оставит 3.
@ -251,7 +193,15 @@ async def _drive_cian_city(*, new: bool) -> _DriveResult:
scraper = _ctx_scraper(fetch_around_multi_room=AsyncMock(return_value=lots))
save_mock = MagicMock(side_effect=[(3, 0)])
cfg = _config()
kwargs = dict(
with (
patch(f"{PFX}.CianScraper", return_value=scraper),
patch(f"{PFX}.save_listings", save_mock),
patch(f"{PFX}.runs", recorder),
):
counters = await run_cian_city_sweep(
db,
config=cfg,
matcher=MagicMock(),
run_id=1,
anchors=[(56.84, 60.60, "A1")],
radius_m=1000,
@ -261,41 +211,24 @@ async def _drive_cian_city(*, new: bool) -> _DriveResult:
enrich_houses=False,
newbuilding_only=True,
)
if new:
with (
patch(f"{NEW_PFX}.CianScraper", return_value=scraper),
patch(f"{NEW_PFX}.save_listings", save_mock),
patch(f"{NEW_PFX}.runs", recorder),
):
counters = await new_cian_city_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
else:
with (
patch("app.services.scrapers.cian.CianScraper", return_value=scraper),
patch(f"{OLD_PFX}.save_listings", save_mock),
patch(f"{OLD_PFX}.scrape_runs", recorder),
patch(f"{OLD_PFX}.settings", cfg),
patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
):
counters = await old_cian_city_sweep(db, **kwargs)
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
async def test_cian_city_sweep_parity() -> None:
async def test_cian_city_sweep() -> None:
"""1 anchor, newbuilding_only отбрасывает вторичку, save новостроек, mark_done."""
counters = await _assert_parity(
await _drive_cian_city(new=False), await _drive_cian_city(new=True)
)
counters, calls = await _drive_cian_city()
assert counters["lots_fetched"] == 5
assert counters["lots_dropped_secondary"] == 2
assert counters["lots_inserted"] == 3
assert counters["anchors_done"] == 1
assert calls[-1][0] == "mark_done"
# ── DomClick city sweep ───────────────────────────────────────────────────────
async def _drive_domclick(*, new: bool, lots_n: int, blocked: bool) -> _DriveResult:
async def _drive_domclick(*, lots_n: int, blocked: bool) -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
lots = [MagicMock() for _ in range(lots_n)]
@ -307,52 +240,39 @@ async def _drive_domclick(*, new: bool, lots_n: int, blocked: bool) -> _DriveRes
)
save_mock = MagicMock(side_effect=[(lots_n, 0)] if lots_n else [])
cfg = _config()
kwargs = dict(run_id=1, city_id=4, pages=1, request_delay_sec=0.0)
if new:
with (
patch(f"{NEW_PFX}.DomClickScraper", return_value=scraper),
patch(f"{NEW_PFX}.save_listings", save_mock),
patch(f"{NEW_PFX}.runs", recorder),
patch(f"{PFX}.DomClickScraper", return_value=scraper),
patch(f"{PFX}.save_listings", save_mock),
patch(f"{PFX}.runs", recorder),
):
counters = await new_domclick_city_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
else:
with (
patch("app.services.scrapers.domclick.DomClickScraper", return_value=scraper),
patch(f"{OLD_PFX}.save_listings", save_mock),
patch(f"{OLD_PFX}.scrape_runs", recorder),
patch(f"{OLD_PFX}.settings", cfg),
patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
):
counters = await old_domclick_city_sweep(db, **kwargs)
counters = await run_domclick_city_sweep(
db, config=cfg, matcher=MagicMock(), run_id=1, city_id=4, pages=1, request_delay_sec=0.0
)
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
async def test_domclick_city_sweep_parity_done() -> None:
async def test_domclick_city_sweep_done() -> None:
"""Citywide fetch_city → 4 lots saved → mark_done."""
counters = await _assert_parity(
await _drive_domclick(new=False, lots_n=4, blocked=False),
await _drive_domclick(new=True, lots_n=4, blocked=False),
)
counters, calls = await _drive_domclick(lots_n=4, blocked=False)
assert counters["lots_fetched"] == 4
assert counters["lots_inserted"] == 4
assert calls[-1][0] == "mark_done"
@pytest.mark.asyncio
async def test_domclick_city_sweep_parity_blocked_failed() -> None:
async def test_domclick_city_sweep_blocked_failed() -> None:
"""QRATOR-блок + 0 lots → mark_failed (честный статус #1968)."""
counters = await _assert_parity(
await _drive_domclick(new=False, lots_n=0, blocked=True),
await _drive_domclick(new=True, lots_n=0, blocked=True),
)
counters, calls = await _drive_domclick(lots_n=0, blocked=True)
assert counters["lots_fetched"] == 0
assert counters["blocked"] == 1
assert calls[-1][0] == "mark_failed"
# ── Avito newbuilding sweep ───────────────────────────────────────────────────
async def _drive_nb_sweep(*, new: bool) -> _DriveResult:
async def _drive_nb_sweep() -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
lots = [MagicMock() for _ in range(6)]
@ -362,40 +282,29 @@ async def _drive_nb_sweep(*, new: bool) -> _DriveResult:
scraper.fetch_newbuildings = AsyncMock(return_value=lots)
save_mock = MagicMock(side_effect=[(5, 1)])
cfg = _config()
kwargs = dict(run_id=1, pages=2, request_delay_sec=0.0)
if new:
with (
patch(f"{NEW_PFX}.AvitoScraper", return_value=scraper),
patch(f"{NEW_PFX}.save_listings", save_mock),
patch(f"{NEW_PFX}.runs", recorder),
patch(f"{NEW_PFX}.AsyncSession", return_value=_async_session_cm()),
patch(f"{PFX}.AvitoScraper", return_value=scraper),
patch(f"{PFX}.save_listings", save_mock),
patch(f"{PFX}.runs", recorder),
patch(f"{PFX}.AsyncSession", return_value=_async_session_cm()),
):
counters = await new_nb_sweep(db, config=cfg, matcher=MagicMock(), **kwargs)
else:
with (
patch(f"{OLD_PFX}.AvitoScraper", return_value=scraper),
patch(f"{OLD_PFX}.save_listings", save_mock),
patch(f"{OLD_PFX}.scrape_runs", recorder),
patch(f"{OLD_PFX}.settings", cfg),
patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
patch(f"{OLD_PFX}.AsyncSession", return_value=_async_session_cm()),
):
counters = await old_nb_sweep(db, **kwargs)
counters = await run_avito_newbuilding_sweep(
db, config=cfg, matcher=MagicMock(), run_id=1, pages=2, request_delay_sec=0.0
)
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
async def test_avito_newbuilding_sweep_parity() -> None:
async def test_avito_newbuilding_sweep() -> None:
"""Citywide novostroyka SERP → save → heartbeat → mark_done."""
counters = await _assert_parity(
await _drive_nb_sweep(new=False), await _drive_nb_sweep(new=True)
)
counters, calls = await _drive_nb_sweep()
assert counters["lots_fetched"] == 6
assert counters["lots_inserted"] == 5
assert counters["lots_updated"] == 1
assert calls[-1][0] == "mark_done"
# ── Full loads (smoke-parity через on_bucket) ─────────────────────────────────
# ── Full loads (smoke через on_bucket) ─────────────────────────────────────────
def _full_load_scraper(buckets: list[tuple[str, list[Any]]]) -> MagicMock:
@ -410,7 +319,7 @@ def _full_load_scraper(buckets: list[tuple[str, list[Any]]]) -> MagicMock:
return scraper
async def _drive_full_load(*, new: bool, source: str) -> _DriveResult:
async def _drive_full_load(*, source: str) -> _DriveResult:
recorder = _RunsRecorder()
db = MagicMock()
buckets = [
@ -421,61 +330,38 @@ async def _drive_full_load(*, new: bool, source: str) -> _DriveResult:
save_mock = MagicMock(side_effect=[(2, 0), (1, 0)])
cfg = _config()
old_map = {
"avito": (old_avito_full_load, new_avito_full_load, f"{OLD_PFX}.AvitoScraper"),
"cian": (old_cian_full_load, new_cian_full_load, "app.services.scrapers.cian.CianScraper"),
"yandex": (
old_yandex_full_load,
new_yandex_full_load,
"app.services.scrapers.yandex_realty.YandexRealtyScraper",
),
fn_map = {
"avito": run_avito_full_load,
"cian": run_cian_full_load,
"yandex": run_yandex_full_load,
}
old_fn, new_fn, old_scraper_target = old_map[source]
if new:
new_scraper_target = {
"avito": f"{NEW_PFX}.AvitoScraper",
"cian": f"{NEW_PFX}.CianScraper",
"yandex": f"{NEW_PFX}.YandexRealtyScraper",
scraper_target = {
"avito": f"{PFX}.AvitoScraper",
"cian": f"{PFX}.CianScraper",
"yandex": f"{PFX}.YandexRealtyScraper",
}[source]
fn = fn_map[source]
extra: dict[str, Any] = {}
if source == "yandex":
enrichment = MagicMock()
enrichment.record_yandex_price_history = MagicMock(return_value=0)
extra["enrichment"] = enrichment
with (
patch(new_scraper_target, return_value=scraper),
patch(f"{NEW_PFX}.save_listings", save_mock),
patch(f"{NEW_PFX}.runs", recorder),
patch(scraper_target, return_value=scraper),
patch(f"{PFX}.save_listings", save_mock),
patch(f"{PFX}.runs", recorder),
):
counters = await new_fn(db, run_id=1, config=cfg, matcher=MagicMock(), **extra)
else:
import contextlib
ctx = [
patch(old_scraper_target, return_value=scraper),
patch(f"{OLD_PFX}.save_listings", save_mock),
patch(f"{OLD_PFX}.scrape_runs", recorder),
patch(f"{OLD_PFX}.settings", cfg),
patch(f"{OLD_PFX}.shutdown_requested", return_value=False),
]
if source == "yandex":
ctx.append(patch(f"{OLD_PFX}.record_yandex_price_history", MagicMock(return_value=0)))
with contextlib.ExitStack() as stack:
for cm in ctx:
stack.enter_context(cm)
counters = await old_fn(db, run_id=1)
counters = await fn(db, run_id=1, config=cfg, matcher=MagicMock(), **extra)
return counters.to_dict(), _normalize(recorder.calls)
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["avito", "cian", "yandex"])
async def test_full_load_smoke_parity(source: str) -> None:
"""Full load: 2 бакета через on_bucket → save + heartbeat → mark_done. Counters ≡."""
counters = await _assert_parity(
await _drive_full_load(new=False, source=source),
await _drive_full_load(new=True, source=source),
)
async def test_full_load_smoke(source: str) -> None:
"""Full load: 2 бакета через on_bucket → save + heartbeat → mark_done."""
counters, calls = await _drive_full_load(source=source)
assert counters["unique_fetched"] == 3
assert counters["saved_inserted"] == 3
assert counters["saved_updated"] == 0
assert calls[-1][0] == "mark_done"

20
tradein-mvp/backend/tests/test_scraper_proxy.py
View file

@ -95,35 +95,33 @@ def test_scraper_proxy_url_empty_string_falls_through_to_avito():
def test_avito_proxies_returns_correct_dict_shape():
from app.services.scrape_pipeline import _avito_proxies
"""Kit `_avito_proxies(config)` — DI параметр вместо module-level settings patch."""
from scraper_kit.orchestration.pipeline import _avito_proxies
proxy_url = "http://user:pass@proxy.example.com:8080"
with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)):
result = _avito_proxies()
result = _avito_proxies(_mock_settings(proxy_url))
assert result == {"http": proxy_url, "https": proxy_url}
def test_avito_proxies_returns_none_when_no_proxy():
from app.services.scrape_pipeline import _avito_proxies
from scraper_kit.orchestration.pipeline import _avito_proxies
with patch("app.services.scrape_pipeline.settings", _mock_settings(None)):
result = _avito_proxies()
result = _avito_proxies(_mock_settings(None))
assert result is None
def test_avito_proxies_fallback_via_avito_proxy_url():
"""_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback.
"""_avito_proxies(config) uses AVITO_PROXY_URL via scraper_proxy_url fallback.
When settings.scraper_proxy_url resolves to the avito URL (fallback path),
When config.scraper_proxy_url resolves to the avito URL (fallback path),
the dict shape must be {"http": url, "https": url}.
"""
from app.services.scrape_pipeline import _avito_proxies
from scraper_kit.orchestration.pipeline import _avito_proxies
avito_url = "http://mobile-proxy.space:9090"
with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)):
result = _avito_proxies()
result = _avito_proxies(_mock_settings(avito_url))
assert result == {"http": avito_url, "https": avito_url}

633
tradein-mvp/backend/tests/test_scraper_resilience_1949_1950.py
View file

@ -1,633 +0,0 @@
"""Тесты устойчивости скрейперов: avito changeip-retry (#1950) + cian anti-zombie (#1949).
Fix A (#1950): _rotate_proxy_ip теперь делает до proxy_rotate_attempts попыток
по proxy_rotate_attempt_timeout_s каждая.
Fix B (#1950): run_avito_city_sweep при AvitoBlockedError/RateLimited с
lots_inserted+lots_updated>0 вызывает mark_done (не mark_banned)
когда avito_serp_ok_not_banned=True.
Fix C (#1949): run_cian_full_load применяет monkey-patch _browser.fetch с
asyncio.wait_for(timeout=cian_full_load_per_fetch_timeout_s).
Fix D (#1949): run_cian_full_load создаёт фоновый heartbeat-task и отменяет
его в finally при любом завершении.
Fix E (#1949): run_cian_city_sweep при defensive abort (consecutive anchor
timeout/failure) вызывает mark_done (не mark_banned) когда
SERP уже сохранил лоты (lots_inserted+lots_updated>0) и
cian_sweep_ok_not_banned=True.
"""
from __future__ import annotations
import asyncio
import os
import time
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.base import ScrapedLot
# ──────────────────────────────────────────────────────────────────────────────
# Fix A: _rotate_proxy_ip retry loop (#1950)
# ──────────────────────────────────────────────────────────────────────────────
def _make_session_mock(get_side_effect: Any) -> AsyncMock:
"""Вспомогательный: AsyncMock-сессия с заданным поведением .get()."""
mock_session = AsyncMock()
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
mock_session.__aexit__ = AsyncMock(return_value=None)
mock_session.get = AsyncMock(side_effect=get_side_effect)
return mock_session
@pytest.mark.asyncio
async def test_changeip_retry_success_on_second_attempt() -> None:
"""#1950-A: первая попытка таймаут, вторая успех → возвращает True; ровно 2 GET-вызова."""
from app.services.scrape_pipeline import _rotate_proxy_ip
get_call_count = 0
async def fake_get(url: str) -> MagicMock:
nonlocal get_call_count
get_call_count += 1
if get_call_count == 1:
raise TimeoutError("connect timeout")
resp = MagicMock()
resp.json.return_value = {"new_ip": "1.2.3.4"}
return resp
mock_session = _make_session_mock(fake_get)
with (
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
s.avito_proxy_rotate_settle_s = 0.0
s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
s.avito_proxy_max_rotations = 3
result = await _rotate_proxy_ip(
source="avito",
reason="test-detail-block",
rotations_done=0,
)
assert result is True
assert get_call_count == 2, f"expected 2 GET calls, got {get_call_count}"
@pytest.mark.asyncio
async def test_changeip_retry_all_fail_returns_false() -> None:
"""#1950-A: все 3 попытки падают с TimeoutError → False; ровно 3 GET-вызова."""
from app.services.scrape_pipeline import _rotate_proxy_ip
get_call_count = 0
async def failing_get(url: str) -> None:
nonlocal get_call_count
get_call_count += 1
raise TimeoutError("connect timeout")
mock_session = _make_session_mock(failing_get)
with (
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
s.avito_proxy_rotate_settle_s = 0.0
s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
s.avito_proxy_max_rotations = 3
result = await _rotate_proxy_ip(
source="avito",
reason="test-detail-block",
rotations_done=0,
)
assert result is False
assert (
get_call_count == 3
), f"expected exactly 3 attempts (proxy_rotate_attempts=3), got {get_call_count}"
@pytest.mark.asyncio
async def test_changeip_retry_bounded_by_setting() -> None:
"""#1950-A: proxy_rotate_attempts=2 → ровно 2 попытки, не больше."""
from app.services.scrape_pipeline import _rotate_proxy_ip
call_count = 0
async def failing_get(url: str) -> None:
nonlocal call_count
call_count += 1
raise TimeoutError("timeout")
mock_session = _make_session_mock(failing_get)
with (
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.proxy_rotate_attempts = 2
s.proxy_rotate_attempt_timeout_s = 8.0
s.avito_proxy_rotate_settle_s = 0.0
s.cian_proxy_rotate_url = "http://changeip.example.com/rotate"
s.avito_proxy_rotate_url = None
s.cian_proxy_max_rotations = 3
result = await _rotate_proxy_ip(
source="cian",
reason="serp-block",
rotations_done=1,
)
assert result is False
assert call_count == 2, f"proxy_rotate_attempts=2 should make exactly 2 calls, got {call_count}"
# ──────────────────────────────────────────────────────────────────────────────
# Fix B: честный статус run_avito_city_sweep (#1950)
# ──────────────────────────────────────────────────────────────────────────────
def _make_city_sweep_mocks() -> tuple[MagicMock, MagicMock, MagicMock]:
"""Возвращает (mock_db, mock_runs, mock_scraper) для тестов run_avito_city_sweep."""
mock_db = MagicMock()
# priority_rows для detail-фазы: 3 строки с source_url
class _DetailRow:
def __getitem__(self, key: str) -> str:
return "https://www.avito.ru/ekaterinburg/kvartiry/test-1"
mock_db.execute.return_value.mappings.return_value.all.return_value = [
_DetailRow(),
_DetailRow(),
_DetailRow(),
]
mock_runs = MagicMock()
mock_runs.is_cancelled.return_value = False
mock_runs.mark_done = MagicMock()
mock_runs.mark_banned = MagicMock()
mock_runs.mark_failed = MagicMock()
mock_runs.update_heartbeat = MagicMock()
mock_scraper = MagicMock()
# AvitoScraper НЕ используется как async CM в _avito_anchor_phases (просто instantiate)
mock_scraper.fetch_around = AsyncMock(
return_value=[
ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/e/k/{i}",
source_id=str(i),
price_rub=5_000_000,
address="ЕКБ test",
listing_segment="vtorichka",
)
for i in range(3)
]
)
return mock_db, mock_runs, mock_scraper
@pytest.mark.asyncio
async def test_avito_city_sweep_serp_ok_marks_done_not_banned() -> None:
"""#1950-B: SERP собрал лоты (ins=3), detail заблокировало → mark_done не mark_banned.
Условие: avito_serp_ok_not_banned=True (default), lots_inserted+lots_updated > 0.
"""
from app.services.scrape_pipeline import run_avito_city_sweep
mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
# Shared AsyncSession mock (browser_mode=False)
mock_session = AsyncMock()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
patch(
"app.services.scrape_pipeline.fetch_detail",
AsyncMock(side_effect=AvitoBlockedError("HTTP 403 banned")),
),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.scraper_fetch_mode = "curl_cffi"
s.scraper_proxy_url = None
s.avito_proxy_max_rotations = 0 # no rotation budget
s.avito_serp_ok_not_banned = True # ← наш флаг
s.avito_proxy_rotate_settle_s = 0.0
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
await run_avito_city_sweep(
mock_db,
run_id=1,
anchors=[(56.84, 60.61, "test-anchor")],
detail_top_n=3,
enrich_houses=False,
enrich_imv=False,
)
assert (
mock_runs.mark_done.called
), "mark_done должен быть вызван: SERP OK (ins=3), detail blocked"
assert (
not mock_runs.mark_banned.called
), "mark_banned НЕ должен быть вызван когда SERP уже сохранил лоты"
# enrichment_abort_note передан в counters
call_args = mock_runs.mark_done.call_args
# mark_done(db, run_id, counters_dict)
passed_counters: dict = call_args[0][2] if call_args[0] else {}
assert (
"enrichment_abort_note" in passed_counters
), "enrichment_abort_note должен присутствовать в counters переданных mark_done"
@pytest.mark.asyncio
async def test_avito_city_sweep_serp_zero_lots_marks_banned() -> None:
"""#1950-B: SERP вернул 0 лотов, SERP-блок → mark_banned (не mark_done).
Условие: lots_inserted+lots_updated == 0.
"""
from app.services.scrape_pipeline import run_avito_city_sweep
mock_db, mock_runs, _ = _make_city_sweep_mocks()
# SERP сам поднимает AvitoBlockedError → anchor_lots не сохраняется
mock_scraper_serp_block = MagicMock()
mock_scraper_serp_block.fetch_around = AsyncMock(side_effect=AvitoBlockedError("SERP HTTP 403"))
mock_session = AsyncMock()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper_serp_block),
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.scraper_fetch_mode = "curl_cffi"
s.scraper_proxy_url = None
s.avito_proxy_max_rotations = 0
s.avito_serp_ok_not_banned = True # включён флаг, но лотов нет
s.avito_proxy_rotate_settle_s = 0.0
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
await run_avito_city_sweep(
mock_db,
run_id=2,
anchors=[(56.84, 60.61, "test-anchor")],
detail_top_n=3,
enrich_houses=False,
enrich_imv=False,
)
assert (
mock_runs.mark_banned.called
), "mark_banned должен быть вызван: SERP сам заблокирован, lots=0"
assert not mock_runs.mark_done.called, "mark_done НЕ должен быть вызван при lots_inserted == 0"
@pytest.mark.asyncio
async def test_avito_city_sweep_flag_off_still_marks_banned() -> None:
"""#1950-B: avito_serp_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
from app.services.scrape_pipeline import run_avito_city_sweep
mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
mock_session = AsyncMock()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
patch(
"app.services.scrape_pipeline.fetch_detail",
AsyncMock(side_effect=AvitoBlockedError("HTTP 403")),
),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.scraper_fetch_mode = "curl_cffi"
s.scraper_proxy_url = None
s.avito_proxy_max_rotations = 0
s.avito_serp_ok_not_banned = False # ← флаг выключен
s.avito_proxy_rotate_settle_s = 0.0
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
await run_avito_city_sweep(
mock_db,
run_id=3,
anchors=[(56.84, 60.61, "test-anchor")],
detail_top_n=3,
enrich_houses=False,
enrich_imv=False,
)
assert (
mock_runs.mark_banned.called
), "avito_serp_ok_not_banned=False → mark_banned (backward-compat)"
assert not mock_runs.mark_done.called
# ──────────────────────────────────────────────────────────────────────────────
# Fix C: cian per-fetch wait_for (#1949)
# ──────────────────────────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_cian_full_load_hanging_fetch_cancelled_by_timeout() -> None:
"""#1949-C: browser-fetch зависший навсегда отменяется per-fetch timeout.
Monkey-patch заменяет _browser.fetch на _timed_fetch(wait_for timeout=0.05s).
Зависший fetch отменяется через ~50ms → TimeoutError → run завершается.
"""
from app.services.scrape_pipeline import run_cian_full_load
fetch_was_called = False
async def hanging_fetch(url: str) -> str:
nonlocal fetch_was_called
fetch_was_called = True
await asyncio.Event().wait() # висит вечно
return ""
mock_browser = MagicMock()
mock_browser.fetch = hanging_fetch
mock_scraper = MagicMock()
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
mock_scraper.__aexit__ = AsyncMock(return_value=None)
mock_scraper._browser = mock_browser
mock_scraper.request_delay_sec = 0.0
# fetch_all_secondary симулирует _fetch_page_html: вызывает fetch и ловит Exception
async def fake_fetch_all_secondary(**kwargs: object) -> None:
try:
# После monkey-patch _browser.fetch = _timed_fetch (с timeout=0.05s)
await mock_scraper._browser.fetch("https://ekb.cian.ru/sale/flat/1/")
except Exception:
pass # TimeoutError → аналог _fetch_page_html → returns None
mock_scraper.fetch_all_secondary = fake_fetch_all_secondary
mock_db = MagicMock()
mock_runs = MagicMock()
mock_runs.is_cancelled.return_value = False
mock_runs.mark_done = MagicMock()
mock_runs.mark_failed = MagicMock()
mock_runs.update_heartbeat = MagicMock()
t0 = time.monotonic()
with (
patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.settings") as s,
):
s.cian_full_load_per_fetch_timeout_s = 0.05 # 50ms — быстрый timeout для теста
s.cian_full_load_detail_top_n = 0
await run_cian_full_load(mock_db, run_id=1)
elapsed = time.monotonic() - t0
assert fetch_was_called, "hanging_fetch должен быть вызван через monkey-patched _browser.fetch"
assert (
elapsed < 3.0
), f"run должен завершиться быстро (fetch timeout 50ms), занял {elapsed:.2f}s"
assert mock_runs.mark_done.called, "run_cian_full_load должен завершиться mark_done"
@pytest.mark.asyncio
async def test_cian_full_load_per_fetch_timeout_disabled_when_zero() -> None:
"""#1949-C: cian_full_load_per_fetch_timeout_s=0 → monkey-patch НЕ применяется."""
from app.services.scrape_pipeline import run_cian_full_load
original_fetch = AsyncMock(return_value="<html/>")
mock_browser = MagicMock()
mock_browser.fetch = original_fetch
mock_scraper = MagicMock()
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
mock_scraper.__aexit__ = AsyncMock(return_value=None)
mock_scraper._browser = mock_browser
mock_scraper.request_delay_sec = 0.0
mock_scraper.fetch_all_secondary = AsyncMock(return_value=None)
mock_db = MagicMock()
mock_runs = MagicMock()
mock_runs.is_cancelled.return_value = False
mock_runs.mark_done = MagicMock()
mock_runs.mark_failed = MagicMock()
mock_runs.update_heartbeat = MagicMock()
with (
patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.settings") as s,
):
s.cian_full_load_per_fetch_timeout_s = 0.0 # отключено
s.cian_full_load_detail_top_n = 0
await run_cian_full_load(mock_db, run_id=2)
# Когда timeout=0 (отключён), monkey-patch не применяется → fetch остаётся original
assert (
mock_browser.fetch is original_fetch
), "cian_full_load_per_fetch_timeout_s=0 → fetch НЕ должен быть заменён monkey-patch"
# ──────────────────────────────────────────────────────────────────────────────
# Fix D: cian background heartbeat created and cancelled (#1949)
# ──────────────────────────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_cian_full_load_background_heartbeat_created_and_cancelled() -> None:
"""#1949-D: background heartbeat task создаётся и отменяется в finally."""
from app.services.scrape_pipeline import run_cian_full_load
mock_scraper = MagicMock()
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
mock_scraper.__aexit__ = AsyncMock(return_value=None)
mock_scraper._browser = None # per-fetch timeout не применяется
mock_scraper.fetch_all_secondary = AsyncMock(return_value=None)
mock_db = MagicMock()
mock_runs = MagicMock()
mock_runs.is_cancelled.return_value = False
mock_runs.mark_done = MagicMock()
mock_runs.mark_failed = MagicMock()
mock_runs.update_heartbeat = MagicMock()
created_tasks: list[asyncio.Task[None]] = []
_real_create_task = asyncio.create_task
def _tracking_create_task(coro: Any, *, name: str | None = None) -> asyncio.Task[None]:
task = _real_create_task(coro, name=name)
created_tasks.append(task)
return task # type: ignore[return-value]
with (
patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch(
"app.services.scrape_pipeline.asyncio.create_task",
side_effect=_tracking_create_task,
),
patch("app.services.scrape_pipeline.settings") as s,
):
s.cian_full_load_per_fetch_timeout_s = 0.0
s.cian_full_load_detail_top_n = 0
await run_cian_full_load(mock_db, run_id=3)
assert (
len(created_tasks) >= 1
), "asyncio.create_task должен быть вызван хотя бы раз (background heartbeat)"
for task in created_tasks:
assert (
task.cancelled() or task.done()
), f"Heartbeat task должен быть отменён в finally: {task}"
@pytest.mark.asyncio
async def test_cian_full_load_heartbeat_cancelled_on_exception() -> None:
"""#1949-D: heartbeat task отменяется в finally даже при исключении в run."""
from app.services.scrape_pipeline import run_cian_full_load
mock_scraper = MagicMock()
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
mock_scraper.__aexit__ = AsyncMock(return_value=None)
mock_scraper._browser = None
mock_scraper.fetch_all_secondary = AsyncMock(side_effect=RuntimeError("fatal-test-error"))
mock_db = MagicMock()
mock_runs = MagicMock()
mock_runs.is_cancelled.return_value = False
mock_runs.mark_done = MagicMock()
mock_runs.mark_failed = MagicMock()
mock_runs.update_heartbeat = MagicMock()
created_tasks: list[asyncio.Task[None]] = []
_real_create_task = asyncio.create_task
def _tracking_create_task(coro: Any, *, name: str | None = None) -> asyncio.Task[None]:
task = _real_create_task(coro, name=name)
created_tasks.append(task)
return task # type: ignore[return-value]
with (
patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch(
"app.services.scrape_pipeline.asyncio.create_task",
side_effect=_tracking_create_task,
),
patch("app.services.scrape_pipeline.settings") as s,
):
s.cian_full_load_per_fetch_timeout_s = 0.0
s.cian_full_load_detail_top_n = 0
with pytest.raises(RuntimeError, match="fatal-test-error"):
await run_cian_full_load(mock_db, run_id=4)
assert len(created_tasks) >= 1, "heartbeat task должен быть создан"
for task in created_tasks:
assert (
task.cancelled() or task.done()
), f"Heartbeat task должен быть отменён в finally даже при RuntimeError: {task}"
assert mock_runs.mark_failed.called, "mark_failed должен быть вызван при RuntimeError"
# ──────────────────────────────────────────────────────────────────────────────
# Fix E: run_cian_city_sweep честный статус при defensive abort (#1949)
# ──────────────────────────────────────────────────────────────────────────────
def test_mark_cian_sweep_abort_lots_present_marks_done() -> None:
"""#1949-E: SERP уже сохранил лоты (ins+upd>0) до abort → mark_done, не mark_banned."""
from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
mock_db = MagicMock()
mock_runs = MagicMock()
counters = CianCitySweepCounters(
anchors_total=5, anchors_done=2, lots_inserted=12, lots_updated=3
)
with (
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.settings") as s,
):
s.cian_sweep_ok_not_banned = True
_mark_cian_sweep_abort(mock_db, 42, counters, "cian sweep aborted: 3 consecutive timeouts")
assert mock_runs.mark_done.called, "mark_done должен быть вызван: SERP OK (ins=12 upd=3)"
assert not mock_runs.mark_banned.called, "mark_banned НЕ должен вызываться когда лоты сохранены"
passed_counters: dict = mock_runs.mark_done.call_args[0][2]
assert "abort_note" in passed_counters, "abort_note должен присутствовать в counters"
def test_mark_cian_sweep_abort_no_lots_marks_banned() -> None:
"""#1949-E: лотов ещё не было (ins=upd=0) на момент abort → mark_banned как раньше."""
from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
mock_db = MagicMock()
mock_runs = MagicMock()
counters = CianCitySweepCounters(
anchors_total=5, anchors_done=1, lots_inserted=0, lots_updated=0
)
with (
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.settings") as s,
):
s.cian_sweep_ok_not_banned = True
_mark_cian_sweep_abort(mock_db, 43, counters, "cian sweep aborted: SERP blocked at anchor")
assert mock_runs.mark_banned.called, "mark_banned должен быть вызван: SERP не собрал лотов"
assert not mock_runs.mark_done.called, "mark_done НЕ должен вызываться при lots ins=upd=0"
def test_mark_cian_sweep_abort_flag_off_still_marks_banned() -> None:
"""#1949-E: cian_sweep_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
from app.services.scrape_pipeline import CianCitySweepCounters, _mark_cian_sweep_abort
mock_db = MagicMock()
mock_runs = MagicMock()
counters = CianCitySweepCounters(
anchors_total=5, anchors_done=3, lots_inserted=20, lots_updated=5
)
with (
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.settings") as s,
):
s.cian_sweep_ok_not_banned = False
_mark_cian_sweep_abort(mock_db, 44, counters, "cian sweep aborted: 3 consecutive timeouts")
assert mock_runs.mark_banned.called, "flag=False → mark_banned (backward-compat)"
assert not mock_runs.mark_done.called

310
tradein-mvp/backend/tests/test_sweep_drain.py
View file

@ -1,310 +0,0 @@
"""Тесты кооперативного SIGTERM-drain в длинных sweep/full-load циклах (#1182 Phase 3a).
Проверяют, что при выставленном shutdown_requested() sweep'ы scrape_pipeline.py
останавливаются ЧИСТО на своих between-unit checkpoint'ах:
- финализируют run как mark_done(partial) — НЕ mark_cancelled (это не user-cancel);
- НЕ обрабатывают последующие anchor'ы / бакеты / дорогие фазы (IMV / detail);
- user-cancel ветка (is_cancelled) не задействуется.
Тесты network-free: внутренние scrape-вызовы и lifecycle скрейперов monkeypatch'атся,
shutdown_requested() патчится контролируемым флагом, переключающимся в True ПОСЛЕ
первого unit'а (а не с самого старта) — чтобы первый unit прошёл, а на втором
сработал drain.
"""
from __future__ import annotations
import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.services import scrape_pipeline
from app.services.scrapers.avito import AvitoScraper
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.cian import CianScraper
# Два anchor'а: первый успешен, второй должен быть отрезан drain'ом.
TWO_ANCHORS: list[tuple[float, float, str]] = [
(56.8400, 60.6050, "First"),
(56.7950, 60.5300, "Second"),
]
# ── Fake scrape_runs (записывает финализаторы) ──────────────────────────────
class _FakeRuns:
def __init__(self) -> None:
self.heartbeats: list[dict[str, int]] = []
self.done: dict[str, int] | None = None
self.failed: tuple[str, dict[str, int]] | None = None
self.banned: tuple[str, dict[str, int]] | None = None
self.cancelled_calls: int = 0
self.is_cancelled_calls: int = 0
def is_cancelled(self, _db: Any, _run_id: int) -> bool:
self.is_cancelled_calls += 1
return False
def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.heartbeats.append(dict(counters))
def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.done = dict(counters)
def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.failed = (error, dict(counters))
def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.banned = (error, dict(counters))
def mark_cancelled(self, _db: Any, _run_id: int) -> bool:
self.cancelled_calls += 1
return True
def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_cancelled", fake.mark_cancelled)
def _fake_lot(source: str, offer_id: str) -> ScrapedLot:
return ScrapedLot(
source=source,
source_url=f"https://example.ru/{offer_id}/",
source_id=offer_id,
address="Екатеринбург, улица Тестовая, 1",
price_rub=4_000_000,
area_m2=42.0,
rooms=1,
)
# ── Avito city sweep: drain на границе anchor'а ─────────────────────────────
async def test_avito_city_sweep_drains_on_shutdown(monkeypatch: pytest.MonkeyPatch) -> None:
"""SIGTERM после первого anchor'а → sweep останавливается на втором anchor'е:
- второй anchor НЕ фетчится (drain до начала его фаз);
- run финализируется mark_done(partial) с anchors_done==1;
- mark_cancelled НЕ вызывается (это drain, не user-cancel);
- mark_failed / mark_banned не вызываются.
"""
import curl_cffi.requests as _cffi_mod
state = {"flag": False}
monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
fetch_calls: list[str] = []
async def fake_fetch_around(
self: AvitoScraper,
lat: float,
lon: float,
radius_m: int,
pages: int = 1,
delay_override_sec: float | None = None,
) -> list[ScrapedLot]:
fetch_calls.append(f"{lat:.4f}")
lots = [_fake_lot("avito", f"{lat:.4f}-{i}") for i in range(3)]
# SIGTERM приходит ПОСЛЕ первого anchor'а — следующий top-of-loop checkpoint
# увидит флаг и сделает drain.
state["flag"] = True
return lots
monkeypatch.setattr(AvitoScraper, "fetch_around", fake_fetch_around)
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda _db, lots, **_kw: (len(lots), 0))
# Stub shared curl_cffi AsyncSession (non-browser fetch mode).
fake_session = AsyncMock()
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
fake_session.__aexit__ = AsyncMock(return_value=None)
monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)
# IMV-фаза не должна вызываться при drain (enrich_imv=False всё равно).
monkeypatch.setattr(
"app.services.house_imv_backfill.process_houses_imv_batch",
AsyncMock(side_effect=AssertionError("IMV phase must not run after drain")),
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_avito_city_sweep(
db=MagicMock(),
run_id=1,
anchors=TWO_ANCHORS,
enrich_houses=False,
detail_top_n=0,
enrich_imv=False,
request_delay_sec=0.0,
)
# Второй anchor не дошёл до SERP — fetch вызван ровно один раз.
assert fetch_calls == ["56.8400"], f"только первый anchor должен фетчиться; got {fetch_calls}"
# Partial-финализация: обработан только первый anchor.
assert counters.anchors_done == 1
assert counters.lots_fetched == 3
# mark_done(partial) вызван (drain-ветка), и счётчики partial.
assert fake.done is not None, "drain должен вызвать mark_done(partial)"
assert fake.done["anchors_done"] == 1
# Это НЕ user-cancel.
assert fake.cancelled_calls == 0, "mark_cancelled не должен вызываться при drain"
assert fake.failed is None
assert fake.banned is None
# ── Cian city sweep: drain на границе anchor'а ──────────────────────────────
@pytest.fixture
def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
def _init(self: CianScraper) -> None:
self.request_delay_sec = 0.0
self._cffi = None
self._browser = None
async def _aenter(self: CianScraper) -> CianScraper:
return self
async def _aexit(self: CianScraper, *_args: Any) -> None:
return None
monkeypatch.setattr(CianScraper, "__init__", _init)
monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
monkeypatch.setattr(CianScraper, "__aexit__", _aexit)
async def test_cian_city_sweep_drains_on_shutdown(
monkeypatch: pytest.MonkeyPatch, _stub_cian_lifecycle: None
) -> None:
"""SIGTERM после первого anchor'а → cian sweep drain на втором anchor'е → mark_done(partial)."""
state = {"flag": False}
monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())
serp_calls: list[str] = []
async def fake_fetch_multi(
self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
) -> list[ScrapedLot]:
serp_calls.append(f"{lat:.4f}")
state["flag"] = True
return [_fake_lot("cian", f"{lat:.4f}-1")]
monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(
scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
)
monkeypatch.setattr(
"app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None)
)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_cian_city_sweep(
db=MagicMock(),
run_id=2,
anchors=TWO_ANCHORS,
detail_top_n=0,
enrich_houses=False,
request_delay_sec=0.0,
)
assert serp_calls == ["56.8400"], f"только первый anchor должен фетчиться; got {serp_calls}"
assert counters.anchors_done == 1
assert fake.done is not None, "drain должен вызвать mark_done(partial)"
assert fake.done["anchors_done"] == 1
assert fake.cancelled_calls == 0
assert fake.failed is None
assert fake.banned is None
# ── Avito full load: drain через sentinel в on_bucket ───────────────────────
@pytest.fixture
def _stub_avito_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
def _init(self: AvitoScraper) -> None:
self.request_delay_sec = 0.0
self._cffi = None
self._browser = None
async def _aenter(self: AvitoScraper) -> AvitoScraper:
return self
async def _aexit(self: AvitoScraper, *_args: Any) -> None:
return None
monkeypatch.setattr(AvitoScraper, "__init__", _init)
monkeypatch.setattr(AvitoScraper, "__aenter__", _aenter)
monkeypatch.setattr(AvitoScraper, "__aexit__", _aexit)
async def test_avito_full_load_drains_on_shutdown(
monkeypatch: pytest.MonkeyPatch, _stub_avito_lifecycle: None
) -> None:
"""SIGTERM после первого бакета → on_bucket поднимает sentinel → mark_done(partial):
- бакет №2 поднимает RuntimeError("shutdown") (НЕ "cancelled"), бакет №3 не достигается;
- done_buckets == ["b1"] (бакет, упавший на drain-checkpoint, в done не попадает);
- run финализируется mark_done(partial) с unique_fetched==1;
- mark_cancelled / mark_failed / mark_banned не вызываются.
"""
state = {"flag": False}
monkeypatch.setattr(scrape_pipeline, "shutdown_requested", lambda: state["flag"])
saved_buckets: list[str] = []
def fake_save(_db: Any, lots: Any, **_kw: Any) -> tuple[int, int]:
saved_buckets.append("x")
return (len(lots), 0)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
async def fake_fetch_all(self: AvitoScraper, **kwargs: Any) -> None:
on_bucket = kwargs["on_bucket"]
# Бакет №1 — сохраняется штатно.
on_bucket("b1", [_fake_lot("avito", "b1-1")])
# SIGTERM приходит после первого бакета.
state["flag"] = True
# Бакет №2 — on_bucket должен поднять RuntimeError("shutdown").
on_bucket("b2", [_fake_lot("avito", "b2-1")])
# Бакет №3 — НЕ должен быть достигнут.
on_bucket("b3", [_fake_lot("avito", "b3-1")])
monkeypatch.setattr(AvitoScraper, "fetch_all_secondary", fake_fetch_all)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_avito_full_load(
db=MagicMock(),
run_id=3,
request_delay_sec=0.0,
)
# Только первый бакет сохранён (b2 упал на checkpoint до save, b3 не достигнут).
assert saved_buckets == ["x"], f"только b1 должен сохраниться; got {saved_buckets}"
assert counters.unique_fetched == 1
assert counters.saved_inserted == 1
# mark_done(partial) с done_buckets только b1.
assert fake.done is not None, "drain должен вызвать mark_done(partial)"
assert fake.done.get("done_buckets") == ["b1"]
assert fake.done["unique_fetched"] == 1
# НЕ user-cancel, не failed/banned.
assert fake.cancelled_calls == 0
assert fake.failed is None
assert fake.banned is None

230
tradein-mvp/backend/tests/test_sweep_imv_phase.py
View file

@ -1,4 +1,4 @@
"""Tests for IMV enrichment phase integrated into Avito city sweep.
"""Tests for IMV enrichment phase integrated into Avito city sweep (kit).
Проверяет:
1. enrich_imv=True — sweep вызывает IMV-фазу для touched house_id.
@ -9,9 +9,12 @@
6. CitySweepStartRequest принимает enrich_imv (API-level param).
7. Cooperative cancel перед IMV-фазой — skip IMV, sweep marked done.
NB: run_avito_city_sweep использует внутренние _avito_anchor_phases (не run_avito_pipeline).
Тесты патчат AvitoScraper.fetch_around + save_listings + fetch_house_catalog +
save_house_catalog_enrichment для контроля touched_house_ids.
Kit — единственная orchestration-копия (#2397 Part E1 удалил legacy
`app.services.scrape_pipeline`): behavior-тесты (пункты 1,2,3,5,7 выше через
`run_avito_city_sweep`) переведены на `scraper_kit.orchestration.pipeline` с DI
(config/matcher/enrichment/shutdown_requested) вместо прямых импортов `app.*`.
`enrichment.process_houses_imv_batch` — инжектируемый Protocol-метод (было прямым
импортом `app.services.house_imv_backfill.process_houses_imv_batch`).
"""
from __future__ import annotations
@ -19,6 +22,7 @@ from __future__ import annotations
import os
from contextlib import ExitStack
from dataclasses import fields
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
@ -26,10 +30,12 @@ import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
PFX = "scraper_kit.orchestration.pipeline"
def _make_house_lot(offer_id: str, house_path: str) -> Any:
"""ScrapedLot с house_url для тестов house-enrich пути."""
from app.services.scrapers.base import ScrapedLot
from scraper_kit.base import ScrapedLot
return ScrapedLot(
source="avito",
@ -43,16 +49,35 @@ def _make_house_lot(offer_id: str, house_path: str) -> Any:
)
def _config() -> SimpleNamespace:
return SimpleNamespace(
scraper_fetch_mode="curl_cffi",
browser_http_endpoint="http://browser.test/fetch",
scraper_proxy_url=None,
avito_proxy_rotate_url=None,
avito_proxy_max_rotations=0,
avito_serp_ok_not_banned=True,
avito_proxy_rotate_settle_s=0.0,
proxy_rotate_attempts=1,
proxy_rotate_attempt_timeout_s=1.0,
cian_proxy_rotate_url=None,
cian_proxy_max_rotations=0,
yandex_proxy_rotate_url=None,
yandex_proxy_max_rotations=0,
scraper_skip_seen_today=False,
)
def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
"""Входит в набор patch'ей для базового Avito sweep через ExitStack:
- AvitoScraper.fetch_around → lots с house_url (по одному на house_id)
- save_listings → (n, 0)
- fetch_house_catalog → MagicMock()
- save_house_catalog_enrichment → {"house_id": X} по очереди
- curl_cffi.requests.AsyncSession → fake
- AsyncSession (kit-модуль) → fake
- asyncio.sleep → no-op
"""
from app.services.scrapers.avito import AvitoScraper
from scraper_kit.providers.avito.serp import AvitoScraper
lots = [_make_house_lot(f"lot-{hid}", f"/ekaterinburg/houses/test/{hid}") for hid in house_ids]
@ -62,7 +87,7 @@ def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
# save_house_catalog_enrichment вызывается по одному разу на каждый дом
house_id_iter = iter(house_ids)
def fake_save_house(db: Any, enrichment: Any) -> dict:
def fake_save_house(db: Any, enrichment: Any, **_kw: Any) -> dict:
try:
return {"house_id": next(house_id_iter)}
except StopIteration:
@ -73,33 +98,29 @@ def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
fake_session.__aexit__ = AsyncMock(return_value=None)
stack.enter_context(patch.object(AvitoScraper, "fetch_around", fake_fetch_around))
stack.enter_context(
patch(
"app.services.scrape_pipeline.save_listings",
lambda _db, _lots, **_kw: (len(_lots), 0),
)
)
stack.enter_context(
patch(
"app.services.scrape_pipeline.fetch_house_catalog",
AsyncMock(return_value=MagicMock()),
)
)
stack.enter_context(
patch(
"app.services.scrape_pipeline.save_house_catalog_enrichment",
fake_save_house,
)
)
stack.enter_context(patch("curl_cffi.requests.AsyncSession", return_value=fake_session))
stack.enter_context(patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()))
stack.enter_context(patch(f"{PFX}.save_listings", lambda _db, _lots, **_kw: (len(_lots), 0)))
stack.enter_context(patch(f"{PFX}.fetch_house_catalog", AsyncMock(return_value=MagicMock())))
stack.enter_context(patch(f"{PFX}.save_house_catalog_enrichment", fake_save_house))
stack.enter_context(patch(f"{PFX}.AsyncSession", return_value=fake_session))
stack.enter_context(patch(f"{PFX}.asyncio.sleep", AsyncMock()))
def _enter_runs_patches(stack: ExitStack, *, mark_done_mock: Any = None) -> Any:
"""Патчит `scraper_kit.orchestration.pipeline.runs` (is_cancelled/update_heartbeat/
mark_done)."""
mock_runs = MagicMock()
mock_runs.is_cancelled.return_value = False
mock_runs.update_heartbeat = MagicMock()
mock_runs.mark_done = mark_done_mock if mark_done_mock is not None else MagicMock()
stack.enter_context(patch(f"{PFX}.runs", mock_runs))
return mock_runs
# ── CitySweepCounters has imv_* fields ────────────────────────────────────────
def test_city_sweep_counters_has_imv_fields() -> None:
from app.services.scrape_pipeline import CitySweepCounters
from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters()
assert hasattr(c, "imv_attempted")
@ -111,7 +132,7 @@ def test_city_sweep_counters_has_imv_fields() -> None:
def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
from app.services.scrape_pipeline import CitySweepCounters
from scraper_kit.orchestration.pipeline import CitySweepCounters
c = CitySweepCounters(imv_attempted=5, imv_enriched=3, imv_failed=2)
d = c.to_dict()
@ -127,7 +148,7 @@ def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
def test_pipeline_result_has_touched_house_ids() -> None:
from app.services.scrape_pipeline import PipelineCounters, PipelineResult
from scraper_kit.orchestration.pipeline import PipelineCounters, PipelineResult
r = PipelineResult(
anchor_lat=56.84,
@ -143,7 +164,7 @@ def test_pipeline_result_has_touched_house_ids() -> None:
def test_pipeline_result_touched_house_ids_set() -> None:
from app.services.scrape_pipeline import PipelineCounters, PipelineResult
from scraper_kit.orchestration.pipeline import PipelineCounters, PipelineResult
r = PipelineResult(
anchor_lat=56.84,
@ -167,36 +188,31 @@ async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
Патчим AvitoScraper.fetch_around → lots с house_url, save_house_catalog_enrichment
→ {house_id: X} чтобы all_touched_house_ids = {10, 20}.
"""
from app.services.house_imv_backfill import HouseIMVBackfillResult
from app.services.scrape_pipeline import run_avito_city_sweep
from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
mock_imv_result = HouseIMVBackfillResult(checked=2, saved=2, skipped=0, errors=0)
imv_result = SimpleNamespace(checked=2, saved=2, errors=0)
enrichment = MagicMock()
enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_result)
with ExitStack() as stack:
_enter_common_patches(stack, [10, 20])
stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
stack.enter_context(patch("app.services.scrape_runs.mark_done"))
mock_imv_batch = stack.enter_context(
patch(
"app.services.house_imv_backfill.process_houses_imv_batch",
new_callable=AsyncMock,
return_value=mock_imv_result,
)
)
_enter_runs_patches(stack)
counters = await run_avito_city_sweep(
mock_db,
run_id=1,
config=_config(),
matcher=MagicMock(),
enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
detail_top_n=0,
)
mock_imv_batch.assert_awaited_once()
enrichment.process_houses_imv_batch.assert_awaited_once()
# Первый позиционный аргумент — db, второй — house_ids
call_args = mock_imv_batch.call_args
call_args = enrichment.process_houses_imv_batch.call_args
assert call_args.args[1] == {10, 20}
assert counters.imv_attempted == 2
assert counters.imv_enriched == 2
@ -209,31 +225,28 @@ async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
@pytest.mark.asyncio
async def test_sweep_enrich_imv_false_skips_imv() -> None:
"""enrich_imv=False → process_houses_imv_batch НЕ вызывается."""
from app.services.scrape_pipeline import run_avito_city_sweep
from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
enrichment = MagicMock()
enrichment.process_houses_imv_batch = AsyncMock()
with ExitStack() as stack:
_enter_common_patches(stack, [99])
stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
stack.enter_context(patch("app.services.scrape_runs.mark_done"))
mock_imv_batch = stack.enter_context(
patch(
"app.services.house_imv_backfill.process_houses_imv_batch",
new_callable=AsyncMock,
)
)
_enter_runs_patches(stack)
counters = await run_avito_city_sweep(
mock_db,
run_id=2,
config=_config(),
matcher=MagicMock(),
enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=False,
enrich_houses=True,
detail_top_n=0,
)
mock_imv_batch.assert_not_awaited()
enrichment.process_houses_imv_batch.assert_not_awaited()
assert counters.imv_attempted == 0
assert counters.imv_enriched == 0
assert counters.imv_failed == 0
@ -248,8 +261,8 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
Лоты без house_url → unique_house_paths пуст → touched_house_ids пуст.
"""
from app.services.scrape_pipeline import run_avito_city_sweep
from app.services.scrapers.avito import AvitoScraper
from scraper_kit.orchestration.pipeline import run_avito_city_sweep
from scraper_kit.providers.avito.serp import AvitoScraper
mock_db = MagicMock()
fake_session = AsyncMock()
@ -260,29 +273,30 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
async def fake_fetch_around(self: Any, lat: float, lon: float, *_a: Any, **_kw: Any) -> list:
return []
enrichment = MagicMock()
enrichment.process_houses_imv_batch = AsyncMock()
with (
patch.object(AvitoScraper, "fetch_around", fake_fetch_around),
patch("app.services.scrape_pipeline.save_listings", lambda _db, _lots, **_kw: (0, 0)),
patch("curl_cffi.requests.AsyncSession", return_value=fake_session),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_runs.is_cancelled", return_value=False),
patch("app.services.scrape_runs.update_heartbeat"),
patch("app.services.scrape_runs.mark_done"),
patch(
"app.services.house_imv_backfill.process_houses_imv_batch",
new_callable=AsyncMock,
) as mock_imv_batch,
patch(f"{PFX}.save_listings", lambda _db, _lots, **_kw: (0, 0)),
patch(f"{PFX}.AsyncSession", return_value=fake_session),
patch(f"{PFX}.asyncio.sleep", AsyncMock()),
patch(f"{PFX}.runs") as mock_runs,
):
mock_runs.is_cancelled.return_value = False
await run_avito_city_sweep(
mock_db,
run_id=3,
config=_config(),
matcher=MagicMock(),
enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
detail_top_n=0,
)
mock_imv_batch.assert_not_awaited()
enrichment.process_houses_imv_batch.assert_not_awaited()
# ── IMV-ошибки не валят sweep ────────────────────────────────────────────────
@ -291,32 +305,28 @@ async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
@pytest.mark.asyncio
async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
"""Если process_houses_imv_batch raises — sweep всё равно завершается (mark_done)."""
from app.services.scrape_pipeline import run_avito_city_sweep
from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
enrichment = MagicMock()
enrichment.process_houses_imv_batch = AsyncMock(side_effect=RuntimeError("IMV network error"))
with ExitStack() as stack:
_enter_common_patches(stack, [55])
stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
mock_mark_done = stack.enter_context(patch("app.services.scrape_runs.mark_done"))
stack.enter_context(
patch(
"app.services.house_imv_backfill.process_houses_imv_batch",
new_callable=AsyncMock,
side_effect=RuntimeError("IMV network error"),
)
)
mock_runs = _enter_runs_patches(stack)
counters = await run_avito_city_sweep(
mock_db,
run_id=4,
config=_config(),
matcher=MagicMock(),
enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
detail_top_n=0,
)
mock_mark_done.assert_called_once()
mock_runs.mark_done.assert_called_once()
# IMV crash → ошибка считается, но sweep завершается как done
assert counters.errors_count >= 1
@ -326,29 +336,24 @@ async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
@pytest.mark.asyncio
async def test_sweep_imv_failed_counter() -> None:
"""imv_failed отражает errors из HouseIMVBackfillResult."""
from app.services.house_imv_backfill import HouseIMVBackfillResult
from app.services.scrape_pipeline import run_avito_city_sweep
"""imv_failed отражает errors из результата enrichment.process_houses_imv_batch."""
from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
# 3 attempted, 1 enriched, 2 failed
mock_imv_result = HouseIMVBackfillResult(checked=3, saved=1, skipped=0, errors=2)
imv_result = SimpleNamespace(checked=3, saved=1, errors=2)
enrichment = MagicMock()
enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_result)
with ExitStack() as stack:
_enter_common_patches(stack, [1, 2, 3])
stack.enter_context(patch("app.services.scrape_runs.is_cancelled", return_value=False))
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
stack.enter_context(patch("app.services.scrape_runs.mark_done"))
stack.enter_context(
patch(
"app.services.house_imv_backfill.process_houses_imv_batch",
new_callable=AsyncMock,
return_value=mock_imv_result,
)
)
_enter_runs_patches(stack)
counters = await run_avito_city_sweep(
mock_db,
run_id=5,
config=_config(),
matcher=MagicMock(),
enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
@ -368,9 +373,11 @@ async def test_sweep_imv_failed_counter() -> None:
@pytest.mark.asyncio
async def test_sweep_cancel_before_imv_skips_imv() -> None:
"""Если is_cancelled=True в IMV-фазе → IMV пропускается, sweep всё равно done."""
from app.services.scrape_pipeline import run_avito_city_sweep
from scraper_kit.orchestration.pipeline import run_avito_city_sweep
mock_db = MagicMock()
enrichment = MagicMock()
enrichment.process_houses_imv_batch = AsyncMock()
# is_cancelled: первый вызов (anchor loop) = False, второй (IMV phase) = True
cancel_side_effects = [False, True]
@ -384,31 +391,26 @@ async def test_sweep_cancel_before_imv_skips_imv() -> None:
with ExitStack() as stack:
_enter_common_patches(stack, [77])
stack.enter_context(
patch(
"app.services.scrape_runs.is_cancelled",
side_effect=is_cancelled_side_effect,
)
)
stack.enter_context(patch("app.services.scrape_runs.update_heartbeat"))
mock_mark_done = stack.enter_context(patch("app.services.scrape_runs.mark_done"))
mock_imv_batch = stack.enter_context(
patch(
"app.services.house_imv_backfill.process_houses_imv_batch",
new_callable=AsyncMock,
)
)
mock_runs = MagicMock()
mock_runs.is_cancelled.side_effect = is_cancelled_side_effect
mock_runs.update_heartbeat = MagicMock()
mock_runs.mark_done = MagicMock()
stack.enter_context(patch(f"{PFX}.runs", mock_runs))
await run_avito_city_sweep(
mock_db,
run_id=6,
config=_config(),
matcher=MagicMock(),
enrichment=enrichment,
anchors=[(56.84, 60.6, "TestAnchor")],
enrich_imv=True,
enrich_houses=True,
detail_top_n=0,
)
mock_imv_batch.assert_not_awaited()
mock_mark_done.assert_called_once()
enrichment.process_houses_imv_batch.assert_not_awaited()
mock_runs.mark_done.assert_called_once()
# ── process_houses_imv_batch с пустым set ────────────────────────────────────

898
tradein-mvp/backend/tests/test_yandex_city_sweep.py
View file

@ -1,21 +1,19 @@
"""Offline unit tests for run_yandex_city_sweep (SERP+address-enrich, #861).
"""Offline unit tests for YandexCitySweepCounters + admin sweep endpoints (#861).
Pure & fast: NO live network, NO DB. We monkeypatch:
- YandexRealtyScraper.__init__ / __aenter__ / __aexit__ — avoid get_scraper_delay
DB read + real curl_cffi session creation.
- YandexRealtyScraper.fetch_around_multi_room — return a fixed list of fake ScrapedLot.
- scrape_pipeline.save_listings — counter/no-op (no real listings table).
- scrape_runs.* (is_cancelled / update_heartbeat / mark_*) — in-memory fakes.
- curl_cffi.requests.AsyncSession — fake for address-enrich HTTP calls.
Pure & fast: NO live network, NO DB.
Asserts orchestration semantics:
- SERP → save (all anchors, counters aggregated)
- address-enrich фаза вызывается если enrich_address=True и есть source_ids
- enrich_address=False → адрес-фаза пропускается
- cancel → останавливает sweep
- N consecutive SERP failures → mark_banned + abort
- per-item address-enrich error не валит sweep
- mark_done вызывается при нормальном завершении
Legacy `run_yandex_city_sweep` orchestration behavior (SERP→save per-combo,
address-enrich phase, cooperative cancel, consecutive-SERP-failure abort, bounded
retry / combo-skip logging) удалён вместе с `app.services.scrape_pipeline` (#2397
Part E1) — эквивалентная regression-coverage теперь в
`test_scraper_kit_pipeline_parity2.py::test_yandex_city_sweep` (kit
`run_yandex_city_sweep`).
Coverage:
- YandexCitySweepCounters корректно заполняются (dataclass shape)
- Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only)
- Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI
(config/matcher/enrichment)
"""
from __future__ import annotations
@ -24,131 +22,16 @@ import os
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from datetime import UTC
from unittest.mock import AsyncMock
import pytest
from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.yandex_realty import YandexRealtyScraper
# 3 anchors keep tests fast and let us probe cancel/fail semantics.
TEST_ANCHORS: list[tuple[float, float, str]] = [
(56.8400, 60.6050, "A1"),
(56.7950, 60.5300, "A2"),
(56.8970, 60.6100, "A3"),
]
def _fake_lot(offer_id: str) -> ScrapedLot:
"""Minimal valid ScrapedLot (source/source_url required, price_rub > 0)."""
return ScrapedLot(
source="yandex",
source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
source_id=offer_id,
address="Екатеринбург, улица Тестовая",
price_rub=5_000_000,
area_m2=50.0,
rooms=2,
)
class _FakeRuns:
"""In-memory stand-in for app.services.scrape_runs (no DB)."""
def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
# cancel_at_anchor: 1-based anchor index at which is_cancelled flips True.
self.cancel_at_anchor = cancel_at_anchor
self.is_cancelled_calls = 0
self.heartbeats: list[dict[str, int]] = []
self.done: dict[str, int] | None = None
self.failed: tuple[str, dict[str, int]] | None = None
self.banned: tuple[str, dict[str, int]] | None = None
def is_cancelled(self, _db: Any, _run_id: int) -> bool:
self.is_cancelled_calls += 1
if self.cancel_at_anchor is None:
return False
return self.is_cancelled_calls >= self.cancel_at_anchor
def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.heartbeats.append(dict(counters))
def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
self.done = dict(counters)
def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.failed = (error, dict(counters))
def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
self.banned = (error, dict(counters))
@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
"""Kill inter-anchor asyncio.sleep so tests are instant."""
async def _instant(_secs: float) -> None:
return None
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
@pytest.fixture(autouse=True)
def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
"""Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no curl_cffi."""
def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
self.city = city
self.request_delay_sec = 0.0
self._cffi_session = None
self._cookies = {}
async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
return self
async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
return None
monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)
def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)
def _make_db_mock(source_ids: list[str]) -> MagicMock:
"""DB mock для address-enrich фазы: возвращает строки без номера дома."""
mock_db = MagicMock()
rows = [
{
"id": i + 1,
"source_id": sid,
"address": "Екатеринбург, улица Тестовая",
"source_url": f"https://realty.yandex.ru/offer/{sid}/",
}
for i, sid in enumerate(source_ids)
]
mock_db.execute.return_value.mappings.return_value.all.return_value = rows
return mock_db
def _fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
return len(lots), 0
# ── YandexCitySweepCounters ─────────────────────────────────────────────────
def test_counters_defaults() -> None:
from app.services.scrape_pipeline import YandexCitySweepCounters
from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
c = YandexCitySweepCounters()
assert c.anchors_total == 0
@ -163,7 +46,7 @@ def test_counters_defaults() -> None:
def test_counters_to_dict_all_keys() -> None:
from dataclasses import fields
from app.services.scrape_pipeline import YandexCitySweepCounters
from scraper_kit.orchestration.pipeline import YandexCitySweepCounters
c = YandexCitySweepCounters()
d = c.to_dict()
@ -171,423 +54,13 @@ def test_counters_to_dict_all_keys() -> None:
assert set(d.keys()) == expected
# ── Happy path: all anchors iterated, counters aggregated ───────────────────
async def test_sweep_iterates_all_anchors_and_aggregates(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""fetch_around_multi_room вызывается для каждого anchor, counters агрегируются.
FIX: fake_fetch_multi вызывает on_combo (инкрементальный save per-combo) —
имитирует реальное поведение scraper'а. save_listings должен вызываться
on_combo'м, а не одним батчем в конце anchor'а.
"""
serp_calls: list[str] = []
save_calls: list[int] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
on_combo: Any = None,
**_: Any,
) -> list[ScrapedLot]:
serp_calls.append(f"{lat:.4f}")
lots = [_fake_lot(f"{lat}-{i}") for i in range(3)]
# Имитируем один combo — вызываем on_combo если предоставлен.
if on_combo is not None:
on_combo(f"vtorichka/combo_{lat:.4f}", lots)
return lots
def fake_save(
_db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
) -> tuple[int, int]:
save_calls.append(len(lots))
return len(lots), 0
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=1,
anchors=TEST_ANCHORS,
pages_per_anchor=2,
request_delay_sec=0.0,
enrich_address=False,
)
assert len(serp_calls) == len(TEST_ANCHORS)
# save вызывается инкрементально через on_combo — по одному разу на anchor
# (fake имитирует один combo per anchor).
assert len(save_calls) == len(TEST_ANCHORS)
assert counters.anchors_total == len(TEST_ANCHORS)
assert counters.anchors_done == len(TEST_ANCHORS)
assert counters.lots_fetched == len(TEST_ANCHORS) * 3
assert counters.lots_inserted == len(TEST_ANCHORS) * 3
assert counters.lots_updated == 0
assert counters.errors_count == 0
assert counters.address_attempted == 0 # enrich_address=False
assert fake.done is not None
assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3
assert fake.failed is None
assert fake.banned is None
# ── enrich_address=False пропускает address-фазу ────────────────────────────
async def test_enrich_address_false_skips_address_phase(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""enrich_address=False → address-enrich фаза не выполняется."""
address_http_calls: list[str] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
**_: Any,
) -> list[ScrapedLot]:
return [_fake_lot(f"{lat}-0")]
async def fake_cffi_get(url: str, **_: Any) -> Any:
address_http_calls.append(url)
return MagicMock(status_code=200, text="<title>test</title>")
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=2,
anchors=TEST_ANCHORS[:1],
enrich_address=False,
request_delay_sec=0.0,
)
assert len(address_http_calls) == 0
assert counters.address_attempted == 0
assert counters.address_enriched == 0
assert fake.done is not None
# ── address-enrich фаза: счётчики заполняются ───────────────────────────────
async def test_address_enrich_fills_counters(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""address-enrich вызывается для листингов без номера дома; counters заполняются."""
# Fake SERP возвращает 2 листинга с source_id
lots = [_fake_lot("id1"), _fake_lot("id2")]
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
on_combo: Any = None,
**_: Any,
) -> list[ScrapedLot]:
# Вызываем on_combo чтобы anchor_lots накопился (инкрементальный save).
if on_combo is not None:
on_combo("vtorichka/combo_enrich_test", lots)
return lots
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
# DB mock: возвращает 2 строки для address-enrich запроса
mock_db = _make_db_mock(["id1", "id2"])
# Title с полным адресом — должен быть извлечён через _extract_address_from_title
fake_response = MagicMock()
fake_response.status_code = 200
fake_response.text = (
"<title>Продажа квартиры — Екатеринбург, улица Тестовая, 36 — id 123 "
"на Яндекс.Недвижимости</title>"
)
# Мок curl_cffi AsyncSession как context manager
mock_session = AsyncMock()
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
mock_session.__aexit__ = AsyncMock(return_value=None)
mock_session.get = AsyncMock(return_value=fake_response)
import curl_cffi.requests as cffi_mod
monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=mock_db,
run_id=3,
anchors=TEST_ANCHORS[:1],
enrich_address=True,
request_delay_sec=0.0,
)
assert counters.address_attempted == 2
assert counters.address_enriched == 2
assert counters.address_failed == 0
assert fake.done is not None
# ── address-enrich: per-item HTTP error не валит sweep ──────────────────────
async def test_address_enrich_http_error_does_not_abort(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""HTTP 403 при address-enrich → address_failed++, sweep продолжается."""
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
on_combo: Any = None,
**_: Any,
) -> list[ScrapedLot]:
lots = [_fake_lot("id-err")]
if on_combo is not None:
on_combo("vtorichka/combo_http_err", lots)
return lots
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
mock_db = _make_db_mock(["id-err"])
fake_response = MagicMock()
fake_response.status_code = 403
fake_response.text = ""
mock_session = AsyncMock()
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
mock_session.__aexit__ = AsyncMock(return_value=None)
mock_session.get = AsyncMock(return_value=fake_response)
import curl_cffi.requests as cffi_mod
monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=mock_db,
run_id=4,
anchors=TEST_ANCHORS[:1],
enrich_address=True,
request_delay_sec=0.0,
)
assert counters.address_attempted == 1
assert counters.address_failed == 1
assert counters.address_enriched == 0
assert fake.done is not None # sweep завершился несмотря на ошибку
# ── Mid-sweep cancel stops further anchors ──────────────────────────────────
async def test_cancel_midway_stops_remaining_anchors(
monkeypatch: pytest.MonkeyPatch,
) -> None:
serp_calls: list[str] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
**_: Any,
) -> list[ScrapedLot]:
serp_calls.append(f"{lat}")
return [_fake_lot(f"{lat}-0")]
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
# is_cancelled flips True on the 2nd check (before anchor #2) → only anchor #1 runs.
fake = _FakeRuns(cancel_at_anchor=2)
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=5,
anchors=TEST_ANCHORS,
pages_per_anchor=1,
enrich_address=False,
request_delay_sec=0.0,
)
# Only the first anchor fetched before cancel.
assert len(serp_calls) == 1
assert counters.anchors_done == 1
# Cancel path returns early: no mark_done, no ban/fail.
assert fake.done is None
assert fake.failed is None
assert fake.banned is None
# ── Consecutive failures → mark_banned + abort ──────────────────────────────
async def test_consecutive_failures_abort_sweep(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""All anchors raise → after MAX_CONSECUTIVE_FAILURES the sweep aborts + bans."""
attempts: list[str] = []
async def fake_fetch_fail(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
**_: Any,
) -> list[ScrapedLot]:
attempts.append(f"{lat}")
raise RuntimeError("yandex blocked")
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_fail)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
# 5 anchors so the 3-consecutive-failure threshold trips before the list ends.
anchors = [
*TEST_ANCHORS,
(56.7700, 60.5500, "A4"),
(56.8650, 60.6200, "A5"),
]
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=6,
anchors=anchors,
enrich_address=False,
request_delay_sec=0.0,
)
# Abort exactly at the threshold (3 consecutive failures).
assert len(attempts) == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
assert counters.errors_count == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
assert counters.anchors_done == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
assert fake.banned is not None
assert "consecutive" in fake.banned[0]
assert fake.done is None # aborted, never marked done
# ── Non-consecutive failures do not abort ───────────────────────────────────
async def test_non_consecutive_failures_do_not_abort(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A success between two failures resets the consecutive counter → no abort."""
def _name_of(lat: float, anchors: list[tuple[float, float, str]]) -> str:
for a_lat, _lon, nm in anchors:
if a_lat == lat:
return nm
return "?"
anchors = [
*TEST_ANCHORS,
(56.7700, 60.5500, "A4"),
(56.8650, 60.6200, "A5"),
]
failing = {"A1", "A3", "A5"}
async def fake_fetch(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
on_combo: Any = None,
**_: Any,
) -> list[ScrapedLot]:
if _name_of(lat, anchors) in failing:
raise RuntimeError("transient")
lots = [_fake_lot(f"{lat}-0")]
# Вызываем on_combo для инкрементального save.
if on_combo is not None:
on_combo(f"vtorichka/combo_{lat:.4f}", lots)
return lots
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=7,
anchors=anchors,
enrich_address=False,
request_delay_sec=0.0,
)
assert counters.errors_count == 3 # A1, A3, A5
assert counters.anchors_done == 5 # walked the whole list, no early abort
assert counters.lots_fetched == 2 # A2 + A4 contributed
assert fake.banned is None
assert fake.done is not None
# ── mark_done вызывается даже при пустом SERP ───────────────────────────────
async def test_mark_done_always_called(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def fake_fetch_empty(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
**_: Any,
) -> list[ScrapedLot]:
return []
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_empty)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=8,
anchors=TEST_ANCHORS,
enrich_address=False,
request_delay_sec=0.0,
)
assert fake.done is not None
assert counters.lots_fetched == 0
assert counters.anchors_done == len(TEST_ANCHORS)
# ── Admin endpoint tests (offline) ──────────────────────────────────────────
@pytest.fixture
def app_with_admin():
from unittest.mock import MagicMock
from fastapi import FastAPI
from fastapi.testclient import TestClient
@ -680,7 +153,7 @@ def test_yandex_cancel_endpoint_not_running(app_with_admin) -> None:
def test_yandex_list_runs_endpoint(app_with_admin) -> None:
"""List runs endpoint возвращает список."""
from datetime import UTC, datetime
from datetime import datetime
from unittest.mock import patch
fake_rows = [
@ -706,144 +179,7 @@ def test_yandex_list_runs_endpoint(app_with_admin) -> None:
assert "2025-05-01" in body[0]["started_at"]
# ── Center-combos wiring: prod (anchors=None) uses EKB center + combos ──────
async def test_prod_mode_uses_center_anchor_with_combos(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""anchors=None (prod mode) → fetch_around_multi_room called with EKB center coords
AND rooms_list + price_ranges (combos mode), NOT with None/None (legacy mode).
anchors_total == 1 (single center sweep).
"""
from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
calls: list[dict] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
rooms_list: list | None = None,
price_ranges: list | None = None,
on_combo: Any = None,
**_: Any,
) -> list[ScrapedLot]:
lots = [_fake_lot("center-1"), _fake_lot("center-2")]
calls.append(
{
"lat": lat,
"lon": lon,
"radius_m": radius_m,
"rooms_list": rooms_list,
"price_ranges": price_ranges,
"max_pages": max_pages,
}
)
# Имитируем on_combo (инкрементальный save) — один combo.
if on_combo is not None:
on_combo("vtorichka/center_combo", lots)
return lots
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=10,
# anchors=None → prod mode (EKB center)
pages_per_anchor=3,
enrich_address=False,
request_delay_sec=0.0,
)
# Single center anchor
assert counters.anchors_total == 1
assert counters.anchors_done == 1
# fetch_around_multi_room called exactly once
assert len(calls) == 1
call = calls[0]
# EKB center coords
assert abs(call["lat"] - 56.8400) < 0.0001
assert abs(call["lon"] - 60.6050) < 0.0001
# radius covers the whole city (25km default)
assert call["radius_m"] >= 10000
# Combos mode: rooms_list and price_ranges are NOT None
assert call["rooms_list"] is not None
assert call["price_ranges"] is not None
# rooms_list covers all ROOM_PATH keys
assert set(call["rooms_list"]) == set(ROOM_PATH.keys())
# price_ranges matches DEFAULT_PRICE_RANGES
assert call["price_ranges"] == DEFAULT_PRICE_RANGES
# max_pages forwarded from pages_per_anchor
assert call["max_pages"] == 3
# lots aggregated correctly via on_combo
assert counters.lots_fetched == 2
assert counters.lots_inserted == 2
assert fake.done is not None
assert fake.failed is None
async def test_explicit_anchors_still_works(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""anchors=[...] (test/manual override) → fetch called for each anchor, combos mode."""
calls: list[dict] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
rooms_list: list | None = None,
price_ranges: list | None = None,
**_: Any,
) -> list[ScrapedLot]:
calls.append({"lat": lat, "rooms_list": rooms_list, "price_ranges": price_ranges})
return [_fake_lot(f"{lat}")]
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
two_anchors = [(56.8400, 60.6050, "A"), (56.7950, 60.5300, "B")]
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=11,
anchors=two_anchors,
pages_per_anchor=2,
enrich_address=False,
request_delay_sec=0.0,
)
# Both anchors iterated
assert len(calls) == 2
assert counters.anchors_total == 2
assert counters.anchors_done == 2
# Each call uses combos (not None/None)
for c in calls:
assert c["rooms_list"] is not None
assert c["price_ranges"] is not None
assert fake.done is not None
# ── Watchdog timeout: combos vs explicit-anchor mode ───────────────────────
# ── Watchdog timeout: combos-mode budget dwarfs ANCHOR_TIMEOUT_SEC ──────────
def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None:
@ -851,13 +187,14 @@ def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None
Проверяет что вычисленный таймаут для дефолтных 30 combos × 3 pages > 1500s,
т.е. не равен ANCHOR_TIMEOUT_SEC=240 и обеспечивает полный прогон sweep'а.
Не нужны мокапы — только публичные константы и арифметика из модуля.
Не нужны мокапы — только публичные константы и арифметика из kit-модуля.
"""
from app.services.scrape_pipeline import (
from scraper_kit.orchestration.pipeline import (
_YANDEX_ADDRESS_ENRICH_BUDGET_S,
_YANDEX_COMBOS_PER_FETCH_S,
ANCHOR_TIMEOUT_SEC,
)
from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
# Параметры prod-режима
@ -884,188 +221,3 @@ def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None
assert (
sweep_timeout > ANCHOR_TIMEOUT_SEC * 4
), f"sweep_timeout={sweep_timeout:.0f}s should be >> ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s"
async def test_explicit_anchor_mode_uses_anchor_timeout_sec(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""anchors=[...] (explicit override) → wait_for используется с ANCHOR_TIMEOUT_SEC.
Проверяет что в legacy/test режиме (explicit anchors) sweep_timeout не раздут
до combos-значения — он равен ANCHOR_TIMEOUT_SEC, т.к. каждый anchor маленький.
"""
import asyncio as _asyncio
from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC
captured_timeouts: list[float] = []
_orig_wait_for = _asyncio.wait_for
async def _capturing_wait_for(coro, *, timeout): # type: ignore[no-untyped-def]
captured_timeouts.append(timeout)
# На самом деле запустить — мокнутый scraper вернёт [] мгновенно.
return await _orig_wait_for(coro, timeout=timeout)
monkeypatch.setattr(_asyncio, "wait_for", _capturing_wait_for)
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
**_: object,
) -> list[ScrapedLot]:
return []
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=20,
anchors=TEST_ANCHORS[:2], # explicit anchors — legacy mode
pages_per_anchor=2,
enrich_address=False,
request_delay_sec=0.0,
)
# В explicit-anchor режиме каждый anchor должен использовать ANCHOR_TIMEOUT_SEC
assert len(captured_timeouts) == 2
for t in captured_timeouts:
assert (
t == ANCHOR_TIMEOUT_SEC
), f"Expected ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s, got {t}s"
# ── FIX A: инкрементальный save per-combo ────────────────────────────────────
async def test_incremental_save_called_per_combo(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""save_listings вызывается on_combo per-combo, не одним батчем в конце anchor'а.
Имитируем 3 combo через on_combo — save_listings должен быть вызван 3 раза
(по одному на combo) с порциями лотов. Проверяем, что при краше anchor'а
уже сохранённые combo не теряются (данные в БД до finish anchor'а).
"""
save_batches: list[list[str]] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
on_combo: Any = None,
**_: Any,
) -> list[ScrapedLot]:
all_lots = []
for combo_idx in range(3):
combo_lots = [_fake_lot(f"{lat}-combo{combo_idx}-lot0")]
all_lots.extend(combo_lots)
if on_combo is not None:
on_combo(f"vtorichka/combo_{combo_idx}", combo_lots)
return all_lots
def fake_save(
_db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
) -> tuple[int, int]:
save_batches.append([lot.source_id for lot in lots if lot.source_id])
return len(lots), 0
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=30,
anchors=TEST_ANCHORS[:1],
pages_per_anchor=2,
request_delay_sec=0.0,
enrich_address=False,
)
# save_listings вызван ровно 3 раза (по одному на combo), не одним батчем.
assert (
len(save_batches) == 3
), f"Ожидалось 3 save вызова (per-combo), получено {len(save_batches)}: {save_batches}"
# Каждый batch содержит ровно 1 лот (размер combo).
for batch in save_batches:
assert len(batch) == 1, f"Каждый combo-batch должен иметь 1 лот, получено {batch}"
# Суммарные counters корректны.
assert counters.lots_fetched == 3
assert counters.lots_inserted == 3
assert fake.done is not None
# heartbeat вызывался после каждого combo (через on_combo).
assert (
len(fake.heartbeats) >= 3
), f"Ожидалось ≥3 heartbeat обновлений (per-combo), получено {len(fake.heartbeats)}"
# ── FIX B: bounded retry — скипает combo после exhaustion ────────────────────
async def test_bounded_retry_combo_skipped_logged(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""on_combo НЕ вызывается если combo пропущен (все retry page-1 исчерпаны).
Имитируем sweep с двумя combos: первый скипается (on_combo не вызван),
второй успешен (on_combo вызван). Проверяем, что save_listings вызван
только для успешного combo, а не скипнутого.
"""
save_batches: list[list[str]] = []
on_combo_calls: list[str] = []
async def fake_fetch_multi(
self: YandexRealtyScraper,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = 2,
on_combo: Any = None,
**_: Any,
) -> list[ScrapedLot]:
# Combo 0: скипается — on_combo НЕ вызывается (симулирует exhausted retries).
# Combo 1: успешен — on_combo вызывается с лотом.
success_lot = _fake_lot(f"{lat}-success")
if on_combo is not None:
on_combo_calls.append("combo_1_success")
on_combo("vtorichka/combo_1", [success_lot])
return [success_lot]
def fake_save(
_db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
) -> tuple[int, int]:
save_batches.append([lot.source_id for lot in lots if lot.source_id])
return len(lots), 0
monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
fake = _FakeRuns()
_install_runs(monkeypatch, fake)
counters = await scrape_pipeline.run_yandex_city_sweep(
db=MagicMock(),
run_id=31,
anchors=TEST_ANCHORS[:1],
pages_per_anchor=2,
request_delay_sec=0.0,
enrich_address=False,
)
# Только успешный combo вызвал on_combo и save.
assert len(on_combo_calls) == 1
assert len(save_batches) == 1
assert counters.lots_fetched == 1
assert counters.lots_inserted == 1
assert fake.done is not None

7
tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/__init__.py
View file

@ -1,4 +1,4 @@
"""Оркестрация sweep-pipeline scraper_kit — strangler-копия `app.services.scrape_pipeline`.
"""Оркестрация sweep-pipeline scraper_kit (#2135 strangler, завершён #2397 Part E1).
Координирует полный прогон скрапа (SERP → save → houses → detail → IMV) поверх
провайдеров `scraper_kit.providers.*` и базового слоя `scraper_kit.base`. Развязана
@ -10,6 +10,7 @@
- `app.core.shutdown.shutdown_requested` → инжектируемый `Callable[[], bool]`
- `app.services.scrape_runs` → локальная копия `scraper_kit.orchestration.runs`
Боевой scheduler по-прежнему зовёт старый `app.services.scrape_pipeline` (strangler:
COPY, не MOVE). Переключение рантайма на этот модуль — отдельный поздний шаг.
Единственный оркестратор в проде: legacy `app.services.scrape_pipeline` (изначальный
strangler-COPY-источник) удалён (#2397 Part E1). Боевой scheduler и admin-роуты
вызывают этот модуль напрямую.
"""

9
tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py
View file

@ -1,8 +1,9 @@
"""pipeline.py — Avito sweep orchestrator (strangler-копия #2135).
"""pipeline.py — Avito sweep orchestrator (#2135 strangler, единственный в проде с #2397 Part E1).
Strangler-COPY боевого `app.services.scrape_pipeline` (шаг оркестрации, #2135).
Логика фаз/бан-ротации/счётчиков идентична оригиналу — единственное отличие в
развязке от `app.*` через инжекцию (см. `scraper_kit.contracts`):
Изначально был strangler-COPY боевого `app.services.scrape_pipeline` (#2135);
legacy-оригинал удалён (#2397 Part E1) — этот модуль теперь единственная
orchestration-реализация. Развязка от `app.*` через инжекцию (см.
`scraper_kit.contracts`):
- `app.core.config.settings` → аргумент `config: ScraperConfig`
- `app.services.matching` → аргумент `matcher: HouseMatcher`

8
tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/avito/detail.py
View file

@ -385,7 +385,7 @@ async def fetch_detail(
reconnect_on_block — если False, 403/firewall поднимает AvitoBlockedError СРАЗУ
(без холодных reconnect'ов — ими займётся outer-loop через rebuild+rewarm прогретой
сессии); 429 после short-retry поднимает AvitoRateLimitedError. Дефолт True
сохраняет старое поведение own-session / scrape_pipeline путей.
сохраняет старое поведение own-session / shared-session (run_avito_city_sweep) путей.
Raises:
httpx.HTTPError — если status != 200 (через raise_for_status-like).
ValueError — если item_id не извлечён из HTML.
@ -427,8 +427,8 @@ async def fetch_detail(
own_session = cffi_session is None
if cffi_session is None:
# Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed,
# e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy
# from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
# e.g. admin endpoint). Caller-provided sessions (run_avito_city_sweep) already have
# proxy from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение.
session = _build_detail_session(config)
else:
session = cffi_session
@ -442,7 +442,7 @@ async def fetch_detail(
# reconnect_on_block=False (warm-batch #1551): не делаем холодных per-card
# reconnect'ов — блок поднимает AvitoBlockedError сразу, outer-loop пересоздаёт+
# прогревает сессию (свежий exit-IP). reconnect_on_block=True (own-session /
# scrape_pipeline) — старое поведение: backconnect-reconnect эфемерной сессией.
# run_avito_city_sweep) — старое поведение: backconnect-reconnect эфемерной сессией.
backconnect = (
bool(config.scraper_proxy_url if config is not None else None) and reconnect_on_block
)