bot-backend
ab4bc229b2
- _walk_price_range open band: depth guard + progressing anchor (lo+_HIGH_ANCHOR) → forward progress, no infinite recursion / tail-loss on >cap open tail - COUNT: raise DomClickBlockedError on missing snippetsCount (schema regression) - _offer_to_lot: "5+" bucket fallback → rooms=5 (was None) - _parse_page: canary when raw>0 and all geo-dropped (ekb=0) - tests: closed-band bisect + real offset-cap (max offset 1980); studio force rooms=0 over present value; real save_listings-error sweep test - 138 migration header filename 132→138
350 lines
12 KiB
Python
350 lines
12 KiB
Python
"""Offline unit tests for run_domclick_city_sweep orchestration (JSON API rewrite).
|
||
|
||
Pure & fast: NO live network, NO DB.
|
||
|
||
Coverage:
|
||
- fetch_city вызывается, save_listings получает lots с source='domklik'
|
||
- DomClickCitySweepCounters корректно заполняется
|
||
- Cooperative cancel прерывает sweep перед SERP
|
||
- fetch_city exception → errors_count++; 0 lots + errors → mark_failed (#1968)
|
||
- Успешный sweep (lots > 0) → mark_done
|
||
- Пустой SERP (0 lots, 0 errors) → mark_done (не ошибка)
|
||
- DomClickBlockedError в фазе → errors_count++; 0 lots → mark_failed
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from typing import Any
|
||
from unittest.mock import MagicMock
|
||
|
||
import pytest
|
||
|
||
from app.services import scrape_pipeline
|
||
from app.services.scrapers.base import ScrapedLot
|
||
from app.services.scrapers.domclick import DomClickScraper
|
||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||
|
||
|
||
def _fake_lot(offer_id: str) -> ScrapedLot:
|
||
return ScrapedLot(
|
||
source="domklik",
|
||
source_url=f"https://domclick.ru/card/sale__flat__{offer_id}",
|
||
source_id=offer_id,
|
||
address="Екатеринбург, улица Тестовая, 1",
|
||
price_rub=5_000_000,
|
||
area_m2=50.0,
|
||
rooms=2,
|
||
lat=56.83,
|
||
lon=60.61,
|
||
)
|
||
|
||
|
||
class _FakeRuns:
|
||
"""Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel."""
|
||
|
||
def __init__(self, *, cancel: bool = False) -> None:
|
||
self.cancel = cancel
|
||
self.heartbeats: list[dict[str, int]] = []
|
||
self.done: dict[str, int] | None = None
|
||
self.failed: tuple[str, dict[str, int]] | None = None
|
||
|
||
def is_cancelled(self, _db: Any, _run_id: int) -> bool:
|
||
return self.cancel
|
||
|
||
def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
|
||
self.heartbeats.append(dict(counters))
|
||
|
||
def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
|
||
self.done = dict(counters)
|
||
|
||
def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
|
||
self.failed = (error, dict(counters))
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
async def _instant(_secs: float) -> None:
|
||
return None
|
||
|
||
monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _stub_domclick_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Нейтрализуем DomClickScraper.__init__/__aenter__/__aexit__ — нет DB/browser."""
|
||
|
||
def _init(self: DomClickScraper) -> None:
|
||
self.request_delay_sec = 0.0
|
||
self.parse_failures = 0
|
||
self._browser = None
|
||
|
||
async def _aenter(self: DomClickScraper) -> DomClickScraper:
|
||
return self
|
||
|
||
async def _aexit(self: DomClickScraper, *_args: Any) -> None:
|
||
return None
|
||
|
||
monkeypatch.setattr(DomClickScraper, "__init__", _init)
|
||
monkeypatch.setattr(DomClickScraper, "__aenter__", _aenter)
|
||
monkeypatch.setattr(DomClickScraper, "__aexit__", _aexit)
|
||
|
||
|
||
def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
|
||
monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
|
||
|
||
|
||
# ── DomClickCitySweepCounters ───────────────────────────────────────────────
|
||
|
||
|
||
def test_domclick_sweep_counters_defaults() -> None:
|
||
from app.services.scrape_pipeline import DomClickCitySweepCounters
|
||
|
||
c = DomClickCitySweepCounters()
|
||
assert c.lots_fetched == 0
|
||
assert c.lots_inserted == 0
|
||
assert c.lots_updated == 0
|
||
assert c.pages_fetched == 0
|
||
assert c.errors_count == 0
|
||
|
||
|
||
def test_domclick_sweep_counters_to_dict_all_keys() -> None:
|
||
from dataclasses import fields
|
||
|
||
from app.services.scrape_pipeline import DomClickCitySweepCounters
|
||
|
||
c = DomClickCitySweepCounters()
|
||
d = c.to_dict()
|
||
expected = {f.name for f in fields(c)}
|
||
assert set(d.keys()) == expected
|
||
|
||
|
||
# ── Full sweep — фазы и агрегация ──────────────────────────────────────────
|
||
|
||
|
||
async def test_sweep_fetches_and_saves_calls_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""fetch_city → 2 lots → save_listings → mark_done (не mark_failed)."""
|
||
fetch_calls: list[tuple[int, int]] = []
|
||
save_calls: list[list[ScrapedLot]] = []
|
||
|
||
async def fake_fetch_city(
|
||
self: DomClickScraper,
|
||
city_id: int,
|
||
rooms: list[int] | None = None,
|
||
pages: int = 100,
|
||
) -> list[ScrapedLot]:
|
||
fetch_calls.append((city_id, pages))
|
||
return [_fake_lot("100"), _fake_lot("101")]
|
||
|
||
def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
|
||
save_calls.append(list(lots))
|
||
return len(lots), 0
|
||
|
||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_city)
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_domclick_city_sweep(
|
||
db=MagicMock(),
|
||
run_id=1,
|
||
city_id=4,
|
||
rooms=[1, 2],
|
||
pages=3,
|
||
request_delay_sec=0.0,
|
||
)
|
||
|
||
assert len(fetch_calls) == 1
|
||
assert fetch_calls[0] == (4, 3)
|
||
assert len(save_calls) == 1
|
||
assert len(save_calls[0]) == 2
|
||
assert all(lot.source == "domklik" for lot in save_calls[0])
|
||
assert counters.lots_fetched == 2
|
||
assert counters.lots_inserted == 2
|
||
assert counters.lots_updated == 0
|
||
assert counters.errors_count == 0
|
||
# Успешный sweep → mark_done, НЕ mark_failed
|
||
assert fake.done is not None
|
||
assert fake.done["lots_fetched"] == 2
|
||
assert fake.failed is None
|
||
|
||
|
||
async def test_sweep_empty_serp_no_errors_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Пустой SERP (0 lots, 0 errors) → mark_done (не ошибка, не mark_failed)."""
|
||
save_called = False
|
||
|
||
async def fake_fetch_empty(
|
||
self: DomClickScraper,
|
||
city_id: int,
|
||
rooms: list[int] | None = None,
|
||
pages: int = 100,
|
||
) -> list[ScrapedLot]:
|
||
return []
|
||
|
||
def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
|
||
nonlocal save_called
|
||
save_called = True
|
||
return 0, 0
|
||
|
||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_empty)
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_domclick_city_sweep(
|
||
db=MagicMock(), run_id=2, request_delay_sec=0.0
|
||
)
|
||
|
||
assert save_called is False
|
||
assert counters.lots_fetched == 0
|
||
assert counters.errors_count == 0
|
||
# 0 lots + 0 errors → mark_done (не mark_failed)
|
||
assert fake.done is not None
|
||
assert fake.failed is None
|
||
|
||
|
||
async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Cooperative cancel перед SERP → fetch_city НЕ вызывается, mark_done НЕ вызывается."""
|
||
fetch_called = False
|
||
|
||
async def fake_fetch(
|
||
self: DomClickScraper,
|
||
city_id: int,
|
||
rooms: list[int] | None = None,
|
||
pages: int = 100,
|
||
) -> list[ScrapedLot]:
|
||
nonlocal fetch_called
|
||
fetch_called = True
|
||
return [_fake_lot("1")]
|
||
|
||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch)
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
|
||
|
||
fake = _FakeRuns(cancel=True)
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_domclick_city_sweep(
|
||
db=MagicMock(), run_id=3, request_delay_sec=0.0
|
||
)
|
||
|
||
assert fetch_called is False
|
||
assert counters.lots_fetched == 0
|
||
assert fake.done is None
|
||
|
||
|
||
async def test_fetch_city_exception_triggers_mark_failed(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Исключение в fetch_city → errors_count++; 0 lots + errors → mark_failed (#1968)."""
|
||
|
||
async def fake_fetch_fail(
|
||
self: DomClickScraper,
|
||
city_id: int,
|
||
rooms: list[int] | None = None,
|
||
pages: int = 100,
|
||
) -> list[ScrapedLot]:
|
||
raise RuntimeError("network error")
|
||
|
||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_fail)
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_domclick_city_sweep(
|
||
db=MagicMock(), run_id=4, request_delay_sec=0.0
|
||
)
|
||
|
||
assert counters.errors_count == 1
|
||
assert counters.lots_fetched == 0
|
||
# 0 lots + errors → mark_failed, НЕ mark_done
|
||
assert fake.failed is not None, "mark_failed должен быть вызван при 0 lots + errors"
|
||
assert fake.done is None
|
||
|
||
|
||
async def test_domclick_blocked_triggers_mark_failed(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""DomClickBlockedError в fetch_city → errors_count++; 0 lots → mark_failed."""
|
||
|
||
async def fake_fetch_blocked(
|
||
self: DomClickScraper,
|
||
city_id: int,
|
||
rooms: list[int] | None = None,
|
||
pages: int = 100,
|
||
) -> list[ScrapedLot]:
|
||
raise DomClickBlockedError("QRATOR blocked")
|
||
|
||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_blocked)
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_domclick_city_sweep(
|
||
db=MagicMock(), run_id=5, request_delay_sec=0.0
|
||
)
|
||
|
||
assert counters.errors_count == 1
|
||
assert counters.lots_fetched == 0
|
||
assert fake.failed is not None
|
||
assert fake.done is None
|
||
|
||
|
||
async def test_save_listings_error_after_fetch_records_error(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""save_listings бросает ПОСЛЕ успешного fetch → ошибка реально учитывается.
|
||
|
||
Раньше этот тест был no-op дубликатом happy-path (errors никогда не возникали).
|
||
Теперь инжектим настоящую ошибку в save_listings и проверяем, что она
|
||
проходит через перехват SERP-фазы (errors_count++).
|
||
|
||
NB: save_listings вызывается ВНУТРИ _domclick_phase под asyncio.wait_for, и
|
||
исключение ловится внутренним `except Exception` (scrape_pipeline.py ~3205),
|
||
БЕЗ re-raise. lots_fetched инкрементируется ДО save (~3187), поэтому
|
||
honest-status (lots_fetched==0 and errors_count>0) ложен и run финализируется
|
||
как mark_done. Реальный путь mark_failed+re-raise — только для исключений,
|
||
утекающих во ВНЕШНИЙ except (например fetch_city), что уже покрыто
|
||
test_fetch_city_exception_triggers_mark_failed.
|
||
"""
|
||
fetch_calls = [0]
|
||
|
||
async def fake_fetch(
|
||
self: DomClickScraper,
|
||
city_id: int,
|
||
rooms: list[int] | None = None,
|
||
pages: int = 100,
|
||
) -> list[ScrapedLot]:
|
||
fetch_calls[0] += 1
|
||
return [_fake_lot("200")]
|
||
|
||
def fake_save_raises(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
|
||
raise RuntimeError("save_listings DB error")
|
||
|
||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch)
|
||
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_raises)
|
||
|
||
fake = _FakeRuns()
|
||
_install_runs(monkeypatch, fake)
|
||
|
||
counters = await scrape_pipeline.run_domclick_city_sweep(
|
||
db=MagicMock(), run_id=6, request_delay_sec=0.0
|
||
)
|
||
|
||
# fetch отдал лот, save упал — ошибка учтена внутренним перехватом фазы.
|
||
assert fetch_calls[0] == 1
|
||
assert counters.lots_fetched == 1
|
||
assert counters.errors_count == 1
|
||
assert counters.lots_inserted == 0
|
||
# lots_fetched>0 → honest-status → mark_done (save-ошибка не утекает в outer except).
|
||
assert fake.done is not None
|
||
assert fake.failed is None
|