"""Offline unit tests for run_domclick_detail_backfill (Layer B orchestrator). Pure & fast: NO live network, NO DB, NO browser. Mirrors test_domclick_sweep mocking style. Mocks fetch_detail (AsyncMock) + save_detail_enrichment + scrape_runs lifecycle + a fake snapshot of listing rows via db.execute(...).mappings().all(). Coverage: - happy path: both rows enriched -> mark_done(attempted=2, enriched=2) - block-abort: DomClickBlockedError x max_consecutive_blocks -> abort, mark_done (not failed) - parse-failure: DomClickParseError for one row -> failed++, db.rollback, other still enriched - cancel before loop -> returns early (no snapshot SELECT, no mark_done) """ from __future__ import annotations import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError from app.tasks import domclick_detail_backfill as mod from app.tasks.domclick_detail_backfill import run_domclick_detail_backfill class _FakeRuns: """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel.""" def __init__(self, *, cancel: bool = False) -> None: self.cancel = cancel self.heartbeats: list[dict[str, int]] = [] self.done: dict[str, int] | None = None self.failed: tuple[str, dict[str, int]] | None = None def is_cancelled(self, _db: Any, _run_id: int) -> bool: return self.cancel def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.heartbeats.append(dict(counters)) def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.done = dict(counters) def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: self.failed = (error, dict(counters)) class _FakeBrowserFetcher: """No-op async context manager standing in for BrowserFetcher (no browser launch).""" def __init__(self, *args: Any, **kwargs: Any) -> None: pass async def __aenter__(self) -> _FakeBrowserFetcher: return self async def __aexit__(self, *_args: Any) -> None: return None def _fake_db(rows: list[dict[str, Any]]) -> MagicMock: """MagicMock Session whose execute(...).mappings().all() returns rows.""" db = MagicMock() db.execute.return_value.mappings.return_value.all.return_value = rows return db def _row(listing_id: int, offer_id: str) -> dict[str, Any]: return { "id": listing_id, "source_id": offer_id, "source_url": f"https://ekaterinburg.domclick.ru/card/sale__flat__{offer_id}", } @pytest.fixture(autouse=True) def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: async def _instant(_secs: float) -> None: return None monkeypatch.setattr(mod.asyncio, "sleep", _instant) @pytest.fixture(autouse=True) def _stub_browser(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(mod, "BrowserFetcher", _FakeBrowserFetcher) def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None: monkeypatch.setattr(mod.runs_mod, "is_cancelled", fake.is_cancelled) monkeypatch.setattr(mod.runs_mod, "update_heartbeat", fake.update_heartbeat) monkeypatch.setattr(mod.runs_mod, "mark_done", fake.mark_done) monkeypatch.setattr(mod.runs_mod, "mark_failed", fake.mark_failed) async def test_happy_path_enriches_all(monkeypatch: pytest.MonkeyPatch) -> None: """Both rows enriched → mark_done(attempted=2, enriched=2), save called per row.""" db = _fake_db([_row(1, "100"), _row(2, "101")]) fetch_mock = AsyncMock(return_value=MagicMock()) save_mock = MagicMock(return_value=True) monkeypatch.setattr(mod, "fetch_detail", fetch_mock) monkeypatch.setattr(mod, "save_detail_enrichment", save_mock) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await run_domclick_detail_backfill(db, run_id=1, params={"request_delay_sec": 0}) assert counters.attempted == 2 assert counters.enriched == 2 assert counters.blocked == 0 assert counters.failed == 0 assert fetch_mock.await_count == 2 assert save_mock.call_count == 2 assert fake.done is not None assert fake.done["attempted"] == 2 assert fake.done["enriched"] == 2 assert fake.failed is None async def test_block_abort_marks_done_not_failed(monkeypatch: pytest.MonkeyPatch) -> None: """DomClickBlockedError max_consecutive_blocks in a row → abort, mark_done (not failed).""" db = _fake_db([_row(i, str(i)) for i in range(1, 11)]) # 10 rows available fetch_mock = AsyncMock(side_effect=DomClickBlockedError("datadome challenge")) save_mock = MagicMock(return_value=True) monkeypatch.setattr(mod, "fetch_detail", fetch_mock) monkeypatch.setattr(mod, "save_detail_enrichment", save_mock) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await run_domclick_detail_backfill( db, run_id=2, params={"request_delay_sec": 0, "max_consecutive_blocks": 5} ) # Abort after 5 consecutive blocks → attempted=5, all blocked, none enriched. assert counters.blocked == 5 assert counters.attempted == 5 assert counters.enriched == 0 assert save_mock.call_count == 0 # Block is transient → mark_done, NOT mark_failed. assert fake.done is not None assert fake.done["blocked"] == 5 assert fake.failed is None async def test_parse_failure_rolls_back_and_continues(monkeypatch: pytest.MonkeyPatch) -> None: """DomClickParseError on row 1 → failed++, db.rollback; row 2 still enriched; mark_done.""" db = _fake_db([_row(1, "100"), _row(2, "101")]) # First row -> parse error; second -> ok. fetch_mock = AsyncMock(side_effect=[DomClickParseError("ssr schema drift"), MagicMock()]) save_mock = MagicMock(return_value=True) monkeypatch.setattr(mod, "fetch_detail", fetch_mock) monkeypatch.setattr(mod, "save_detail_enrichment", save_mock) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await run_domclick_detail_backfill(db, run_id=3, params={"request_delay_sec": 0}) assert counters.attempted == 2 assert counters.failed == 1 assert counters.enriched == 1 db.rollback.assert_called() # parse-failure rolled back the un-committed row assert save_mock.call_count == 1 # only the second (parsed) row persisted assert fake.done is not None assert fake.failed is None async def test_cancel_before_loop_returns_early(monkeypatch: pytest.MonkeyPatch) -> None: """Cooperative cancel before loop → no snapshot SELECT, no fetch, no mark_done.""" db = _fake_db([_row(1, "100")]) fetch_mock = AsyncMock(return_value=MagicMock()) save_mock = MagicMock(return_value=True) monkeypatch.setattr(mod, "fetch_detail", fetch_mock) monkeypatch.setattr(mod, "save_detail_enrichment", save_mock) fake = _FakeRuns(cancel=True) _install_runs(monkeypatch, fake) counters = await run_domclick_detail_backfill(db, run_id=4, params={"request_delay_sec": 0}) assert counters.attempted == 0 assert fetch_mock.await_count == 0 db.execute.assert_not_called() # cancel short-circuits before the snapshot SELECT assert fake.done is None assert fake.failed is None