gendesign/tradein-mvp/backend/tests/test_domclick_detail_backfill.py

"""Offline unit tests for run_domclick_detail_backfill (Layer B orchestrator).

Pure & fast: NO live network, NO DB, NO browser. Mirrors test_domclick_sweep mocking
style. Mocks fetch_detail (AsyncMock) + save_detail_enrichment + scrape_runs lifecycle
+ a fake snapshot of listing rows via db.execute(...).mappings().all().

Coverage:
  - happy path: both rows enriched -> mark_done(attempted=2, enriched=2)
  - block-abort: DomClickBlockedError x max_consecutive_blocks -> abort, mark_done (not failed)
  - parse-failure: DomClickParseError for one row -> failed++, db.rollback, other still enriched
  - cancel before loop -> returns early (no snapshot SELECT, no mark_done)
"""

from __future__ import annotations

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from typing import Any
from unittest.mock import AsyncMock, MagicMock

import pytest

from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError
from app.tasks import domclick_detail_backfill as mod
from app.tasks.domclick_detail_backfill import run_domclick_detail_backfill


class _FakeRuns:
    """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel."""

    def __init__(self, *, cancel: bool = False) -> None:
        self.cancel = cancel
        self.heartbeats: list[dict[str, int]] = []
        self.done: dict[str, int] | None = None
        self.failed: tuple[str, dict[str, int]] | None = None

    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
        return self.cancel

    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.heartbeats.append(dict(counters))

    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.done = dict(counters)

    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.failed = (error, dict(counters))


class _FakeBrowserFetcher:
    """No-op async context manager standing in for BrowserFetcher (no browser launch)."""

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        pass

    async def __aenter__(self) -> _FakeBrowserFetcher:
        return self

    async def __aexit__(self, *_args: Any) -> None:
        return None


def _fake_db(rows: list[dict[str, Any]]) -> MagicMock:
    """MagicMock Session whose execute(...).mappings().all() returns rows."""
    db = MagicMock()
    db.execute.return_value.mappings.return_value.all.return_value = rows
    return db


def _row(listing_id: int, offer_id: str) -> dict[str, Any]:
    return {
        "id": listing_id,
        "source_id": offer_id,
        "source_url": f"https://ekaterinburg.domclick.ru/card/sale__flat__{offer_id}",
    }


@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
    async def _instant(_secs: float) -> None:
        return None

    monkeypatch.setattr(mod.asyncio, "sleep", _instant)


@pytest.fixture(autouse=True)
def _stub_browser(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(mod, "BrowserFetcher", _FakeBrowserFetcher)


def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
    monkeypatch.setattr(mod.runs_mod, "is_cancelled", fake.is_cancelled)
    monkeypatch.setattr(mod.runs_mod, "update_heartbeat", fake.update_heartbeat)
    monkeypatch.setattr(mod.runs_mod, "mark_done", fake.mark_done)
    monkeypatch.setattr(mod.runs_mod, "mark_failed", fake.mark_failed)


async def test_happy_path_enriches_all(monkeypatch: pytest.MonkeyPatch) -> None:
    """Both rows enriched → mark_done(attempted=2, enriched=2), save called per row."""
    db = _fake_db([_row(1, "100"), _row(2, "101")])

    fetch_mock = AsyncMock(return_value=MagicMock())
    save_mock = MagicMock(return_value=True)
    monkeypatch.setattr(mod, "fetch_detail", fetch_mock)
    monkeypatch.setattr(mod, "save_detail_enrichment", save_mock)

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await run_domclick_detail_backfill(db, run_id=1, params={"request_delay_sec": 0})

    assert counters.attempted == 2
    assert counters.enriched == 2
    assert counters.blocked == 0
    assert counters.failed == 0
    assert fetch_mock.await_count == 2
    assert save_mock.call_count == 2
    assert fake.done is not None
    assert fake.done["attempted"] == 2
    assert fake.done["enriched"] == 2
    assert fake.failed is None


async def test_block_abort_marks_done_not_failed(monkeypatch: pytest.MonkeyPatch) -> None:
    """DomClickBlockedError max_consecutive_blocks in a row → abort, mark_done (not failed)."""
    db = _fake_db([_row(i, str(i)) for i in range(1, 11)])  # 10 rows available

    fetch_mock = AsyncMock(side_effect=DomClickBlockedError("datadome challenge"))
    save_mock = MagicMock(return_value=True)
    monkeypatch.setattr(mod, "fetch_detail", fetch_mock)
    monkeypatch.setattr(mod, "save_detail_enrichment", save_mock)

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await run_domclick_detail_backfill(
        db, run_id=2, params={"request_delay_sec": 0, "max_consecutive_blocks": 5}
    )

    # Abort after 5 consecutive blocks → attempted=5, all blocked, none enriched.
    assert counters.blocked == 5
    assert counters.attempted == 5
    assert counters.enriched == 0
    assert save_mock.call_count == 0
    # Block is transient → mark_done, NOT mark_failed.
    assert fake.done is not None
    assert fake.done["blocked"] == 5
    assert fake.failed is None


async def test_parse_failure_rolls_back_and_continues(monkeypatch: pytest.MonkeyPatch) -> None:
    """DomClickParseError on row 1 → failed++, db.rollback; row 2 still enriched; mark_done."""
    db = _fake_db([_row(1, "100"), _row(2, "101")])

    # First row -> parse error; second -> ok.
    fetch_mock = AsyncMock(side_effect=[DomClickParseError("ssr schema drift"), MagicMock()])
    save_mock = MagicMock(return_value=True)
    monkeypatch.setattr(mod, "fetch_detail", fetch_mock)
    monkeypatch.setattr(mod, "save_detail_enrichment", save_mock)

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await run_domclick_detail_backfill(db, run_id=3, params={"request_delay_sec": 0})

    assert counters.attempted == 2
    assert counters.failed == 1
    assert counters.enriched == 1
    db.rollback.assert_called()  # parse-failure rolled back the un-committed row
    assert save_mock.call_count == 1  # only the second (parsed) row persisted
    assert fake.done is not None
    assert fake.failed is None


async def test_cancel_before_loop_returns_early(monkeypatch: pytest.MonkeyPatch) -> None:
    """Cooperative cancel before loop → no snapshot SELECT, no fetch, no mark_done."""
    db = _fake_db([_row(1, "100")])

    fetch_mock = AsyncMock(return_value=MagicMock())
    save_mock = MagicMock(return_value=True)
    monkeypatch.setattr(mod, "fetch_detail", fetch_mock)
    monkeypatch.setattr(mod, "save_detail_enrichment", save_mock)

    fake = _FakeRuns(cancel=True)
    _install_runs(monkeypatch, fake)

    counters = await run_domclick_detail_backfill(db, run_id=4, params={"request_delay_sec": 0})

    assert counters.attempted == 0
    assert fetch_mock.await_count == 0
    db.execute.assert_not_called()  # cancel short-circuits before the snapshot SELECT
    assert fake.done is None
    assert fake.failed is None