gendesign/tradein-mvp/backend/tests/tasks/test_domclick_detail_backfill.py

"""Tests для DomClick detail-backfill orchestrator (issue #2000 Layer B).

Зеркалит конвенции test_avito_detail_backfill.py: module-level patch-target строки,
_mock_db(snapshot) helper, runs = MagicMock() assertions. DomClick-специфика:
единственный BrowserFetcher (нет curl-fallback), cookies из domclick_session_svc
(мокается целиком как модуль), эксепшн-триада DomClickBlockedError/DomClickParseError
вместо Avito's Blocked/RateLimited/ListingGone.
"""

from __future__ import annotations

import logging
import os
import sys
from datetime import UTC, datetime, timedelta
from unittest.mock import AsyncMock, MagicMock, patch

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

_wp_mock = MagicMock()
sys.modules.setdefault("weasyprint", _wp_mock)

import pytest  # noqa: E402
from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError  # noqa: E402

from app.core import shutdown as _sd  # noqa: E402
from app.tasks.domclick_detail_backfill import (  # noqa: E402
    DomClickDetailBackfillResult,
    run_domclick_detail_backfill,
)


@pytest.fixture(autouse=True)
def _reset_shutdown() -> None:
    """shutdown — module-global Event: чистим вокруг каждого теста (изоляция #1182)."""
    _sd.reset_shutdown()
    yield
    _sd.reset_shutdown()


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _make_snapshot(n: int) -> list[dict]:
    return [
        {
            "id": i + 1,
            "source_url": f"https://ekaterinburg.domclick.ru/card/sale__flat__{i + 1}",
        }
        for i in range(n)
    ]


def _mock_db(snapshot: list[dict]) -> MagicMock:
    """Fake Session: first execute() returns snapshot via .mappings().all()."""
    db = MagicMock()
    sel = MagicMock()
    sel.mappings.return_value.all.return_value = snapshot
    db.execute.return_value = sel
    return db


def _mock_session_svc(cookies: dict[str, str] | None) -> MagicMock:
    """Fake domclick_session модуль: load_session + срок годности кук (#2674).

    session_expires_at по умолчанию далеко в будущем — иначе каждый тест ловил бы
    предупреждение «куки скоро протухнут». Отдельно оно проверяется в
    tests/test_alerts_become_events.py.
    """
    svc = MagicMock()
    svc.load_session.return_value = cookies
    svc.COOKIE_EXPIRY_WARN_DAYS = 5
    svc.session_expires_at.return_value = datetime.now(tz=UTC) + timedelta(days=30)
    return svc


def _mock_browser_fetcher_cls() -> MagicMock:
    """MagicMock class whose instance is a working async context manager."""
    instance = AsyncMock()
    instance.__aenter__ = AsyncMock(return_value=instance)
    instance.__aexit__ = AsyncMock(return_value=False)
    # request_context_reset (#3118) — синхронный метод, не awaited caller'ом; явный
    # MagicMock вместо auto-AsyncMock-атрибута, чтобы вызов не оставлял неawaited
    # coroutine (RuntimeWarning в тестах с блоками).
    instance.request_context_reset = MagicMock()
    return MagicMock(return_value=instance)


_FETCH = "app.tasks.domclick_detail_backfill.fetch_detail"
_SAVE = "app.tasks.domclick_detail_backfill.save_detail_enrichment"
_RUNS = "app.tasks.domclick_detail_backfill.runs_mod"
_SLEEP = "app.tasks.domclick_detail_backfill.asyncio.sleep"
_SETTINGS = "app.tasks.domclick_detail_backfill.settings"
_SHUTDOWN = "app.tasks.domclick_detail_backfill.shutdown_requested"
_BROWSER_FETCHER = "app.tasks.domclick_detail_backfill.BrowserFetcher"
_SESSION_SVC = "app.tasks.domclick_detail_backfill.domclick_session_svc"

# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_backfill_empty_snapshot_marks_done() -> None:
    """Empty snapshot -> mark_done immediately, no fetch calls, no BrowserFetcher open."""
    db = _mock_db([])
    runs = MagicMock()
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc({"CAS_ID": "123"})
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch(_FETCH) as mock_fetch,
    ):
        result = await run_domclick_detail_backfill(
            db, run_id=1, params={"batch_size": 10, "budget_sec": 60}
        )

    assert isinstance(result, DomClickDetailBackfillResult)
    assert result.attempted == 0
    assert result.enriched == 0
    mock_fetch.assert_not_called()
    mock_bf_cls.assert_not_called()
    runs.mark_done.assert_called_once()
    runs.mark_failed.assert_not_called()


@pytest.mark.asyncio
async def test_backfill_processes_snapshot_with_cookies_threaded() -> None:
    """3 listings -> all fetched+enriched, cookies from load_session() threaded into
    every fetch_detail() call kwargs, mark_done called."""
    snapshot = _make_snapshot(3)
    db = _mock_db(snapshot)
    runs = MagicMock()
    mock_enrichment = MagicMock()
    mock_fetch = AsyncMock(return_value=mock_enrichment)
    mock_save = MagicMock(return_value=True)
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    fake_cookies = {"CAS_ID": "999", "qrator_jsid2": "abc"}
    mock_svc = _mock_session_svc(fake_cookies)
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch(_FETCH, mock_fetch),
        patch(_SAVE, mock_save),
        patch(_SLEEP, new_callable=AsyncMock),
    ):
        result = await run_domclick_detail_backfill(
            db, run_id=2, params={"batch_size": 10, "budget_sec": 3600}
        )

    assert result.attempted == 3
    assert result.enriched == 3
    assert result.blocked == 0
    assert result.failed == 0
    assert mock_fetch.call_count == 3
    # #3197: конструктор с 08.2026 несёт ещё proxy_provider/use_pool/environment
    # (проводка прокси-пула) -- здесь проверяем только то, что нужно этому тесту,
    # полная проводка покрыта tests/test_3197_domclick_proxy_pool_wiring.py.
    mock_bf_cls.assert_called_once()
    _, bf_kwargs = mock_bf_cls.call_args
    assert bf_kwargs["source"] == "domclick"
    assert bf_kwargs["endpoint"] == fake_settings.browser_http_endpoint
    assert bf_kwargs["reuse_context"] is True
    for call in mock_fetch.call_args_list:
        _, kwargs = call
        assert kwargs.get("cookies") == fake_cookies
        assert kwargs.get("browser_fetcher") is not None
    runs.mark_backfill_finished.assert_called_once()
    runs.mark_failed.assert_not_called()


@pytest.mark.asyncio
async def test_backfill_cookies_none_still_proceeds_with_error_alert(caplog) -> None:
    """No valid session (load_session()->None) -> run still proceeds (fetch_detail
    called with cookies=None), но сигнал теперь ERROR, а не WARNING (#2674): в
    скрапер-контейнере событием GlitchTip становится только ERROR.
    """
    snapshot = _make_snapshot(1)
    db = _mock_db(snapshot)
    runs = MagicMock()
    mock_enrichment = MagicMock()
    mock_fetch = AsyncMock(return_value=mock_enrichment)
    mock_save = MagicMock(return_value=True)
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc(None)
    mock_svc.session_expires_at.return_value = None  # кук никогда не загружали
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        caplog.at_level("WARNING"),
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch(_FETCH, mock_fetch),
        patch(_SAVE, mock_save),
        patch(_SLEEP, new_callable=AsyncMock),
    ):
        result = await run_domclick_detail_backfill(
            db, run_id=3, params={"batch_size": 10, "budget_sec": 3600}
        )

    assert result.enriched == 1
    _, kwargs = mock_fetch.call_args
    assert kwargs.get("cookies") is None
    assert "кук DomClick нет в БД" in caplog.text
    assert [r for r in caplog.records if r.levelno >= logging.ERROR]
    runs.mark_backfill_finished.assert_called_once()
    runs.mark_failed.assert_not_called()


@pytest.mark.asyncio
async def test_backfill_blocked_abort_after_max_consecutive() -> None:
    """3 consecutive DomClickBlockedError -> abort с пометкой aborted_by_blocks (#2674).

    Раньше здесь стоял mark_done — на проде это дало 16 «успешных» прогонов подряд
    с attempted=3 blocked=3 enriched=0. Теперь финал уходит в mark_backfill_finished
    с флагом обрыва → статус 'banned' (ветвление проверено в test_backfill_honest_status).

    No IP-rotation recovery step exists for DomClick (single dedicated proxy) --
    abort happens on the SAME iteration the threshold is hit, no extra recovery calls.
    """
    snapshot = _make_snapshot(10)
    db = _mock_db(snapshot)
    runs = MagicMock()
    blocked_exc = DomClickBlockedError("QRATOR challenge page detected")
    mock_fetch = AsyncMock(side_effect=blocked_exc)
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc({"CAS_ID": "123"})
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch(_FETCH, mock_fetch),
        patch(_SLEEP, new_callable=AsyncMock),
    ):
        result = await run_domclick_detail_backfill(
            db,
            run_id=4,
            params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 3},
        )

    assert result.blocked == 3
    assert result.attempted == 3
    assert result.enriched == 0
    runs.mark_backfill_finished.assert_called_once()
    assert runs.mark_backfill_finished.call_args.kwargs["aborted_by_blocks"] is True
    runs.mark_failed.assert_not_called()


@pytest.mark.asyncio
async def test_backfill_abort_log_has_no_qrator_literal(caplog: pytest.LogCaptureFixture) -> None:
    """ABORT-лог не утверждает конкретную причину, которую задача не устанавливает (#3178).

    DomClickBlockedError поднимается и на распознанном QRATOR-маркере (площадка), и
    на любом другом сбое браузерного fetch (наш тракт) -- см. докстринг класса выше
    (#2764: диагноз здесь НЕ установлен). До правки ABORT всегда писал 'QRATOR
    reputation likely burned for the session/proxy' -- утверждение, для которого нет
    основания в этом прогоне.

    #3196: причину теперь устанавливает HTTP-статус ответа. Здесь его нет (блок без
    `.status`), поэтому лог печатает перепись диагнозов с честным 'unknown' -- и
    по-прежнему НЕ называет QRATOR.
    """
    snapshot = _make_snapshot(10)
    db = _mock_db(snapshot)
    runs = MagicMock()
    blocked_exc = DomClickBlockedError("browser fetch failed")
    mock_fetch = AsyncMock(side_effect=blocked_exc)
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc({"CAS_ID": "123"})
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch(_FETCH, mock_fetch),
        patch(_SLEEP, new_callable=AsyncMock),
        # серия подтверждённых блоков площадки -- штатный исход скрапинга, не
        # инцидент; понижено до warning, чтобы не шуметь в GlitchTip.
        caplog.at_level("WARNING"),
    ):
        await run_domclick_detail_backfill(
            db,
            run_id=4,
            params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 3},
        )

    abort_records = [r.message for r in caplog.records if "ABORT" in r.message]
    assert abort_records, "ожидался ABORT-лог"
    assert "QRATOR reputation likely burned" not in abort_records[0]
    assert "диагнозы: {'unknown': 3}" in abort_records[0]


@pytest.mark.asyncio
async def test_backfill_parse_error_counts_failed_no_abort() -> None:
    """DomClickParseError (schema drift, not a block) -> failed++, consecutive-block
    breaker NOT touched, run does NOT abort, continues to next listing.
    """
    snapshot = _make_snapshot(2)
    db = _mock_db(snapshot)
    runs = MagicMock()
    mock_enrichment = MagicMock()
    mock_fetch = AsyncMock(
        side_effect=[DomClickParseError("__SSR_STATE__ not found"), mock_enrichment]
    )
    mock_save = MagicMock(return_value=True)
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc({"CAS_ID": "123"})
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch(_FETCH, mock_fetch),
        patch(_SAVE, mock_save),
        patch(_SLEEP, new_callable=AsyncMock),
    ):
        result = await run_domclick_detail_backfill(
            db,
            run_id=5,
            params={"batch_size": 10, "budget_sec": 3600, "max_consecutive_blocks": 3},
        )

    assert result.failed == 1
    assert result.enriched == 1
    assert result.attempted == 2
    assert result.blocked == 0
    runs.mark_backfill_finished.assert_called_once()
    runs.mark_failed.assert_not_called()


@pytest.mark.asyncio
async def test_backfill_sigterm_drain_breaks_and_marks_done_partial() -> None:
    """#1182 Phase 2: shutdown_requested() True -> loop выходит на границе карточки,
    mark_done вызывается с ЧАСТИЧНЫМИ счётчиками (не mark_failed).
    """
    snapshot = _make_snapshot(3)
    db = _mock_db(snapshot)
    runs = MagicMock()
    mock_enrichment = MagicMock()
    mock_fetch = AsyncMock(return_value=mock_enrichment)
    mock_save = MagicMock(return_value=True)
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc({"CAS_ID": "123"})
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch(_FETCH, mock_fetch),
        patch(_SAVE, mock_save),
        patch(_SLEEP, new_callable=AsyncMock),
        patch(_SHUTDOWN, side_effect=[False, True]),
    ):
        result = await run_domclick_detail_backfill(
            db, run_id=6, params={"batch_size": 10, "budget_sec": 3600}
        )

    assert result.attempted == 1
    assert result.enriched == 1
    assert mock_fetch.call_count == 1
    runs.mark_backfill_finished.assert_called_once()
    runs.mark_failed.assert_not_called()
    runs.mark_cancelled.assert_not_called()
    done_counters = runs.mark_backfill_finished.call_args.args[2]
    assert done_counters["attempted"] == 1


@pytest.mark.asyncio
async def test_backfill_budget_guard_stops_loop() -> None:
    """Budget expired before first listing -> fetch_detail not called."""
    snapshot = _make_snapshot(5)
    db = _mock_db(snapshot)
    runs = MagicMock()
    mono_values = iter([0.0, 999.0, 999.0])
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc({"CAS_ID": "123"})
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch("app.tasks.domclick_detail_backfill.time.monotonic", side_effect=mono_values),
        patch(_FETCH) as mock_fetch,
    ):
        await run_domclick_detail_backfill(db, run_id=7, params={"batch_size": 5, "budget_sec": 1})

    mock_fetch.assert_not_called()
    runs.mark_backfill_finished.assert_called_once()


@pytest.mark.asyncio
async def test_backfill_top_level_exception_marks_failed() -> None:
    """db.execute raises -> mark_failed called, exception re-raised."""
    db = MagicMock()
    db.execute.side_effect = RuntimeError("DB connection lost")
    runs = MagicMock()
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc({"CAS_ID": "123"})
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
    ):
        with pytest.raises(RuntimeError, match="DB connection lost"):
            await run_domclick_detail_backfill(
                db, run_id=8, params={"batch_size": 5, "budget_sec": 60}
            )

    runs.mark_failed.assert_called_once()
    runs.mark_backfill_finished.assert_not_called()


@pytest.mark.asyncio
async def test_backfill_generic_exception_continues_and_rolls_back() -> None:
    """Unexpected exception on one listing -> failed++, db.rollback(), loop continues."""
    snapshot = _make_snapshot(2)
    db = _mock_db(snapshot)
    runs = MagicMock()
    mock_enrichment = MagicMock()
    mock_fetch = AsyncMock(side_effect=[RuntimeError("unexpected"), mock_enrichment])
    fake_settings = MagicMock(browser_http_endpoint="http://browser:9000")
    mock_svc = _mock_session_svc({"CAS_ID": "123"})
    mock_bf_cls = _mock_browser_fetcher_cls()
    with (
        patch(_SETTINGS, fake_settings),
        patch(_SESSION_SVC, mock_svc),
        patch(_RUNS, runs),
        patch(_BROWSER_FETCHER, mock_bf_cls),
        patch(_FETCH, mock_fetch),
        patch(_SAVE, return_value=True),
        patch(_SLEEP, new_callable=AsyncMock),
    ):
        result = await run_domclick_detail_backfill(
            db, run_id=9, params={"batch_size": 10, "budget_sec": 3600}
        )

    assert result.failed == 1
    assert result.enriched == 1
    assert result.attempted == 2
    db.rollback.assert_called()
    runs.mark_backfill_finished.assert_called_once()