gendesign/tradein-mvp/backend/tests/test_scraper_kit_pipeline_parity.py

"""Golden-parity: `scraper_kit.orchestration.pipeline` ≡ `app.services.scrape_pipeline`.

Strangler-инвариант (#2135): новая scraper_kit-копия оркестрации sweep-pipeline
должна давать ТОТ ЖЕ наблюдаемый эффект, что и старый боевой orchestrator, на
одинаковом мокнутом I/O. Развязка `app.*` (config/matcher/enrichment/runs/shutdown
инжектируются) не должна менять ветвление.

Фокус — КРИТИЧНАЯ логика оркестрации (то, ради чего PR):
  - ban/rotation state-machine: SERP-блок на anchor'е → abort sweep;
  - partial-ban intake (#1950): SERP intake сохранён + detail заблокирован →
    mark_done (не mark_banned) под флагом avito_serp_ok_not_banned;
  - detail-фаза: N подряд блоков → propagate → anchor-handler;
  - counters aggregation по anchor'ам + IMV-фаза;
  - последовательность вызовов scrape_runs (heartbeat / mark_done / mark_banned).

Метод: гоняем ОБА orchestrator'а (`_drive_old` / `_drive_new`) на идентичном
описании сценария, перехватывая scrape_runs recording-mock'ом. Сравниваем:
  - итоговый counters.to_dict();
  - последовательность (method, numeric-counters) вызовов scrape_runs.

Без сети, без БД.
"""

from __future__ import annotations

import os
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")

from scraper_kit.avito_exceptions import (
    AvitoBlockedError as NewAvitoBlockedError,
)
from scraper_kit.orchestration.pipeline import (
    run_avito_city_sweep as new_city_sweep,
)

from app.services.scrape_pipeline import run_avito_city_sweep as old_city_sweep
from app.services.scrapers.avito_exceptions import (
    AvitoBlockedError as OldAvitoBlockedError,
)

# ── recording scrape_runs ────────────────────────────────────────────────────

_NON_NUMERIC_KEYS = {"enrichment_abort_note"}


class _RunsRecorder:
    """Записывает вызовы scrape_runs-финализаторов в общий список.

    is_cancelled всегда False (кооп-cancel не тестируем здесь). Каждый терминальный
    вызов пишет (method, counters_dict) — сравниваем последовательности old/new.
    """

    def __init__(self) -> None:
        self.calls: list[tuple[str, dict[str, Any]]] = []

    def is_cancelled(self, db: Any, run_id: int) -> bool:
        return False

    def update_heartbeat(self, db: Any, run_id: int, counters: dict[str, Any]) -> None:
        self.calls.append(("update_heartbeat", dict(counters)))

    def mark_done(self, db: Any, run_id: int, counters: dict[str, Any]) -> None:
        self.calls.append(("mark_done", dict(counters)))

    def mark_banned(self, db: Any, run_id: int, error: str, counters: dict[str, Any]) -> None:
        self.calls.append(("mark_banned", dict(counters)))

    def mark_failed(self, db: Any, run_id: int, error: str, counters: dict[str, Any]) -> None:
        self.calls.append(("mark_failed", dict(counters)))


_DriveResult = tuple[dict[str, int], list[tuple[str, dict[str, int]]]]


def _normalize(calls: list[tuple[str, dict[str, Any]]]) -> list[tuple[str, dict[str, int]]]:
    """Оставить только имя метода + числовые counters (убрать note-строки)."""
    out: list[tuple[str, dict[str, int]]] = []
    for method, counters in calls:
        numeric = {k: v for k, v in counters.items() if k not in _NON_NUMERIC_KEYS}
        out.append((method, numeric))
    return out


# ── scenario description ─────────────────────────────────────────────────────


class _Scenario:
    """Декларативное описание одного прогона city-sweep для обоих orchestrator'ов."""

    def __init__(
        self,
        *,
        anchors: list[tuple[float, float, str]],
        # per-anchor: ("lots", n_lots, ins, upd) | ("block",)
        per_anchor: list[tuple[Any, ...]],
        enrich_houses: bool = False,
        detail_top_n: int = 0,
        detail_rows: int = 0,
        detail_behavior: str = "ok",  # "ok" | "block_all"
        enrich_imv: bool = False,
        imv_result: tuple[int, int, int] | None = None,
        avito_serp_ok_not_banned: bool = True,
        avito_proxy_max_rotations: int = 0,
        lots_have_house_url: bool = False,
    ) -> None:
        self.anchors = anchors
        self.per_anchor = per_anchor
        self.enrich_houses = enrich_houses
        self.detail_top_n = detail_top_n
        self.detail_rows = detail_rows
        self.detail_behavior = detail_behavior
        self.enrich_imv = enrich_imv
        self.imv_result = imv_result
        self.avito_serp_ok_not_banned = avito_serp_ok_not_banned
        self.avito_proxy_max_rotations = avito_proxy_max_rotations
        self.lots_have_house_url = lots_have_house_url

    def _config(self) -> SimpleNamespace:
        return SimpleNamespace(
            scraper_fetch_mode="curl_cffi",
            browser_http_endpoint="http://browser.test/fetch",
            scraper_proxy_url=None,
            avito_proxy_rotate_url=None,
            avito_proxy_max_rotations=self.avito_proxy_max_rotations,
            avito_serp_ok_not_banned=self.avito_serp_ok_not_banned,
            avito_proxy_rotate_settle_s=0.0,
            proxy_rotate_attempts=1,
            proxy_rotate_attempt_timeout_s=1.0,
            cian_proxy_rotate_url=None,
            cian_proxy_max_rotations=0,
            yandex_proxy_rotate_url=None,
            yandex_proxy_max_rotations=0,
            scraper_skip_seen_today=False,
        )

    def _fetch_around_side_effects(self, blocked_exc: type[Exception]) -> list[Any]:
        effects: list[Any] = []
        for spec in self.per_anchor:
            if spec[0] == "block":
                effects.append(blocked_exc("SERP blocked"))
            else:
                _, n_lots, _ins, _upd = spec
                house_url = "/catalog/houses/ekb/h1/100" if self.lots_have_house_url else None
                effects.append([MagicMock(house_url=house_url) for _ in range(n_lots)])
        return effects

    def _save_side_effects(self) -> list[tuple[int, int]]:
        return [(spec[2], spec[3]) for spec in self.per_anchor if spec[0] == "lots" and spec[1] > 0]

    def _detail_rows(self) -> list[dict[str, str]]:
        return [{"source_url": f"https://www.avito.ru/x/{i}"} for i in range(self.detail_rows)]

    def _fetch_detail_side_effects(self, blocked_exc: type[Exception]) -> Any:
        if self.detail_behavior == "block_all":
            return blocked_exc("detail blocked")
        return MagicMock(house_catalog_url=None)


def _make_db(scenario: _Scenario) -> MagicMock:
    db = MagicMock()
    # detail-фаза: db.execute(...).mappings().all() → priority rows
    db.execute.return_value.mappings.return_value.all.return_value = scenario._detail_rows()
    return db


def _make_scraper(scenario: _Scenario, blocked_exc: type[Exception]) -> MagicMock:
    scraper = MagicMock()
    scraper._cffi = None
    scraper._browser = None
    scraper.fetch_around = AsyncMock(side_effect=scenario._fetch_around_side_effects(blocked_exc))
    return scraper


def _async_session_cm() -> MagicMock:
    sess = MagicMock()
    sess.__aenter__ = AsyncMock(return_value=sess)
    sess.__aexit__ = AsyncMock(return_value=None)
    sess.close = AsyncMock()
    return sess


async def _drive_old(scenario: _Scenario) -> _DriveResult:
    recorder = _RunsRecorder()
    db = _make_db(scenario)
    scraper = _make_scraper(scenario, OldAvitoBlockedError)
    save_mock = MagicMock(side_effect=scenario._save_side_effects())

    imv_res = None
    if scenario.imv_result is not None:
        checked, saved, errors = scenario.imv_result
        imv_res = SimpleNamespace(checked=checked, saved=saved, errors=errors)

    with (
        patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
        patch("app.services.scrape_pipeline.save_listings", save_mock),
        patch(
            "app.services.scrape_pipeline.fetch_house_catalog",
            AsyncMock(return_value=MagicMock()),
        ),
        patch(
            "app.services.scrape_pipeline.save_house_catalog_enrichment",
            return_value={"house_id": 1},
        ),
        patch(
            "app.services.scrape_pipeline.fetch_detail",
            AsyncMock(side_effect=scenario._fetch_detail_side_effects(OldAvitoBlockedError)),
        ),
        patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
        patch("app.services.scrape_pipeline.scrape_runs", recorder),
        patch(
            "app.services.house_imv_backfill.process_houses_imv_batch",
            AsyncMock(return_value=imv_res),
        ),
        patch("app.services.scrape_pipeline.settings", scenario._config()),
        patch("app.services.scrape_pipeline.shutdown_requested", return_value=False),
        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
        patch("app.services.scrape_pipeline.AsyncSession", return_value=_async_session_cm()),
    ):
        counters = await old_city_sweep(
            db,
            run_id=1,
            radius_m=1000,
            anchors=scenario.anchors,
            pages_per_anchor=1,
            enrich_houses=scenario.enrich_houses,
            detail_top_n=scenario.detail_top_n,
            request_delay_sec=0.0,
            enrich_imv=scenario.enrich_imv,
        )
    return counters.to_dict(), _normalize(recorder.calls)


async def _drive_new(scenario: _Scenario) -> _DriveResult:
    recorder = _RunsRecorder()
    db = _make_db(scenario)
    scraper = _make_scraper(scenario, NewAvitoBlockedError)
    save_mock = MagicMock(side_effect=scenario._save_side_effects())

    imv_res = None
    if scenario.imv_result is not None:
        checked, saved, errors = scenario.imv_result
        imv_res = SimpleNamespace(checked=checked, saved=saved, errors=errors)

    enrichment = MagicMock()
    enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_res)

    pfx = "scraper_kit.orchestration.pipeline"
    with (
        patch(f"{pfx}.AvitoScraper", return_value=scraper),
        patch(f"{pfx}.save_listings", save_mock),
        patch(f"{pfx}.fetch_house_catalog", AsyncMock(return_value=MagicMock())),
        patch(f"{pfx}.save_house_catalog_enrichment", return_value={"house_id": 1}),
        patch(
            f"{pfx}.fetch_detail",
            AsyncMock(side_effect=scenario._fetch_detail_side_effects(NewAvitoBlockedError)),
        ),
        patch(f"{pfx}.save_detail_enrichment", return_value=True),
        patch(f"{pfx}.runs", recorder),
        patch(f"{pfx}.asyncio.sleep", AsyncMock()),
        patch(f"{pfx}.AsyncSession", return_value=_async_session_cm()),
    ):
        counters = await new_city_sweep(
            db,
            run_id=1,
            config=scenario._config(),
            matcher=MagicMock(),
            enrichment=enrichment,
            shutdown_requested=lambda: False,
            radius_m=1000,
            anchors=scenario.anchors,
            pages_per_anchor=1,
            enrich_houses=scenario.enrich_houses,
            detail_top_n=scenario.detail_top_n,
            request_delay_sec=0.0,
            enrich_imv=scenario.enrich_imv,
        )
    return counters.to_dict(), _normalize(recorder.calls)


async def _assert_parity(scenario: _Scenario) -> dict[str, int]:
    old_counters, old_calls = await _drive_old(scenario)
    new_counters, new_calls = await _drive_new(scenario)
    assert (
        new_counters == old_counters
    ), f"counters diverged:\n old={old_counters}\n new={new_counters}"
    assert (
        new_calls == old_calls
    ), f"scrape_runs calls diverged:\n old={old_calls}\n new={new_calls}"
    return new_counters


# ── scenarios ────────────────────────────────────────────────────────────────

_ANCHORS_2 = [(56.84, 60.60, "A1"), (56.79, 60.53, "A2")]


@pytest.mark.asyncio
async def test_parity_happy_path_counters_aggregation() -> None:
    """2 anchor'а, SERP+save, без houses/detail/imv → mark_done + агрегированные counters."""
    scenario = _Scenario(
        anchors=_ANCHORS_2,
        per_anchor=[("lots", 10, 8, 2), ("lots", 6, 5, 1)],
    )
    counters = await _assert_parity(scenario)
    # Убеждаемся что сценарий действительно прогнал агрегацию (не пустой прогон).
    assert counters["lots_fetched"] == 16
    assert counters["lots_inserted"] == 13
    assert counters["lots_updated"] == 3
    assert counters["anchors_done"] == 2


@pytest.mark.asyncio
async def test_parity_full_ban_no_intake_marks_banned() -> None:
    """Первый же anchor SERP-блок, лоты не сохранены → mark_banned (не done)."""
    scenario = _Scenario(
        anchors=_ANCHORS_2,
        per_anchor=[("block",), ("block",)],
    )
    await _assert_parity(scenario)


@pytest.mark.asyncio
async def test_parity_partial_ban_intake_marks_done() -> None:
    """Anchor1 intake сохранён, anchor2 SERP-блок → partial-intake → mark_done не banned."""
    scenario = _Scenario(
        anchors=_ANCHORS_2,
        per_anchor=[("lots", 10, 9, 1), ("block",)],
        avito_serp_ok_not_banned=True,
    )
    await _assert_parity(scenario)


@pytest.mark.asyncio
async def test_parity_partial_ban_flag_off_marks_banned() -> None:
    """avito_serp_ok_not_banned=False: даже при intake SERP-блок → mark_banned."""
    scenario = _Scenario(
        anchors=_ANCHORS_2,
        per_anchor=[("lots", 10, 9, 1), ("block",)],
        avito_serp_ok_not_banned=False,
    )
    await _assert_parity(scenario)


@pytest.mark.asyncio
async def test_parity_detail_consecutive_block_propagates() -> None:
    """detail-фаза: 3 подряд блока (rotations=0) → propagate → anchor-handler.

    Лоты сохранены на этом же anchor'е → partial-intake → mark_done.
    """
    scenario = _Scenario(
        anchors=[(56.84, 60.60, "A1")],
        per_anchor=[("lots", 5, 5, 0)],
        detail_top_n=5,
        detail_rows=3,
        detail_behavior="block_all",
        avito_serp_ok_not_banned=True,
        avito_proxy_max_rotations=0,
    )
    await _assert_parity(scenario)


@pytest.mark.asyncio
async def test_parity_imv_phase_counters() -> None:
    """IMV-фаза: touched houses → process_houses_imv_batch → imv-counters агрегированы."""
    scenario = _Scenario(
        anchors=[(56.84, 60.60, "A1")],
        per_anchor=[("lots", 4, 4, 0)],
        enrich_houses=True,
        lots_have_house_url=True,
        enrich_imv=True,
        imv_result=(3, 2, 1),
    )
    counters = await _assert_parity(scenario)
    # touched house → IMV-фаза отработала, imv-counters агрегированы одинаково.
    assert counters["imv_attempted"] == 3
    assert counters["imv_enriched"] == 2
    assert counters["imv_failed"] == 1