gendesign/tradein-mvp/backend/tests/test_sweep_imv_phase.py

"""Tests for IMV enrichment phase integrated into Avito city sweep (kit).

Проверяет:
1. enrich_imv=True — sweep вызывает IMV-фазу для touched house_id.
2. enrich_imv=False — IMV не вызывается.
3. IMV counters (imv_attempted, imv_enriched, imv_failed) заполняются.
4. CitySweepCounters содержит новые imv_* поля.
5. process_houses_imv_batch пропускает пустой набор house_ids.
6. CitySweepStartRequest принимает enrich_imv (API-level param).
7. Cooperative cancel перед IMV-фазой — skip IMV, sweep marked done.

Kit — единственная orchestration-копия (#2397 Part E1 удалил legacy
`app.services.scrape_pipeline`): behavior-тесты (пункты 1,2,3,5,7 выше через
`run_avito_city_sweep`) переведены на `scraper_kit.orchestration.pipeline` с DI
(config/matcher/enrichment/shutdown_requested) вместо прямых импортов `app.*`.
`enrichment.process_houses_imv_batch` — инжектируемый Protocol-метод (было прямым
импортом `app.services.house_imv_backfill.process_houses_imv_batch`).
"""

from __future__ import annotations

import os
from contextlib import ExitStack
from dataclasses import fields
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

PFX = "scraper_kit.orchestration.pipeline"


def _make_house_lot(offer_id: str, house_path: str) -> Any:
    """ScrapedLot с house_url для тестов house-enrich пути."""
    from scraper_kit.base import ScrapedLot

    return ScrapedLot(
        source="avito",
        source_url=f"https://www.avito.ru{house_path}/{offer_id}",
        source_id=offer_id,
        address="Тест",
        price_rub=3_000_000,
        area_m2=40.0,
        rooms=1,
        house_url=f"https://www.avito.ru{house_path}",
    )


def _config() -> SimpleNamespace:
    return SimpleNamespace(
        scraper_fetch_mode="curl_cffi",
        browser_http_endpoint="http://browser.test/fetch",
        scraper_proxy_url=None,
        avito_proxy_rotate_url=None,
        avito_proxy_max_rotations=0,
        avito_serp_ok_not_banned=True,
        avito_proxy_rotate_settle_s=0.0,
        proxy_rotate_attempts=1,
        proxy_rotate_attempt_timeout_s=1.0,
        cian_proxy_rotate_url=None,
        cian_proxy_max_rotations=0,
        yandex_proxy_rotate_url=None,
        yandex_proxy_max_rotations=0,
        scraper_skip_seen_today=False,
    )


def _enter_common_patches(stack: ExitStack, house_ids: list[int]) -> None:
    """Входит в набор patch'ей для базового Avito sweep через ExitStack:
    - AvitoScraper.fetch_around → lots с house_url (по одному на house_id)
    - save_listings → (n, 0)
    - fetch_house_catalog → MagicMock()
    - save_house_catalog_enrichment → {"house_id": X} по очереди
    - AsyncSession (kit-модуль) → fake
    - asyncio.sleep → no-op
    """
    from scraper_kit.providers.avito.serp import AvitoScraper

    lots = [_make_house_lot(f"lot-{hid}", f"/ekaterinburg/houses/test/{hid}") for hid in house_ids]

    async def fake_fetch_around(self: Any, lat: float, lon: float, *_a: Any, **_kw: Any) -> list:
        return lots

    # save_house_catalog_enrichment вызывается по одному разу на каждый дом
    house_id_iter = iter(house_ids)

    def fake_save_house(db: Any, enrichment: Any, **_kw: Any) -> dict:
        try:
            return {"house_id": next(house_id_iter)}
        except StopIteration:
            return {}

    fake_session = AsyncMock()
    fake_session.__aenter__ = AsyncMock(return_value=fake_session)
    fake_session.__aexit__ = AsyncMock(return_value=None)

    stack.enter_context(patch.object(AvitoScraper, "fetch_around", fake_fetch_around))
    stack.enter_context(patch(f"{PFX}.save_listings", lambda _db, _lots, **_kw: (len(_lots), 0)))
    stack.enter_context(patch(f"{PFX}.fetch_house_catalog", AsyncMock(return_value=MagicMock())))
    stack.enter_context(patch(f"{PFX}.save_house_catalog_enrichment", fake_save_house))
    stack.enter_context(patch(f"{PFX}.AsyncSession", return_value=fake_session))
    stack.enter_context(patch(f"{PFX}.asyncio.sleep", AsyncMock()))


def _enter_runs_patches(stack: ExitStack, *, mark_done_mock: Any = None) -> Any:
    """Патчит `scraper_kit.orchestration.pipeline.runs` (is_cancelled/update_heartbeat/
    mark_done)."""
    mock_runs = MagicMock()
    mock_runs.is_cancelled.return_value = False
    mock_runs.update_heartbeat = MagicMock()
    mock_runs.mark_done = mark_done_mock if mark_done_mock is not None else MagicMock()
    stack.enter_context(patch(f"{PFX}.runs", mock_runs))
    return mock_runs


# ── CitySweepCounters has imv_* fields ────────────────────────────────────────


def test_city_sweep_counters_has_imv_fields() -> None:
    from scraper_kit.orchestration.pipeline import CitySweepCounters

    c = CitySweepCounters()
    assert hasattr(c, "imv_attempted")
    assert hasattr(c, "imv_enriched")
    assert hasattr(c, "imv_failed")
    assert c.imv_attempted == 0
    assert c.imv_enriched == 0
    assert c.imv_failed == 0


def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
    from scraper_kit.orchestration.pipeline import CitySweepCounters

    c = CitySweepCounters(imv_attempted=5, imv_enriched=3, imv_failed=2)
    d = c.to_dict()
    assert d["imv_attempted"] == 5
    assert d["imv_enriched"] == 3
    assert d["imv_failed"] == 2
    # Все dataclass-поля должны быть в to_dict
    expected_keys = {f.name for f in fields(c)}
    assert set(d.keys()) == expected_keys


# ── PipelineResult имеет touched_house_ids ────────────────────────────────────


def test_pipeline_result_has_touched_house_ids() -> None:
    from scraper_kit.orchestration.pipeline import PipelineCounters, PipelineResult

    r = PipelineResult(
        anchor_lat=56.84,
        anchor_lon=60.6,
        radius_m=1500,
        counters=PipelineCounters(),
        enrich_houses=True,
        enrich_detail_top_n=10,
    )
    assert hasattr(r, "touched_house_ids")
    assert isinstance(r.touched_house_ids, set)
    assert len(r.touched_house_ids) == 0


def test_pipeline_result_touched_house_ids_set() -> None:
    from scraper_kit.orchestration.pipeline import PipelineCounters, PipelineResult

    r = PipelineResult(
        anchor_lat=56.84,
        anchor_lon=60.6,
        radius_m=1500,
        counters=PipelineCounters(),
        enrich_houses=True,
        enrich_detail_top_n=10,
        touched_house_ids={1, 2, 3},
    )
    assert r.touched_house_ids == {1, 2, 3}


# ── enrich_imv=True → IMV phase вызывается ───────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
    """enrich_imv=True + есть touched house_ids → process_houses_imv_batch вызывается.

    Патчим AvitoScraper.fetch_around → lots с house_url, save_house_catalog_enrichment
    → {house_id: X} чтобы all_touched_house_ids = {10, 20}.
    """
    from scraper_kit.orchestration.pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    imv_result = SimpleNamespace(checked=2, saved=2, errors=0)
    enrichment = MagicMock()
    enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_result)

    with ExitStack() as stack:
        _enter_common_patches(stack, [10, 20])
        _enter_runs_patches(stack)
        counters = await run_avito_city_sweep(
            mock_db,
            run_id=1,
            config=_config(),
            matcher=MagicMock(),
            enrichment=enrichment,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
            enrich_houses=True,
            detail_top_n=0,
        )

    enrichment.process_houses_imv_batch.assert_awaited_once()
    # Первый позиционный аргумент — db, второй — house_ids
    call_args = enrichment.process_houses_imv_batch.call_args
    assert call_args.args[1] == {10, 20}
    assert counters.imv_attempted == 2
    assert counters.imv_enriched == 2
    assert counters.imv_failed == 0


# ── enrich_imv=False → IMV не вызывается ────────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_enrich_imv_false_skips_imv() -> None:
    """enrich_imv=False → process_houses_imv_batch НЕ вызывается."""
    from scraper_kit.orchestration.pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    enrichment = MagicMock()
    enrichment.process_houses_imv_batch = AsyncMock()

    with ExitStack() as stack:
        _enter_common_patches(stack, [99])
        _enter_runs_patches(stack)
        counters = await run_avito_city_sweep(
            mock_db,
            run_id=2,
            config=_config(),
            matcher=MagicMock(),
            enrichment=enrichment,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=False,
            enrich_houses=True,
            detail_top_n=0,
        )

    enrichment.process_houses_imv_batch.assert_not_awaited()
    assert counters.imv_attempted == 0
    assert counters.imv_enriched == 0
    assert counters.imv_failed == 0


# ── enrich_imv=True но нет touched house_ids → IMV не вызывается ────────────


@pytest.mark.asyncio
async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
    """enrich_imv=True, но touched_house_ids пустой → IMV не вызывается.

    Лоты без house_url → unique_house_paths пуст → touched_house_ids пуст.
    """
    from scraper_kit.orchestration.pipeline import run_avito_city_sweep
    from scraper_kit.providers.avito.serp import AvitoScraper

    mock_db = MagicMock()
    fake_session = AsyncMock()
    fake_session.__aenter__ = AsyncMock(return_value=fake_session)
    fake_session.__aexit__ = AsyncMock(return_value=None)

    # Лоты БЕЗ house_url → нет домов → IMV не вызывается
    async def fake_fetch_around(self: Any, lat: float, lon: float, *_a: Any, **_kw: Any) -> list:
        return []

    enrichment = MagicMock()
    enrichment.process_houses_imv_batch = AsyncMock()

    with (
        patch.object(AvitoScraper, "fetch_around", fake_fetch_around),
        patch(f"{PFX}.save_listings", lambda _db, _lots, **_kw: (0, 0)),
        patch(f"{PFX}.AsyncSession", return_value=fake_session),
        patch(f"{PFX}.asyncio.sleep", AsyncMock()),
        patch(f"{PFX}.runs") as mock_runs,
    ):
        mock_runs.is_cancelled.return_value = False
        await run_avito_city_sweep(
            mock_db,
            run_id=3,
            config=_config(),
            matcher=MagicMock(),
            enrichment=enrichment,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
            enrich_houses=True,
            detail_top_n=0,
        )

    enrichment.process_houses_imv_batch.assert_not_awaited()


# ── IMV-ошибки не валят sweep ────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
    """Если process_houses_imv_batch raises — sweep всё равно завершается (mark_done)."""
    from scraper_kit.orchestration.pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    enrichment = MagicMock()
    enrichment.process_houses_imv_batch = AsyncMock(side_effect=RuntimeError("IMV network error"))

    with ExitStack() as stack:
        _enter_common_patches(stack, [55])
        mock_runs = _enter_runs_patches(stack)
        counters = await run_avito_city_sweep(
            mock_db,
            run_id=4,
            config=_config(),
            matcher=MagicMock(),
            enrichment=enrichment,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
            enrich_houses=True,
            detail_top_n=0,
        )

    mock_runs.mark_done.assert_called_once()
    # IMV crash → ошибка считается, но sweep завершается как done
    assert counters.errors_count >= 1


# ── IMV counters накапливают failed ────────────────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_imv_failed_counter() -> None:
    """imv_failed отражает errors из результата enrichment.process_houses_imv_batch."""
    from scraper_kit.orchestration.pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    # 3 attempted, 1 enriched, 2 failed
    imv_result = SimpleNamespace(checked=3, saved=1, errors=2)
    enrichment = MagicMock()
    enrichment.process_houses_imv_batch = AsyncMock(return_value=imv_result)

    with ExitStack() as stack:
        _enter_common_patches(stack, [1, 2, 3])
        _enter_runs_patches(stack)
        counters = await run_avito_city_sweep(
            mock_db,
            run_id=5,
            config=_config(),
            matcher=MagicMock(),
            enrichment=enrichment,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
            enrich_houses=True,
            detail_top_n=0,
        )

    assert counters.imv_attempted == 3
    assert counters.imv_enriched == 1
    assert counters.imv_failed == 2
    # errors_count должен включить imv_failed
    assert counters.errors_count >= 2


# ── Cancel перед IMV-фазой ────────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_cancel_before_imv_skips_imv() -> None:
    """Если is_cancelled=True в IMV-фазе → IMV пропускается, sweep всё равно done."""
    from scraper_kit.orchestration.pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    enrichment = MagicMock()
    enrichment.process_houses_imv_batch = AsyncMock()

    # is_cancelled: первый вызов (anchor loop) = False, второй (IMV phase) = True
    cancel_side_effects = [False, True]
    call_idx = 0

    def is_cancelled_side_effect(db: Any, run_id: int) -> bool:
        nonlocal call_idx
        val = cancel_side_effects[call_idx] if call_idx < len(cancel_side_effects) else True
        call_idx += 1
        return val

    with ExitStack() as stack:
        _enter_common_patches(stack, [77])
        mock_runs = MagicMock()
        mock_runs.is_cancelled.side_effect = is_cancelled_side_effect
        mock_runs.update_heartbeat = MagicMock()
        mock_runs.mark_done = MagicMock()
        stack.enter_context(patch(f"{PFX}.runs", mock_runs))

        await run_avito_city_sweep(
            mock_db,
            run_id=6,
            config=_config(),
            matcher=MagicMock(),
            enrichment=enrichment,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
            enrich_houses=True,
            detail_top_n=0,
        )

    enrichment.process_houses_imv_batch.assert_not_awaited()
    mock_runs.mark_done.assert_called_once()


# ── process_houses_imv_batch с пустым set ────────────────────────────────────


@pytest.mark.asyncio
async def test_process_houses_imv_batch_empty_ids() -> None:
    """process_houses_imv_batch с пустым house_ids → result.checked=0, быстрый return."""
    from app.services.house_imv_backfill import process_houses_imv_batch

    mock_db = MagicMock()

    result = await process_houses_imv_batch(mock_db, set(), request_delay_sec=1.0)

    assert result.checked == 0
    assert result.saved == 0
    assert result.errors == 0
    # DB не должна вызываться при пустом наборе
    mock_db.execute.assert_not_called()


# ── CitySweepStartRequest принимает enrich_imv ───────────────────────────────


def test_city_sweep_start_request_enrich_imv_default_true() -> None:
    """enrich_imv по умолчанию True."""
    from app.api.v1.admin import CitySweepStartRequest

    req = CitySweepStartRequest()
    assert req.enrich_imv is True


def test_city_sweep_start_request_enrich_imv_false() -> None:
    """enrich_imv=False принимается."""
    from app.api.v1.admin import CitySweepStartRequest

    req = CitySweepStartRequest(enrich_imv=False)
    assert req.enrich_imv is False


# ── API endpoint передаёт enrich_imv в sweep ────────────────────────────────


def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None:
    """POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV.

    kit sweep вызван с DI (config/matcher/enrichment) — mock должен перехватывать
    имя, забинженное в app.api.v1.admin (kit-импорт), не устаревший scrape_pipeline.
    """
    from app.services.scraper_adapters import (
        RealEnrichmentJobs,
        RealMatcherAdapter,
        RealScraperConfig,
    )

    with (
        patch("app.api.v1.admin.has_running_run", return_value=False),
        patch("app.services.scrape_runs.create_run", return_value=42),
        patch(
            "app.api.v1.admin.run_avito_city_sweep",
            new_callable=AsyncMock,
        ) as mock_sweep,
    ):
        r = app_with_admin.post(
            "/api/v1/admin/scrape/avito-city-sweep",
            json={"pages_per_anchor": 2, "enrich_imv": False},
        )
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 42

    mock_sweep.assert_called_once()
    kw = mock_sweep.call_args.kwargs
    assert isinstance(kw["config"], RealScraperConfig)
    assert isinstance(kw["matcher"], RealMatcherAdapter)
    assert isinstance(kw["enrichment"], RealEnrichmentJobs)
    assert "proxy_provider" in kw
    assert kw["enrich_imv"] is False


@pytest.fixture
def app_with_admin():
    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from app.api.v1 import admin as admin_module
    from app.core.db import get_db

    app = FastAPI()
    app.include_router(admin_module.router, prefix="/api/v1/admin")

    def fake_db():
        yield MagicMock()

    app.dependency_overrides[get_db] = fake_db
    return TestClient(app)