gendesign/tradein-mvp/backend/tests/test_sweep_imv_phase.py

"""Tests for IMV enrichment phase integrated into Avito city sweep.

Проверяет:
1. enrich_imv=True — sweep вызывает IMV-фазу для touched house_id.
2. enrich_imv=False — IMV не вызывается.
3. IMV counters (imv_attempted, imv_enriched, imv_failed) заполняются.
4. CitySweepCounters содержит новые imv_* поля.
5. process_houses_imv_batch пропускает пустой набор house_ids.
6. CitySweepStartRequest принимает enrich_imv (API-level param).
7. Cooperative cancel перед IMV-фазой — skip IMV, sweep marked done.
"""

from __future__ import annotations

import os
from dataclasses import fields
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")


# ── CitySweepCounters has imv_* fields ────────────────────────────────────────


def test_city_sweep_counters_has_imv_fields() -> None:
    from app.services.scrape_pipeline import CitySweepCounters

    c = CitySweepCounters()
    assert hasattr(c, "imv_attempted")
    assert hasattr(c, "imv_enriched")
    assert hasattr(c, "imv_failed")
    assert c.imv_attempted == 0
    assert c.imv_enriched == 0
    assert c.imv_failed == 0


def test_city_sweep_counters_to_dict_has_imv_keys() -> None:
    from app.services.scrape_pipeline import CitySweepCounters

    c = CitySweepCounters(imv_attempted=5, imv_enriched=3, imv_failed=2)
    d = c.to_dict()
    assert d["imv_attempted"] == 5
    assert d["imv_enriched"] == 3
    assert d["imv_failed"] == 2
    # Все dataclass-поля должны быть в to_dict
    expected_keys = {f.name for f in fields(c)}
    assert set(d.keys()) == expected_keys


# ── PipelineResult имеет touched_house_ids ────────────────────────────────────


def test_pipeline_result_has_touched_house_ids() -> None:
    from app.services.scrape_pipeline import PipelineCounters, PipelineResult

    r = PipelineResult(
        anchor_lat=56.84,
        anchor_lon=60.6,
        radius_m=1500,
        counters=PipelineCounters(),
        enrich_houses=True,
        enrich_detail_top_n=10,
    )
    assert hasattr(r, "touched_house_ids")
    assert isinstance(r.touched_house_ids, set)
    assert len(r.touched_house_ids) == 0


def test_pipeline_result_touched_house_ids_set() -> None:
    from app.services.scrape_pipeline import PipelineCounters, PipelineResult

    r = PipelineResult(
        anchor_lat=56.84,
        anchor_lon=60.6,
        radius_m=1500,
        counters=PipelineCounters(),
        enrich_houses=True,
        enrich_detail_top_n=10,
        touched_house_ids={1, 2, 3},
    )
    assert r.touched_house_ids == {1, 2, 3}


# ── enrich_imv=True → IMV phase вызывается ───────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_enrich_imv_true_calls_imv_batch() -> None:
    """enrich_imv=True + есть touched house_ids → process_houses_imv_batch вызывается."""
    from app.services.house_imv_backfill import HouseIMVBackfillResult
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    mock_pipeline_result = MagicMock()
    mock_pipeline_result.counters.lots_fetched = 5
    mock_pipeline_result.counters.lots_inserted = 5
    mock_pipeline_result.counters.lots_updated = 0
    mock_pipeline_result.counters.unique_houses = 2
    mock_pipeline_result.counters.houses_enriched = 2
    mock_pipeline_result.counters.houses_failed = 0
    mock_pipeline_result.counters.detail_attempted = 2
    mock_pipeline_result.counters.detail_enriched = 2
    mock_pipeline_result.counters.detail_failed = 0
    mock_pipeline_result.counters.errors = []
    mock_pipeline_result.touched_house_ids = {10, 20}

    mock_imv_result = HouseIMVBackfillResult(checked=2, saved=2, skipped=0, errors=0)

    with (
        patch("app.services.scrape_runs.is_cancelled", return_value=False),
        patch("app.services.scrape_runs.update_heartbeat"),
        patch("app.services.scrape_runs.mark_done"),
        patch(
            "app.services.scrape_pipeline.run_avito_pipeline",
            new_callable=AsyncMock,
            return_value=mock_pipeline_result,
        ),
        patch(
            "app.services.house_imv_backfill.process_houses_imv_batch",
            new_callable=AsyncMock,
            return_value=mock_imv_result,
        ) as mock_imv_batch,
    ):
        # Один anchor — минимальный sweep
        counters = await run_avito_city_sweep(
            mock_db,
            run_id=1,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
        )

    mock_imv_batch.assert_awaited_once()
    # Первый позиционный аргумент — db, второй — house_ids
    call_args = mock_imv_batch.call_args
    assert call_args.args[1] == {10, 20}
    assert counters.imv_attempted == 2
    assert counters.imv_enriched == 2
    assert counters.imv_failed == 0


# ── enrich_imv=False → IMV не вызывается ────────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_enrich_imv_false_skips_imv() -> None:
    """enrich_imv=False → process_houses_imv_batch НЕ вызывается."""
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    mock_pipeline_result = MagicMock()
    mock_pipeline_result.counters.lots_fetched = 3
    mock_pipeline_result.counters.lots_inserted = 3
    mock_pipeline_result.counters.lots_updated = 0
    mock_pipeline_result.counters.unique_houses = 1
    mock_pipeline_result.counters.houses_enriched = 1
    mock_pipeline_result.counters.houses_failed = 0
    mock_pipeline_result.counters.detail_attempted = 1
    mock_pipeline_result.counters.detail_enriched = 1
    mock_pipeline_result.counters.detail_failed = 0
    mock_pipeline_result.counters.errors = []
    mock_pipeline_result.touched_house_ids = {99}

    with (
        patch("app.services.scrape_runs.is_cancelled", return_value=False),
        patch("app.services.scrape_runs.update_heartbeat"),
        patch("app.services.scrape_runs.mark_done"),
        patch(
            "app.services.scrape_pipeline.run_avito_pipeline",
            new_callable=AsyncMock,
            return_value=mock_pipeline_result,
        ),
        patch(
            "app.services.house_imv_backfill.process_houses_imv_batch",
            new_callable=AsyncMock,
        ) as mock_imv_batch,
    ):
        counters = await run_avito_city_sweep(
            mock_db,
            run_id=2,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=False,
        )

    mock_imv_batch.assert_not_awaited()
    assert counters.imv_attempted == 0
    assert counters.imv_enriched == 0
    assert counters.imv_failed == 0


# ── enrich_imv=True но нет touched house_ids → IMV не вызывается ────────────


@pytest.mark.asyncio
async def test_sweep_enrich_imv_no_touched_ids_skips() -> None:
    """enrich_imv=True, но touched_house_ids пустой → IMV не вызывается."""
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    mock_pipeline_result = MagicMock()
    mock_pipeline_result.counters.lots_fetched = 0
    mock_pipeline_result.counters.lots_inserted = 0
    mock_pipeline_result.counters.lots_updated = 0
    mock_pipeline_result.counters.unique_houses = 0
    mock_pipeline_result.counters.houses_enriched = 0
    mock_pipeline_result.counters.houses_failed = 0
    mock_pipeline_result.counters.detail_attempted = 0
    mock_pipeline_result.counters.detail_enriched = 0
    mock_pipeline_result.counters.detail_failed = 0
    mock_pipeline_result.counters.errors = []
    mock_pipeline_result.touched_house_ids = set()  # пустой!

    with (
        patch("app.services.scrape_runs.is_cancelled", return_value=False),
        patch("app.services.scrape_runs.update_heartbeat"),
        patch("app.services.scrape_runs.mark_done"),
        patch(
            "app.services.scrape_pipeline.run_avito_pipeline",
            new_callable=AsyncMock,
            return_value=mock_pipeline_result,
        ),
        patch(
            "app.services.house_imv_backfill.process_houses_imv_batch",
            new_callable=AsyncMock,
        ) as mock_imv_batch,
    ):
        await run_avito_city_sweep(
            mock_db,
            run_id=3,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
        )

    mock_imv_batch.assert_not_awaited()


# ── IMV-ошибки не валят sweep ────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_imv_crash_does_not_abort_sweep() -> None:
    """Если process_houses_imv_batch raises — sweep всё равно завершается (mark_done)."""
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    mock_pipeline_result = MagicMock()
    mock_pipeline_result.counters.lots_fetched = 2
    mock_pipeline_result.counters.lots_inserted = 2
    mock_pipeline_result.counters.lots_updated = 0
    mock_pipeline_result.counters.unique_houses = 1
    mock_pipeline_result.counters.houses_enriched = 1
    mock_pipeline_result.counters.houses_failed = 0
    mock_pipeline_result.counters.detail_attempted = 1
    mock_pipeline_result.counters.detail_enriched = 1
    mock_pipeline_result.counters.detail_failed = 0
    mock_pipeline_result.counters.errors = []
    mock_pipeline_result.touched_house_ids = {55}

    with (
        patch("app.services.scrape_runs.is_cancelled", return_value=False),
        patch("app.services.scrape_runs.update_heartbeat"),
        patch("app.services.scrape_runs.mark_done") as mock_mark_done,
        patch(
            "app.services.scrape_pipeline.run_avito_pipeline",
            new_callable=AsyncMock,
            return_value=mock_pipeline_result,
        ),
        patch(
            "app.services.house_imv_backfill.process_houses_imv_batch",
            new_callable=AsyncMock,
            side_effect=RuntimeError("IMV network error"),
        ),
    ):
        counters = await run_avito_city_sweep(
            mock_db,
            run_id=4,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
        )

    mock_mark_done.assert_called_once()
    # IMV crash → ошибка считается, но sweep завершается как done
    assert counters.errors_count >= 1


# ── IMV counters накапливают failed ────────────────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_imv_failed_counter() -> None:
    """imv_failed отражает errors из HouseIMVBackfillResult."""
    from app.services.house_imv_backfill import HouseIMVBackfillResult
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    mock_pipeline_result = MagicMock()
    mock_pipeline_result.counters.lots_fetched = 3
    mock_pipeline_result.counters.lots_inserted = 3
    mock_pipeline_result.counters.lots_updated = 0
    mock_pipeline_result.counters.unique_houses = 3
    mock_pipeline_result.counters.houses_enriched = 3
    mock_pipeline_result.counters.houses_failed = 0
    mock_pipeline_result.counters.detail_attempted = 0
    mock_pipeline_result.counters.detail_enriched = 0
    mock_pipeline_result.counters.detail_failed = 0
    mock_pipeline_result.counters.errors = []
    mock_pipeline_result.touched_house_ids = {1, 2, 3}

    # 3 attempted, 1 enriched, 2 failed
    mock_imv_result = HouseIMVBackfillResult(checked=3, saved=1, skipped=0, errors=2)

    with (
        patch("app.services.scrape_runs.is_cancelled", return_value=False),
        patch("app.services.scrape_runs.update_heartbeat"),
        patch("app.services.scrape_runs.mark_done"),
        patch(
            "app.services.scrape_pipeline.run_avito_pipeline",
            new_callable=AsyncMock,
            return_value=mock_pipeline_result,
        ),
        patch(
            "app.services.house_imv_backfill.process_houses_imv_batch",
            new_callable=AsyncMock,
            return_value=mock_imv_result,
        ),
    ):
        counters = await run_avito_city_sweep(
            mock_db,
            run_id=5,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
        )

    assert counters.imv_attempted == 3
    assert counters.imv_enriched == 1
    assert counters.imv_failed == 2
    # errors_count должен включить imv_failed
    assert counters.errors_count >= 2


# ── Cancel перед IMV-фазой ────────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_sweep_cancel_before_imv_skips_imv() -> None:
    """Если is_cancelled=True в IMV-фазе → IMV пропускается, sweep всё равно done."""
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db = MagicMock()
    mock_pipeline_result = MagicMock()
    mock_pipeline_result.counters.lots_fetched = 1
    mock_pipeline_result.counters.lots_inserted = 1
    mock_pipeline_result.counters.lots_updated = 0
    mock_pipeline_result.counters.unique_houses = 1
    mock_pipeline_result.counters.houses_enriched = 1
    mock_pipeline_result.counters.houses_failed = 0
    mock_pipeline_result.counters.detail_attempted = 0
    mock_pipeline_result.counters.detail_enriched = 0
    mock_pipeline_result.counters.detail_failed = 0
    mock_pipeline_result.counters.errors = []
    mock_pipeline_result.touched_house_ids = {77}

    # is_cancelled: первый вызов (anchor loop) = False, второй (IMV phase) = True
    cancel_side_effects = [False, True]
    call_idx = 0

    def is_cancelled_side_effect(db, run_id):
        nonlocal call_idx
        val = cancel_side_effects[call_idx] if call_idx < len(cancel_side_effects) else True
        call_idx += 1
        return val

    with (
        patch(
            "app.services.scrape_runs.is_cancelled",
            side_effect=is_cancelled_side_effect,
        ),
        patch("app.services.scrape_runs.update_heartbeat"),
        patch("app.services.scrape_runs.mark_done") as mock_mark_done,
        patch(
            "app.services.scrape_pipeline.run_avito_pipeline",
            new_callable=AsyncMock,
            return_value=mock_pipeline_result,
        ),
        patch(
            "app.services.house_imv_backfill.process_houses_imv_batch",
            new_callable=AsyncMock,
        ) as mock_imv_batch,
    ):
        await run_avito_city_sweep(
            mock_db,
            run_id=6,
            anchors=[(56.84, 60.6, "TestAnchor")],
            enrich_imv=True,
        )

    mock_imv_batch.assert_not_awaited()
    mock_mark_done.assert_called_once()


# ── process_houses_imv_batch с пустым set ────────────────────────────────────


@pytest.mark.asyncio
async def test_process_houses_imv_batch_empty_ids() -> None:
    """process_houses_imv_batch с пустым house_ids → result.checked=0, быстрый return."""
    from app.services.house_imv_backfill import process_houses_imv_batch

    mock_db = MagicMock()

    result = await process_houses_imv_batch(mock_db, set(), request_delay_sec=1.0)

    assert result.checked == 0
    assert result.saved == 0
    assert result.errors == 0
    # DB не должна вызываться при пустом наборе
    mock_db.execute.assert_not_called()


# ── CitySweepStartRequest принимает enrich_imv ───────────────────────────────


def test_city_sweep_start_request_enrich_imv_default_true() -> None:
    """enrich_imv по умолчанию True."""
    from app.api.v1.admin import CitySweepStartRequest

    req = CitySweepStartRequest()
    assert req.enrich_imv is True


def test_city_sweep_start_request_enrich_imv_false() -> None:
    """enrich_imv=False принимается."""
    from app.api.v1.admin import CitySweepStartRequest

    req = CitySweepStartRequest(enrich_imv=False)
    assert req.enrich_imv is False


# ── API endpoint передаёт enrich_imv в sweep ────────────────────────────────


def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None:
    """POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV."""
    with (
        patch("app.api.v1.admin.has_running_run", return_value=False),
        patch("app.services.scrape_runs.create_run", return_value=42),
        patch(
            "app.services.scrape_pipeline.run_avito_city_sweep",
            new_callable=AsyncMock,
        ),
    ):
        r = app_with_admin.post(
            "/api/v1/admin/scrape/avito-city-sweep",
            json={"pages_per_anchor": 2, "enrich_imv": False},
        )
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 42


@pytest.fixture
def app_with_admin():
    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from app.api.v1 import admin as admin_module
    from app.core.db import get_db

    app = FastAPI()
    app.include_router(admin_module.router, prefix="/api/v1/admin")

    def fake_db():
        yield MagicMock()

    app.dependency_overrides[get_db] = fake_db
    return TestClient(app)