gendesign/tradein-mvp/backend/tests/test_scraper_resilience_1949_1950.py

"""Тесты устойчивости скрейперов: avito changeip-retry (#1950) + cian anti-zombie (#1949).

Fix A (#1950): _rotate_proxy_ip теперь делает до proxy_rotate_attempts попыток
               по proxy_rotate_attempt_timeout_s каждая.

Fix B (#1950): run_avito_city_sweep при AvitoBlockedError/RateLimited с
               lots_inserted+lots_updated>0 вызывает mark_done (не mark_banned)
               когда avito_serp_ok_not_banned=True.

Fix C (#1949): run_cian_full_load применяет monkey-patch _browser.fetch с
               asyncio.wait_for(timeout=cian_full_load_per_fetch_timeout_s).

Fix D (#1949): run_cian_full_load создаёт фоновый heartbeat-task и отменяет
               его в finally при любом завершении.
"""

from __future__ import annotations

import asyncio
import os
import time
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")

from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.base import ScrapedLot

# ──────────────────────────────────────────────────────────────────────────────
# Fix A: _rotate_proxy_ip retry loop (#1950)
# ──────────────────────────────────────────────────────────────────────────────


def _make_session_mock(get_side_effect: Any) -> AsyncMock:
    """Вспомогательный: AsyncMock-сессия с заданным поведением .get()."""
    mock_session = AsyncMock()
    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
    mock_session.__aexit__ = AsyncMock(return_value=None)
    mock_session.get = AsyncMock(side_effect=get_side_effect)
    return mock_session


@pytest.mark.asyncio
async def test_changeip_retry_success_on_second_attempt() -> None:
    """#1950-A: первая попытка таймаут, вторая успех → возвращает True; ровно 2 GET-вызова."""
    from app.services.scrape_pipeline import _rotate_proxy_ip

    get_call_count = 0

    async def fake_get(url: str) -> MagicMock:
        nonlocal get_call_count
        get_call_count += 1
        if get_call_count == 1:
            raise TimeoutError("connect timeout")
        resp = MagicMock()
        resp.json.return_value = {"new_ip": "1.2.3.4"}
        return resp

    mock_session = _make_session_mock(fake_get)

    with (
        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.proxy_rotate_attempts = 3
        s.proxy_rotate_attempt_timeout_s = 8.0
        s.avito_proxy_rotate_settle_s = 0.0
        s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
        s.avito_proxy_max_rotations = 3

        result = await _rotate_proxy_ip(
            source="avito",
            reason="test-detail-block",
            rotations_done=0,
        )

    assert result is True
    assert get_call_count == 2, f"expected 2 GET calls, got {get_call_count}"


@pytest.mark.asyncio
async def test_changeip_retry_all_fail_returns_false() -> None:
    """#1950-A: все 3 попытки падают с TimeoutError → False; ровно 3 GET-вызова."""
    from app.services.scrape_pipeline import _rotate_proxy_ip

    get_call_count = 0

    async def failing_get(url: str) -> None:
        nonlocal get_call_count
        get_call_count += 1
        raise TimeoutError("connect timeout")

    mock_session = _make_session_mock(failing_get)

    with (
        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.proxy_rotate_attempts = 3
        s.proxy_rotate_attempt_timeout_s = 8.0
        s.avito_proxy_rotate_settle_s = 0.0
        s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
        s.avito_proxy_max_rotations = 3

        result = await _rotate_proxy_ip(
            source="avito",
            reason="test-detail-block",
            rotations_done=0,
        )

    assert result is False
    assert (
        get_call_count == 3
    ), f"expected exactly 3 attempts (proxy_rotate_attempts=3), got {get_call_count}"


@pytest.mark.asyncio
async def test_changeip_retry_bounded_by_setting() -> None:
    """#1950-A: proxy_rotate_attempts=2 → ровно 2 попытки, не больше."""
    from app.services.scrape_pipeline import _rotate_proxy_ip

    call_count = 0

    async def failing_get(url: str) -> None:
        nonlocal call_count
        call_count += 1
        raise TimeoutError("timeout")

    mock_session = _make_session_mock(failing_get)

    with (
        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.proxy_rotate_attempts = 2
        s.proxy_rotate_attempt_timeout_s = 8.0
        s.avito_proxy_rotate_settle_s = 0.0
        s.cian_proxy_rotate_url = "http://changeip.example.com/rotate"
        s.avito_proxy_rotate_url = None
        s.cian_proxy_max_rotations = 3

        result = await _rotate_proxy_ip(
            source="cian",
            reason="serp-block",
            rotations_done=1,
        )

    assert result is False
    assert call_count == 2, f"proxy_rotate_attempts=2 should make exactly 2 calls, got {call_count}"


# ──────────────────────────────────────────────────────────────────────────────
# Fix B: честный статус run_avito_city_sweep (#1950)
# ──────────────────────────────────────────────────────────────────────────────


def _make_city_sweep_mocks() -> tuple[MagicMock, MagicMock, MagicMock]:
    """Возвращает (mock_db, mock_runs, mock_scraper) для тестов run_avito_city_sweep."""
    mock_db = MagicMock()

    # priority_rows для detail-фазы: 3 строки с source_url
    class _DetailRow:
        def __getitem__(self, key: str) -> str:
            return "https://www.avito.ru/ekaterinburg/kvartiry/test-1"

    mock_db.execute.return_value.mappings.return_value.all.return_value = [
        _DetailRow(),
        _DetailRow(),
        _DetailRow(),
    ]

    mock_runs = MagicMock()
    mock_runs.is_cancelled.return_value = False
    mock_runs.mark_done = MagicMock()
    mock_runs.mark_banned = MagicMock()
    mock_runs.mark_failed = MagicMock()
    mock_runs.update_heartbeat = MagicMock()

    mock_scraper = MagicMock()
    # AvitoScraper НЕ используется как async CM в _avito_anchor_phases (просто instantiate)
    mock_scraper.fetch_around = AsyncMock(
        return_value=[
            ScrapedLot(
                source="avito",
                source_url=f"https://www.avito.ru/e/k/{i}",
                source_id=str(i),
                price_rub=5_000_000,
                address="ЕКБ test",
                listing_segment="vtorichka",
            )
            for i in range(3)
        ]
    )

    return mock_db, mock_runs, mock_scraper


@pytest.mark.asyncio
async def test_avito_city_sweep_serp_ok_marks_done_not_banned() -> None:
    """#1950-B: SERP собрал лоты (ins=3), detail заблокировало → mark_done не mark_banned.

    Условие: avito_serp_ok_not_banned=True (default), lots_inserted+lots_updated > 0.
    """
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()

    # Shared AsyncSession mock (browser_mode=False)
    mock_session = AsyncMock()

    with (
        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
        patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
        patch(
            "app.services.scrape_pipeline.fetch_detail",
            AsyncMock(side_effect=AvitoBlockedError("HTTP 403 banned")),
        ),
        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.scraper_fetch_mode = "curl_cffi"
        s.scraper_proxy_url = None
        s.avito_proxy_max_rotations = 0  # no rotation budget
        s.avito_serp_ok_not_banned = True  # ← наш флаг
        s.avito_proxy_rotate_settle_s = 0.0
        s.proxy_rotate_attempts = 3
        s.proxy_rotate_attempt_timeout_s = 8.0

        await run_avito_city_sweep(
            mock_db,
            run_id=1,
            anchors=[(56.84, 60.61, "test-anchor")],
            detail_top_n=3,
            enrich_houses=False,
            enrich_imv=False,
        )

    assert (
        mock_runs.mark_done.called
    ), "mark_done должен быть вызван: SERP OK (ins=3), detail blocked"
    assert (
        not mock_runs.mark_banned.called
    ), "mark_banned НЕ должен быть вызван когда SERP уже сохранил лоты"
    # enrichment_abort_note передан в counters
    call_args = mock_runs.mark_done.call_args
    # mark_done(db, run_id, counters_dict)
    passed_counters: dict = call_args[0][2] if call_args[0] else {}
    assert (
        "enrichment_abort_note" in passed_counters
    ), "enrichment_abort_note должен присутствовать в counters переданных mark_done"


@pytest.mark.asyncio
async def test_avito_city_sweep_serp_zero_lots_marks_banned() -> None:
    """#1950-B: SERP вернул 0 лотов, SERP-блок → mark_banned (не mark_done).

    Условие: lots_inserted+lots_updated == 0.
    """
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db, mock_runs, _ = _make_city_sweep_mocks()

    # SERP сам поднимает AvitoBlockedError → anchor_lots не сохраняется
    mock_scraper_serp_block = MagicMock()
    mock_scraper_serp_block.fetch_around = AsyncMock(side_effect=AvitoBlockedError("SERP HTTP 403"))

    mock_session = AsyncMock()

    with (
        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper_serp_block),
        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.scraper_fetch_mode = "curl_cffi"
        s.scraper_proxy_url = None
        s.avito_proxy_max_rotations = 0
        s.avito_serp_ok_not_banned = True  # включён флаг, но лотов нет
        s.avito_proxy_rotate_settle_s = 0.0
        s.proxy_rotate_attempts = 3
        s.proxy_rotate_attempt_timeout_s = 8.0

        await run_avito_city_sweep(
            mock_db,
            run_id=2,
            anchors=[(56.84, 60.61, "test-anchor")],
            detail_top_n=3,
            enrich_houses=False,
            enrich_imv=False,
        )

    assert (
        mock_runs.mark_banned.called
    ), "mark_banned должен быть вызван: SERP сам заблокирован, lots=0"
    assert not mock_runs.mark_done.called, "mark_done НЕ должен быть вызван при lots_inserted == 0"


@pytest.mark.asyncio
async def test_avito_city_sweep_flag_off_still_marks_banned() -> None:
    """#1950-B: avito_serp_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
    from app.services.scrape_pipeline import run_avito_city_sweep

    mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
    mock_session = AsyncMock()

    with (
        patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
        patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
        patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
        patch(
            "app.services.scrape_pipeline.fetch_detail",
            AsyncMock(side_effect=AvitoBlockedError("HTTP 403")),
        ),
        patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.scraper_fetch_mode = "curl_cffi"
        s.scraper_proxy_url = None
        s.avito_proxy_max_rotations = 0
        s.avito_serp_ok_not_banned = False  # ← флаг выключен
        s.avito_proxy_rotate_settle_s = 0.0
        s.proxy_rotate_attempts = 3
        s.proxy_rotate_attempt_timeout_s = 8.0

        await run_avito_city_sweep(
            mock_db,
            run_id=3,
            anchors=[(56.84, 60.61, "test-anchor")],
            detail_top_n=3,
            enrich_houses=False,
            enrich_imv=False,
        )

    assert (
        mock_runs.mark_banned.called
    ), "avito_serp_ok_not_banned=False → mark_banned (backward-compat)"
    assert not mock_runs.mark_done.called


# ──────────────────────────────────────────────────────────────────────────────
# Fix C: cian per-fetch wait_for (#1949)
# ──────────────────────────────────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_cian_full_load_hanging_fetch_cancelled_by_timeout() -> None:
    """#1949-C: browser-fetch зависший навсегда отменяется per-fetch timeout.

    Monkey-patch заменяет _browser.fetch на _timed_fetch(wait_for timeout=0.05s).
    Зависший fetch отменяется через ~50ms → TimeoutError → run завершается.
    """
    from app.services.scrape_pipeline import run_cian_full_load

    fetch_was_called = False

    async def hanging_fetch(url: str) -> str:
        nonlocal fetch_was_called
        fetch_was_called = True
        await asyncio.Event().wait()  # висит вечно
        return ""

    mock_browser = MagicMock()
    mock_browser.fetch = hanging_fetch

    mock_scraper = MagicMock()
    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
    mock_scraper.__aexit__ = AsyncMock(return_value=None)
    mock_scraper._browser = mock_browser
    mock_scraper.request_delay_sec = 0.0

    # fetch_all_secondary симулирует _fetch_page_html: вызывает fetch и ловит Exception
    async def fake_fetch_all_secondary(**kwargs: object) -> None:
        try:
            # После monkey-patch _browser.fetch = _timed_fetch (с timeout=0.05s)
            await mock_scraper._browser.fetch("https://ekb.cian.ru/sale/flat/1/")
        except Exception:
            pass  # TimeoutError → аналог _fetch_page_html → returns None

    mock_scraper.fetch_all_secondary = fake_fetch_all_secondary

    mock_db = MagicMock()
    mock_runs = MagicMock()
    mock_runs.is_cancelled.return_value = False
    mock_runs.mark_done = MagicMock()
    mock_runs.mark_failed = MagicMock()
    mock_runs.update_heartbeat = MagicMock()

    t0 = time.monotonic()
    with (
        patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.cian_full_load_per_fetch_timeout_s = 0.05  # 50ms — быстрый timeout для теста
        s.cian_full_load_detail_top_n = 0

        await run_cian_full_load(mock_db, run_id=1)

    elapsed = time.monotonic() - t0

    assert fetch_was_called, "hanging_fetch должен быть вызван через monkey-patched _browser.fetch"
    assert (
        elapsed < 3.0
    ), f"run должен завершиться быстро (fetch timeout 50ms), занял {elapsed:.2f}s"
    assert mock_runs.mark_done.called, "run_cian_full_load должен завершиться mark_done"


@pytest.mark.asyncio
async def test_cian_full_load_per_fetch_timeout_disabled_when_zero() -> None:
    """#1949-C: cian_full_load_per_fetch_timeout_s=0 → monkey-patch НЕ применяется."""
    from app.services.scrape_pipeline import run_cian_full_load

    original_fetch = AsyncMock(return_value="<html/>")

    mock_browser = MagicMock()
    mock_browser.fetch = original_fetch

    mock_scraper = MagicMock()
    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
    mock_scraper.__aexit__ = AsyncMock(return_value=None)
    mock_scraper._browser = mock_browser
    mock_scraper.request_delay_sec = 0.0
    mock_scraper.fetch_all_secondary = AsyncMock(return_value=None)

    mock_db = MagicMock()
    mock_runs = MagicMock()
    mock_runs.is_cancelled.return_value = False
    mock_runs.mark_done = MagicMock()
    mock_runs.mark_failed = MagicMock()
    mock_runs.update_heartbeat = MagicMock()

    with (
        patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.cian_full_load_per_fetch_timeout_s = 0.0  # отключено
        s.cian_full_load_detail_top_n = 0

        await run_cian_full_load(mock_db, run_id=2)

    # Когда timeout=0 (отключён), monkey-patch не применяется → fetch остаётся original
    assert (
        mock_browser.fetch is original_fetch
    ), "cian_full_load_per_fetch_timeout_s=0 → fetch НЕ должен быть заменён monkey-patch"


# ──────────────────────────────────────────────────────────────────────────────
# Fix D: cian background heartbeat created and cancelled (#1949)
# ──────────────────────────────────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_cian_full_load_background_heartbeat_created_and_cancelled() -> None:
    """#1949-D: background heartbeat task создаётся и отменяется в finally."""
    from app.services.scrape_pipeline import run_cian_full_load

    mock_scraper = MagicMock()
    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
    mock_scraper.__aexit__ = AsyncMock(return_value=None)
    mock_scraper._browser = None  # per-fetch timeout не применяется
    mock_scraper.fetch_all_secondary = AsyncMock(return_value=None)

    mock_db = MagicMock()
    mock_runs = MagicMock()
    mock_runs.is_cancelled.return_value = False
    mock_runs.mark_done = MagicMock()
    mock_runs.mark_failed = MagicMock()
    mock_runs.update_heartbeat = MagicMock()

    created_tasks: list[asyncio.Task[None]] = []
    _real_create_task = asyncio.create_task

    def _tracking_create_task(coro: Any, *, name: str | None = None) -> asyncio.Task[None]:
        task = _real_create_task(coro, name=name)
        created_tasks.append(task)
        return task  # type: ignore[return-value]

    with (
        patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
        patch(
            "app.services.scrape_pipeline.asyncio.create_task",
            side_effect=_tracking_create_task,
        ),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.cian_full_load_per_fetch_timeout_s = 0.0
        s.cian_full_load_detail_top_n = 0

        await run_cian_full_load(mock_db, run_id=3)

    assert (
        len(created_tasks) >= 1
    ), "asyncio.create_task должен быть вызван хотя бы раз (background heartbeat)"
    for task in created_tasks:
        assert (
            task.cancelled() or task.done()
        ), f"Heartbeat task должен быть отменён в finally: {task}"


@pytest.mark.asyncio
async def test_cian_full_load_heartbeat_cancelled_on_exception() -> None:
    """#1949-D: heartbeat task отменяется в finally даже при исключении в run."""
    from app.services.scrape_pipeline import run_cian_full_load

    mock_scraper = MagicMock()
    mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
    mock_scraper.__aexit__ = AsyncMock(return_value=None)
    mock_scraper._browser = None
    mock_scraper.fetch_all_secondary = AsyncMock(side_effect=RuntimeError("fatal-test-error"))

    mock_db = MagicMock()
    mock_runs = MagicMock()
    mock_runs.is_cancelled.return_value = False
    mock_runs.mark_done = MagicMock()
    mock_runs.mark_failed = MagicMock()
    mock_runs.update_heartbeat = MagicMock()

    created_tasks: list[asyncio.Task[None]] = []
    _real_create_task = asyncio.create_task

    def _tracking_create_task(coro: Any, *, name: str | None = None) -> asyncio.Task[None]:
        task = _real_create_task(coro, name=name)
        created_tasks.append(task)
        return task  # type: ignore[return-value]

    with (
        patch("app.services.scrapers.cian.CianScraper", return_value=mock_scraper),
        patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
        patch(
            "app.services.scrape_pipeline.asyncio.create_task",
            side_effect=_tracking_create_task,
        ),
        patch("app.services.scrape_pipeline.settings") as s,
    ):
        s.cian_full_load_per_fetch_timeout_s = 0.0
        s.cian_full_load_detail_top_n = 0

        with pytest.raises(RuntimeError, match="fatal-test-error"):
            await run_cian_full_load(mock_db, run_id=4)

    assert len(created_tasks) >= 1, "heartbeat task должен быть создан"
    for task in created_tasks:
        assert (
            task.cancelled() or task.done()
        ), f"Heartbeat task должен быть отменён в finally даже при RuntimeError: {task}"
    assert mock_runs.mark_failed.called, "mark_failed должен быть вызван при RuntimeError"