"""#901/#1805 — avito SERP fetch via BrowserFetcher (camoufox), curl_cffi fallback.

Tests:
1. browser branch default (#1805): __aenter__ → _browser set AND _cffi fallback armed.
2. curl branch unchanged: _browser is None → _cffi session used directly.
3. firewall-detect in browser-mode: firewall HTML + no rotate URL + no cffi → AvitoBlockedError.
3b. #1805 curl_cffi fallback: browser firewall → curl retry returns HTML / re-raises if also banned.
4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks.

Без сети, без БД, без curl_cffi.
"""

from __future__ import annotations

import os
from unittest.mock import AsyncMock, MagicMock, patch

import httpx
import pytest

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")

from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
from app.services.scrapers.base import ScrapedLot

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

_NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
_FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"


def _sidecar_error(status: int, error: str) -> httpx.HTTPStatusError:
    """Build a tradein-browser sidecar HTTPStatusError with a real JSON body.

    Mirrors what BrowserFetcher.fetch re-raises: an httpx.HTTPStatusError whose
    `.response` is intact so `.response.json()["error"]` and `.response.status_code`
    work the same as in production.
    """
    request = httpx.Request("POST", "http://tradein-browser:3000/fetch")
    response = httpx.Response(status, json={"error": error}, request=request)
    return httpx.HTTPStatusError(f"{status} error", request=request, response=response)


def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
    """Return a mock BrowserFetcher context manager that yields html from fetch()."""
    mock = MagicMock()
    mock.__aenter__ = AsyncMock(return_value=mock)
    mock.__aexit__ = AsyncMock(return_value=None)
    mock.fetch = AsyncMock(return_value=html)
    return mock


# ---------------------------------------------------------------------------
# 1. browser branch chosen
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_aenter_browser_mode_arms_cffi_fallback() -> None:
    """#1805: mode=browser → __aenter__ поднимает BrowserFetcher И curl_cffi-fallback.

    Раньше browser-mode оставлял _cffi=None; теперь curl_cffi-сессия строится тут
    же как fallback на firewall/пусто (см. _fetch_serp_html_browser).
    """
    mock_browser = _make_mock_browser()
    scraper = AvitoScraper()

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.scraper_fetch_mode = "browser"
        mock_settings.scraper_proxy_url = None
        mock_settings.avito_proxy_rotate_url = None
        mock_settings.avito_proxy_max_rotations = 2

        with patch(
            "app.services.scrapers.avito.BrowserFetcher",
            return_value=mock_browser,
        ):
            result = await scraper.__aenter__()

    assert result is scraper
    assert scraper._browser is mock_browser
    # curl_cffi fallback armed (real AsyncSession built, no network/proxy).
    assert scraper._cffi is not None
    await scraper._cffi.close()


@pytest.mark.asyncio
async def test_aenter_proactive_rotate_when_rotate_url_set() -> None:
    """#1731: rotate_url configured → __aenter__ calls _rotate_ip exactly once at start.

    _rotate_ip is mocked (no real changeip HTTP / no 9s sleep). Browser mode used
    to keep BaseScraper setup out of the picture.
    """
    mock_browser = _make_mock_browser()
    scraper = AvitoScraper()
    mock_rotate = AsyncMock(return_value=True)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.scraper_fetch_mode = "browser"
        mock_settings.scraper_proxy_url = None
        mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
        mock_settings.avito_proxy_max_rotations = 4

        with patch(
            "app.services.scrapers.avito.BrowserFetcher",
            return_value=mock_browser,
        ):
            with patch.object(scraper, "_rotate_ip", new=mock_rotate):
                result = await scraper.__aenter__()

    assert result is scraper
    mock_rotate.assert_awaited_once()


@pytest.mark.asyncio
async def test_aenter_no_proactive_rotate_when_rotate_url_none() -> None:
    """#1731: rotate_url is None → __aenter__ must NOT call _rotate_ip."""
    mock_browser = _make_mock_browser()
    scraper = AvitoScraper()
    mock_rotate = AsyncMock(return_value=True)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.scraper_fetch_mode = "browser"
        mock_settings.scraper_proxy_url = None
        mock_settings.avito_proxy_rotate_url = None
        mock_settings.avito_proxy_max_rotations = 4

        with patch(
            "app.services.scrapers.avito.BrowserFetcher",
            return_value=mock_browser,
        ):
            with patch.object(scraper, "_rotate_ip", new=mock_rotate):
                await scraper.__aenter__()

    mock_rotate.assert_not_awaited()


@pytest.mark.asyncio
async def test_aenter_proactive_rotate_failure_does_not_abort() -> None:
    """#1731: a failed proactive rotate (returns False / raises) must NOT abort __aenter__."""
    mock_browser = _make_mock_browser()
    scraper = AvitoScraper()

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.scraper_fetch_mode = "browser"
        mock_settings.scraper_proxy_url = None
        mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
        mock_settings.avito_proxy_max_rotations = 4

        with patch(
            "app.services.scrapers.avito.BrowserFetcher",
            return_value=mock_browser,
        ):
            # _rotate_ip raises — sweep must proceed (browser still attached)
            with patch.object(
                scraper, "_rotate_ip", new=AsyncMock(side_effect=RuntimeError("changeip down"))
            ):
                result = await scraper.__aenter__()

    assert result is scraper
    assert scraper._browser is mock_browser


@pytest.mark.asyncio
async def test_fetch_serp_html_browser_mode_returns_html() -> None:
    """_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""
    scraper = AvitoScraper()
    mock_browser = _make_mock_browser(_NORMAL_HTML)
    scraper._browser = mock_browser

    result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)

    assert result == _NORMAL_HTML
    mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
    # _cffi not armed (direct _browser assignment bypasses __aenter__, which would
    # otherwise build the curl_cffi fallback session); the browser path never touches it.
    assert scraper._cffi is None


# ---------------------------------------------------------------------------
# 2. curl branch unchanged
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_fetch_serp_html_curl_mode_uses_cffi() -> None:
    """Default mode (_browser is None) → _fetch_serp_html goes through curl path."""
    scraper = AvitoScraper()

    # Mock the curl session returning HTTP 200 with normal HTML
    mock_response = MagicMock()
    mock_response.status_code = 200
    mock_response.text = _NORMAL_HTML

    mock_cffi = MagicMock()
    mock_cffi.get = AsyncMock(return_value=mock_response)
    scraper._cffi = mock_cffi

    # _browser must be None (default curl path)
    assert scraper._browser is None

    result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)

    assert result == _NORMAL_HTML
    mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")


# ---------------------------------------------------------------------------
# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL)
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_browser_mode_firewall_raises_blocked_error() -> None:
    """Browser firewall + no rotate URL + no curl fallback (_cffi=None) → AvitoBlockedError."""
    scraper = AvitoScraper()
    mock_browser = _make_mock_browser(_FIREWALL_HTML)
    scraper._browser = mock_browser
    scraper._cffi = None  # no curl_cffi fallback available (pipeline shared-browser path)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = None  # no rotation configured
        mock_settings.avito_proxy_max_rotations = 2

        with pytest.raises(AvitoBlockedError, match="browser-mode"):
            await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)


# ---------------------------------------------------------------------------
# 3b. #1805 — curl_cffi fallback when browser returns firewall
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_browser_firewall_falls_back_to_cffi_returns_html() -> None:
    """#1805: browser firewall + curl_cffi fallback armed → curl returns clean HTML."""
    scraper = AvitoScraper()
    scraper._browser = _make_mock_browser(_FIREWALL_HTML)

    # curl_cffi fallback: HTTP 200 with clean (non-firewall) HTML.
    mock_resp = MagicMock()
    mock_resp.status_code = 200
    mock_resp.text = _NORMAL_HTML
    mock_cffi = MagicMock()
    mock_cffi.get = AsyncMock(return_value=mock_resp)
    scraper._cffi = mock_cffi

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = None
        mock_settings.avito_proxy_max_rotations = 0
        mock_settings.scraper_proxy_url = None

        result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)

    assert result == _NORMAL_HTML
    mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")


@pytest.mark.asyncio
async def test_browser_firewall_cffi_fallback_also_firewall_raises() -> None:
    """#1805: browser firewall AND curl_cffi fallback also firewall → AvitoBlockedError."""
    scraper = AvitoScraper()
    scraper._browser = _make_mock_browser(_FIREWALL_HTML)

    # curl_cffi fallback: HTTP 200 but ALSO firewall HTML → no escape.
    mock_resp = MagicMock()
    mock_resp.status_code = 200
    mock_resp.text = _FIREWALL_HTML
    mock_cffi = MagicMock()
    mock_cffi.get = AsyncMock(return_value=mock_resp)
    scraper._cffi = mock_cffi

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = None
        mock_settings.avito_proxy_max_rotations = 0
        mock_settings.scraper_proxy_url = None  # no backconnect reconnect retries

        # curl fallback raises its own AvitoBlockedError (firewall на HTTP 200);
        # browser branch re-raises it as a hard-block.
        with pytest.raises(AvitoBlockedError):
            await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)


@pytest.mark.asyncio
async def test_browser_mode_firewall_rotates_and_retries() -> None:
    """Browser returns firewall on first call, normal HTML after rotation → returns HTML."""
    scraper = AvitoScraper()

    # First call → firewall; second call → normal HTML
    mock_browser = MagicMock()
    mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML])
    scraper._browser = mock_browser

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate"
        mock_settings.avito_proxy_max_rotations = 1

        with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)):
            result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2)

    assert result == _NORMAL_HTML
    assert mock_browser.fetch.await_count == 2


# ---------------------------------------------------------------------------
# 4. pagination terminates via empty-parse
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
    """fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops.

    Verifies the documented invariant: callers break on `not lots` regardless of
    fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is
    already covered by unit tests above); here we confirm the caller loop behaviour.
    """
    scraper = AvitoScraper()

    html_p1 = "<html>page1</html>"
    html_p2 = "<html>page2</html>"

    lot = ScrapedLot(
        source="avito",
        source_url="https://www.avito.ru/ekaterinburg/kvartiry/test",
        source_id="LOT_1",
        price_rub=5_000_000,
    )

    call_count = 0

    async def mock_fetch_html(url: str, page: int) -> str:
        return html_p1 if page == 1 else html_p2

    def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
        nonlocal call_count
        call_count += 1
        return [lot] if "page1" in html else []

    with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html):
        with patch.object(scraper, "_parse_html", side_effect=mock_parse):
            with patch.object(scraper, "sleep_between_requests", new=AsyncMock()):
                result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0)

    assert len(result) == 1
    assert result[0].source_id == "LOT_1"
    # page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
    assert call_count == 2


# ---------------------------------------------------------------------------
# 5. rotation returns False → immediate AvitoBlockedError
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
    """rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.

    Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
    (e.g. rotation API timeout/failure) → we cannot recover → blocked.
    """
    scraper = AvitoScraper()
    scraper._browser = _make_mock_browser(_FIREWALL_HTML)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
        mock_settings.avito_proxy_max_rotations = 2
        with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
            with pytest.raises(AvitoBlockedError, match="browser-mode"):
                await scraper._fetch_serp_html("https://avito.ru/test", page=1)


# ---------------------------------------------------------------------------
# 6. #356 — sidecar HTTPStatusError (500/503) → graceful AvitoRateLimitedError
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_sidecar_proxy_too_many_requests_rotates_then_succeeds() -> None:
    """#356: sidecar 500 NS_ERROR_PROXY_TOO_MANY_REQUESTS, rotation available →
    after IP rotation a retry succeeds and the clean HTML is returned."""
    scraper = AvitoScraper()

    mock_browser = MagicMock()
    # First fetch raises the soft-ban 500; after rotation the second fetch is clean.
    mock_browser.fetch = AsyncMock(
        side_effect=[
            _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
            _NORMAL_HTML,
        ]
    )
    scraper._browser = mock_browser
    mock_rotate = AsyncMock(return_value=True)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
        mock_settings.avito_proxy_max_rotations = 2
        with patch.object(scraper, "_rotate_ip", new=mock_rotate):
            result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)

    assert result == _NORMAL_HTML
    mock_rotate.assert_awaited_once()
    assert mock_browser.fetch.await_count == 2


@pytest.mark.asyncio
async def test_sidecar_proxy_too_many_requests_exhausted_raises_rate_limited() -> None:
    """#356: sidecar 500 proxy-too-many with rotation budget exhausted →
    AvitoRateLimitedError (NOT a bare HTTPStatusError, NOT AvitoBlockedError)."""
    scraper = AvitoScraper()

    mock_browser = MagicMock()
    mock_browser.fetch = AsyncMock(
        side_effect=_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS")
    )
    scraper._browser = mock_browser
    mock_rotate = AsyncMock(return_value=True)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
        mock_settings.avito_proxy_max_rotations = 2
        with patch.object(scraper, "_rotate_ip", new=mock_rotate):
            with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
                await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)

    # 2 rotations consumed + 1 final fetch that exhausts the budget.
    assert mock_rotate.await_count == 2
    assert mock_browser.fetch.await_count == 3


@pytest.mark.asyncio
async def test_sidecar_transient_timeout_exhausts_budget_no_infinite_loop() -> None:
    """#356: sidecar 500 transient (Timeout) repeated past the transient budget with
    NO rotations → AvitoRateLimitedError after bounded retries (no infinite loop)."""
    scraper = AvitoScraper()

    mock_browser = MagicMock()
    mock_browser.fetch = AsyncMock(
        side_effect=_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded")
    )
    scraper._browser = mock_browser
    mock_rotate = AsyncMock(return_value=True)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = None  # no rotation → transient budget only
        mock_settings.avito_proxy_max_rotations = 0
        # Patch sleep so the bounded backoff doesn't slow the test.
        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
            with patch.object(scraper, "_rotate_ip", new=mock_rotate):
                with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
                    await scraper._fetch_serp_html("https://www.avito.ru/test", page=5)

    # Transient is NOT a soft-ban → never rotates.
    mock_rotate.assert_not_awaited()
    # Bounded: _AVITO_SIDECAR_TRANSIENT_RETRIES (2) retries + final attempt = 3.
    assert mock_browser.fetch.await_count == 3


@pytest.mark.asyncio
async def test_sidecar_503_browser_unavailable_exhausted_raises_rate_limited() -> None:
    """#356: sidecar 503 'browser unavailable (proxy may be down)' with rotation
    exhausted → AvitoRateLimitedError (soft-ban classification via status==503)."""
    scraper = AvitoScraper()

    mock_browser = MagicMock()
    mock_browser.fetch = AsyncMock(
        side_effect=_sidecar_error(503, "browser unavailable (proxy may be down)")
    )
    scraper._browser = mock_browser
    mock_rotate = AsyncMock(return_value=True)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
        mock_settings.avito_proxy_max_rotations = 1
        with patch.object(scraper, "_rotate_ip", new=mock_rotate):
            with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
                await scraper._fetch_serp_html("https://www.avito.ru/test", page=7)

    # 1 rotation consumed then exhausted.
    assert mock_rotate.await_count == 1
    assert mock_browser.fetch.await_count == 2


@pytest.mark.asyncio
async def test_sidecar_transient_then_soft_ban_independent_budgets() -> None:
    """#356: смешанная последовательность transient → soft-ban пинит, что бюджеты
    независимы и корректно чередуются в одном while-цикле.

    attempt 1: transient Timeout → тратит transient_left (1 из 2), backoff, continue.
    attempt 2: too_many_requests → soft-ban, тратит ротацию (1 из 1), continue.
    attempt 3: too_many_requests, ротация исчерпана → AvitoRateLimitedError.
    """
    scraper = AvitoScraper()

    mock_browser = MagicMock()
    mock_browser.fetch = AsyncMock(
        side_effect=[
            _sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded"),
            _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
            _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
        ]
    )
    scraper._browser = mock_browser
    mock_rotate = AsyncMock(return_value=True)

    with patch("app.services.scrapers.avito.settings") as mock_settings:
        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
        mock_settings.avito_proxy_max_rotations = 1
        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
            with patch.object(scraper, "_rotate_ip", new=mock_rotate):
                with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
                    await scraper._fetch_serp_html("https://www.avito.ru/test", page=9)

    # 1 transient retry + 1 rotation, оба бюджета тронуты до исчерпания.
    assert mock_rotate.await_count == 1
    assert mock_browser.fetch.await_count == 3
