gendesign/tradein-mvp/backend/tests/test_scraper_proxy.py

"""Tests for scraper proxy wiring (#806 + follow-up).

Covers:
  - scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL
  - AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
  - None/empty → direct connection (no proxies dict)
  - _avito_proxies() helper returns correct dict shape
  - per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation,
    CianNewbuilding (fetch_newbuilding — now BrowserFetcher, proxy is browser-level),
    AvitoIMV, AvitoDetail
"""

from __future__ import annotations

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch

import pytest


def _mock_settings(
    scraper_proxy_url: str | None,
    yandex_cookies_file: str | None = None,
) -> SimpleNamespace:
    """Minimal settings stand-in with fields read by proxy helpers and scraper __aenter__."""
    return SimpleNamespace(
        scraper_proxy_url=scraper_proxy_url,
        # Cian scrapers read settings.cian_proxy_url (property: CIAN_PROXY_URL or
        # scraper_proxy_url fallback). Mirror scraper_proxy_url here so the existing
        # cian proxy-wiring assertions keep passing (#923 follow-up).
        cian_proxy_url=scraper_proxy_url,
        # Yandex scrapers read settings.yandex_proxy_url (property: YANDEX_PROXY_URL or
        # scraper_proxy_url fallback). Mirror scraper_proxy_url here for test compat.
        yandex_proxy_url=scraper_proxy_url,
        yandex_cookies_file=yandex_cookies_file,
    )


# ── Settings.scraper_proxy_url property ──────────────────────────────────────
# Test the property logic directly via inline replica — avoids instantiating
# pydantic-settings (requires DATABASE_URL) for pure unit tests of the rule.


def _resolve_scraper_proxy_url(
    scraper_proxy_url_env: str | None, avito_proxy_url: str | None
) -> str | None:
    """Inline replica of Settings.scraper_proxy_url property logic."""
    return scraper_proxy_url_env or avito_proxy_url


def test_scraper_proxy_url_uses_scraper_env_when_set():
    result = _resolve_scraper_proxy_url(
        scraper_proxy_url_env="http://scraper-proxy:1234",
        avito_proxy_url="http://avito-proxy:5678",
    )
    assert result == "http://scraper-proxy:1234"


def test_scraper_proxy_url_falls_back_to_avito_proxy_url():
    """AVITO_PROXY_URL used when SCRAPER_PROXY_URL absent — zero env change on prod."""
    result = _resolve_scraper_proxy_url(
        scraper_proxy_url_env=None,
        avito_proxy_url="http://avito-proxy:5678",
    )
    assert result == "http://avito-proxy:5678"


def test_scraper_proxy_url_none_when_both_absent():
    result = _resolve_scraper_proxy_url(scraper_proxy_url_env=None, avito_proxy_url=None)
    assert result is None


def test_scraper_proxy_url_empty_string_falls_through_to_avito():
    """Empty string is falsy — falls through to avito_proxy_url."""
    result = _resolve_scraper_proxy_url(
        scraper_proxy_url_env="",
        avito_proxy_url="http://avito-proxy:5678",
    )
    assert result == "http://avito-proxy:5678"


# ── _avito_proxies() dict shape ───────────────────────────────────────────────


def test_avito_proxies_returns_correct_dict_shape():
    from app.services.scrape_pipeline import _avito_proxies

    proxy_url = "http://user:pass@proxy.example.com:8080"
    with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)):
        result = _avito_proxies()

    assert result == {"http": proxy_url, "https": proxy_url}


def test_avito_proxies_returns_none_when_no_proxy():
    from app.services.scrape_pipeline import _avito_proxies

    with patch("app.services.scrape_pipeline.settings", _mock_settings(None)):
        result = _avito_proxies()

    assert result is None


def test_avito_proxies_fallback_via_avito_proxy_url():
    """_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback.

    When settings.scraper_proxy_url resolves to the avito URL (fallback path),
    the dict shape must be {"http": url, "https": url}.
    """
    from app.services.scrape_pipeline import _avito_proxies

    avito_url = "http://mobile-proxy.space:9090"
    with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)):
        result = _avito_proxies()

    assert result == {"http": avito_url, "https": avito_url}


# ── Real pydantic env-binding (regression for #806 fixup) ─────────────────────
# The inline-replica tests above prove the OR-logic but NOT that the field
# actually binds to env SCRAPER_PROXY_URL. `validation_alias` is what makes that
# work — without it pydantic-settings reads SCRAPER_PROXY_URL_ENV (the field
# name), so setting SCRAPER_PROXY_URL would be a silent no-op. These tests
# construct a real Settings and assert end-to-end env binding.


def _fresh_settings():
    from app.core.config import Settings

    return Settings(_env_file=None)


def test_settings_binds_scraper_proxy_url_env(monkeypatch):
    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234")
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234"


def test_settings_scraper_env_overrides_avito(monkeypatch):
    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234")
    monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678")
    assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234"


def test_settings_falls_back_to_avito_env(monkeypatch):
    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
    monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678")
    assert _fresh_settings().scraper_proxy_url == "http://avito-proxy:5678"


def test_settings_ignores_misnamed_scraper_proxy_url_env(monkeypatch):
    """Regression: the old wrong env name SCRAPER_PROXY_URL_ENV must NOT bind."""
    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    monkeypatch.setenv("SCRAPER_PROXY_URL_ENV", "http://wrong-name:1111")
    assert _fresh_settings().scraper_proxy_url is None


# ── Per-scraper proxy wiring (follow-up: yandex/valuation/imv/newbuilding) ────
# Each test: patch settings.scraper_proxy_url, mock AsyncSession constructor,
# invoke __aenter__ / create-session path, assert proxies= kwarg is forwarded.
# proxy=None path: assert proxies kwarg is None / not passed as non-None.


_PROXY_URL = "http://mobile.proxy:8080"
_EXPECTED_PROXIES = {"http": _PROXY_URL, "https": _PROXY_URL}


# ── YandexRealtyScraper ──────────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_yandex_realty_session_receives_proxies(monkeypatch):
    """YandexRealtyScraper.__aenter__ forwards scraper_proxy_url as proxies= kwarg.

    settings is imported deferred inside __aenter__ via `from app.core.config import settings`,
    so we patch app.core.config.settings (module attribute) which the deferred import reads.
    """
    from app.services.scrapers.yandex_realty import YandexRealtyScraper

    mock_session = MagicMock()
    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
    mock_session.__aexit__ = AsyncMock(return_value=None)
    mock_session.close = AsyncMock()

    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    monkeypatch.setattr("app.services.scrapers.yandex_realty._CurlCffiSession", _fake_session)
    with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
        scraper = YandexRealtyScraper()
        await scraper.__aenter__()

    assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES


@pytest.mark.asyncio
async def test_yandex_realty_session_no_proxies_when_none(monkeypatch):
    """YandexRealtyScraper.__aenter__: proxies=None when scraper_proxy_url is None."""
    from app.services.scrapers.yandex_realty import YandexRealtyScraper

    mock_session = MagicMock()
    mock_session.close = AsyncMock()
    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    monkeypatch.setattr("app.services.scrapers.yandex_realty._CurlCffiSession", _fake_session)
    with patch("app.core.config.settings", _mock_settings(None)):
        scraper = YandexRealtyScraper()
        await scraper.__aenter__()

    assert captured_kwargs.get("proxies") is None


# ── YandexValuationScraper ───────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_yandex_valuation_session_receives_proxies():
    """YandexValuationScraper.__aenter__ forwards proxies= kwarg."""
    from app.services.scrapers.yandex_valuation import YandexValuationScraper

    mock_session = MagicMock()
    mock_session.close = AsyncMock()
    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session):
        with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
            scraper = YandexValuationScraper()
            await scraper.__aenter__()

    assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES


@pytest.mark.asyncio
async def test_yandex_valuation_session_no_proxies_when_none():
    """YandexValuationScraper.__aenter__: proxies=None when no proxy configured."""
    from app.services.scrapers.yandex_valuation import YandexValuationScraper

    mock_session = MagicMock()
    mock_session.close = AsyncMock()
    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session):
        with patch("app.core.config.settings", _mock_settings(None)):
            scraper = YandexValuationScraper()
            await scraper.__aenter__()

    assert captured_kwargs.get("proxies") is None


# ── CianNewbuilding — fetch_newbuilding ───────────────────────────────────────
# После #972 fetch_newbuilding тянет страницу ЖК через BrowserFetcher (camoufox),
# а НЕ через curl_cffi AsyncSession. Прокси применяются на уровне браузер-контейнера,
# а не через proxies= kwarg. AsyncSession для page-fetch больше НЕ создаётся.


@pytest.mark.asyncio
async def test_cian_newbuilding_own_session_receives_proxies():
    """fetch_newbuilding использует BrowserFetcher для page-fetch, не AsyncSession.

    Проверяем:
    - BrowserFetcher.fetch(url) вызывается с правильным URL
    - curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level)
    """
    from app.services.scrapers import cian_newbuilding

    zhk_url = "https://zhk-test.cian.ru/"
    fetch_calls: list[str] = []
    session_ctor_calls: list[dict] = []

    mock_fetcher = MagicMock()
    mock_fetcher.fetch = AsyncMock(
        side_effect=lambda url: fetch_calls.append(url) or "<html></html>"
    )
    mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
    mock_fetcher.__aexit__ = AsyncMock(return_value=None)

    def _fake_async_session(*args, **kwargs):
        session_ctor_calls.append(kwargs)
        return MagicMock()

    with patch.object(cian_newbuilding, "BrowserFetcher", lambda: mock_fetcher):
        with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
            await cian_newbuilding.fetch_newbuilding(zhk_url)

    # BrowserFetcher.fetch вызван с ЖК-url
    assert fetch_calls == [zhk_url]
    # curl_cffi AsyncSession НЕ создавался для page-fetch
    assert session_ctor_calls == []


@pytest.mark.asyncio
async def test_cian_newbuilding_shared_session_not_recreated():
    """fetch_newbuilding принимает session= для backward-compat, но использует BrowserFetcher.

    Даже когда передан внешний session — страница ЖК тянется через BrowserFetcher,
    AsyncSession не пересоздаётся, session.get не вызывается.
    """
    from app.services.scrapers import cian_newbuilding

    zhk_url = "https://zhk-test.cian.ru/"
    fetch_calls: list[str] = []
    session_ctor_calls: list[dict] = []

    mock_fetcher = MagicMock()
    mock_fetcher.fetch = AsyncMock(
        side_effect=lambda url: fetch_calls.append(url) or "<html></html>"
    )
    mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
    mock_fetcher.__aexit__ = AsyncMock(return_value=None)

    def _fake_async_session(*args, **kwargs):
        session_ctor_calls.append(kwargs)
        return MagicMock()

    sentinel_session = MagicMock(name="external_session")

    with patch.object(cian_newbuilding, "BrowserFetcher", lambda: mock_fetcher):
        with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
            await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session)

    # BrowserFetcher.fetch вызван — именно через него тянется страница
    assert fetch_calls == [zhk_url]
    # Ни нового AsyncSession, ни вызова переданного session.get — proxy browser-level
    assert session_ctor_calls == []
    sentinel_session.get.assert_not_called()


# ── AvitoDetail — fetch_detail own-session ───────────────────────────────────


@pytest.mark.asyncio
async def test_avito_detail_own_session_receives_proxies():
    """fetch_detail creates own session with proxies= when no cffi_session passed."""
    from app.services.scrapers import avito_detail

    mock_session = MagicMock()
    mock_resp = MagicMock()
    mock_resp.status_code = 200
    # Minimal HTML that parse_detail_html won't raise ValueError (item_id required)
    mock_resp.text = "<html><body></body></html>"
    mock_session.get = AsyncMock(return_value=mock_resp)
    mock_session.close = AsyncMock()

    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    with patch.object(avito_detail, "AsyncSession", _fake_session):
        with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
            try:
                await avito_detail.fetch_detail("/ekaterinburg/kvartiry/test-1234")
            except ValueError:
                pass  # parse_detail_html raises ValueError (no item_id) — that's ok

    assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES


@pytest.mark.asyncio
async def test_avito_detail_shared_session_not_patched():
    """fetch_detail does NOT build own session when cffi_session is provided."""
    from app.services.scrapers import avito_detail

    mock_session = MagicMock()
    mock_resp = MagicMock()
    mock_resp.status_code = 200
    mock_resp.text = "<html><body></body></html>"
    mock_session.get = AsyncMock(return_value=mock_resp)
    mock_session.close = AsyncMock()

    ctor_calls = []

    def _fake_session(*args, **kwargs):
        ctor_calls.append(kwargs)
        return MagicMock()

    with patch.object(avito_detail, "AsyncSession", _fake_session):
        with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
            try:
                await avito_detail.fetch_detail(
                    "/ekaterinburg/kvartiry/test-1234",
                    cffi_session=mock_session,
                )
            except ValueError:
                pass

    # No own session created — shared session was used as-is
    assert ctor_calls == []


# ── AvitoIMV — evaluate_via_imv own-session ──────────────────────────────────


@pytest.mark.asyncio
async def test_avito_imv_own_session_receives_proxies():
    """evaluate_via_imv creates own session with proxies= when none passed.

    `CffiAsyncSession` is imported locally inside evaluate_via_imv as
    `from curl_cffi.requests import AsyncSession as CffiAsyncSession`, so we
    patch `curl_cffi.requests.AsyncSession` (the source) to intercept it.
    """
    from app.services.scrapers import avito_imv

    mock_session = MagicMock()
    mock_warmup_resp = MagicMock()
    mock_warmup_resp.status_code = 200
    mock_geo_resp = MagicMock()
    mock_geo_resp.status_code = 200
    mock_geo_resp.json = MagicMock(return_value={"result": {"point": None}})
    mock_session.get = AsyncMock(return_value=mock_warmup_resp)
    mock_session.post = AsyncMock(return_value=mock_geo_resp)
    mock_session.close = AsyncMock()

    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    # Patch the source so the local `from curl_cffi.requests import AsyncSession` picks it up
    with patch("curl_cffi.requests.AsyncSession", _fake_session):
        with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
            try:
                await avito_imv.evaluate_via_imv(
                    address="ул. Тургенева, 4",
                    rooms=2,
                    area_m2=50.0,
                    floor=3,
                    floor_at_home=9,
                    house_type="panel",
                    renovation_type="cosmetic",
                    has_balcony=False,
                    has_loggia=False,
                )
            except Exception:
                pass  # network/parse failures expected in unit test

    assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES


# ── Settings.cian_proxy_url property ─────────────────────────────────────────
# cian_proxy_url: CIAN_PROXY_URL > scraper_proxy_url (fallback) > None.


def _resolve_cian_proxy_url(
    cian_proxy_url_env: str | None, scraper_proxy_url: str | None
) -> str | None:
    """Inline replica of Settings.cian_proxy_url property logic."""
    return cian_proxy_url_env or scraper_proxy_url


def test_cian_proxy_url_uses_cian_env_when_set():
    """CIAN_PROXY_URL takes precedence over shared scraper_proxy_url."""
    result = _resolve_cian_proxy_url(
        cian_proxy_url_env="http://cian-proxy:2222",
        scraper_proxy_url="http://shared-proxy:1111",
    )
    assert result == "http://cian-proxy:2222"


def test_cian_proxy_url_falls_back_to_scraper_proxy_url():
    """Falls back to shared scraper_proxy_url when CIAN_PROXY_URL is absent."""
    result = _resolve_cian_proxy_url(
        cian_proxy_url_env=None,
        scraper_proxy_url="http://shared-proxy:1111",
    )
    assert result == "http://shared-proxy:1111"


def test_cian_proxy_url_none_when_both_absent():
    result = _resolve_cian_proxy_url(cian_proxy_url_env=None, scraper_proxy_url=None)
    assert result is None


def test_settings_binds_cian_proxy_url_env(monkeypatch):
    """CIAN_PROXY_URL env binds to cian_proxy_url_env via validation_alias."""
    monkeypatch.setenv("CIAN_PROXY_URL", "http://cian-proxy:2222")
    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    assert _fresh_settings().cian_proxy_url == "http://cian-proxy:2222"


def test_settings_cian_env_overrides_scraper_proxy(monkeypatch):
    """CIAN_PROXY_URL overrides shared SCRAPER_PROXY_URL for Cian scrapers."""
    monkeypatch.setenv("CIAN_PROXY_URL", "http://cian-proxy:2222")
    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://shared-proxy:1111")
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    s = _fresh_settings()
    assert s.cian_proxy_url == "http://cian-proxy:2222"
    assert s.scraper_proxy_url == "http://shared-proxy:1111"


def test_settings_cian_proxy_falls_back_to_scraper(monkeypatch):
    """Without CIAN_PROXY_URL, cian_proxy_url falls back to scraper_proxy_url."""
    monkeypatch.delenv("CIAN_PROXY_URL", raising=False)
    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://shared-proxy:1111")
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    assert _fresh_settings().cian_proxy_url == "http://shared-proxy:1111"