gendesign/tradein-mvp/backend/tests/test_scraper_proxy.py

"""Tests for scraper proxy wiring (#806 + follow-up).

Covers:
  - scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL
  - AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
  - None/empty → direct connection (no proxies dict)
  - _avito_proxies() helper returns correct dict shape
  - per-scraper proxy wiring: YandexRealty, YandexValuation (kit), CianValuation,
    CianNewbuilding (fetch_newbuilding — now BrowserFetcher, proxy is browser-level),
    AvitoDetail

Легаси `app.services.scrapers.yandex_valuation` / `avito_imv` удалены (#2277
финальный шаг scraper_kit-миграции). YandexValuation proxy-wiring тесты переведены
на kit-эквивалент (`config=` DI вместо патча `app.core.config.settings`).
`test_avito_imv_own_session_receives_proxies` (легаси, patch-based) удалён —
эквивалентная и более точная kit-регрессия (config footgun: с/без `config=`) уже
покрыта в `tests/scrapers/test_avito_imv_kit_parity.py::
test_kit_evaluate_via_imv_{with,without}_config_{uses,drops}_proxy`.
"""

from __future__ import annotations

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch

import pytest


def _mock_settings(
    scraper_proxy_url: str | None,
    yandex_cookies_file: str | None = None,
) -> SimpleNamespace:
    """Minimal settings stand-in with fields read by proxy helpers and scraper __aenter__."""
    return SimpleNamespace(
        scraper_proxy_url=scraper_proxy_url,
        # Cian scrapers read settings.cian_proxy_url (property: CIAN_PROXY_URL or
        # scraper_proxy_url fallback). Mirror scraper_proxy_url here so the existing
        # cian proxy-wiring assertions keep passing (#923 follow-up).
        cian_proxy_url=scraper_proxy_url,
        # Yandex scrapers read settings.yandex_proxy_url (property: YANDEX_PROXY_URL or
        # scraper_proxy_url fallback). Mirror scraper_proxy_url here for test compat.
        yandex_proxy_url=scraper_proxy_url,
        yandex_cookies_file=yandex_cookies_file,
    )


# ── Settings.scraper_proxy_url property ──────────────────────────────────────
# Test the property logic directly via inline replica — avoids instantiating
# pydantic-settings (requires DATABASE_URL) for pure unit tests of the rule.


def _resolve_scraper_proxy_url(
    scraper_proxy_url_env: str | None, avito_proxy_url: str | None
) -> str | None:
    """Inline replica of Settings.scraper_proxy_url property logic."""
    return scraper_proxy_url_env or avito_proxy_url


def test_scraper_proxy_url_uses_scraper_env_when_set():
    result = _resolve_scraper_proxy_url(
        scraper_proxy_url_env="http://scraper-proxy:1234",
        avito_proxy_url="http://avito-proxy:5678",
    )
    assert result == "http://scraper-proxy:1234"


def test_scraper_proxy_url_falls_back_to_avito_proxy_url():
    """AVITO_PROXY_URL used when SCRAPER_PROXY_URL absent — zero env change on prod."""
    result = _resolve_scraper_proxy_url(
        scraper_proxy_url_env=None,
        avito_proxy_url="http://avito-proxy:5678",
    )
    assert result == "http://avito-proxy:5678"


def test_scraper_proxy_url_none_when_both_absent():
    result = _resolve_scraper_proxy_url(scraper_proxy_url_env=None, avito_proxy_url=None)
    assert result is None


def test_scraper_proxy_url_empty_string_falls_through_to_avito():
    """Empty string is falsy — falls through to avito_proxy_url."""
    result = _resolve_scraper_proxy_url(
        scraper_proxy_url_env="",
        avito_proxy_url="http://avito-proxy:5678",
    )
    assert result == "http://avito-proxy:5678"


# ── _avito_proxies() dict shape ───────────────────────────────────────────────


def test_avito_proxies_returns_correct_dict_shape():
    """Kit `_avito_proxies(config)` — DI параметр вместо module-level settings patch."""
    from scraper_kit.orchestration.pipeline import _avito_proxies

    proxy_url = "http://user:pass@proxy.example.com:8080"
    result = _avito_proxies(_mock_settings(proxy_url))

    assert result == {"http": proxy_url, "https": proxy_url}


def test_avito_proxies_returns_none_when_no_proxy():
    from scraper_kit.orchestration.pipeline import _avito_proxies

    result = _avito_proxies(_mock_settings(None))

    assert result is None


def test_avito_proxies_fallback_via_avito_proxy_url():
    """_avito_proxies(config) uses AVITO_PROXY_URL via scraper_proxy_url fallback.

    When config.scraper_proxy_url resolves to the avito URL (fallback path),
    the dict shape must be {"http": url, "https": url}.
    """
    from scraper_kit.orchestration.pipeline import _avito_proxies

    avito_url = "http://mobile-proxy.space:9090"
    result = _avito_proxies(_mock_settings(avito_url))

    assert result == {"http": avito_url, "https": avito_url}


# ── Real pydantic env-binding (regression for #806 fixup) ─────────────────────
# The inline-replica tests above prove the OR-logic but NOT that the field
# actually binds to env SCRAPER_PROXY_URL. `validation_alias` is what makes that
# work — without it pydantic-settings reads SCRAPER_PROXY_URL_ENV (the field
# name), so setting SCRAPER_PROXY_URL would be a silent no-op. These tests
# construct a real Settings and assert end-to-end env binding.


def _fresh_settings():
    from app.core.config import Settings

    return Settings(_env_file=None)


def test_settings_binds_scraper_proxy_url_env(monkeypatch):
    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234")
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234"


def test_settings_scraper_env_overrides_avito(monkeypatch):
    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234")
    monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678")
    assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234"


def test_settings_falls_back_to_avito_env(monkeypatch):
    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
    monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678")
    assert _fresh_settings().scraper_proxy_url == "http://avito-proxy:5678"


def test_settings_ignores_misnamed_scraper_proxy_url_env(monkeypatch):
    """Regression: the old wrong env name SCRAPER_PROXY_URL_ENV must NOT bind."""
    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    monkeypatch.setenv("SCRAPER_PROXY_URL_ENV", "http://wrong-name:1111")
    assert _fresh_settings().scraper_proxy_url is None


# ── Per-scraper proxy wiring (follow-up: yandex/valuation/imv/newbuilding) ────
# Each test: patch settings.scraper_proxy_url, mock AsyncSession constructor,
# invoke __aenter__ / create-session path, assert proxies= kwarg is forwarded.
# proxy=None path: assert proxies kwarg is None / not passed as non-None.


_PROXY_URL = "http://mobile.proxy:8080"
_EXPECTED_PROXIES = {"http": _PROXY_URL, "https": _PROXY_URL}


# ── YandexRealtyScraper ──────────────────────────────────────────────────────
#
# `test_yandex_realty_opens_browser_fetcher` (legacy `app.services.scrapers.
# yandex_realty.YandexRealtyScraper.__aenter__` opens ONE BrowserFetcher session)
# removed (#2397 Part E2) — `yandex_realty` deleted, 0 runtime importers. Kit
# equivalent already covered (opens exactly one BrowserFetcher session via
# `build_browser_fetcher`) by `test_kit_serp_proxy_pool.py::
# test_yandex_serp_no_pool_by_default` (and the pool-wiring variants alongside it).


# ── YandexValuationScraper ───────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_yandex_valuation_session_receives_proxies():
    """Kit YandexValuationScraper.__aenter__ forwards proxies= kwarg from config."""
    from scraper_kit.providers.yandex.valuation import YandexValuationScraper

    mock_session = MagicMock()
    mock_session.close = AsyncMock()
    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    cfg = SimpleNamespace(use_proxy_pool_curl=False, scraper_proxy_url=_PROXY_URL)
    with patch("scraper_kit.providers.yandex.valuation._CurlCffiSession", _fake_session):
        async with YandexValuationScraper(cfg):
            pass

    assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES


@pytest.mark.asyncio
async def test_yandex_valuation_session_no_proxies_when_none():
    """Kit YandexValuationScraper.__aenter__: proxies=None when no proxy configured."""
    from scraper_kit.providers.yandex.valuation import YandexValuationScraper

    mock_session = MagicMock()
    mock_session.close = AsyncMock()
    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    cfg = SimpleNamespace(use_proxy_pool_curl=False, scraper_proxy_url=None)
    with patch("scraper_kit.providers.yandex.valuation._CurlCffiSession", _fake_session):
        async with YandexValuationScraper(cfg):
            pass

    assert captured_kwargs.get("proxies") is None


# ── CianNewbuilding — fetch_newbuilding ───────────────────────────────────────
# После #972 fetch_newbuilding тянет страницу ЖК через BrowserFetcher (camoufox),
# а НЕ через curl_cffi AsyncSession. Прокси применяются на уровне браузер-контейнера,
# а не через proxies= kwarg. AsyncSession для page-fetch больше НЕ создаётся.


@pytest.mark.asyncio
async def test_cian_newbuilding_own_session_receives_proxies():
    """fetch_newbuilding использует BrowserFetcher для page-fetch, не AsyncSession.

    Проверяем:
    - BrowserFetcher.fetch(url) вызывается с правильным URL
    - curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level)
    """
    from app.services.scrapers import cian_newbuilding

    zhk_url = "https://zhk-test.cian.ru/"
    fetch_calls: list[str] = []
    session_ctor_calls: list[dict] = []

    mock_fetcher = MagicMock()
    mock_fetcher.fetch = AsyncMock(
        side_effect=lambda url: fetch_calls.append(url) or "<html></html>"
    )
    mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
    mock_fetcher.__aexit__ = AsyncMock(return_value=None)

    def _fake_async_session(*args, **kwargs):
        session_ctor_calls.append(kwargs)
        return MagicMock()

    with patch.object(cian_newbuilding, "BrowserFetcher", lambda *a, **k: mock_fetcher):
        with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
            await cian_newbuilding.fetch_newbuilding(zhk_url)

    # BrowserFetcher.fetch вызван с ЖК-url
    assert fetch_calls == [zhk_url]
    # curl_cffi AsyncSession НЕ создавался для page-fetch
    assert session_ctor_calls == []


@pytest.mark.asyncio
async def test_cian_newbuilding_shared_session_not_recreated():
    """fetch_newbuilding принимает session= для backward-compat, но использует BrowserFetcher.

    Даже когда передан внешний session — страница ЖК тянется через BrowserFetcher,
    AsyncSession не пересоздаётся, session.get не вызывается.
    """
    from app.services.scrapers import cian_newbuilding

    zhk_url = "https://zhk-test.cian.ru/"
    fetch_calls: list[str] = []
    session_ctor_calls: list[dict] = []

    mock_fetcher = MagicMock()
    mock_fetcher.fetch = AsyncMock(
        side_effect=lambda url: fetch_calls.append(url) or "<html></html>"
    )
    mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
    mock_fetcher.__aexit__ = AsyncMock(return_value=None)

    def _fake_async_session(*args, **kwargs):
        session_ctor_calls.append(kwargs)
        return MagicMock()

    sentinel_session = MagicMock(name="external_session")

    with patch.object(cian_newbuilding, "BrowserFetcher", lambda *a, **k: mock_fetcher):
        with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
            await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session)

    # BrowserFetcher.fetch вызван — именно через него тянется страница
    assert fetch_calls == [zhk_url]
    # Ни нового AsyncSession, ни вызова переданного session.get — proxy browser-level
    assert session_ctor_calls == []
    sentinel_session.get.assert_not_called()


# ── AvitoDetail — fetch_detail own-session ───────────────────────────────────


@pytest.mark.asyncio
async def test_avito_detail_own_session_receives_proxies():
    """fetch_detail creates own session with proxies= when no cffi_session passed."""
    from app.services.scrapers import avito_detail

    mock_session = MagicMock()
    mock_resp = MagicMock()
    mock_resp.status_code = 200
    # Minimal HTML that parse_detail_html won't raise ValueError (item_id required)
    mock_resp.text = "<html><body></body></html>"
    mock_session.get = AsyncMock(return_value=mock_resp)
    mock_session.close = AsyncMock()

    captured_kwargs: dict = {}

    def _fake_session(*args, **kwargs):
        captured_kwargs.update(kwargs)
        return mock_session

    with patch.object(avito_detail, "AsyncSession", _fake_session):
        with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
            try:
                await avito_detail.fetch_detail("/ekaterinburg/kvartiry/test-1234")
            except ValueError:
                pass  # parse_detail_html raises ValueError (no item_id) — that's ok

    assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES


@pytest.mark.asyncio
async def test_avito_detail_shared_session_not_patched():
    """fetch_detail does NOT build own session when cffi_session is provided."""
    from app.services.scrapers import avito_detail

    mock_session = MagicMock()
    mock_resp = MagicMock()
    mock_resp.status_code = 200
    mock_resp.text = "<html><body></body></html>"
    mock_session.get = AsyncMock(return_value=mock_resp)
    mock_session.close = AsyncMock()

    ctor_calls = []

    def _fake_session(*args, **kwargs):
        ctor_calls.append(kwargs)
        return MagicMock()

    with patch.object(avito_detail, "AsyncSession", _fake_session):
        with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
            try:
                await avito_detail.fetch_detail(
                    "/ekaterinburg/kvartiry/test-1234",
                    cffi_session=mock_session,
                )
            except ValueError:
                pass

    # No own session created — shared session was used as-is
    assert ctor_calls == []


# ── Settings.cian_proxy_url property ─────────────────────────────────────────
# cian_proxy_url: CIAN_PROXY_URL > scraper_proxy_url (fallback) > None.


def _resolve_cian_proxy_url(
    cian_proxy_url_env: str | None, scraper_proxy_url: str | None
) -> str | None:
    """Inline replica of Settings.cian_proxy_url property logic."""
    return cian_proxy_url_env or scraper_proxy_url


def test_cian_proxy_url_uses_cian_env_when_set():
    """CIAN_PROXY_URL takes precedence over shared scraper_proxy_url."""
    result = _resolve_cian_proxy_url(
        cian_proxy_url_env="http://cian-proxy:2222",
        scraper_proxy_url="http://shared-proxy:1111",
    )
    assert result == "http://cian-proxy:2222"


def test_cian_proxy_url_falls_back_to_scraper_proxy_url():
    """Falls back to shared scraper_proxy_url when CIAN_PROXY_URL is absent."""
    result = _resolve_cian_proxy_url(
        cian_proxy_url_env=None,
        scraper_proxy_url="http://shared-proxy:1111",
    )
    assert result == "http://shared-proxy:1111"


def test_cian_proxy_url_none_when_both_absent():
    result = _resolve_cian_proxy_url(cian_proxy_url_env=None, scraper_proxy_url=None)
    assert result is None


def test_settings_binds_cian_proxy_url_env(monkeypatch):
    """CIAN_PROXY_URL env binds to cian_proxy_url_env via validation_alias."""
    monkeypatch.setenv("CIAN_PROXY_URL", "http://cian-proxy:2222")
    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    assert _fresh_settings().cian_proxy_url == "http://cian-proxy:2222"


def test_settings_cian_env_overrides_scraper_proxy(monkeypatch):
    """CIAN_PROXY_URL overrides shared SCRAPER_PROXY_URL for Cian scrapers."""
    monkeypatch.setenv("CIAN_PROXY_URL", "http://cian-proxy:2222")
    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://shared-proxy:1111")
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    s = _fresh_settings()
    assert s.cian_proxy_url == "http://cian-proxy:2222"
    assert s.scraper_proxy_url == "http://shared-proxy:1111"


def test_settings_cian_proxy_falls_back_to_scraper(monkeypatch):
    """Without CIAN_PROXY_URL, cian_proxy_url falls back to scraper_proxy_url."""
    monkeypatch.delenv("CIAN_PROXY_URL", raising=False)
    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://shared-proxy:1111")
    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
    assert _fresh_settings().cian_proxy_url == "http://shared-proxy:1111"