gendesign/tradein-mvp/backend/tests/scrapers/test_yandex_valuation_kit_migration.py

"""Issue #2336 (Group E3 of scraper_kit migration epic #2277/#2308).

`scraper_kit.providers.yandex.valuation.YandexValuationScraper` is already a
strangler-copy (#2133) of the (now deleted, #2277 final step)
`app.services.scrapers.yandex_valuation`, and already used via the kit path in
`admin.py::scrape_yandex_valuation` (#2305, Group A) with the established DI
convention: `config=RealScraperConfig()`, `delay_provider=get_scraper_delay`,
`proxy_provider=_kit_proxy_provider()`.

This file proves #2336's two confirmed risk patterns for any future call site that
constructs the kit scraper directly, so a regression is caught in CI rather than
silently in prod:

1. **Mandatory `config: ScraperConfig`** (same shape as the cian_valuation footgun,
   #2335) — omitting it raises `TypeError`.

2. **Optional `delay_provider`** — if omitted, silently falls back to the hardcoded
   class-default `request_delay_sec = 5.0`, discarding whatever an admin configured via
   the DB-backed `scraper_settings` table (`get_scraper_delay`, cached 60s,
   `max(per_source, global)`). No exception, no log — just quietly weaker anti-ban
   throttling if an admin had raised it after ban trouble.

Legacy-vs-kit parity coverage (previously here, using
`app.services.scrapers.yandex_valuation`) was removed together with the legacy module
deletion — `.parse()` correctness itself is still covered kit-only by
`tests/test_yandex_valuation.py` / `tests/test_yandex_valuation_save.py`.
"""

from __future__ import annotations

import os
from unittest.mock import patch

import pytest

# app.services.scraper_adapters imports app.core.config.settings=Settings(), which
# requires DATABASE_URL. Offline construction never touches a real DB — a fake DSN
# is enough.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")

from scraper_kit.providers.yandex.valuation import (
    YandexValuationScraper as KitYandexValuationScraper,
)

from app.services import scraper_settings
from app.services.scraper_adapters import RealScraperConfig

# ---------------------------------------------------------------------------
# Footgun #1 — mandatory `config`
# ---------------------------------------------------------------------------


def test_kit_valuation_requires_config() -> None:
    """Omitting `config=` raises TypeError — documents the mandatory-DI contract.

    Guards the exact risk flagged for #2336: if a future call site (#2337, switching
    estimator.py to the kit class) forgets `config=`, this must fail loud in CI, not
    silently in prod inside estimator.py's blanket except-block.
    """
    with pytest.raises(TypeError):
        KitYandexValuationScraper()  # type: ignore[call-arg]


# ---------------------------------------------------------------------------
# Footgun #2 — `delay_provider` silently falls back to hardcoded 5.0s
# ---------------------------------------------------------------------------


def test_kit_valuation_without_delay_provider_uses_hardcoded_default() -> None:
    """No `delay_provider=` -> `request_delay_sec` stays at the class-default 5.0s.

    This is the regression this test guards: a real call site that forgets to wire
    `delay_provider=get_scraper_delay` silently reverts an admin-tuned anti-ban delay
    (e.g. raised after a ban) to 5.0s -- no error, no log, just quietly weaker
    throttling.
    """
    scraper = KitYandexValuationScraper(config=RealScraperConfig())
    assert scraper.request_delay_sec == 5.0
    assert KitYandexValuationScraper.request_delay_sec == 5.0  # class default untouched


def test_kit_valuation_with_delay_provider_uses_configured_value() -> None:
    """`delay_provider=get_scraper_delay` -> instance picks up whatever it returns.

    Controls the return value via patch so the assertion doesn't depend on a real DB
    row / DB availability -- this proves the WIRING (delay_provider is actually called
    and its return value actually lands in request_delay_sec), not
    `scraper_settings`'s own DB-fallback behaviour (covered elsewhere, e.g.
    `tests/test_yandex_scrapers_delay_wiring.py` for the legacy module).
    """
    with patch("app.services.scraper_settings.get_scraper_delay", return_value=12.5) as mock_delay:
        # Attribute lookup on the MODULE (not the bare name imported at file-top)
        # so this resolves to the patched mock, not the original function object
        # `from ... import get_scraper_delay` would have bound at import time.
        scraper = KitYandexValuationScraper(
            config=RealScraperConfig(), delay_provider=scraper_settings.get_scraper_delay
        )
    mock_delay.assert_called_once_with("yandex_valuation")
    assert scraper.request_delay_sec == 12.5
    assert scraper.request_delay_sec != 5.0  # explicitly NOT the hardcoded fallback


def test_kit_valuation_delay_provider_receives_scraper_name() -> None:
    """delay_provider is invoked with `self.name` ('yandex_valuation'), not a typo'd string.

    `get_scraper_delay` aliases 'yandex_valuation' to the shared 'yandex' DB row today
    (`_KEY_ALIASES` in scraper_settings.py), but a hardcoded/mismatched source string
    passed at the call site would be a latent bug if that alias ever changes.
    """
    calls: list[str] = []

    def _spy_delay_provider(source: str) -> float:
        calls.append(source)
        return 9.0

    scraper = KitYandexValuationScraper(
        config=RealScraperConfig(), delay_provider=_spy_delay_provider
    )
    assert calls == ["yandex_valuation"]
    assert scraper.request_delay_sec == 9.0