gendesign/tradein-mvp/backend/tests/test_estimator_yandex_integration.py

"""Tests for Yandex Valuation integration in estimator.py."""

import os

# Settings requires DATABASE_URL at init time. Set dummy DSN before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db")

from datetime import date
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from scraper_kit.providers.yandex.valuation import (
    ValuationHistoryItem,
    ValuationHouseMeta,
    YandexValuationResult,
)

from app.services.estimator import (
    YANDEX_VALUATION_CACHE_TTL_HOURS,
    _get_or_fetch_yandex_valuation_cached,
    _save_yandex_history_items,
    _yandex_valuation_cache_key,
)
from app.services.scraper_settings import get_scraper_delay


def _history_rows(db) -> list[dict]:
    """Строки батча house_placement_history из мока сессии (фильтр по SQL, не по позиции)."""
    for call in db.execute.call_args_list:
        if "INSERT INTO house_placement_history" in str(call.args[0]):
            return call.args[1]
    return []


def _sample_result(address: str = "Екатеринбург, ул. Учителей, 18") -> YandexValuationResult:
    return YandexValuationResult(
        address=address,
        offer_category="APARTMENT",
        offer_type="SELL",
        page=1,
        source_url=(
            f"https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address={address}"
        ),
        house=ValuationHouseMeta(
            year_built=1981,
            total_floors=9,
            house_type="panel",
            ceiling_height=2.5,
            has_lift=True,
            total_objects=12,
        ),
        history_items=[
            ValuationHistoryItem(
                area_m2=36.8,
                rooms=1,
                floor=8,
                start_price=4_500_000,
                start_price_per_m2=122_283,
                last_price=4_400_000,
                last_price_per_m2=119_566,
                publish_date=date(2026, 2, 17),
                exposure_days=96,
                status="В продаже",
            ),
            ValuationHistoryItem(
                area_m2=64.4,
                rooms=2,
                floor=1,
                start_price=6_580_000,
                start_price_per_m2=102_174,
                last_price=6_100_000,
                last_price_per_m2=94_721,
                publish_date=date(2026, 2, 2),
                exposure_days=111,
                status="В продаже",
            ),
        ],
    )


def test_cache_key_deterministic():
    k1 = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
    k2 = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
    assert k1 == k2
    assert len(k1) == 64


def test_cache_key_different_for_different_inputs():
    a = _yandex_valuation_cache_key("addr X", "APARTMENT", "SELL")
    b = _yandex_valuation_cache_key("addr Y", "APARTMENT", "SELL")
    c = _yandex_valuation_cache_key("addr X", "HOUSE", "SELL")
    d = _yandex_valuation_cache_key("addr X", "APARTMENT", "RENT")
    assert len({a, b, c, d}) == 4


def test_ttl_default_24h():
    assert YANDEX_VALUATION_CACHE_TTL_HOURS == 24


@pytest.mark.asyncio
async def test_cache_hit_returns_deserialized_result():
    """When cache row exists and hasn't expired, return parsed YandexValuationResult."""
    db = MagicMock()
    result = _sample_result()
    db.execute.return_value.mappings.return_value.first.return_value = {
        "raw_payload": result.model_dump(mode="json"),
        "fetched_at": None,
    }
    with patch("app.services.estimator.YandexValuationScraper") as mock_scraper_cls:
        out = await _get_or_fetch_yandex_valuation_cached(db, address=result.address)
    mock_scraper_cls.assert_not_called()  # no fetch on hit
    assert out is not None
    assert out.address == result.address
    assert len(out.history_items) == 2


@pytest.mark.asyncio
async def test_cache_miss_triggers_fetch_and_persist():
    """On cache miss, fetch via scraper + INSERT into external_valuations."""
    from app.services.scraper_adapters import RealScraperConfig

    db = MagicMock()
    db.execute.return_value.mappings.return_value.first.return_value = None  # miss
    fresh = _sample_result()
    fake_scraper = MagicMock()
    fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
    fake_scraper.__aexit__ = AsyncMock(return_value=False)
    fake_scraper.fetch_house_history = AsyncMock(return_value=fresh)
    with patch(
        "app.services.estimator.YandexValuationScraper", return_value=fake_scraper
    ) as mock_scraper_cls:
        out = await _get_or_fetch_yandex_valuation_cached(db, address=fresh.address)
    assert out is fresh
    fake_scraper.fetch_house_history.assert_awaited_once()
    # #2337 regression guard (Group E4): kit YandexValuationScraper requires config=
    # (TypeError if omitted) AND silently falls back to a hardcoded 5.0s throttle
    # instead of the DB-configured anti-ban delay unless delay_provider= is passed.
    # assert_called alone wouldn't catch either kwarg being dropped later.
    call_args, call_kwargs = mock_scraper_cls.call_args
    assert isinstance(call_args[0], RealScraperConfig)
    assert call_kwargs.get("delay_provider") is get_scraper_delay
    # 2 DB calls: 1 SELECT (miss) + 1 INSERT/UPSERT
    assert db.execute.call_count >= 2


@pytest.mark.asyncio
async def test_fetch_error_returns_none_gracefully():
    db = MagicMock()
    db.execute.return_value.mappings.return_value.first.return_value = None
    fake_scraper = MagicMock()
    fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
    fake_scraper.__aexit__ = AsyncMock(return_value=False)
    fake_scraper.fetch_house_history = AsyncMock(side_effect=RuntimeError("network"))
    with patch("app.services.estimator.YandexValuationScraper", return_value=fake_scraper):
        out = await _get_or_fetch_yandex_valuation_cached(db, address="addr")
    assert out is None


@pytest.mark.asyncio
async def test_fetch_returns_none_propagates_none():
    db = MagicMock()
    db.execute.return_value.mappings.return_value.first.return_value = None
    fake_scraper = MagicMock()
    fake_scraper.__aenter__ = AsyncMock(return_value=fake_scraper)
    fake_scraper.__aexit__ = AsyncMock(return_value=False)
    fake_scraper.fetch_house_history = AsyncMock(return_value=None)
    with patch("app.services.estimator.YandexValuationScraper", return_value=fake_scraper):
        out = await _get_or_fetch_yandex_valuation_cached(db, address="addr")
    assert out is None


def test_save_history_items_inserts_each():
    db = MagicMock()
    result = _sample_result()
    with patch(
        "app.services.estimator.match_or_create_house",
        return_value=(1, 0.9, "fingerprint"),
    ):
        saved = _save_yandex_history_items(db, result)
    assert saved == 2
    # 1 batch INSERT (executemany). #2674 добавил вторым вызовом UPDATE
    # houses.has_panorama — считаем именно вставки истории, а не все execute.
    rows = _history_rows(db)
    assert isinstance(rows, list) and len(rows) == 2
    # Два коммита: панорама (до истории) + батч истории. Раньше был один.
    assert db.commit.call_count == 2


def test_save_history_items_empty_no_commit():
    """Пустая история + НЕподтверждённая страница → дом резолвится, но не пишется ничего.

    #2674 (ревью): ранний возврат по пустой истории раньше стоял ПЕРВЫМ и заодно
    отрезал запись houses.has_panorama для отрисованных страниц без объявлений (~10%).
    Теперь резолв дома идёт до возврата, поэтому match_or_create_house вызывается —
    а вот записей по-прежнему ноль: мета пустая, гейт панорамы не пропускает.
    """
    db = MagicMock()
    result = YandexValuationResult(
        address="x",
        offer_category="APARTMENT",
        offer_type="SELL",
        page=1,
        source_url="https://x",
        house=ValuationHouseMeta(),
        history_items=[],
    )
    with patch(
        "app.services.estimator.match_or_create_house",
        return_value=(1, 0.9, "fingerprint"),
    ) as m:
        saved = _save_yandex_history_items(db, result)
    assert saved == 0
    m.assert_called_once()
    db.execute.assert_not_called()
    db.commit.assert_not_called()


def test_save_history_items_ext_id_stable_across_calls():
    """Same result → same ext_item_id sequence → idempotent UPSERT."""
    db1 = MagicMock()
    db2 = MagicMock()
    result = _sample_result()
    with patch(
        "app.services.estimator.match_or_create_house",
        return_value=(1, 0.9, "fingerprint"),
    ):
        _save_yandex_history_items(db1, result)
        _save_yandex_history_items(db2, result)
    # Both calls pass a list-of-dicts (batch executemany); ext_id extracted from each row
    ext_ids_1 = [
        r["ext_id"]
        for c in db1.execute.call_args_list
        if isinstance(c.args[1], list)
        for r in c.args[1]
    ]
    ext_ids_2 = [
        r["ext_id"]
        for c in db2.execute.call_args_list
        if isinstance(c.args[1], list)
        for r in c.args[1]
    ]
    assert ext_ids_1 == ext_ids_2
    assert len(ext_ids_1) == 2


def test_save_history_items_db_error_rolls_back_batch():
    """Any item failing rolls back the whole batch — batch semantics (finding #5).

    #2674: side_effect адресуем по SQL, а не по позиции вызова — иначе исключение
    доставалось бы UPDATE houses.has_panorama (он идёт первым и свои ошибки глотает),
    а батч истории проходил бы успешно, и тест молча проверял бы не тот путь.
    """
    db = MagicMock()

    def _fail_history(sql, *args, **kwargs):
        if "INSERT INTO house_placement_history" in str(sql):
            raise RuntimeError("first row fails")
        return MagicMock()

    db.execute.side_effect = _fail_history
    result = _sample_result()
    with patch(
        "app.services.estimator.match_or_create_house",
        return_value=(1, 0.9, "fingerprint"),
    ):
        saved = _save_yandex_history_items(db, result)
    assert saved == 0  # whole batch rolled back
    db.rollback.assert_called_once()
    # Панорама коммитится отдельно и раньше — её успех не отменяет отката истории.
    assert db.commit.call_count == 1