gendesign/tradein-mvp/backend/tests/test_estimator_hedonic.py

"""Focused hedonic year+area correction tests (#2002).

Exercises the multiplicative hedonic factor on the expected_sold POINT directly
via ``_price_from_inputs`` (hermetic — no DB, no network). Verifies:
  * the factor magnitude vs the hedonic-OFF baseline (mid case);
  * both clamp boundaries (≤ factor_min via huge area, ≥ factor_max via small+new);
  * the neutral year term when ``target_year`` is None (≡ year 2000);
  * the le_asking invariant — the corrected expected_sold never exceeds the asking
    headline (median) — the expected_sold ≤ asking clamp has no switch (#2380).

NOTE: importing app.services.estimator pulls app.core.config.Settings which
requires DATABASE_URL. Set it BEFORE importing app modules.
"""

from __future__ import annotations

import math
import os

import pytest

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from app.services import estimator
from app.services.geocoder import GeocodeResult

# ── helpers ──────────────────────────────────────────────────────────────────


def _geo() -> GeocodeResult:
    return GeocodeResult(
        lat=56.838,
        lon=60.597,
        full_address="ул. Тестовая, 1",
        provider="nominatim",
        confidence="approximate",
    )


def _lots(ppm2: float, n: int = 7) -> list[dict]:
    """n unique-address lots all at the same ₽/m² → median_ppm2 == ppm2."""
    return [
        {"price_per_m2": ppm2, "address": f"ул. Тестовая, {i + 1}", "source": "avito"}
        for i in range(n)
    ]


def _price(
    *,
    area_m2: float,
    target_year: int | None,
    ratio: float,
    ppm2: float = 100_000.0,
    floor: int | None = 5,
) -> estimator.PricingResult:
    """Pure radius-only spine call (no anchor / dkp / imv) with a forced ratio."""

    def ratio_resolver(appm2: float | None) -> tuple[float | None, str | None]:
        return ratio, "per_rooms"

    return estimator._price_from_inputs(
        listings=_lots(ppm2),
        area_m2=area_m2,
        rooms=2,
        repair_state=None,
        floor=floor,
        total_floors=10,
        target_year=target_year,
        analog_tier="W",
        fallback_used=False,
        area_widened=False,
        anchor_comps=[],
        anchor_tier_fetched=None,
        dkp_raw=None,
        imv_anchor=None,
        imv_eval=None,
        yandex_val_present=False,
        cian_val_present=False,
        ratio_resolver=ratio_resolver,
        quarter_index_lookup=lambda q: None,
        quarter_indexes_lookup=lambda qs: {},
        target_house_cadnum=None,
        dadata_coarse=False,
        geo=_geo(),
        dadata_qc_geo=None,
    )


def _expected_factor(area_m2: float, target_year: int | None, floor: int | None = 5) -> float:
    """Reproduce the production factor from the live settings (no hard-coding).

    ``floor`` mirrors the production gate: the ground-floor term is added only when
    ``floor == 1`` (default 5 ⇒ no first-floor term, the year+area factor).
    """
    s = estimator.settings
    yr = ((target_year - 2000) / 20.0) if target_year else 0.0
    first = s.estimate_hedonic_first_floor_coef if floor == 1 else 0.0
    raw = math.exp(
        s.estimate_hedonic_b0
        + s.estimate_hedonic_year_coef * yr
        + s.estimate_hedonic_larea_coef * math.log(area_m2)
        + first
    )
    return max(s.estimate_hedonic_factor_min, min(s.estimate_hedonic_factor_max, raw))


# ── tests ────────────────────────────────────────────────────────────────────


def test_mid_case_shifts_by_expected_factor(monkeypatch: pytest.MonkeyPatch) -> None:
    """year≈2010, area≈50 → expected_sold shifts by the hedonic factor vs OFF."""
    # OFF baseline (exact legacy expected_sold).
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", False)
    off = _price(area_m2=50.0, target_year=2010, ratio=0.85)
    # ON.
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
    on = _price(area_m2=50.0, target_year=2010, ratio=0.85)

    factor = _expected_factor(50.0, 2010)
    # 2010 + 50 m² → mild uplift, strictly inside the clamp band.
    assert 1.0 < factor < estimator.settings.estimate_hedonic_factor_max
    assert off.expected_sold_price is not None and on.expected_sold_price is not None
    # ratio 0.85 × factor < 1.0 → le_asking re-clamp is a no-op here (no confound).
    assert on.expected_sold_price == round(off.expected_sold_price * factor)
    assert on.expected_sold_per_m2 == round(off.expected_sold_per_m2 * factor)


def test_factor_clamps_to_min_for_huge_area(monkeypatch: pytest.MonkeyPatch) -> None:
    """Very large area → raw factor < factor_min → clamped to the floor.

    #3248: тест проверяет МЕХАНИЗМ клэмпа, а не конкретную подгонку. С момента
    перефита `estimate_hedonic_larea_coef` = 0 (площадь несёт area-бакетный ratio),
    поэтому площадь сама по себе фактор вниз больше не гонит. Задаём площадной
    коэффициент явно — иначе тест молча перестаёт проверять клэмп при каждом
    перефите вместо того, чтобы падать.
    """
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_larea_coef", -0.1603)
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", False)
    off = _price(area_m2=10_000.0, target_year=None, ratio=0.85)
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
    on = _price(area_m2=10_000.0, target_year=None, ratio=0.85)

    factor = _expected_factor(10_000.0, None)
    assert factor == estimator.settings.estimate_hedonic_factor_min
    assert off.expected_sold_price is not None and on.expected_sold_price is not None
    assert on.expected_sold_price == round(
        off.expected_sold_price * estimator.settings.estimate_hedonic_factor_min
    )


def test_factor_clamps_to_max_for_small_new_lot(monkeypatch: pytest.MonkeyPatch) -> None:
    """Small area + new year → raw factor > factor_max → clamped to the ceiling.

    ratio 0.70 keeps 0.70 × factor_max (1.30) = 0.91 < 1.0, so the always-on
    expected_sold ≤ asking re-clamp does not bind and the raw ceiling factor is
    observable on the point (#2380: the clamp no longer has an OFF switch).

    #3248: коэффициенты задаются явно — тест про МЕХАНИЗМ потолка, а не про
    текущую подгонку (после перефита b0 = -0.0140 и потолка сам по себе не
    достаёт).
    """
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_b0", 0.6146)
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_larea_coef", -0.1603)
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_year_coef", 0.1220)
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", False)
    off = _price(area_m2=15.0, target_year=2025, ratio=0.70)
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
    on = _price(area_m2=15.0, target_year=2025, ratio=0.70)

    factor = _expected_factor(15.0, 2025)
    assert factor == estimator.settings.estimate_hedonic_factor_max
    assert off.expected_sold_price is not None and on.expected_sold_price is not None
    assert on.expected_sold_price == round(
        off.expected_sold_price * estimator.settings.estimate_hedonic_factor_max
    )


def test_target_year_none_is_neutral(monkeypatch: pytest.MonkeyPatch) -> None:
    """target_year=None → year term is 0 → identical to year 2000 (intercept+area)."""
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
    none_year = _price(area_m2=50.0, target_year=None, ratio=0.85)
    year_2000 = _price(area_m2=50.0, target_year=2000, ratio=0.85)

    assert none_year.expected_sold_price == year_2000.expected_sold_price
    assert none_year.expected_sold_per_m2 == year_2000.expected_sold_per_m2
    assert _expected_factor(50.0, None) == _expected_factor(50.0, 2000)


def test_le_asking_invariant_holds_under_hedonic(monkeypatch: pytest.MonkeyPatch) -> None:
    """The hedonic-corrected expected_sold never exceeds asking."""
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
    # small area + new year → factor 1.30; ratio 0.95 → 0.95×1.30 ≈ 1.235 > 1 →
    # uncorrected the point would exceed the asking headline; the re-clamp must bind.
    res = _price(area_m2=15.0, target_year=2025, ratio=0.95)

    assert res.expected_sold_price is not None
    assert res.expected_sold_price <= res.median_price
    assert res.expected_sold_per_m2 is not None
    assert res.expected_sold_per_m2 <= res.median_ppm2
    # The clamp binds exactly at the asking headline (proves it actually fired).
    assert res.expected_sold_price == res.median_price


def test_ground_floor_applies_extra_discount(monkeypatch: pytest.MonkeyPatch) -> None:
    """floor==1 → дополнительная скидка exp(first_floor_coef) против floor=3.

    Same year/area/ratio; only the floor differs. The mid case sits strictly inside
    the clamp band so the first-floor term is fully observable (no clamp confound).
    ``floor`` only feeds the (empty) same-building anchor here, so the hedonic-OFF
    base expected_sold is identical for floor=1 and floor=3 — the only delta is the
    ground-floor coefficient in the factor.
    """
    # OFF baseline (no factor at all) — floor-independent in the radius-only spine.
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", False)
    off = _price(area_m2=50.0, target_year=2010, ratio=0.85, floor=3)
    # ON.
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
    ground = _price(area_m2=50.0, target_year=2010, ratio=0.85, floor=1)
    upper = _price(area_m2=50.0, target_year=2010, ratio=0.85, floor=3)

    f_ground = _expected_factor(50.0, 2010, floor=1)
    f_upper = _expected_factor(50.0, 2010, floor=3)
    extra = math.exp(estimator.settings.estimate_hedonic_first_floor_coef)

    # ground-floor multiplies the year+area factor by the extra discount.
    # #3248: сверяем со ЗНАЧЕНИЕМ НАСТРОЙКИ, а не с зашитым 0.8827 — коэффициент
    # подгоняемый и меняется при каждом перефите, а проверяем мы связь термина с
    # фактором. Границы держат тест осмысленным: скидка, но не обвал.
    assert 0.80 < extra < 1.0
    assert f_ground == pytest.approx(f_upper * extra)
    # both factors strictly inside the clamp band → the term is fully observable.
    assert estimator.settings.estimate_hedonic_factor_min < f_ground < f_upper
    assert f_upper < estimator.settings.estimate_hedonic_factor_max
    # expected_sold tracks each factor vs the shared OFF baseline (ratio<1 → le_asking
    # re-clamp is a no-op, no confound).
    assert off.expected_sold_price is not None
    assert ground.expected_sold_price == round(off.expected_sold_price * f_ground)
    assert upper.expected_sold_price == round(off.expected_sold_price * f_upper)
    assert ground.expected_sold_per_m2 == round(off.expected_sold_per_m2 * f_ground)
    # the headline mechanic: a ground-floor unit sells below the upper-floor estimate.
    assert upper.expected_sold_price is not None
    assert ground.expected_sold_price < upper.expected_sold_price


def test_non_ground_floors_unchanged_vs_year_area_factor(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """floor in {None, 2, 3, 99} stays byte-identical to the prior 2-term factor."""
    monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
    ref = _price(area_m2=50.0, target_year=2010, ratio=0.85, floor=5)
    for fl in (None, 2, 3, 99):
        other = _price(area_m2=50.0, target_year=2010, ratio=0.85, floor=fl)
        assert other.expected_sold_price == ref.expected_sold_price
        assert other.expected_sold_per_m2 == ref.expected_sold_per_m2