bot-backend
f999011b30
Review found the hedonic factor (≤1.30) was applied AFTER the estimate_expected_sold_le_asking clamp, so expected_sold could exceed the asking headline (median) by up to ~26% for new/small lots — an overpay risk that violated the default-True le_asking invariant. Fix: re-assert le_asking right after the hedonic block and before the calibrated-PI range block. Final order: ratio clamp -> hedonic factor -> le_asking re-clamp -> calibrated PI range (range still wraps the clamped point). round(median_ppm2) keeps expected_sold_per_m2 an int. The re-clamp does NOT materially erode the новостройка/бизнес benefit (бизнес bias -9.15->-10.19, MAPE held 12.87) and actually improves overall expected_sold MAPE on the frozen fixture (14.24->13.23) by capping over-corrections that overshot actual sold prices. Adds tests/test_estimator_hedonic.py (hermetic, no DB): factor magnitude vs OFF, both clamp boundaries, neutral year term for target_year=None, and the le_asking invariant under hedonic ON. Baseline regenerated; gate green. Also clarifies the config comment: the quoted held-out fit metrics (18.5->16.1 etc., n=2366) are distinct from the 277-deal backtest fixture (18.63->13.23).
191 lines
8.3 KiB
Python
191 lines
8.3 KiB
Python
"""Focused hedonic year+area correction tests (#2002).
|
||
|
||
Exercises the multiplicative hedonic factor on the expected_sold POINT directly
|
||
via ``_price_from_inputs`` (hermetic — no DB, no network). Verifies:
|
||
* the factor magnitude vs the hedonic-OFF baseline (mid case);
|
||
* both clamp boundaries (≤ factor_min via huge area, ≥ factor_max via small+new);
|
||
* the neutral year term when ``target_year`` is None (≡ year 2000);
|
||
* the le_asking invariant — the corrected expected_sold never exceeds the asking
|
||
headline (median) when ``estimate_expected_sold_le_asking`` is on.
|
||
|
||
NOTE: importing app.services.estimator pulls app.core.config.Settings which
|
||
requires DATABASE_URL. Set it BEFORE importing app modules.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import math
|
||
import os
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from app.services import estimator
|
||
from app.services.geocoder import GeocodeResult
|
||
|
||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _geo() -> GeocodeResult:
|
||
return GeocodeResult(
|
||
lat=56.838,
|
||
lon=60.597,
|
||
full_address="ул. Тестовая, 1",
|
||
provider="nominatim",
|
||
confidence="approximate",
|
||
)
|
||
|
||
|
||
def _lots(ppm2: float, n: int = 7) -> list[dict]:
|
||
"""n unique-address lots all at the same ₽/m² → median_ppm2 == ppm2."""
|
||
return [
|
||
{"price_per_m2": ppm2, "address": f"ул. Тестовая, {i + 1}", "source": "avito"}
|
||
for i in range(n)
|
||
]
|
||
|
||
|
||
def _price(
|
||
*,
|
||
area_m2: float,
|
||
target_year: int | None,
|
||
ratio: float,
|
||
ppm2: float = 100_000.0,
|
||
) -> estimator.PricingResult:
|
||
"""Pure radius-only spine call (no anchor / dkp / imv) with a forced ratio."""
|
||
|
||
def ratio_resolver(appm2: float | None) -> tuple[float | None, str | None]:
|
||
return ratio, "per_rooms"
|
||
|
||
return estimator._price_from_inputs(
|
||
listings=_lots(ppm2),
|
||
area_m2=area_m2,
|
||
rooms=2,
|
||
repair_state=None,
|
||
floor=5,
|
||
total_floors=10,
|
||
target_year=target_year,
|
||
analog_tier="W",
|
||
fallback_used=False,
|
||
area_widened=False,
|
||
anchor_comps=[],
|
||
anchor_tier_fetched=None,
|
||
dkp_raw=None,
|
||
imv_anchor=None,
|
||
imv_eval=None,
|
||
yandex_val_present=False,
|
||
cian_val_present=False,
|
||
ratio_resolver=ratio_resolver,
|
||
quarter_index_lookup=lambda q: None,
|
||
quarter_indexes_lookup=lambda qs: {},
|
||
target_house_cadnum=None,
|
||
dadata_coarse=False,
|
||
geo=_geo(),
|
||
dadata_qc_geo=None,
|
||
)
|
||
|
||
|
||
def _expected_factor(area_m2: float, target_year: int | None) -> float:
|
||
"""Reproduce the production factor from the live settings (no hard-coding)."""
|
||
s = estimator.settings
|
||
yr = ((target_year - 2000) / 20.0) if target_year else 0.0
|
||
raw = math.exp(
|
||
s.estimate_hedonic_b0
|
||
+ s.estimate_hedonic_year_coef * yr
|
||
+ s.estimate_hedonic_larea_coef * math.log(area_m2)
|
||
)
|
||
return max(s.estimate_hedonic_factor_min, min(s.estimate_hedonic_factor_max, raw))
|
||
|
||
|
||
# ── tests ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_mid_case_shifts_by_expected_factor(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""year≈2010, area≈50 → expected_sold shifts by the hedonic factor vs OFF."""
|
||
# OFF baseline (exact legacy expected_sold).
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", False)
|
||
off = _price(area_m2=50.0, target_year=2010, ratio=0.85)
|
||
# ON.
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
|
||
on = _price(area_m2=50.0, target_year=2010, ratio=0.85)
|
||
|
||
factor = _expected_factor(50.0, 2010)
|
||
# 2010 + 50 m² → mild uplift, strictly inside the clamp band.
|
||
assert 1.0 < factor < estimator.settings.estimate_hedonic_factor_max
|
||
assert off.expected_sold_price is not None and on.expected_sold_price is not None
|
||
# ratio 0.85 × factor < 1.0 → le_asking re-clamp is a no-op here (no confound).
|
||
assert on.expected_sold_price == round(off.expected_sold_price * factor)
|
||
assert on.expected_sold_per_m2 == round(off.expected_sold_per_m2 * factor)
|
||
|
||
|
||
def test_factor_clamps_to_min_for_huge_area(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Very large area → raw factor < factor_min → clamped to the floor."""
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", False)
|
||
off = _price(area_m2=10_000.0, target_year=None, ratio=0.85)
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
|
||
on = _price(area_m2=10_000.0, target_year=None, ratio=0.85)
|
||
|
||
factor = _expected_factor(10_000.0, None)
|
||
assert factor == estimator.settings.estimate_hedonic_factor_min
|
||
assert off.expected_sold_price is not None and on.expected_sold_price is not None
|
||
assert on.expected_sold_price == round(
|
||
off.expected_sold_price * estimator.settings.estimate_hedonic_factor_min
|
||
)
|
||
|
||
|
||
def test_factor_clamps_to_max_for_small_new_lot(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Small area + new year → raw factor > factor_max → clamped to the ceiling.
|
||
|
||
le_asking is held OFF so the raw ceiling factor is observable on the point
|
||
(otherwise the re-clamp would cap it at the asking headline).
|
||
"""
|
||
monkeypatch.setattr(estimator.settings, "estimate_expected_sold_le_asking", False)
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", False)
|
||
off = _price(area_m2=15.0, target_year=2025, ratio=0.85)
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
|
||
on = _price(area_m2=15.0, target_year=2025, ratio=0.85)
|
||
|
||
factor = _expected_factor(15.0, 2025)
|
||
assert factor == estimator.settings.estimate_hedonic_factor_max
|
||
assert off.expected_sold_price is not None and on.expected_sold_price is not None
|
||
assert on.expected_sold_price == round(
|
||
off.expected_sold_price * estimator.settings.estimate_hedonic_factor_max
|
||
)
|
||
|
||
|
||
def test_target_year_none_is_neutral(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""target_year=None → year term is 0 → identical to year 2000 (intercept+area)."""
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
|
||
none_year = _price(area_m2=50.0, target_year=None, ratio=0.85)
|
||
year_2000 = _price(area_m2=50.0, target_year=2000, ratio=0.85)
|
||
|
||
assert none_year.expected_sold_price == year_2000.expected_sold_price
|
||
assert none_year.expected_sold_per_m2 == year_2000.expected_sold_per_m2
|
||
assert _expected_factor(50.0, None) == _expected_factor(50.0, 2000)
|
||
|
||
|
||
def test_le_asking_invariant_holds_under_hedonic(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""With le_asking on, the hedonic-corrected expected_sold never exceeds asking."""
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
|
||
monkeypatch.setattr(estimator.settings, "estimate_expected_sold_le_asking", True)
|
||
# small area + new year → factor 1.30; ratio 0.95 → 0.95×1.30 ≈ 1.235 > 1 →
|
||
# uncorrected the point would exceed the asking headline; the re-clamp must bind.
|
||
res = _price(area_m2=15.0, target_year=2025, ratio=0.95)
|
||
|
||
assert res.expected_sold_price is not None
|
||
assert res.expected_sold_price <= res.median_price
|
||
assert res.expected_sold_per_m2 is not None
|
||
assert res.expected_sold_per_m2 <= res.median_ppm2
|
||
# The clamp binds exactly at the asking headline (proves it actually fired).
|
||
assert res.expected_sold_price == res.median_price
|
||
|
||
|
||
def test_le_asking_off_allows_hedonic_above_asking(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Control: with le_asking OFF, the hedonic uplift may exceed asking (no clamp)."""
|
||
monkeypatch.setattr(estimator.settings, "estimate_hedonic_correction_enabled", True)
|
||
monkeypatch.setattr(estimator.settings, "estimate_expected_sold_le_asking", False)
|
||
res = _price(area_m2=15.0, target_year=2025, ratio=0.95)
|
||
|
||
assert res.expected_sold_price is not None
|
||
# 0.95 × 1.30 ≈ 1.235 → point is allowed above the asking headline.
|
||
assert res.expected_sold_price > res.median_price
|