diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index 0b34d440..bba9be84 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -111,6 +111,25 @@ class Settings(BaseSettings): # (tier-C) больше воздуха над sold-коридором, не возвращая исходную 2× инфляцию # (tier-C гейт ×1.5 ловит явную контаминацию выше). estimate_corridor_clamp_slack: float = 0.40 + # Нижний floor для radius-пути: симметрично corridor-clamp сверху, но снизу. + # Если итоговый median_ppm2 < dkp_low_ppm2 × factor — поднимаем до floor. + # Применяется ТОЛЬКО на radius-пути (anchor_tier is None) и при dkp_raw. + # factor=0.8: 20% зазор ниже P10 коридора → floor достаточно мягкий для эконома + # (избегаем ложных подъёмов) и ловит явный undershoot. False → no-op. + # ENV: ESTIMATE_RADIUS_FLOOR_ENABLED, ESTIMATE_RADIUS_FLOOR_FACTOR. + estimate_radius_floor_enabled: bool = True + estimate_radius_floor_factor: float = 0.8 + # Шаг 5 — clamp expected_sold <= asking: ratio > 1.0 физически невозможен для + # trade-in (ожидаемая цена сделки не должна превышать цену объявления). + # Диагноз: в high-price tier asking->sold ratio > 1.0 (product artefact, не реальные + # сделки выше прайса) -> expected_sold = headline x ratio > headline. + # При флаге True: если ratio > 1.0 — клампаем до 1.0 и логируем. Применяется + # к point И range (expected_sold_low/high/price) консистентно. + # False -> старое поведение без clamp (backward-compat). + # ENV: ESTIMATE_EXPECTED_SOLD_LE_ASKING. + estimate_expected_sold_le_asking: bool = Field( + default=True, validation_alias="ESTIMATE_EXPECTED_SOLD_LE_ASKING" + ) # Шаг 2 — ужесточённый MAD-clip на малых выборках в same-building anchor: # при n < small_n_threshold используем mad_k_small вместо estimate_sb_mad_k # (3.5 слишком мягкий при n=7 → элитные хвосты не срезаются, mean тянется вверх). diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index 084f4d6e..dc33da79 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -772,7 +772,14 @@ def _save_yandex_history_items( ) rows = [] + skipped_area = 0 for item in result.history_items: + # Фильтруем items с нулевой/отрицательной/отсутствующей площадью — битый парс + # («0,5 м²» и пр.) сохраняет мусор в house_placement_history и искажает + # price_trend. Estimator защищён NULLIF, но грязь копится → чистим на входе. + if item.area_m2 is None or item.area_m2 <= 0: + skipped_area += 1 + continue ext_seed = ( f"{result.address}|{item.publish_date}|{item.area_m2}|{item.floor}|" f"{item.start_price}|{item.last_price}" @@ -797,6 +804,14 @@ def _save_yandex_history_items( } ) + if skipped_area > 0: + logger.info( + "yandex_valuation: skipped %d/%d history items with area_m2 <= 0 or None" " (addr=%r)", + skipped_area, + len(result.history_items), + result.address, + ) + sql = text( """ INSERT INTO house_placement_history ( @@ -2160,28 +2175,23 @@ async def estimate_quality( # NOTE: actual_deals (#564) остаётся ИНФОРМАЦИОННЫМ и НЕ подмешивается в # headline — sold-коррекция здесь единственный sold-сигнал (без double-count). # NOTE: expected_sold_* (= asking × ratio) выводятся НЕ здесь, а ПОСЛЕ #651 - # IMV-blend (ниже), который мутирует median_price/median_ppm2/range_high. Иначе - # expected_sold остаётся pre-blend → asking 75M / sold 45M (бессмысленная скидка - # в HeroSummary) и stale-значения persist'ятся в trade_in_estimates. Здесь только - # резолвим ratio/basis (нужны для confidence/explanation и null-guard). - # #928: pass median_ppm2 (best proxy available at this point — anchor_ppm2 from - # same-building anchor computed below, but that's post-call). median_ppm2 = 0 when - # no radius analogs yet; tier lookup uses it as the ppm2 placement signal. - # NOTE(#928): tier placement uses the pre-anchor radius median (median_ppm2), not the - # same-building Tukey anchor that #928 ideally specifies (anchor is computed post-call). - # Coarse 3-tier bucketing is robust to this for the golden case; revisit (resolve ratio - # after anchor) before flipping tier_aware_ratio_enabled ON — validate in the held-out - # backtest. - asking_to_sold_ratio, ratio_basis = _get_asking_sold_ratio( - db, payload.rooms, anchor_ppm2=median_ppm2 if median_ppm2 > 0 else None - ) + # IMV-blend / SB-anchor / #1795 corridor-clamp / radius-floor (ниже), которые + # мутируют median_price/median_ppm2/range_high. Иначе expected_sold остаётся + # pre-blend → asking 75M / sold 45M (бессмысленная скидка в HeroSummary) и + # stale-значения persist'ятся в trade_in_estimates. + # FIX(ratio-tier-mismatch): _get_asking_sold_ratio вызывается ПОСЛЕ всех + # headline-мутаций (anchor/IMV-blend/quarter-index/corridor-clamp), передавая + # ФИНАЛЬНЫЙ median_ppm2 для tier-placement. Это устраняет tier-несовпадение + # когда anchor/quarter-index двигают headline в другой tier — ratio теперь + # соответствует тому ppm², к которому он будет применён. + # Инициализируем переменные здесь; реальный _get_asking_sold_ratio вызов — после + # corridor-clamp / radius-floor (ниже, ~строка 4c-cont). + asking_to_sold_ratio: float | None = None + ratio_basis: str | None = None expected_sold_per_m2: int | None = None expected_sold_price: int | None = None expected_sold_range_low: int | None = None expected_sold_range_high: int | None = None - # Не было ratio (нет таблицы/бакета) — не вводим в заблуждение пустым basis. - if asking_to_sold_ratio is None: - ratio_basis = None confidence, explanation = _compute_confidence( n_analogs, @@ -2799,22 +2809,74 @@ async def estimate_quality( " Оценка ограничена коридором реальных сделок Росреестра по улице." ) - # 4c (cont.). expected_sold_* выводим ЗДЕСЬ — ПОСЛЕ #651 IMV-blend / SB-anchor / - # #1795 corridor-clamp, которые могли поднять/опустить median_price/median_ppm2 - # и range_high. Применяем ratio к POST-якорным значениям → asking - # (median_price_rub) и sold (expected_sold_price_rub) консистентны в HeroSummary, - # и в DB persist'ятся свежие значения (no stale «скидки»). range_low берём как есть. - # Headline (median_price/ppm²/range) — ASKING-space во ВСЕХ ветках (радиус, - # SB-якорь, IMV-blend): якорь теперь берёт anchor_ppm2 (pre-haircut), blend - # работает в asking. asking→sold скидка — ЕДИНСТВЕННЫМ механизмом per-rooms - # ratio: expected_sold = headline × ratio → DISTINCT, строго ниже median (когда - # ratio<1). Null-guard: нет ratio (нет migration-080 строки) → expected_sold_* + # ── FIX: radius-path нижний floor от DKP-коридора (анти-undershoot) ──────── + # Anchor-путь имеет hard floor (comp_min_ppm2×(1-tol)) — radius-путь аналогичной + # защиты снизу не имел: quarter-index-вниз или corridor-clamp могли занизить + # median неоправданно (асимметрия с anchor-path). Если dkp_raw доступен и + # итоговый median_ppm2 < dkp_low_ppm2 × factor → поднимаем до floor и логируем. + # Только radius-путь (anchor_tier is None); без dkp_raw → no-op. + # За флагом estimate_radius_floor_enabled (дефолт True). + if ( + settings.estimate_radius_floor_enabled + and anchor_tier is None + and dkp_raw is not None + and dkp_raw.get("low_ppm2", 0) > 0 + and median_ppm2 > 0 + and dkp_raw.get("count", 0) >= settings.estimate_corridor_clamp_min_n + ): + radius_floor_ppm2 = float(dkp_raw["low_ppm2"]) * settings.estimate_radius_floor_factor + if median_ppm2 < radius_floor_ppm2: + floor_factor = radius_floor_ppm2 / median_ppm2 + logger.info( + "radius_floor: median_ppm2=%d < dkp_low=%d × factor=%.2f = floor=%d" + " → lifting (factor=%.3f)", + int(median_ppm2), + dkp_raw["low_ppm2"], + settings.estimate_radius_floor_factor, + int(radius_floor_ppm2), + floor_factor, + ) + median_ppm2 = radius_floor_ppm2 + median_price = round(median_price * floor_factor) + range_low = round(range_low * floor_factor) + range_high = round(range_high * floor_factor) + explanation = (explanation or "") + ( + " Оценка поднята до нижней границы коридора реальных сделок Росреестра." + ) + + # 4c (cont.). expected_sold_* выводим ЗДЕСЬ — ПОСЛЕ всех headline-мутаций: + # #651 IMV-blend / SB-anchor / quarter-index / #1795 corridor-clamp / radius-floor. + # FIX(ratio-tier-mismatch): _get_asking_sold_ratio вызывается с ФИНАЛЬНЫМ + # median_ppm2 (а не с pre-anchor radius-медианой). Это устраняет tier-несовпадение + # когда anchor/quarter-index двигают headline в другой ppm2-tier. + # Сохраняем per_rooms/global_fallback семантику ratio_basis из самой функции. + # Graceful: нет ratio (нет migration-080 строки / БД-ошибка) → expected_sold_* # остаются None → UI не показывает «−N%» badge (не фабрикуем). + if median_ppm2 > 0: + asking_to_sold_ratio, ratio_basis = _get_asking_sold_ratio( + db, payload.rooms, anchor_ppm2=median_ppm2 + ) + # Не было ratio — не вводим в заблуждение пустым basis. + if asking_to_sold_ratio is None: + ratio_basis = None + if asking_to_sold_ratio is not None and median_price > 0: - expected_sold_per_m2 = round(median_ppm2 * asking_to_sold_ratio) - expected_sold_price = round(median_price * asking_to_sold_ratio) - expected_sold_range_low = round(range_low * asking_to_sold_ratio) - expected_sold_range_high = round(range_high * asking_to_sold_ratio) + # FIX(expected_sold-le-asking): clamp ratio <= 1.0 чтобы expected_sold не + # превышала объявление. Smoke: 3к/27.0М -> expected_sold 31.4М > asking. + # Причина: high-price tier ratio > 1.0 (product artefact, не реальные сделки + # выше прайса). Флаг OFF -> старое поведение без clamp. + effective_ratio = asking_to_sold_ratio + if settings.estimate_expected_sold_le_asking and effective_ratio > 1.0: + logger.info( + "expected_sold ratio clamped %.3f->1.0 (rooms=%s)", + effective_ratio, + payload.rooms, + ) + effective_ratio = 1.0 + expected_sold_per_m2 = round(median_ppm2 * effective_ratio) + expected_sold_price = round(median_price * effective_ratio) + expected_sold_range_low = round(range_low * effective_ratio) + expected_sold_range_high = round(range_high * effective_ratio) # ── #652: ДКП-коридор реальных сделок (ADVISORY + soft sanity-bound) ───── # dkp_raw уже зафетчен выше (#1795) — переиспользуем, не фетчим повторно. diff --git a/tradein-mvp/backend/tests/test_estimator_expected_sold_clamp.py b/tradein-mvp/backend/tests/test_estimator_expected_sold_clamp.py new file mode 100644 index 00000000..8545d460 --- /dev/null +++ b/tradein-mvp/backend/tests/test_estimator_expected_sold_clamp.py @@ -0,0 +1,181 @@ +"""Tests for Fix 4: clamp expected_sold <= asking (ratio cap 1.0). + +Smoke-диагноз: 3к/27.0М -> expected_sold 31.4М > asking. +Причина: _get_asking_sold_ratio для high-price tier возвращает ratio > 1.0 +(product artefact) -> expected_sold = headline x ratio > headline. + +Проверяем: +1. ratio=1.15 + флаг on -> expected_sold == headline (не выше); лог clamp. +2. ratio=0.8 -> expected_sold = headline x 0.8 (норм, без clamp). +3. ratio None -> expected_sold не выводится (graceful). +4. Флаг off + ratio > 1 -> expected_sold > headline (старое поведение). +""" + +from __future__ import annotations + +import logging +import os +from datetime import UTC, datetime +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import anyio +import pytest + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + + +def _make_listing(*, price_per_m2: float, area_m2: float = 50.0) -> dict[str, Any]: + return { + "source": "cian", + "source_url": "https://cian.ru/offer/1", + "address": "ЕКБ, ул. Малышева, 30", + "lat": 56.838, + "lon": 60.595, + "rooms": 3, + "area_m2": area_m2, + "floor": 10, + "total_floors": 20, + "price_rub": price_per_m2 * area_m2, + "price_per_m2": price_per_m2, + "listing_date": datetime(2026, 5, 1), + "days_on_market": 5, + "photo_urls": [], + "scraped_at": datetime(2026, 5, 20, tzinfo=UTC), + "distance_m": 100.0, + "relevance_score": 0.1, + } + + +_ANALOGS: list[dict[str, Any]] = [ + _make_listing(price_per_m2=260_000.0), + _make_listing(price_per_m2=270_000.0), + _make_listing(price_per_m2=280_000.0), +] + + +def _make_geo(): + from app.services.geocoder import GeocodeResult + + return GeocodeResult( + lat=56.838, + lon=60.595, + full_address="Свердловская обл., Екатеринбург, ул. Малышева, 30", + provider="nominatim", + ) + + +def _make_payload(rooms: int = 3): + from app.schemas.trade_in import TradeInEstimateInput + + return TradeInEstimateInput( + address="ЕКБ, ул. Малышева, 30", area_m2=50.0, rooms=rooms, floor=10, total_floors=20 + ) + + +def _run_estimate( + ratio_tuple: tuple[float | None, str | None], + *, + clamp_enabled: bool = True, +) -> Any: + from app.services.estimator import estimate_quality + + db = MagicMock() + payload = _make_payload() + + async def _run() -> Any: + with ( + patch("app.core.config.settings.estimate_expected_sold_le_asking", new=clamp_enabled), + patch("app.services.estimator.geocode", new=AsyncMock(return_value=_make_geo())), + patch("app.services.estimator.dadata_clean_address", new=AsyncMock(return_value=None)), + patch("app.services.estimator.match_house_readonly", return_value=None), + patch( + "app.services.estimator.get_house_metadata", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator._fetch_analogs", + return_value=(list(_ANALOGS), False, "S"), + ), + patch("app.services.estimator._fetch_deals", return_value=[]), + patch( + "app.services.estimator._get_or_fetch_imv_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator._get_or_fetch_yandex_valuation_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator.estimate_via_cian_valuation", + new=AsyncMock(return_value=None), + ), + patch("app.services.estimator._fetch_dkp_corridor", return_value=None), + patch("app.services.estimator._get_asking_sold_ratio", return_value=ratio_tuple), + ): + return await estimate_quality(payload, db) + + return anyio.run(_run) + + +def test_expected_sold_clamped_to_headline_when_ratio_above_1( + caplog: pytest.LogCaptureFixture, +) -> None: + ratio = 1.15 + with caplog.at_level(logging.INFO, logger="app.services.estimator"): + est = _run_estimate((ratio, "per_rooms"), clamp_enabled=True) + assert est.median_price_rub > 0, "headline должен быть задан аналогами" + assert est.expected_sold_price_rub is not None + assert ( + est.expected_sold_price_rub <= est.median_price_rub + ), f"expected_sold {est.expected_sold_price_rub} > asking {est.median_price_rub}" + assert est.expected_sold_per_m2 is not None + assert ( + est.expected_sold_per_m2 <= est.median_price_per_m2 + ), "expected_sold_per_m2 превышает median_price_per_m2" + assert est.expected_sold_range_high_rub is not None + assert ( + est.expected_sold_range_high_rub <= est.range_high_rub + ), "expected_sold_range_high > range_high" + assert est.expected_sold_range_low_rub is not None + assert ( + est.expected_sold_range_low_rub <= est.range_low_rub + ), "expected_sold_range_low > range_low" + assert est.expected_sold_price_rub == est.median_price_rub + assert est.expected_sold_per_m2 == est.median_price_per_m2 + clamp_logs = [r for r in caplog.records if "clamped" in r.getMessage()] + assert clamp_logs, "Ожидался log-message о clamp ratio" + assert "1.150" in clamp_logs[0].getMessage() or "1.15" in clamp_logs[0].getMessage() + + +def test_expected_sold_not_clamped_when_ratio_below_1() -> None: + ratio = 0.8 + est = _run_estimate((ratio, "per_rooms"), clamp_enabled=True) + assert est.median_price_rub > 0 + assert est.expected_sold_price_rub == round(est.median_price_rub * ratio) + assert est.expected_sold_per_m2 == round(est.median_price_per_m2 * ratio) + assert est.expected_sold_range_low_rub == round(est.range_low_rub * ratio) + assert est.expected_sold_range_high_rub == round(est.range_high_rub * ratio) + assert est.expected_sold_price_rub < est.median_price_rub + + +def test_expected_sold_none_when_ratio_none() -> None: + est = _run_estimate((None, None), clamp_enabled=True) + assert est.expected_sold_price_rub is None + assert est.expected_sold_per_m2 is None + assert est.expected_sold_range_low_rub is None + assert est.expected_sold_range_high_rub is None + assert est.asking_to_sold_ratio is None + assert est.median_price_rub > 0 + + +def test_expected_sold_exceeds_asking_when_clamp_flag_off() -> None: + ratio = 1.15 + est = _run_estimate((ratio, "per_rooms"), clamp_enabled=False) + assert est.median_price_rub > 0 + assert est.expected_sold_price_rub is not None + assert est.expected_sold_price_rub > est.median_price_rub, ( + f"Ожидалось expected_sold > asking при clamp=False, " + f"но {est.expected_sold_price_rub} <= {est.median_price_rub}" + ) + assert est.expected_sold_price_rub == round(est.median_price_rub * ratio) diff --git a/tradein-mvp/backend/tests/test_estimator_radius_floor.py b/tradein-mvp/backend/tests/test_estimator_radius_floor.py new file mode 100644 index 00000000..21f39e1c --- /dev/null +++ b/tradein-mvp/backend/tests/test_estimator_radius_floor.py @@ -0,0 +1,249 @@ +"""Tests for Fix 2: radius-path нижний floor от DKP-коридора (анти-undershoot). + +Симметрично corridor-clamp (верхний потолок): если median_ppm2 < dkp_low_ppm2 × factor +и anchor_tier is None → поднимаем до floor. Только radius-путь. Без dkp_raw → no-op. + +Тесты: +- radius median ниже dkp_low × factor → median поднята до floor +- radius median выше dkp_low × factor → no-op (медиана не изменена) +- dkp_raw is None → no-op (нет базы для floor) +- anchor-путь (anchor_tier != None) → не затронут floor'ом +- флаг estimate_radius_floor_enabled=False → no-op +""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import anyio + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + + +# ── helpers ────────────────────────────────────────────────────────────────── + + +def _make_listing(*, price_per_m2: float, area_m2: float = 50.0) -> dict[str, Any]: + return { + "source": "cian", + "source_url": "https://cian.ru/offer/1", + "address": "ЕКБ, ул. Учителей, 18", + "lat": 56.838, + "lon": 60.595, + "rooms": 2, + "area_m2": area_m2, + "floor": 5, + "total_floors": 16, + "price_rub": price_per_m2 * area_m2, + "price_per_m2": price_per_m2, + "listing_date": datetime(2026, 5, 1), + "days_on_market": 10, + "photo_urls": [], + "scraped_at": datetime(2026, 5, 20, tzinfo=UTC), + "distance_m": 150.0, + "relevance_score": 0.1, + } + + +def _make_geo(): + from app.services.geocoder import GeocodeResult + + return GeocodeResult( + lat=56.838, + lon=60.595, + full_address="Свердловская обл., Екатеринбург, ул. Учителей, 18", + provider="nominatim", + ) + + +def _make_payload(): + from app.schemas.trade_in import TradeInEstimateInput + + return TradeInEstimateInput( + address="ЕКБ, ул. Учителей, 18", + area_m2=50.0, + rooms=2, + floor=5, + total_floors=16, + ) + + +def _run_estimate( + analogs: list[dict[str, Any]], + dkp_raw: dict[str, Any] | None, + *, + radius_floor_enabled: bool = True, + radius_floor_factor: float = 0.8, +) -> Any: + from app.core.config import settings + from app.services.estimator import estimate_quality + + db = MagicMock() + payload = _make_payload() + + async def _run() -> Any: + with ( + patch("app.services.estimator.geocode", new=AsyncMock(return_value=_make_geo())), + patch("app.services.estimator.dadata_clean_address", new=AsyncMock(return_value=None)), + patch("app.services.estimator.match_house_readonly", return_value=None), + patch("app.services.estimator.get_house_metadata", new=AsyncMock(return_value=None)), + patch( + "app.services.estimator._fetch_analogs", + return_value=(list(analogs), False, "S"), + ), + patch("app.services.estimator._fetch_deals", return_value=[]), + patch( + "app.services.estimator._get_or_fetch_imv_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator._get_or_fetch_yandex_valuation_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator.estimate_via_cian_valuation", + new=AsyncMock(return_value=None), + ), + patch("app.services.estimator._fetch_dkp_corridor", return_value=dkp_raw), + patch("app.services.estimator._get_asking_sold_ratio", return_value=(None, None)), + patch.object(settings, "estimate_radius_floor_enabled", radius_floor_enabled), + patch.object(settings, "estimate_radius_floor_factor", radius_floor_factor), + ): + return await estimate_quality(payload, db) + + return anyio.run(_run) + + +# ── тест 1: radius median ниже floor → поднята ─────────────────────────────── + + +def test_radius_median_below_dkp_floor_is_lifted() -> None: + """Radius median = 80k, dkp_low = 150k, factor 0.8 → floor = 120k → median поднята.""" + # analogs с median ~80k + analogs = [ + _make_listing(price_per_m2=75_000.0), + _make_listing(price_per_m2=80_000.0), + _make_listing(price_per_m2=85_000.0), + ] + dkp_raw = { + "count": 15, + "low_ppm2": 150_000, + "median_ppm2": 180_000, + "high_ppm2": 220_000, + "period_months": 12, + } + est = _run_estimate(analogs, dkp_raw, radius_floor_factor=0.8) + + floor_ppm2 = 150_000 * 0.8 # 120_000 + assert ( + est.median_price_per_m2 >= floor_ppm2 + ), f"median_ppm2={est.median_price_per_m2} должна быть >= floor={floor_ppm2}" + + +# ── тест 2: radius median выше floor → no-op ───────────────────────────────── + + +def test_radius_median_above_dkp_floor_unchanged() -> None: + """Radius median = 200k > dkp_low × factor = 150k × 0.8 = 120k → no-op.""" + analogs = [ + _make_listing(price_per_m2=190_000.0), + _make_listing(price_per_m2=200_000.0), + _make_listing(price_per_m2=210_000.0), + ] + dkp_raw = { + "count": 15, + "low_ppm2": 150_000, + "median_ppm2": 180_000, + "high_ppm2": 220_000, + "period_months": 12, + } + est = _run_estimate(analogs, dkp_raw, radius_floor_factor=0.8) + + # floor = 150k × 0.8 = 120k; median = 200k > floor → no-op, медиана не изменяется + # Медиана должна остаться в диапазоне аналогов (190-210k), а не подняться к floor. + floor_ppm2 = 150_000 * 0.8 # 120_000 + assert est.median_price_per_m2 > floor_ppm2, "median должна быть выше floor (no-op)" + # Медиана соответствует аналогам (~200k), а не floor + assert ( + 180_000 <= est.median_price_per_m2 <= 220_000 + ), f"median_ppm2={est.median_price_per_m2} должна остаться в диапазоне аналогов (no-op)" + + +# ── тест 3: dkp_raw is None → no-op ───────────────────────────────────────── + + +def test_no_dkp_raw_no_floor() -> None: + """Без dkp_raw floor не применяется — median остаётся radius-значением.""" + analogs = [ + _make_listing(price_per_m2=75_000.0), + _make_listing(price_per_m2=80_000.0), + _make_listing(price_per_m2=85_000.0), + ] + est = _run_estimate(analogs, dkp_raw=None, radius_floor_factor=0.8) + + # median ~80k, без dkp_raw floor не поднимает + assert ( + est.median_price_per_m2 < 100_000 + ), f"median_ppm2={est.median_price_per_m2} без dkp_raw не должна расти" + + +# ── тест 4: флаг OFF → no-op ───────────────────────────────────────────────── + + +def test_radius_floor_disabled_no_op() -> None: + """estimate_radius_floor_enabled=False → floor не применяется.""" + analogs = [ + _make_listing(price_per_m2=75_000.0), + _make_listing(price_per_m2=80_000.0), + _make_listing(price_per_m2=85_000.0), + ] + dkp_raw = { + "count": 15, + "low_ppm2": 150_000, + "median_ppm2": 180_000, + "high_ppm2": 220_000, + "period_months": 12, + } + # Без флага — median не должна подняться + from app.core.config import settings + from app.services.estimator import estimate_quality + + db = MagicMock() + payload = _make_payload() + + async def _run() -> Any: + with ( + patch("app.services.estimator.geocode", new=AsyncMock(return_value=_make_geo())), + patch("app.services.estimator.dadata_clean_address", new=AsyncMock(return_value=None)), + patch("app.services.estimator.match_house_readonly", return_value=None), + patch("app.services.estimator.get_house_metadata", new=AsyncMock(return_value=None)), + patch( + "app.services.estimator._fetch_analogs", + return_value=(list(analogs), False, "S"), + ), + patch("app.services.estimator._fetch_deals", return_value=[]), + patch( + "app.services.estimator._get_or_fetch_imv_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator._get_or_fetch_yandex_valuation_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator.estimate_via_cian_valuation", + new=AsyncMock(return_value=None), + ), + patch("app.services.estimator._fetch_dkp_corridor", return_value=dkp_raw), + patch("app.services.estimator._get_asking_sold_ratio", return_value=(None, None)), + patch.object(settings, "estimate_radius_floor_enabled", False), + ): + return await estimate_quality(payload, db) + + est = anyio.run(_run) + assert ( + est.median_price_per_m2 < 100_000 + ), f"Флаг OFF: median_ppm2={est.median_price_per_m2} не должна расти" diff --git a/tradein-mvp/backend/tests/test_estimator_ratio_tier_fix.py b/tradein-mvp/backend/tests/test_estimator_ratio_tier_fix.py new file mode 100644 index 00000000..069d4bc1 --- /dev/null +++ b/tradein-mvp/backend/tests/test_estimator_ratio_tier_fix.py @@ -0,0 +1,311 @@ +"""Tests for Fix 1: asking_to_sold ratio tier resolved from FINAL headline ppm². + +Проверяем, что _get_asking_sold_ratio вызывается ПОСЛЕ всех headline-мутаций +(anchor/IMV-blend/quarter-index/corridor-clamp) с финальным median_ppm2, а не с +pre-anchor радиусной медианой. Это устраняет tier-несовпадение (#928 audit fix). + +Два слоя: +1. Стохастический: _get_asking_sold_ratio не вызывается в ранней части estimate_quality + (только после corridor-clamp/radius-floor). +2. Функциональный: когда anchor поднял headline в другой tier, ratio берётся + по финальному ppm² (из нового tier), а не по исходной радиусной медиане. + Graceful: нет ratio → expected_sold_* == None, headline не изменён. +""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import anyio + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + + +# ── helpers ────────────────────────────────────────────────────────────────── + + +def _make_listing(*, price_per_m2: float, area_m2: float = 50.0) -> dict[str, Any]: + return { + "source": "cian", + "source_url": "https://cian.ru/offer/1", + "address": "ЕКБ, ул. Учителей, 18", + "lat": 56.838, + "lon": 60.595, + "rooms": 2, + "area_m2": area_m2, + "floor": 5, + "total_floors": 16, + "price_rub": price_per_m2 * area_m2, + "price_per_m2": price_per_m2, + "listing_date": datetime(2026, 5, 1), + "days_on_market": 10, + "photo_urls": [], + "scraped_at": datetime(2026, 5, 20, tzinfo=UTC), + "distance_m": 150.0, + "relevance_score": 0.1, + } + + +_ANALOGS_LOW: list[dict[str, Any]] = [ + _make_listing(price_per_m2=100_000.0), + _make_listing(price_per_m2=110_000.0), + _make_listing(price_per_m2=105_000.0), +] + +_ANALOGS_HIGH: list[dict[str, Any]] = [ + _make_listing(price_per_m2=300_000.0), + _make_listing(price_per_m2=310_000.0), + _make_listing(price_per_m2=305_000.0), +] + + +def _make_geo(): + from app.services.geocoder import GeocodeResult + + return GeocodeResult( + lat=56.838, + lon=60.595, + full_address="Свердловская обл., Екатеринбург, ул. Учителей, 18", + provider="nominatim", + ) + + +def _make_payload(rooms: int = 2): + from app.schemas.trade_in import TradeInEstimateInput + + return TradeInEstimateInput( + address="ЕКБ, ул. Учителей, 18", + area_m2=50.0, + rooms=rooms, + floor=5, + total_floors=16, + ) + + +def _run_estimate_with_ratio_spy( + analogs: list[dict[str, Any]], + ratio_return: tuple[float | None, str | None], +) -> tuple[Any, list[Any]]: + """Запускает estimate_quality с отслеживанием вызовов _get_asking_sold_ratio. + + Возвращает (estimate_result, list_of_call_args). + """ + from app.services.estimator import estimate_quality + + db = MagicMock() + payload = _make_payload() + + ratio_calls: list[Any] = [] + + def _spy_ratio(db_inner: Any, rooms: Any, anchor_ppm2: Any = None) -> Any: + ratio_calls.append({"rooms": rooms, "anchor_ppm2": anchor_ppm2}) + return ratio_return + + async def _run() -> Any: + with ( + patch("app.services.estimator.geocode", new=AsyncMock(return_value=_make_geo())), + patch("app.services.estimator.dadata_clean_address", new=AsyncMock(return_value=None)), + patch("app.services.estimator.match_house_readonly", return_value=None), + patch("app.services.estimator.get_house_metadata", new=AsyncMock(return_value=None)), + patch( + "app.services.estimator._fetch_analogs", + return_value=(list(analogs), False, "S"), + ), + patch("app.services.estimator._fetch_deals", return_value=[]), + patch( + "app.services.estimator._get_or_fetch_imv_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator._get_or_fetch_yandex_valuation_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator.estimate_via_cian_valuation", + new=AsyncMock(return_value=None), + ), + patch("app.services.estimator._fetch_dkp_corridor", return_value=None), + patch("app.services.estimator._get_asking_sold_ratio", side_effect=_spy_ratio), + ): + return await estimate_quality(payload, db) + + est = anyio.run(_run) + return est, ratio_calls + + +# ── тест 1: ratio вызывается РОВНО ОДИН РАЗ (не дважды — до и после headline) ── + + +def test_ratio_called_exactly_once() -> None: + """_get_asking_sold_ratio должен вызываться ровно один раз — после headline.""" + _est, calls = _run_estimate_with_ratio_spy(_ANALOGS_LOW, (0.80, "per_rooms")) + assert ( + len(calls) == 1 + ), f"_get_asking_sold_ratio должен вызываться 1 раз, вызван {len(calls)} раз" + + +# ── тест 2: anchor поднял headline → ratio вызван с финальным (high) ppm² ──── + + +def test_ratio_tier_uses_final_headline_after_anchor() -> None: + """Когда anchor поднял median_ppm2 с ~105k до ~300k, ratio вызывается с ~300k.""" + from app.services.estimator import estimate_quality + + db = MagicMock() + payload = _make_payload() + + # Симулируем: radius median = 105k, anchor поднимает до 300k. + # _get_asking_sold_ratio должен получить anchor_ppm2 ≈ 300k (не 105k). + captured_anchor_ppm2: list[float | None] = [] + + def _spy(db_inner: Any, rooms: Any, anchor_ppm2: Any = None) -> tuple: + captured_anchor_ppm2.append(anchor_ppm2) + return (0.78, "per_rooms_tier:high") + + async def _run() -> Any: + with ( + patch("app.services.estimator.geocode", new=AsyncMock(return_value=_make_geo())), + patch("app.services.estimator.dadata_clean_address", new=AsyncMock(return_value=None)), + patch("app.services.estimator.match_house_readonly", return_value=None), + patch("app.services.estimator.get_house_metadata", new=AsyncMock(return_value=None)), + # Radius analogs дают медиану ~105k. + patch( + "app.services.estimator._fetch_analogs", + return_value=(list(_ANALOGS_LOW), False, "S"), + ), + patch("app.services.estimator._fetch_deals", return_value=[]), + patch( + "app.services.estimator._get_or_fetch_imv_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator._get_or_fetch_yandex_valuation_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator.estimate_via_cian_valuation", + new=AsyncMock(return_value=None), + ), + patch("app.services.estimator._fetch_dkp_corridor", return_value=None), + # Same-building anchor поднимает headline до ~300k. + patch( + "app.services.estimator._fetch_anchor_comps", + return_value=( + [ + { + "price_per_m2": 300_000, + "area_m2": 50.0, + "rooms": 2, + "source": "cian", + "source_url": "u", + "address": "a", + "lat": 56.838, + "lon": 60.595, + "floor": 5, + "total_floors": 16, + "listing_date": datetime(2026, 5, 1), + "photo_urls": [], + "scraped_at": datetime(2026, 5, 20, tzinfo=UTC), + "distance_m": 5.0, + "relevance_score": 0.9, + "price_rub": 15_000_000, + "building_cadastral_number": None, + "days_on_market": 5, + }, + { + "price_per_m2": 305_000, + "area_m2": 50.0, + "rooms": 2, + "source": "cian", + "source_url": "u2", + "address": "a", + "lat": 56.838, + "lon": 60.595, + "floor": 6, + "total_floors": 16, + "listing_date": datetime(2026, 5, 1), + "photo_urls": [], + "scraped_at": datetime(2026, 5, 20, tzinfo=UTC), + "distance_m": 6.0, + "relevance_score": 0.9, + "price_rub": 15_250_000, + "building_cadastral_number": None, + "days_on_market": 5, + }, + { + "price_per_m2": 298_000, + "area_m2": 50.0, + "rooms": 2, + "source": "cian", + "source_url": "u3", + "address": "a", + "lat": 56.838, + "lon": 60.595, + "floor": 4, + "total_floors": 16, + "listing_date": datetime(2026, 5, 1), + "photo_urls": [], + "scraped_at": datetime(2026, 5, 20, tzinfo=UTC), + "distance_m": 7.0, + "relevance_score": 0.9, + "price_rub": 14_900_000, + "building_cadastral_number": None, + "days_on_market": 5, + }, + { + "price_per_m2": 302_000, + "area_m2": 50.0, + "rooms": 2, + "source": "cian", + "source_url": "u4", + "address": "a", + "lat": 56.838, + "lon": 60.595, + "floor": 7, + "total_floors": 16, + "listing_date": datetime(2026, 5, 1), + "photo_urls": [], + "scraped_at": datetime(2026, 5, 20, tzinfo=UTC), + "distance_m": 8.0, + "relevance_score": 0.9, + "price_rub": 15_100_000, + "building_cadastral_number": None, + "days_on_market": 5, + }, + ], + "A", + ), + ), + patch("app.services.estimator._get_asking_sold_ratio", side_effect=_spy), + ): + return await estimate_quality(payload, db) + + anyio.run(_run) + + assert len(captured_anchor_ppm2) == 1 + ppm2_used = captured_anchor_ppm2[0] + assert ppm2_used is not None + # Финальный headline должен быть в зоне anchor (~300k), а НЕ в зоне radius (~105k). + assert ( + ppm2_used > 200_000 + ), f"ratio должен вызываться с anchor ppm2 (~300k), получено {ppm2_used}" + + +# ── тест 3: graceful — нет ratio → expected_sold_* = None, headline не изменён ── + + +def test_ratio_graceful_none_no_expected_sold() -> None: + """Если ratio = None → все expected_sold_* == None, headline в норме.""" + est, _calls = _run_estimate_with_ratio_spy(_ANALOGS_LOW, (None, None)) + + assert est.expected_sold_price_rub is None + assert est.expected_sold_per_m2 is None + assert est.expected_sold_range_low_rub is None + assert est.expected_sold_range_high_rub is None + assert est.asking_to_sold_ratio is None + assert est.ratio_basis is None + # Headline не изменён. + assert est.median_price_rub > 0 diff --git a/tradein-mvp/backend/tests/test_yandex_history_area_filter.py b/tradein-mvp/backend/tests/test_yandex_history_area_filter.py new file mode 100644 index 00000000..ce31b8f3 --- /dev/null +++ b/tradein-mvp/backend/tests/test_yandex_history_area_filter.py @@ -0,0 +1,201 @@ +"""Tests for Fix 3: yandex history items с area_m2 <= 0 или None отброшены при сохранении. + +_save_yandex_history_items должен: +- пропустить item если area_m2 is None или area_m2 <= 0 +- сохранить валидные items (area_m2 > 0) без потерь +- не ронять батч из-за одного битого item +- лого кол-во отброшенных items +""" + +from __future__ import annotations + +import os +from datetime import date +from unittest.mock import MagicMock, patch + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from app.services.estimator import _save_yandex_history_items +from app.services.scrapers.yandex_valuation import ( + ValuationHistoryItem, + ValuationHouseMeta, + YandexValuationResult, +) + + +def _make_result(items: list[ValuationHistoryItem]) -> YandexValuationResult: + return YandexValuationResult( + address="Россия, Свердловская область, Екатеринбург, ул. Куйбышева, 106", + offer_category="APARTMENT", + offer_type="SELL", + page=1, + source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address=test", + house=ValuationHouseMeta(year_built=2010, total_floors=16), + history_items=items, + ) + + +def _valid_item(area_m2: float = 50.0) -> ValuationHistoryItem: + return ValuationHistoryItem( + area_m2=area_m2, + rooms=2, + floor=5, + start_price=9_000_000, + last_price=8_800_000, + publish_date=date(2024, 4, 10), + exposure_days=30, + status="Снято", + ) + + +def _item_with_area(area_m2: float | None) -> ValuationHistoryItem: + return ValuationHistoryItem( + area_m2=area_m2, + rooms=1, + floor=2, + start_price=6_000_000, + last_price=6_000_000, + publish_date=date(2024, 1, 1), + exposure_days=10, + ) + + +# ── тест 1: item с area=None отброшен ──────────────────────────────────────── + + +def test_item_with_area_none_is_skipped() -> None: + """Item с area_m2=None должен быть пропущен, валидный item сохранён.""" + items = [ + _item_with_area(None), # битый — пропускаем + _valid_item(50.0), # валидный — сохраняем + ] + db = MagicMock() + result = _make_result(items) + + with patch( + "app.services.estimator.match_or_create_house", + return_value=(99, 0.9, "fp"), + ): + saved = _save_yandex_history_items(db, result) + + assert saved == 1, f"Ожидали 1 сохранённый item, получили {saved}" + rows = db.execute.call_args_list[0].args[1] + assert len(rows) == 1 + assert rows[0]["area"] == 50.0 + + +# ── тест 2: item с area=0 отброшен ─────────────────────────────────────────── + + +def test_item_with_area_zero_is_skipped() -> None: + """Item с area_m2=0 должен быть пропущен.""" + items = [ + _item_with_area(0.0), # area=0 — пропускаем + _item_with_area(0), # area=0 (int) — пропускаем + _valid_item(55.0), # валидный + ] + db = MagicMock() + result = _make_result(items) + + with patch( + "app.services.estimator.match_or_create_house", + return_value=(1, 1.0, "new"), + ): + saved = _save_yandex_history_items(db, result) + + assert saved == 1 + rows = db.execute.call_args_list[0].args[1] + assert rows[0]["area"] == 55.0 + + +# ── тест 3: item с area < 0 отброшен ───────────────────────────────────────── + + +def test_item_with_negative_area_is_skipped() -> None: + """Item с area_m2 < 0 должен быть пропущен.""" + items = [ + _item_with_area(-5.0), # отрицательная площадь + _valid_item(42.0), + ] + db = MagicMock() + result = _make_result(items) + + with patch( + "app.services.estimator.match_or_create_house", + return_value=(2, 0.7, "geo"), + ): + saved = _save_yandex_history_items(db, result) + + assert saved == 1 + + +# ── тест 4: все битые → 0 сохранено, но батч не падает ─────────────────────── + + +def test_all_invalid_area_returns_zero_no_crash() -> None: + """Все items с area=None или 0 → saved=0, ни одного db.execute INSERT-вызова с данными.""" + items = [ + _item_with_area(None), + _item_with_area(0.0), + _item_with_area(None), + ] + db = MagicMock() + result = _make_result(items) + + with patch( + "app.services.estimator.match_or_create_house", + return_value=(3, 0.9, "fp"), + ): + saved = _save_yandex_history_items(db, result) + + assert saved == 0 + # db.execute не должен вызываться для пустого rows (нет INSERT) + # Commit вызывается, rollback — нет. + db.commit.assert_called_once() + db.rollback.assert_not_called() + + +# ── тест 5: все валидные — всё сохранено ───────────────────────────────────── + + +def test_all_valid_items_all_saved() -> None: + """Все items с корректной площадью сохраняются без потерь.""" + items = [_valid_item(float(40 + i * 5)) for i in range(5)] + db = MagicMock() + result = _make_result(items) + + with patch( + "app.services.estimator.match_or_create_house", + return_value=(10, 0.9, "fp"), + ): + saved = _save_yandex_history_items(db, result) + + assert saved == 5 + + +# ── тест 6: микс валидных и битых ───────────────────────────────────────────── + + +def test_mixed_items_only_valid_saved() -> None: + """Микс: 2 валидных, 3 битых → saved=2.""" + items = [ + _valid_item(40.0), + _item_with_area(None), + _valid_item(60.0), + _item_with_area(0.0), + _item_with_area(-1.0), + ] + db = MagicMock() + result = _make_result(items) + + with patch( + "app.services.estimator.match_or_create_house", + return_value=(20, 0.9, "fp"), + ): + saved = _save_yandex_history_items(db, result) + + assert saved == 2 + rows = db.execute.call_args_list[0].args[1] + assert len(rows) == 2 + areas = {r["area"] for r in rows} + assert areas == {40.0, 60.0}