Merge pull request 'fix(tradein/estimator): non-EKB headline из deals + geo-bound Tier-S analog leak' (#2492) from fix/tradein-oblast-nonekb-headline into main

tradein-mvp/backend/app/services/estimator.py

@@ -92,6 +92,15 @@ MIN_ANALOGS_PER_SOURCE = 5  # гарантированный минимум на
 LISTINGS_FRESH_DAYS = 14  # объявления не старше 14 дней
 DEALS_PERIOD_MONTHS = 12  # сделки за последний год
 
+# #oblast-D (non-EKB deals-headline-fallback): минимум ДКП-сделок, чтобы
+# _fetch_dkp_corridor доверял СВОЕЙ street-scoped выборке — иначе (тонкая
+# конкретная улица небольшого города) виджет расширяется до city-wide (см.
+# _fetch_dkp_corridor). ОТДЕЛЬНО — минимум, чтобы _price_from_inputs счёл
+# коридор пригодным как HEADLINE (не просто advisory/clamp/floor) при
+# отсутствии листинговых аналогов — см. deals-headline-fallback ниже.
+DKP_CORRIDOR_CITY_WIDE_MIN_N = 3
+DEALS_HEADLINE_FALLBACK_MIN_N = 3
+
 # #794: СберИндекс time-adjustment of frozen Rosreestr ДКП deals.
 # Rosreestr deals freeze ~2026-01; the sber monthly index re-bases a stale deal's ppm²
 # to the latest available month. Region fixed to Свердловская обл. (tradein MVP = ЕКБ).
@@ -1423,6 +1432,78 @@ def _fetch_dkp_corridor(
         adjusted.append(float(ppm2) * factor)
         factors_applied.append(factor)
     ppm2_values = sorted(adjusted)
+
+    # #oblast-D widen: a single street in a small non-EKB town can easily have
+    # <3 (or 0) ДКП deals in the last 12 months even though the CITY overall
+    # has plenty (migration 177 loaded 47k+ oblast-wide deals across ~368
+    # cities) — the street-only corridor was silently unusable for exactly the
+    # towns that most need it (Нижний Тагил/Серов/Каменск-Уральский/...).
+    # Widen to a CITY-scoped (no street filter) sample when the street sample
+    # is too thin. EKB is explicitly EXCLUDED (city == "екатеринбург" is
+    # always dense at street level) — this widen path NEVER engages for EKB,
+    # so live EKB behaviour is unaffected. The frozen regression gate never
+    # calls this function at all (it replays a captured `dkp_raw` dict), so
+    # this change carries zero risk to the gate either way.
+    if len(ppm2_values) < DKP_CORRIDOR_CITY_WIDE_MIN_N and city and city != "екатеринбург":
+        try:
+            city_rows = (
+                db.execute(
+                    text(
+                        """
+                        SELECT d.price_per_m2, d.deal_date
+                        FROM deals d
+                        LEFT JOIN deal_city_price_bands b ON b.city = d.city
+                        WHERE d.source = 'rosreestr'
+                          AND d.city IS NOT NULL
+                          AND LOWER(d.city) = CAST(:target_city AS text)
+                          AND d.rooms = CAST(:rooms AS integer)
+                          AND d.area_m2 BETWEEN :area_min AND :area_max
+                          AND d.deal_date > NOW()
+                              - (CAST(:period_months AS integer) || ' months')::interval
+                          AND d.price_per_m2 > 0
+                          AND d.price_per_m2 BETWEEN COALESCE(b.ppm2_min, CAST(:ppm_min AS int))
+                                                  AND COALESCE(b.ppm2_max, CAST(:ppm_max AS int))
+                        """
+                    ),
+                    {
+                        "target_city": city.lower(),
+                        "rooms": rooms,
+                        "area_min": area_min,
+                        "area_max": area_max,
+                        "period_months": period_months,
+                        "ppm_min": DEAL_MIN_PPM2,
+                        "ppm_max": DEAL_MAX_PPM2,
+                    },
+                )
+                .mappings()
+                .all()
+            )
+        except Exception as exc:  # pragma: no cover — defensive
+            logger.warning("dkp_corridor city-wide widen failed (graceful): %s", exc)
+            city_rows = []
+
+        city_adjusted: list[float] = []
+        for r in city_rows:
+            ppm2 = r["price_per_m2"]
+            if not ppm2:
+                continue
+            dd = r.get("deal_date")
+            factor = 1.0
+            if series and dd is not None:
+                deal_month = date(dd.year, dd.month, 1)
+                factor = _sber_time_factor(series, deal_month)
+            city_adjusted.append(float(ppm2) * factor)
+
+        if len(city_adjusted) > len(ppm2_values):
+            logger.info(
+                "dkp_corridor widen #oblast-D: street n=%d < %d → city-wide n=%d (city=%s)",
+                len(ppm2_values),
+                DKP_CORRIDOR_CITY_WIDE_MIN_N,
+                len(city_adjusted),
+                city,
+            )
+            ppm2_values = sorted(city_adjusted)
+
     if not ppm2_values:
         return None
     if series and factors_applied:
@@ -2823,6 +2904,57 @@ def _price_from_inputs(
             if abs(honest_ratio - asking_to_sold_ratio) > _RATIO_DESCRIPTOR_EPS:
                 asking_to_sold_ratio = honest_ratio
 
+    # ── #oblast-D: deals-headline-fallback ───────────────────────────────────
+    # When the radius/anchor analog pipeline found NOTHING usable (median_ppm2
+    # is still 0 here — happens for non-EKB towns where scraped `listings`
+    # coverage is ~0; see the Part-1 geo-bound fix in the Tier S fallback
+    # query above) but we DO have a ДКП deal corridor (dkp_raw — city+street
+    # scoped via _resolve_target_city/_fetch_dkp_corridor, widened to
+    # city-wide when the street sample is too thin, see _fetch_dkp_corridor),
+    # build the headline FROM THE DEAL SIGNAL instead of surfacing an empty
+    # ("n/a") estimate — or, pre-Part-1, a distant EKB-leaked listing median.
+    #
+    # Placed AFTER the expected_sold block above (guarded by the SAME
+    # `median_ppm2 <= 0` state it ran in) so ratio_resolver/hedonic/le_asking
+    # never touch this value: dkp_raw's ppm² are ALREADY sold prices — running
+    # them through the asking→sold ratio would double-discount. expected_sold_*
+    # stays None (an existing, already-supported state — see the Prediction
+    # docstring in scripts/backtest_estimator.py: "may be None when the spine
+    # produced a headline but no asking→sold ratio resolved").
+    #
+    # n_analogs stays 0 (true — zero scraped-listing analogs), so
+    # _enforce_zero_analog_low (estimate_quality, downstream) forces
+    # confidence 'low' regardless — we set it explicitly here too so the
+    # explanation text stays coherent with the actual reason (deals-only, not
+    # generic ghost-anchor). No repair_state adjustment: the deal corridor
+    # mixes conditions across sold units — unlike the listings comp pool,
+    # there is no per-unit signal to correct against.
+    if (
+        median_ppm2 <= 0
+        and anchor_tier is None
+        and dkp_raw is not None
+        and dkp_raw.get("count", 0) >= DEALS_HEADLINE_FALLBACK_MIN_N
+        and dkp_raw.get("median_ppm2", 0) > 0
+    ):
+        median_ppm2 = float(dkp_raw["median_ppm2"])
+        median_price = int(median_ppm2 * area_m2)
+        range_low = int(dkp_raw["low_ppm2"] * area_m2)
+        range_high = int(dkp_raw["high_ppm2"] * area_m2)
+        n_analogs = 0
+        confidence = "low"
+        cv = None
+        explanation = (explanation or "") + (
+            " Рядом нет актуальных объявлений — оценка построена по реальным "
+            f"сделкам Росреестра ({dkp_raw['count']} шт. за {dkp_raw['period_months']} мес.),"
+            " точность ориентировочная."
+        )
+        logger.info(
+            "deals_headline_fallback #oblast-D: dkp median=%d (n=%d) → headline"
+            " (listings=0, anchor=None)",
+            int(median_ppm2),
+            dkp_raw["count"],
+        )
+
     # ── #652: ДКП-коридор реальных сделок (advisory) ─────────────────────────
     dkp_corridor: DkpCorridor | None = None
     if dkp_raw is not None:
@@ -4490,12 +4622,28 @@ def _fetch_analogs(
             return _stratify_candidates(tier_sc), radius_m > DEFAULT_RADIUS_M, "S"
 
     # ── Tier S (fallback): same building via address prefix ───────────────────
+    # #oblast-D geo-bound fix: _extract_short_addr strips the city/admin prefix
+    # ("Нижний Тагил, улица Ленина, 100" → "улица Ленина, 100"), so an
+    # address-only ILIKE prefix match can collide with an IDENTICALLY-NAMED
+    # street+house-number in a COMPLETELY DIFFERENT city (e.g. "улица Ленина,
+    # 100" exists verbatim in Берёзовский — ~40 191 of ~40 200 active listings
+    # are EKB-region, so any such collision silently resolves to a distant EKB
+    # listing). A Нижний Тагил / Серов / Каменск subject then got "same
+    # building" comps from a listing ~100+ km away with distance_m hardcoded to
+    # 0.0 (masking the gap). Tier S's OWN semantics ("same building") already
+    # imply the match must be near the subject, so we now require ST_DWithin on
+    # the SAME radius_m the caller passed in (mirrors Tier H/W below) — this
+    # can only DROP false-positive cross-city matches, never legitimate
+    # same-building EKB hits (>99.9% of active listings carry lat/lon).
     short_addr = _extract_short_addr(full_address)
 
     if short_addr:
         tier_s_params = {
             **base_params,
             "short_addr_prefix": short_addr + "%",
+            "lat": lat,
+            "lon": lon,
+            "radius": radius_m,
         }
 
         tier_s_rows = (
@@ -4513,6 +4661,7 @@ def _fetch_analogs(
 {_RN_DUP_WINDOW}
                     FROM listings
                     WHERE address ILIKE :short_addr_prefix
+                      AND ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius)
                     {_COMMON_WHERE}
                 )
                 SELECT

tradein-mvp/backend/tests/test_estimator_deals_headline_fallback_oblast_d.py

@@ -0,0 +1,238 @@
+"""Tests for the oblast-D deals-headline-fallback (non-EKB accuracy gap).
+
+Context: `listings` coverage is ~40k EKB / ~0 non-EKB (Нижний Тагил / Серов /
+Каменск-Уральский). Before this fix, a non-EKB subject either:
+  (a) leaked a DISTANT EKB listing median via the Tier S address-prefix
+      fallback (no geo bound — see `_fetch_analogs` Tier S fix), or
+  (b) surfaced an empty ("n/a") headline once (a) was fixed and the
+      geo-bound radius tiers legitimately found 0 local listings.
+
+This fallback builds the headline from the ДКП deal corridor (`dkp_raw`,
+already city+street-scoped via `_resolve_target_city`/`_fetch_dkp_corridor`)
+instead — honestly flagged 'low' confidence, deals-only.
+
+These tests exercise `estimate_quality` end-to-end (real spine code, mocked
+DB-facing helpers only) — the same pattern as test_estimator_radius_floor.py.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import UTC, datetime
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import anyio
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
+
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+
+def _make_listing(*, price_per_m2: float, area_m2: float = 45.0) -> dict[str, Any]:
+    """An EKB analog — used only in the "EKB unaffected" control test."""
+    return {
+        "source": "cian",
+        "source_url": "https://cian.ru/offer/1",
+        "address": "ЕКБ, ул. Малышева, 30",
+        "lat": 56.838,
+        "lon": 60.595,
+        "rooms": 2,
+        "area_m2": area_m2,
+        "floor": 5,
+        "total_floors": 16,
+        "price_rub": price_per_m2 * area_m2,
+        "price_per_m2": price_per_m2,
+        "listing_date": datetime(2026, 5, 1),
+        "days_on_market": 10,
+        "photo_urls": [],
+        "scraped_at": datetime(2026, 5, 20, tzinfo=UTC),
+        "distance_m": 150.0,
+        "relevance_score": 0.1,
+    }
+
+
+def _make_geo_tagil():
+    from app.services.geocoder import GeocodeResult
+
+    return GeocodeResult(
+        lat=57.9094,
+        lon=59.9789,
+        full_address="Свердловская обл., Нижний Тагил, ул. Ленина, 5",
+        provider="nominatim",
+    )
+
+
+def _make_payload_tagil():
+    from app.schemas.trade_in import TradeInEstimateInput
+
+    return TradeInEstimateInput(
+        address="Нижний Тагил, ул. Ленина, 5",
+        area_m2=45.0,
+        rooms=2,
+        floor=5,
+        total_floors=9,
+    )
+
+
+def _run_estimate(
+    *,
+    analogs: list[dict[str, Any]],
+    dkp_raw: dict[str, Any] | None,
+    geo: Any,
+    payload: Any,
+) -> Any:
+    from app.services.estimator import estimate_quality
+
+    db = MagicMock()
+
+    async def _run() -> Any:
+        with (
+            patch("app.services.estimator.geocode", new=AsyncMock(return_value=geo)),
+            patch("app.services.estimator.dadata_clean_address", new=AsyncMock(return_value=None)),
+            patch("app.services.estimator.match_house_readonly", return_value=None),
+            patch("app.services.estimator.get_house_metadata", new=AsyncMock(return_value=None)),
+            patch(
+                "app.services.estimator._fetch_analogs",
+                # Post Part-1 (geo-bound Tier S) reality for a non-EKB town: the
+                # radius/tier ladder legitimately returns NOTHING — tier='W' is
+                # what the always-executed final fallback tier returns.
+                return_value=(list(analogs), False, "W"),
+            ),
+            patch("app.services.estimator._fetch_deals", return_value=[]),
+            patch(
+                "app.services.estimator._get_or_fetch_imv_cached",
+                new=AsyncMock(return_value=None),
+            ),
+            patch(
+                "app.services.estimator._get_or_fetch_yandex_valuation_cached",
+                new=AsyncMock(return_value=None),
+            ),
+            patch(
+                "app.services.estimator.estimate_via_cian_valuation",
+                new=AsyncMock(return_value=None),
+            ),
+            patch("app.services.estimator._fetch_dkp_corridor", return_value=dkp_raw),
+            patch("app.services.estimator._get_asking_sold_ratio", return_value=(None, None)),
+        ):
+            return await estimate_quality(payload, db)
+
+    return anyio.run(_run)
+
+
+# ── Нижний Тагил: no local listings, deal corridor present ───────────────────
+
+
+def test_non_ekb_empty_listings_uses_deals_headline() -> None:
+    """0 local listings + a usable ДКП corridor → headline comes from deals.
+
+    Mirrors the reported Нижний Тагил gap: deal_median ≈ 85 911 ₽/м² (accurate)
+    vs the old EKB-leaked asking headline ≈ 186 461 (~6x over). After the fix,
+    the headline must equal the deal corridor's median — nowhere near the
+    EKB-range figure — and confidence must be honestly 'low' (deals-only, zero
+    scraped analogs).
+    """
+    dkp_raw = {
+        "count": 12,
+        "low_ppm2": 70_000,
+        "median_ppm2": 85_911,
+        "high_ppm2": 100_000,
+        "period_months": 12,
+    }
+    est = _run_estimate(
+        analogs=[],  # 0 listings — the honest post-geo-bound-fix reality
+        dkp_raw=dkp_raw,
+        geo=_make_geo_tagil(),
+        payload=_make_payload_tagil(),
+    )
+
+    assert est.median_price_per_m2 == 85_911, (
+        f"headline={est.median_price_per_m2} must equal the deal corridor "
+        f"median, not 0/n-a and nowhere near an EKB-range figure (~186k)"
+    )
+    assert est.median_price_per_m2 < 120_000, "must NOT be EKB-leaked (~186k)"
+    assert est.n_analogs == 0, "honest: zero scraped-listing analogs were used"
+    assert est.confidence == "low", "deals-only headline must be honestly low-confidence"
+    assert est.median_price_rub == round(85_911 * 45.0)
+    # Range should bracket the corridor's P10/P90, not collapse to a point.
+    assert est.range_low_rub <= est.median_price_rub <= est.range_high_rub
+
+
+def test_non_ekb_empty_listings_no_deals_stays_insufficient() -> None:
+    """0 listings + NO deal corridor either → stays honest n/a (median=0).
+
+    Guards against the fallback inventing a number when there is truly no
+    signal at all (e.g. Каменск-Уральский with an unresolvable street).
+    """
+    est = _run_estimate(
+        analogs=[],
+        dkp_raw=None,
+        geo=_make_geo_tagil(),
+        payload=_make_payload_tagil(),
+    )
+    assert est.median_price_per_m2 == 0
+    assert est.n_analogs == 0
+    assert est.confidence == "low"
+
+
+def test_non_ekb_thin_deal_corridor_below_min_n_stays_insufficient() -> None:
+    """Deal corridor exists but below DEALS_HEADLINE_FALLBACK_MIN_N → no fallback.
+
+    A single stale sold price should not become the town's headline.
+    """
+    dkp_raw = {
+        "count": 1,
+        "low_ppm2": 70_000,
+        "median_ppm2": 85_911,
+        "high_ppm2": 100_000,
+        "period_months": 12,
+    }
+    est = _run_estimate(
+        analogs=[],
+        dkp_raw=dkp_raw,
+        geo=_make_geo_tagil(),
+        payload=_make_payload_tagil(),
+    )
+    assert est.median_price_per_m2 == 0
+    assert est.n_analogs == 0
+
+
+# ── EKB control: dense local listings → deals-fallback must NOT engage ───────
+
+
+def test_ekb_with_dense_listings_ignores_deals_fallback() -> None:
+    """EKB has plenty of local listings — the radius-path headline must win,
+    NOT the deal corridor, even though dkp_raw is present (byte-green guard:
+    EKB must stay on the existing listings-median path unconditionally).
+    """
+    from app.schemas.trade_in import TradeInEstimateInput
+    from app.services.geocoder import GeocodeResult
+
+    analogs = [
+        _make_listing(price_per_m2=140_000.0),
+        _make_listing(price_per_m2=145_000.0),
+        _make_listing(price_per_m2=150_000.0),
+    ]
+    dkp_raw = {
+        "count": 20,
+        "low_ppm2": 120_000,
+        "median_ppm2": 144_000,
+        "high_ppm2": 160_000,
+        "period_months": 12,
+    }
+    geo = GeocodeResult(
+        lat=56.838,
+        lon=60.595,
+        full_address="Свердловская обл., Екатеринбург, ул. Малышева, 30",
+        provider="nominatim",
+    )
+    payload = TradeInEstimateInput(
+        address="ЕКБ, ул. Малышева, 30", area_m2=45.0, rooms=2, floor=5, total_floors=16
+    )
+    est = _run_estimate(analogs=analogs, dkp_raw=dkp_raw, geo=geo, payload=payload)
+
+    # Headline built from the LISTINGS median (~145k), not silently replaced —
+    # n_analogs must reflect the real listing count (deals-fallback never ran).
+    assert est.n_analogs == len(analogs)
+    assert 140_000 <= est.median_price_per_m2 <= 150_000