From f9ae6f0c539e69b5142e26191c93d5a6502bd16f Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sun, 12 Jul 2026 21:46:38 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein/estimator):=20non-EKB=20headline=20?= =?UTF-8?q?=D0=B8=D0=B7=20deals=20+=20geo-bound=20Tier-S=20analog=20leak?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Валидация вскрыла: не-ЕКБ headline over-priced — analog Tier-S (same-building) fallback БЕЗ geo-предиката матчил одноимённую улицу+дом в ДРУГОМ городе → тянул ЕКБ-листинги (40191 EKB/~0 non-EKB) с distance_m=0. Серов ask-head 186k vs sold 30k; Н.Тагил 187k vs deal-corridor 89k. Part 1 (Tier-S geo-bound, ~4624): + ST_DWithin(geom, subject, radius) в address-prefix fallback (зеркалит Tier-W). >99.97% листингов с coords → режет только cross-city false-positives. Part 2 (deals-headline-fallback): _fetch_dkp_corridor widen на city-wide при street-sample <3 (gated city!=екатеринбург); _price_from_inputs: при median_ppm2<=0 (нет листингов) + anchor_tier None + dkp_raw>=3 → headline из deal-corridor, confidence=low, n_analogs=0, expected_sold=None. Deep-review ✅ APPROVE (no 🔴/🟠; ~10 downstream stages traced coherent). EKB byte-green regression-гейт (fallback требует median_ppm2<=0, невозможно для frozen fixture; Part 1 SQL не в replay). Тесты: Н.Тагил 0-листингов+corridor85911 → headline=85911 (не 0/186k); EKB-control dense → headline из листингов (fallback не срабатывает). Full suite 2418 passed. Follow-ups (🟡): fallback без city-gate (sparse EKB n/a→deals, safe); api_analog_tier мислейбл 'city'; Part1/2a integration-test; EKB backtest post-deploy (rule #7). --- tradein-mvp/backend/app/services/estimator.py | 149 +++++++++++ ...imator_deals_headline_fallback_oblast_d.py | 238 ++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 tradein-mvp/backend/tests/test_estimator_deals_headline_fallback_oblast_d.py diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index afe8adc4..838de0c8 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -92,6 +92,15 @@ MIN_ANALOGS_PER_SOURCE = 5 # гарантированный минимум на LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней DEALS_PERIOD_MONTHS = 12 # сделки за последний год +# #oblast-D (non-EKB deals-headline-fallback): минимум ДКП-сделок, чтобы +# _fetch_dkp_corridor доверял СВОЕЙ street-scoped выборке — иначе (тонкая +# конкретная улица небольшого города) виджет расширяется до city-wide (см. +# _fetch_dkp_corridor). ОТДЕЛЬНО — минимум, чтобы _price_from_inputs счёл +# коридор пригодным как HEADLINE (не просто advisory/clamp/floor) при +# отсутствии листинговых аналогов — см. deals-headline-fallback ниже. +DKP_CORRIDOR_CITY_WIDE_MIN_N = 3 +DEALS_HEADLINE_FALLBACK_MIN_N = 3 + # #794: СберИндекс time-adjustment of frozen Rosreestr ДКП deals. # Rosreestr deals freeze ~2026-01; the sber monthly index re-bases a stale deal's ppm² # to the latest available month. Region fixed to Свердловская обл. (tradein MVP = ЕКБ). @@ -1423,6 +1432,78 @@ def _fetch_dkp_corridor( adjusted.append(float(ppm2) * factor) factors_applied.append(factor) ppm2_values = sorted(adjusted) + + # #oblast-D widen: a single street in a small non-EKB town can easily have + # <3 (or 0) ДКП deals in the last 12 months even though the CITY overall + # has plenty (migration 177 loaded 47k+ oblast-wide deals across ~368 + # cities) — the street-only corridor was silently unusable for exactly the + # towns that most need it (Нижний Тагил/Серов/Каменск-Уральский/...). + # Widen to a CITY-scoped (no street filter) sample when the street sample + # is too thin. EKB is explicitly EXCLUDED (city == "екатеринбург" is + # always dense at street level) — this widen path NEVER engages for EKB, + # so live EKB behaviour is unaffected. The frozen regression gate never + # calls this function at all (it replays a captured `dkp_raw` dict), so + # this change carries zero risk to the gate either way. + if len(ppm2_values) < DKP_CORRIDOR_CITY_WIDE_MIN_N and city and city != "екатеринбург": + try: + city_rows = ( + db.execute( + text( + """ + SELECT d.price_per_m2, d.deal_date + FROM deals d + LEFT JOIN deal_city_price_bands b ON b.city = d.city + WHERE d.source = 'rosreestr' + AND d.city IS NOT NULL + AND LOWER(d.city) = CAST(:target_city AS text) + AND d.rooms = CAST(:rooms AS integer) + AND d.area_m2 BETWEEN :area_min AND :area_max + AND d.deal_date > NOW() + - (CAST(:period_months AS integer) || ' months')::interval + AND d.price_per_m2 > 0 + AND d.price_per_m2 BETWEEN COALESCE(b.ppm2_min, CAST(:ppm_min AS int)) + AND COALESCE(b.ppm2_max, CAST(:ppm_max AS int)) + """ + ), + { + "target_city": city.lower(), + "rooms": rooms, + "area_min": area_min, + "area_max": area_max, + "period_months": period_months, + "ppm_min": DEAL_MIN_PPM2, + "ppm_max": DEAL_MAX_PPM2, + }, + ) + .mappings() + .all() + ) + except Exception as exc: # pragma: no cover — defensive + logger.warning("dkp_corridor city-wide widen failed (graceful): %s", exc) + city_rows = [] + + city_adjusted: list[float] = [] + for r in city_rows: + ppm2 = r["price_per_m2"] + if not ppm2: + continue + dd = r.get("deal_date") + factor = 1.0 + if series and dd is not None: + deal_month = date(dd.year, dd.month, 1) + factor = _sber_time_factor(series, deal_month) + city_adjusted.append(float(ppm2) * factor) + + if len(city_adjusted) > len(ppm2_values): + logger.info( + "dkp_corridor widen #oblast-D: street n=%d < %d → city-wide n=%d (city=%s)", + len(ppm2_values), + DKP_CORRIDOR_CITY_WIDE_MIN_N, + len(city_adjusted), + city, + ) + ppm2_values = sorted(city_adjusted) + if not ppm2_values: return None if series and factors_applied: @@ -2823,6 +2904,57 @@ def _price_from_inputs( if abs(honest_ratio - asking_to_sold_ratio) > _RATIO_DESCRIPTOR_EPS: asking_to_sold_ratio = honest_ratio + # ── #oblast-D: deals-headline-fallback ─────────────────────────────────── + # When the radius/anchor analog pipeline found NOTHING usable (median_ppm2 + # is still 0 here — happens for non-EKB towns where scraped `listings` + # coverage is ~0; see the Part-1 geo-bound fix in the Tier S fallback + # query above) but we DO have a ДКП deal corridor (dkp_raw — city+street + # scoped via _resolve_target_city/_fetch_dkp_corridor, widened to + # city-wide when the street sample is too thin, see _fetch_dkp_corridor), + # build the headline FROM THE DEAL SIGNAL instead of surfacing an empty + # ("n/a") estimate — or, pre-Part-1, a distant EKB-leaked listing median. + # + # Placed AFTER the expected_sold block above (guarded by the SAME + # `median_ppm2 <= 0` state it ran in) so ratio_resolver/hedonic/le_asking + # never touch this value: dkp_raw's ppm² are ALREADY sold prices — running + # them through the asking→sold ratio would double-discount. expected_sold_* + # stays None (an existing, already-supported state — see the Prediction + # docstring in scripts/backtest_estimator.py: "may be None when the spine + # produced a headline but no asking→sold ratio resolved"). + # + # n_analogs stays 0 (true — zero scraped-listing analogs), so + # _enforce_zero_analog_low (estimate_quality, downstream) forces + # confidence 'low' regardless — we set it explicitly here too so the + # explanation text stays coherent with the actual reason (deals-only, not + # generic ghost-anchor). No repair_state adjustment: the deal corridor + # mixes conditions across sold units — unlike the listings comp pool, + # there is no per-unit signal to correct against. + if ( + median_ppm2 <= 0 + and anchor_tier is None + and dkp_raw is not None + and dkp_raw.get("count", 0) >= DEALS_HEADLINE_FALLBACK_MIN_N + and dkp_raw.get("median_ppm2", 0) > 0 + ): + median_ppm2 = float(dkp_raw["median_ppm2"]) + median_price = int(median_ppm2 * area_m2) + range_low = int(dkp_raw["low_ppm2"] * area_m2) + range_high = int(dkp_raw["high_ppm2"] * area_m2) + n_analogs = 0 + confidence = "low" + cv = None + explanation = (explanation or "") + ( + " Рядом нет актуальных объявлений — оценка построена по реальным " + f"сделкам Росреестра ({dkp_raw['count']} шт. за {dkp_raw['period_months']} мес.)," + " точность ориентировочная." + ) + logger.info( + "deals_headline_fallback #oblast-D: dkp median=%d (n=%d) → headline" + " (listings=0, anchor=None)", + int(median_ppm2), + dkp_raw["count"], + ) + # ── #652: ДКП-коридор реальных сделок (advisory) ───────────────────────── dkp_corridor: DkpCorridor | None = None if dkp_raw is not None: @@ -4490,12 +4622,28 @@ def _fetch_analogs( return _stratify_candidates(tier_sc), radius_m > DEFAULT_RADIUS_M, "S" # ── Tier S (fallback): same building via address prefix ─────────────────── + # #oblast-D geo-bound fix: _extract_short_addr strips the city/admin prefix + # ("Нижний Тагил, улица Ленина, 100" → "улица Ленина, 100"), so an + # address-only ILIKE prefix match can collide with an IDENTICALLY-NAMED + # street+house-number in a COMPLETELY DIFFERENT city (e.g. "улица Ленина, + # 100" exists verbatim in Берёзовский — ~40 191 of ~40 200 active listings + # are EKB-region, so any such collision silently resolves to a distant EKB + # listing). A Нижний Тагил / Серов / Каменск subject then got "same + # building" comps from a listing ~100+ km away with distance_m hardcoded to + # 0.0 (masking the gap). Tier S's OWN semantics ("same building") already + # imply the match must be near the subject, so we now require ST_DWithin on + # the SAME radius_m the caller passed in (mirrors Tier H/W below) — this + # can only DROP false-positive cross-city matches, never legitimate + # same-building EKB hits (>99.9% of active listings carry lat/lon). short_addr = _extract_short_addr(full_address) if short_addr: tier_s_params = { **base_params, "short_addr_prefix": short_addr + "%", + "lat": lat, + "lon": lon, + "radius": radius_m, } tier_s_rows = ( @@ -4513,6 +4661,7 @@ def _fetch_analogs( {_RN_DUP_WINDOW} FROM listings WHERE address ILIKE :short_addr_prefix + AND ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius) {_COMMON_WHERE} ) SELECT diff --git a/tradein-mvp/backend/tests/test_estimator_deals_headline_fallback_oblast_d.py b/tradein-mvp/backend/tests/test_estimator_deals_headline_fallback_oblast_d.py new file mode 100644 index 00000000..93b7f502 --- /dev/null +++ b/tradein-mvp/backend/tests/test_estimator_deals_headline_fallback_oblast_d.py @@ -0,0 +1,238 @@ +"""Tests for the oblast-D deals-headline-fallback (non-EKB accuracy gap). + +Context: `listings` coverage is ~40k EKB / ~0 non-EKB (Нижний Тагил / Серов / +Каменск-Уральский). Before this fix, a non-EKB subject either: + (a) leaked a DISTANT EKB listing median via the Tier S address-prefix + fallback (no geo bound — see `_fetch_analogs` Tier S fix), or + (b) surfaced an empty ("n/a") headline once (a) was fixed and the + geo-bound radius tiers legitimately found 0 local listings. + +This fallback builds the headline from the ДКП deal corridor (`dkp_raw`, +already city+street-scoped via `_resolve_target_city`/`_fetch_dkp_corridor`) +instead — honestly flagged 'low' confidence, deals-only. + +These tests exercise `estimate_quality` end-to-end (real spine code, mocked +DB-facing helpers only) — the same pattern as test_estimator_radius_floor.py. +""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import anyio + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + + +# ── helpers ────────────────────────────────────────────────────────────────── + + +def _make_listing(*, price_per_m2: float, area_m2: float = 45.0) -> dict[str, Any]: + """An EKB analog — used only in the "EKB unaffected" control test.""" + return { + "source": "cian", + "source_url": "https://cian.ru/offer/1", + "address": "ЕКБ, ул. Малышева, 30", + "lat": 56.838, + "lon": 60.595, + "rooms": 2, + "area_m2": area_m2, + "floor": 5, + "total_floors": 16, + "price_rub": price_per_m2 * area_m2, + "price_per_m2": price_per_m2, + "listing_date": datetime(2026, 5, 1), + "days_on_market": 10, + "photo_urls": [], + "scraped_at": datetime(2026, 5, 20, tzinfo=UTC), + "distance_m": 150.0, + "relevance_score": 0.1, + } + + +def _make_geo_tagil(): + from app.services.geocoder import GeocodeResult + + return GeocodeResult( + lat=57.9094, + lon=59.9789, + full_address="Свердловская обл., Нижний Тагил, ул. Ленина, 5", + provider="nominatim", + ) + + +def _make_payload_tagil(): + from app.schemas.trade_in import TradeInEstimateInput + + return TradeInEstimateInput( + address="Нижний Тагил, ул. Ленина, 5", + area_m2=45.0, + rooms=2, + floor=5, + total_floors=9, + ) + + +def _run_estimate( + *, + analogs: list[dict[str, Any]], + dkp_raw: dict[str, Any] | None, + geo: Any, + payload: Any, +) -> Any: + from app.services.estimator import estimate_quality + + db = MagicMock() + + async def _run() -> Any: + with ( + patch("app.services.estimator.geocode", new=AsyncMock(return_value=geo)), + patch("app.services.estimator.dadata_clean_address", new=AsyncMock(return_value=None)), + patch("app.services.estimator.match_house_readonly", return_value=None), + patch("app.services.estimator.get_house_metadata", new=AsyncMock(return_value=None)), + patch( + "app.services.estimator._fetch_analogs", + # Post Part-1 (geo-bound Tier S) reality for a non-EKB town: the + # radius/tier ladder legitimately returns NOTHING — tier='W' is + # what the always-executed final fallback tier returns. + return_value=(list(analogs), False, "W"), + ), + patch("app.services.estimator._fetch_deals", return_value=[]), + patch( + "app.services.estimator._get_or_fetch_imv_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator._get_or_fetch_yandex_valuation_cached", + new=AsyncMock(return_value=None), + ), + patch( + "app.services.estimator.estimate_via_cian_valuation", + new=AsyncMock(return_value=None), + ), + patch("app.services.estimator._fetch_dkp_corridor", return_value=dkp_raw), + patch("app.services.estimator._get_asking_sold_ratio", return_value=(None, None)), + ): + return await estimate_quality(payload, db) + + return anyio.run(_run) + + +# ── Нижний Тагил: no local listings, deal corridor present ─────────────────── + + +def test_non_ekb_empty_listings_uses_deals_headline() -> None: + """0 local listings + a usable ДКП corridor → headline comes from deals. + + Mirrors the reported Нижний Тагил gap: deal_median ≈ 85 911 ₽/м² (accurate) + vs the old EKB-leaked asking headline ≈ 186 461 (~6x over). After the fix, + the headline must equal the deal corridor's median — nowhere near the + EKB-range figure — and confidence must be honestly 'low' (deals-only, zero + scraped analogs). + """ + dkp_raw = { + "count": 12, + "low_ppm2": 70_000, + "median_ppm2": 85_911, + "high_ppm2": 100_000, + "period_months": 12, + } + est = _run_estimate( + analogs=[], # 0 listings — the honest post-geo-bound-fix reality + dkp_raw=dkp_raw, + geo=_make_geo_tagil(), + payload=_make_payload_tagil(), + ) + + assert est.median_price_per_m2 == 85_911, ( + f"headline={est.median_price_per_m2} must equal the deal corridor " + f"median, not 0/n-a and nowhere near an EKB-range figure (~186k)" + ) + assert est.median_price_per_m2 < 120_000, "must NOT be EKB-leaked (~186k)" + assert est.n_analogs == 0, "honest: zero scraped-listing analogs were used" + assert est.confidence == "low", "deals-only headline must be honestly low-confidence" + assert est.median_price_rub == round(85_911 * 45.0) + # Range should bracket the corridor's P10/P90, not collapse to a point. + assert est.range_low_rub <= est.median_price_rub <= est.range_high_rub + + +def test_non_ekb_empty_listings_no_deals_stays_insufficient() -> None: + """0 listings + NO deal corridor either → stays honest n/a (median=0). + + Guards against the fallback inventing a number when there is truly no + signal at all (e.g. Каменск-Уральский with an unresolvable street). + """ + est = _run_estimate( + analogs=[], + dkp_raw=None, + geo=_make_geo_tagil(), + payload=_make_payload_tagil(), + ) + assert est.median_price_per_m2 == 0 + assert est.n_analogs == 0 + assert est.confidence == "low" + + +def test_non_ekb_thin_deal_corridor_below_min_n_stays_insufficient() -> None: + """Deal corridor exists but below DEALS_HEADLINE_FALLBACK_MIN_N → no fallback. + + A single stale sold price should not become the town's headline. + """ + dkp_raw = { + "count": 1, + "low_ppm2": 70_000, + "median_ppm2": 85_911, + "high_ppm2": 100_000, + "period_months": 12, + } + est = _run_estimate( + analogs=[], + dkp_raw=dkp_raw, + geo=_make_geo_tagil(), + payload=_make_payload_tagil(), + ) + assert est.median_price_per_m2 == 0 + assert est.n_analogs == 0 + + +# ── EKB control: dense local listings → deals-fallback must NOT engage ─────── + + +def test_ekb_with_dense_listings_ignores_deals_fallback() -> None: + """EKB has plenty of local listings — the radius-path headline must win, + NOT the deal corridor, even though dkp_raw is present (byte-green guard: + EKB must stay on the existing listings-median path unconditionally). + """ + from app.schemas.trade_in import TradeInEstimateInput + from app.services.geocoder import GeocodeResult + + analogs = [ + _make_listing(price_per_m2=140_000.0), + _make_listing(price_per_m2=145_000.0), + _make_listing(price_per_m2=150_000.0), + ] + dkp_raw = { + "count": 20, + "low_ppm2": 120_000, + "median_ppm2": 144_000, + "high_ppm2": 160_000, + "period_months": 12, + } + geo = GeocodeResult( + lat=56.838, + lon=60.595, + full_address="Свердловская обл., Екатеринбург, ул. Малышева, 30", + provider="nominatim", + ) + payload = TradeInEstimateInput( + address="ЕКБ, ул. Малышева, 30", area_m2=45.0, rooms=2, floor=5, total_floors=16 + ) + est = _run_estimate(analogs=analogs, dkp_raw=dkp_raw, geo=geo, payload=payload) + + # Headline built from the LISTINGS median (~145k), not silently replaced — + # n_analogs must reflect the real listing count (deals-fallback never ran). + assert est.n_analogs == len(analogs) + assert 140_000 <= est.median_price_per_m2 <= 150_000