From ac9988620153ed2cd17aab2744d7cea9f519b240 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 27 Jun 2026 20:31:49 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein/estimator):=20remove=20dead=20tier-?= =?UTF-8?q?aware-ratio=20path=20=E2=80=94=20truncation=20artifact=20+=20fo?= =?UTF-8?q?otgun=20(#2002)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tier_aware_ratio_enabled path (#928) binned SOLD deals by sold-ppm² against ASKING-derived percentile bounds, then divided within-tier medians. That is a ratio-of-truncated-medians ARTIFACT: a Monte-Carlo with a CONSTANT true sold/asking=0.84 reproduced the prod tier values (0.94/0.99/0.96) and the skewed deal split (57/27/15%) exactly — the "premium sells closer to asking" gradient is 100% spurious. Flipping the flag ON made prod WORSE (overall MAPE 14.6→17.2, эконом bias +5.8→+18.1). It shipped dark (default False) but is a latent footgun. A valid within-price-tier sold/asking is uncomputable from asking-less ДКП deals; the per-rooms blend + the shipped hedonic (year+area) are the correct conditioning. So the dead path is removed entirely, not just disabled. - estimator._get_asking_sold_ratio: drop the tier branch (bounds read, t33/t66, asking_to_sold_ratios_tiered read, empirical-Bayes shrink) + the _legacy/tier cache-key split; keep only the legacy per-rooms → global -1 lookup. Cache key collapses to rooms bucket. anchor_ppm2 param retained for call-site compat (now unused). - config: remove tier_aware_ratio_enabled + tier_ratio_shrink_k settings. - tasks/asking_to_sold_ratio: drop the tiered refresh (DELETE/re-derive bounds + tier rows, tiered counters); the daily task no longer touches the dead tier tables. Legacy asking_to_sold_ratios DELETE+re-derive intact. - tests: delete the tier-ratio unit test file (legacy path covered by test_estimator_expected_sold.py Layer 1); fix the daily-refresh fake-db (8→3 execute calls, 3 counter keys); drop the two segment-guard tests that asserted the removed tiered SQL; clean a dead per_rooms_tier basis in fixtures. Tier tables asking_to_sold_ratios_tiered / asking_to_sold_tier_bounds + migration 098 are left in place (inert once nothing reads/writes them); a DROP migration is a separate, optional follow-up. Regression gate stays byte-identical (flag was False in prod → active path unchanged). Refs #2002 --- tradein-mvp/backend/app/core/config.py | 12 - tradein-mvp/backend/app/services/estimator.py | 220 +++---------- .../backend/app/tasks/asking_to_sold_ratio.py | 255 +-------------- .../services/test_asking_sold_tier_ratio.py | 291 ------------------ .../tests/test_asking_to_sold_ratio.py | 28 +- .../tests/test_backtest_fixture_roundtrip.py | 2 +- .../tests/test_estimator_ratio_tier_fix.py | 2 +- .../backend/tests/test_segment_guard_1186.py | 22 +- 8 files changed, 53 insertions(+), 779 deletions(-) delete mode 100644 tradein-mvp/backend/tests/services/test_asking_sold_tier_ratio.py diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index df2ac627..9c28ff12 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -323,18 +323,6 @@ class Settings(BaseSettings): # ENV: ASKING_RATIO_PPM2_MAX. asking_ratio_ppm2_max: int = 1_200_000 - # ppm2-tier asking->sold lookup (#928) -- gate + empirical-Bayes shrink parameter. - # tier_aware_ratio_enabled=False (default) -> byte-identical flag-OFF path via - # asking_to_sold_ratios (old table). True -> tiered lookup from - # asking_to_sold_ratios_tiered (requires migration 098 applied). - # ENV: TIER_AWARE_RATIO_ENABLED. - tier_aware_ratio_enabled: bool = False - # Empirical-Bayes shrink weight denominator: w = n_deals / (n_deals + K). - # K=150 -> cell with 150 deals gets w=0.5 (50% own tier ratio, 50% rooms-'all'). - # Higher K -> more shrinkage toward 'all' ratio (conservative); lower -> more tier. - # ENV: TIER_RATIO_SHRINK_K. - tier_ratio_shrink_k: int = 150 - # SSRF-защита для admin scrape endpoints (#756). # Список хостов которым разрешено передавать абсолютные URL в параметрах *_url. # Относительные пути (без netloc) проходят без проверки — хост подставляется diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index e4047a25..17802b7e 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -202,9 +202,18 @@ def _repair_coefficient(repair_state: str | None) -> float: # Ratio дрейфует медленно (refresh-задача раз в сутки, Stage 4), поэтому 300с # TTL более чем достаточно и снимает по SELECT'у с каждой оценки. Single-worker # uvicorn/scheduler — GIL делает dict-доступ atomic enough (без явного lock). +# +# Tier-aware ppm²-путь (#928) УДАЛЁН (#2002): per-price-tier sold/asking — это +# артефакт ratio-of-truncated-medians. Бинирование SOLD-сделок по ASKING-перцентилям +# с делением within-tier медиан рождает мнимый «премиум продаётся ближе к asking» +# градиент (Monte-Carlo с КОНСТАНТНЫМ true sold/asking=0.84 точно воспроизводил +# прод-тиры 0.94/0.99/0.96 и перекошенный сплит сделок 57/27/15%; включение флага +# ухудшало прод — overall MAPE 14.6→17.2, эконом-bias +5.8→+18.1). Валидный +# within-price-tier sold/asking НЕВЫЧИСЛИМ из asking-less ДКП-сделок; корректное +# обусловливание дают per-rooms blend + захардженный hedonic (год+площадь). _ASKING_SOLD_RATIO_CACHE_TTL_S = 300.0 -# Cache key: (bucket, tier_or_None) — tier=None for flag-OFF / no-anchor path (#928). -_asking_sold_ratio_cache: dict[tuple[int, str | None], tuple[float | None, str | None, float]] = {} +# Cache key: rooms bucket (единственный legacy per-rooms путь после #2002). +_asking_sold_ratio_cache: dict[int, tuple[float | None, str | None, float]] = {} def _get_asking_sold_ratio( @@ -216,210 +225,65 @@ def _get_asking_sold_ratio( bucket = min(max(rooms or 0, 0), 4). - Flag-OFF (settings.tier_aware_ratio_enabled=False) или anchor_ppm2=None: - Запрос к asking_to_sold_ratios (старая таблица) — байт-в-байт как до #928. - Fallback: per-rooms строка → global -1. + Запрос к asking_to_sold_ratios (migration 080): per-rooms строка + (WHERE rooms_bucket = bucket AND district = '') → fallback на global -1 + (basis='global_fallback'). Ничего нет → (None, None). - Flag-ON и anchor_ppm2 задан (#928 ppm2-tier path): - 1. Читаем t33/t66 из asking_to_sold_tier_bounds для (bucket, ''). - Нет bounds-строки → fallback к 'all' строке (шаг 3). - 2. tier = 'low'/'mid'/'high' по anchor_ppm2 vs t33/t66. - Читаем asking_to_sold_ratios_tiered (bucket,'',tier) → tier_ratio, n_deals. - Shrink: w = n_deals/(n_deals + settings.tier_ratio_shrink_k). - Также читаем (bucket,'','all') как shrink-target. - shrunk = tier_ratio * w + all_ratio * (1 - w). - basis = f'per_rooms_tier:{tier}'. - 3. Fallback (bucket,'','all') → basis='per_rooms_all'. - 4. Fallback (-1,'','all') → basis='global_all'. - 5. Ничего → (None, None). + anchor_ppm2 сохранён в сигнатуре для совместимости с call-site (резолвер + вызывается с ФИНАЛЬНЫМ headline ppm² после anchor/blend-мутаций — см. + _ratio_resolver), но БОЛЬШЕ НЕ используется: tier-aware путь удалён как + статистический артефакт (#2002, см. module-level комментарий выше). Таблицы нет / любая ошибка → (None, None), НЕ raise (graceful). - Кэшируется на ключ (bucket, tier|None) с TTL _ASKING_SOLD_RATIO_CACHE_TTL_S. + Кэшируется на ключ bucket с TTL _ASKING_SOLD_RATIO_CACHE_TTL_S. """ bucket = min(max(rooms or 0, 0), 4) - # Determine tier key for cache: - # flag-OFF or no anchor_ppm2 → use None (legacy path, reads old table). - tier_key: str | None = None - use_tier_path = settings.tier_aware_ratio_enabled and anchor_ppm2 is not None + cached = _asking_sold_ratio_cache.get(bucket) + if cached is not None: + ratio, basis, fetched = cached + if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: + return ratio, basis - cache_key: tuple[int, str | None] = (bucket, tier_key) # may be updated below - - if not use_tier_path: - # ── Flag-OFF path: byte-identical behavior (legacy asking_to_sold_ratios) ── - # Use a distinct key "_legacy" to avoid collision with the flag-ON Step-3 'all' - # fallback which also caches under (bucket, None). Without this, a process that - # flips tier_aware_ratio_enabled ON without restart could serve the old-table ratio - # from the flag-OFF entry for up to the TTL — wrong table. - cache_key = (bucket, "_legacy") - cached = _asking_sold_ratio_cache.get(cache_key) - if cached is not None: - ratio, basis, fetched = cached - if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: - return ratio, basis - - ratio: float | None = None - basis: str | None = None - try: - row = db.execute( - text( - """ - SELECT ratio, basis FROM asking_to_sold_ratios - WHERE rooms_bucket = CAST(:b AS int) AND district = '' - """ - ), - {"b": bucket}, - ).fetchone() - if row is None: - # Бакет тонкий (n<30 при seed'е) или отсутствует → global fallback (-1). - row = db.execute( - text( - """ - SELECT ratio, basis FROM asking_to_sold_ratios - WHERE rooms_bucket = -1 AND district = '' - """ - ), - ).fetchone() - if row is not None and row.ratio is not None: - ratio = float(row.ratio) - basis = row.basis - except Exception as exc: - # Таблицы может не быть на свежей/старой БД (миграция 080 не применена), - # либо транзакция в сбойном состоянии — graceful: без sold-коррекции. - # ОБЯЗАТЕЛЬНО rollback: неудачный SELECT помечает транзакцию - # InFailedSqlTransaction, и без отката следующий statement упал бы → 500. - logger.debug("asking_to_sold_ratio lookup skipped (graceful): %s", exc) - try: - db.rollback() - except Exception: - pass - ratio, basis = None, None - - _asking_sold_ratio_cache[cache_key] = (ratio, basis, time.monotonic()) - return ratio, basis - - # ── Flag-ON path: ppm2-tier lookup (#928) ─────────────────────────────────── ratio: float | None = None basis: str | None = None try: - # Step 1: read bounds for this bucket. - bounds_row = db.execute( + row = db.execute( text( """ - SELECT t33, t66 FROM asking_to_sold_tier_bounds + SELECT ratio, basis FROM asking_to_sold_ratios WHERE rooms_bucket = CAST(:b AS int) AND district = '' """ ), {"b": bucket}, ).fetchone() - - if bounds_row is not None: - t33 = float(bounds_row.t33) - t66 = float(bounds_row.t66) - tier_key = "low" if anchor_ppm2 < t33 else ("mid" if anchor_ppm2 < t66 else "high") - cache_key = (bucket, tier_key) - - # Check cache for this (bucket, tier) key. - cached = _asking_sold_ratio_cache.get(cache_key) - if cached is not None: - c_ratio, c_basis, fetched = cached - if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: - return c_ratio, c_basis - - # Step 2: look up tier row + 'all' row for shrink. - tier_row = db.execute( + if row is None: + # Бакет тонкий (n<30 при seed'е) или отсутствует → global fallback (-1). + row = db.execute( text( """ - SELECT ratio, n_deals FROM asking_to_sold_ratios_tiered - WHERE rooms_bucket = CAST(:b AS int) - AND district = '' - AND ppm2_tier = CAST(:tier AS text) + SELECT ratio, basis FROM asking_to_sold_ratios + WHERE rooms_bucket = -1 AND district = '' """ ), - {"b": bucket, "tier": tier_key}, ).fetchone() - all_row = db.execute( - text( - """ - SELECT ratio FROM asking_to_sold_ratios_tiered - WHERE rooms_bucket = CAST(:b AS int) - AND district = '' - AND ppm2_tier = 'all' - """ - ), - {"b": bucket}, - ).fetchone() - - if tier_row is not None and all_row is not None: - tier_ratio = float(tier_row.ratio) - n_deals = int(tier_row.n_deals) - all_ratio = float(all_row.ratio) - k = settings.tier_ratio_shrink_k - w = n_deals / (n_deals + k) - ratio = tier_ratio * w + all_ratio * (1.0 - w) - basis = f"per_rooms_tier:{tier_key}" - _asking_sold_ratio_cache[cache_key] = (ratio, basis, time.monotonic()) - return ratio, basis - - # Step 3: fallback to 'all' for this bucket. - cache_key = (bucket, None) - cached = _asking_sold_ratio_cache.get(cache_key) - if cached is not None: - c_ratio, c_basis, fetched = cached - if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: - return c_ratio, c_basis - - all_row = db.execute( - text( - """ - SELECT ratio FROM asking_to_sold_ratios_tiered - WHERE rooms_bucket = CAST(:b AS int) - AND district = '' - AND ppm2_tier = 'all' - """ - ), - {"b": bucket}, - ).fetchone() - if all_row is not None: - ratio = float(all_row.ratio) - basis = "per_rooms_all" - _asking_sold_ratio_cache[cache_key] = (ratio, basis, time.monotonic()) - return ratio, basis - - # Step 4: global fallback (-1, 'all'). - global_key: tuple[int, str | None] = (-1, None) - cached = _asking_sold_ratio_cache.get(global_key) - if cached is not None: - c_ratio, c_basis, fetched = cached - if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: - return c_ratio, c_basis - - global_row = db.execute( - text( - """ - SELECT ratio FROM asking_to_sold_ratios_tiered - WHERE rooms_bucket = -1 - AND district = '' - AND ppm2_tier = 'all' - """ - ), - ).fetchone() - if global_row is not None: - ratio = float(global_row.ratio) - basis = "global_all" - _asking_sold_ratio_cache[global_key] = (ratio, basis, time.monotonic()) - return ratio, basis - + if row is not None and row.ratio is not None: + ratio = float(row.ratio) + basis = row.basis except Exception as exc: - # Таблицы нет / ошибка → graceful (migration 098 ещё не применена). - logger.debug("asking_to_sold_ratio tiered lookup skipped (graceful): %s", exc) + # Таблицы может не быть на свежей/старой БД (миграция 080 не применена), + # либо транзакция в сбойном состоянии — graceful: без sold-коррекции. + # ОБЯЗАТЕЛЬНО rollback: неудачный SELECT помечает транзакцию + # InFailedSqlTransaction, и без отката следующий statement упал бы → 500. + logger.debug("asking_to_sold_ratio lookup skipped (graceful): %s", exc) try: db.rollback() except Exception: pass + ratio, basis = None, None - # Step 5: nothing found. - return None, None + _asking_sold_ratio_cache[bucket] = (ratio, basis, time.monotonic()) + return ratio, basis # ── Avito IMV cache lookup (Stage 3) ──────────────────────────────────────── diff --git a/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py b/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py index 661cf8c9..b21ad247 100644 --- a/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py +++ b/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py @@ -189,237 +189,6 @@ _COUNTERS_SQL = text( ) -# ── Tiered refresh (#928): DELETE + re-derive for tier tables ──────────────── -# атомарно DELETE WHERE district='' ПЕРЕД re-derive INSERT (те же семантика, что -# для asking_to_sold_ratios выше: stale строки не оставляем). -_DELETE_TIERED_ROWS_SQL = text( - """ - DELETE FROM asking_to_sold_ratios_tiered WHERE district = '' - """ -) -_DELETE_TIER_BOUNDS_SQL = text( - """ - DELETE FROM asking_to_sold_tier_bounds WHERE district = '' - """ -) - -# ── Tiered derivation (BYTE-CONSISTENT с migration 098) ────────────────────── -# Те же CTE что в 098: bounds -> deal_tiered/ask_tiered -> tier_rows + all_rows + global_row. -# Bind params: :ppm2_min / :ppm2_max (как в _REDERIVE_SQL выше). -# CAST не нужен -- psycopg v3 передаёт int напрямую. -_REDERIVE_BOUNDS_SQL = text( - """ - WITH bounds AS ( - SELECT - LEAST(GREATEST(rooms, 0), 4) AS rooms_bucket, - percentile_cont(0.3333) WITHIN GROUP (ORDER BY price_per_m2)::bigint AS t33, - percentile_cont(0.6667) WITHIN GROUP (ORDER BY price_per_m2)::bigint AS t66, - COUNT(*)::int AS n_listings - FROM listings - WHERE is_active - AND rooms IS NOT NULL - AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max - -- novostroyki guard (#1186): NULL = legacy вторичка до м.011 - AND (listing_segment IS NULL OR listing_segment = 'vtorichka') - GROUP BY LEAST(GREATEST(rooms, 0), 4) - HAVING COUNT(*) > 0 - ) - INSERT INTO asking_to_sold_tier_bounds (rooms_bucket, district, t33, t66, n_listings) - SELECT rooms_bucket, ''::text, t33, t66, n_listings FROM bounds - """ -) - -_REDERIVE_TIERED_SQL = text( - """ - WITH - bnd AS ( - SELECT rooms_bucket, t33, t66 FROM asking_to_sold_tier_bounds WHERE district = '' - ), - deal_tiered AS ( - SELECT - LEAST(GREATEST(d.rooms, 0), 4) AS rooms_bucket, - CASE - WHEN d.price_per_m2 < b.t33 THEN 'low' - WHEN d.price_per_m2 < b.t66 THEN 'mid' - ELSE 'high' - END AS ppm2_tier, - d.price_per_m2 - FROM deals d - JOIN bnd b ON b.rooms_bucket = LEAST(GREATEST(d.rooms, 0), 4) - WHERE d.source = 'rosreestr' - AND d.rooms IS NOT NULL - AND d.price_per_m2 BETWEEN :ppm2_min AND :ppm2_max - AND d.deal_date >= CURRENT_DATE - INTERVAL '12 months' - ), - ask_tiered AS ( - SELECT - LEAST(GREATEST(l.rooms, 0), 4) AS rooms_bucket, - CASE - WHEN l.price_per_m2 < b.t33 THEN 'low' - WHEN l.price_per_m2 < b.t66 THEN 'mid' - ELSE 'high' - END AS ppm2_tier, - l.price_per_m2 - FROM listings l - JOIN bnd b ON b.rooms_bucket = LEAST(GREATEST(l.rooms, 0), 4) - WHERE l.is_active - AND l.rooms IS NOT NULL - AND l.price_per_m2 BETWEEN :ppm2_min AND :ppm2_max - -- novostroyki guard (#1186): NULL = legacy вторичка до м.011 - AND (l.listing_segment IS NULL OR l.listing_segment = 'vtorichka') - ), - deal_side_tier AS ( - SELECT - rooms_bucket, - ppm2_tier, - percentile_cont(0.5) WITHIN GROUP (ORDER BY price_per_m2) AS sold_median, - COUNT(*)::int AS n_deals - FROM deal_tiered - GROUP BY rooms_bucket, ppm2_tier - ), - ask_side_tier AS ( - SELECT - rooms_bucket, - ppm2_tier, - percentile_cont(0.5) WITHIN GROUP (ORDER BY price_per_m2) AS ask_median, - COUNT(*)::int AS n_listings - FROM ask_tiered - GROUP BY rooms_bucket, ppm2_tier - ), - tier_rows AS ( - SELECT - d.rooms_bucket, - ''::text AS district, - d.ppm2_tier, - (d.sold_median / a.ask_median)::numeric AS ratio, - round(d.sold_median)::bigint AS sold_median, - round(a.ask_median)::bigint AS ask_median, - d.n_deals, - a.n_listings, - 12 AS window_months, - 'per_rooms_tier'::text AS basis - FROM deal_side_tier d - JOIN ask_side_tier a USING (rooms_bucket, ppm2_tier) - WHERE d.n_deals >= 30 - AND a.n_listings >= 30 - AND a.ask_median IS NOT NULL - AND a.ask_median > 0 - AND d.sold_median IS NOT NULL - AND d.sold_median > 0 - ), - deal_side_all AS ( - SELECT - LEAST(GREATEST(rooms, 0), 4) AS rooms_bucket, - percentile_cont(0.5) WITHIN GROUP (ORDER BY price_per_m2) AS sold_median, - COUNT(*)::int AS n_deals - FROM deals - WHERE source = 'rosreestr' - AND rooms IS NOT NULL - AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max - AND deal_date >= CURRENT_DATE - INTERVAL '12 months' - GROUP BY LEAST(GREATEST(rooms, 0), 4) - ), - ask_side_all AS ( - SELECT - LEAST(GREATEST(rooms, 0), 4) AS rooms_bucket, - percentile_cont(0.5) WITHIN GROUP (ORDER BY price_per_m2) AS ask_median, - COUNT(*)::int AS n_listings - FROM listings - WHERE is_active - AND rooms IS NOT NULL - AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max - -- novostroyki guard (#1186): NULL = legacy вторичка до м.011 - AND (listing_segment IS NULL OR listing_segment = 'vtorichka') - GROUP BY LEAST(GREATEST(rooms, 0), 4) - ), - all_rows AS ( - SELECT - d.rooms_bucket, - ''::text AS district, - 'all'::text AS ppm2_tier, - (d.sold_median / a.ask_median)::numeric AS ratio, - round(d.sold_median)::bigint AS sold_median, - round(a.ask_median)::bigint AS ask_median, - d.n_deals, - a.n_listings, - 12 AS window_months, - 'per_rooms_all'::text AS basis - FROM deal_side_all d - JOIN ask_side_all a USING (rooms_bucket) - WHERE d.n_deals >= 30 - AND a.n_listings >= 30 - AND a.ask_median IS NOT NULL - AND a.ask_median > 0 - AND d.sold_median IS NOT NULL - AND d.sold_median > 0 - ), - deal_global AS ( - SELECT - percentile_cont(0.5) WITHIN GROUP (ORDER BY price_per_m2) AS sold_median, - COUNT(*)::int AS n_deals - FROM deals - WHERE source = 'rosreestr' - AND rooms IS NOT NULL - AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max - AND deal_date >= CURRENT_DATE - INTERVAL '12 months' - ), - ask_global AS ( - SELECT - percentile_cont(0.5) WITHIN GROUP (ORDER BY price_per_m2) AS ask_median, - COUNT(*)::int AS n_listings - FROM listings - WHERE is_active - AND rooms IS NOT NULL - AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max - -- novostroyki guard (#1186): NULL = legacy вторичка до м.011 - AND (listing_segment IS NULL OR listing_segment = 'vtorichka') - ), - global_row AS ( - SELECT - -1 AS rooms_bucket, - ''::text AS district, - 'all'::text AS ppm2_tier, - (d.sold_median / a.ask_median)::numeric AS ratio, - round(d.sold_median)::bigint AS sold_median, - round(a.ask_median)::bigint AS ask_median, - d.n_deals, - a.n_listings, - 12 AS window_months, - 'global_all'::text AS basis - FROM deal_global d - CROSS JOIN ask_global a - WHERE a.ask_median IS NOT NULL - AND a.ask_median > 0 - AND d.sold_median IS NOT NULL - AND d.sold_median > 0 - ) - INSERT INTO asking_to_sold_ratios_tiered ( - rooms_bucket, district, ppm2_tier, ratio, sold_median, ask_median, - n_deals, n_listings, window_months, basis - ) - SELECT rooms_bucket, district, ppm2_tier, ratio, sold_median, ask_median, - n_deals, n_listings, window_months, basis FROM global_row - UNION ALL - SELECT rooms_bucket, district, ppm2_tier, ratio, sold_median, ask_median, - n_deals, n_listings, window_months, basis FROM all_rows - UNION ALL - SELECT rooms_bucket, district, ppm2_tier, ratio, sold_median, ask_median, - n_deals, n_listings, window_months, basis FROM tier_rows - """ -) - -# ── Post-insert counters (tiered) ───────────────────────────────────────────── -_TIERED_COUNTERS_SQL = text( - """ - SELECT - (SELECT COUNT(*) FROM asking_to_sold_ratios_tiered WHERE district = '') - AS tiered_rows_written, - (SELECT COUNT(*) FROM asking_to_sold_tier_bounds WHERE district = '') - AS tier_bounds_written - """ -) - - def recompute_asking_to_sold_ratios(db: Session, run_id: int) -> dict[str, int]: """Пересчитать asking_to_sold_ratios (TRUE-MIRROR refresh, #648 Stage 4). @@ -437,8 +206,6 @@ def recompute_asking_to_sold_ratios(db: Session, run_id: int) -> dict[str, int]: "rows_written": 0, "per_rooms_rows": 0, "used_global_fallback": 0, - "tiered_rows_written": 0, - "tier_bounds_written": 0, } try: # DELETE + re-derive INSERT в одной транзакции (НЕ коммитим между ними — @@ -455,35 +222,15 @@ def recompute_asking_to_sold_ratios(db: Session, run_id: int) -> dict[str, int]: counters["per_rooms_rows"] = int(row["per_rooms_rows"] or 0) counters["used_global_fallback"] = int(row["used_global_fallback"] or 0) - # Tiered refresh (#928): DELETE + re-derive bounds + tier rows в той же транзакции. - # Атомарно — таблицы никогда не пустые mid-refresh (всё в одном commit ниже). - db.execute(_DELETE_TIERED_ROWS_SQL) - db.execute(_DELETE_TIER_BOUNDS_SQL) - db.execute( - _REDERIVE_BOUNDS_SQL, - {"ppm2_min": _PPM2_MIN, "ppm2_max": settings.asking_ratio_ppm2_max}, - ) - db.execute( - _REDERIVE_TIERED_SQL, - {"ppm2_min": _PPM2_MIN, "ppm2_max": settings.asking_ratio_ppm2_max}, - ) - tier_row = db.execute(_TIERED_COUNTERS_SQL).mappings().first() - if tier_row is not None: - counters["tiered_rows_written"] = int(tier_row["tiered_rows_written"] or 0) - counters["tier_bounds_written"] = int(tier_row["tier_bounds_written"] or 0) - db.commit() runs_mod.mark_done(db, run_id, counters) logger.info( "recompute_asking_to_sold_ratios run_id=%d done: " - "rows_written=%d per_rooms_rows=%d used_global_fallback=%d " - "tiered_rows_written=%d tier_bounds_written=%d", + "rows_written=%d per_rooms_rows=%d used_global_fallback=%d", run_id, counters["rows_written"], counters["per_rooms_rows"], counters["used_global_fallback"], - counters["tiered_rows_written"], - counters["tier_bounds_written"], ) return counters except Exception as exc: diff --git a/tradein-mvp/backend/tests/services/test_asking_sold_tier_ratio.py b/tradein-mvp/backend/tests/services/test_asking_sold_tier_ratio.py deleted file mode 100644 index ecf922ee..00000000 --- a/tradein-mvp/backend/tests/services/test_asking_sold_tier_ratio.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Unit tests for ppm2-tier asking->sold ratio lookup (#928). - -Tests for the _get_asking_sold_ratio helper with tier_aware_ratio_enabled flag. - -Five scenarios: -1. flag OFF -> uses old asking_to_sold_ratios table (byte-identical to pre-#928). -2. flag ON, anchor in low band -> returns shrunk tier ratio, basis startswith 'per_rooms_tier'. -3. flag ON, no tier row for that cell -> fallback to 'all' (basis 'per_rooms_all'). -4. flag ON, no 'all' row either -> global (-1) fallback (basis 'global_all'). -5. flag ON, no tables / empty -> (None, None), no raise. - -Static callers audit: _get_asking_sold_ratio gained anchor_ppm2 param (default None), -so all existing call sites using positional (db, rooms) continue to work unchanged. -""" - -from __future__ import annotations - -import os -from typing import Any -from unittest.mock import MagicMock, patch - -# Settings requires DATABASE_URL at init time. Set dummy DSN before any app import. -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - - -class _FakeRow: - """Stand-in for a SQLAlchemy Row.""" - - def __init__(self, **kw: Any) -> None: - for k, v in kw.items(): - setattr(self, k, v) - - -def _clear_ratio_cache() -> None: - from app.services import estimator - - estimator._asking_sold_ratio_cache.clear() - - -# ── helper: build a MagicMock db returning consecutive fetchone() results ───── - - -def _db_sequence(rows: list[Any]) -> MagicMock: - """MagicMock db whose execute().fetchone() calls return `rows` in order.""" - db = MagicMock() - db.execute.return_value.fetchone.side_effect = rows - return db - - -# ── 1. Flag OFF -> old table, ignores anchor_ppm2 ──────────────────────────── - - -def test_flag_off_uses_old_table_ignores_anchor_ppm2() -> None: - """Flag OFF: _get_asking_sold_ratio(db, 2, anchor_ppm2=90000) hits asking_to_sold_ratios.""" - from app.core.config import settings - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - db = _db_sequence([_FakeRow(ratio=0.80, basis="per_rooms")]) - - with patch.object(settings, "tier_aware_ratio_enabled", False): - ratio, basis = _get_asking_sold_ratio(db, 2, anchor_ppm2=90_000.0) - - assert ratio == 0.80 - assert basis == "per_rooms" - # Only one DB query (old path, no tier tables touched) - assert db.execute.call_count == 1 - # The query must target asking_to_sold_ratios (old table) - sql_str = str(db.execute.call_args_list[0].args[0]) - assert "asking_to_sold_ratios" in sql_str - assert "tiered" not in sql_str - - -def test_flag_off_no_anchor_ppm2_still_works() -> None: - """Flag OFF with no anchor_ppm2 (positional call) still works as before.""" - from app.core.config import settings - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - db = _db_sequence([_FakeRow(ratio=0.82, basis="per_rooms")]) - - with patch.object(settings, "tier_aware_ratio_enabled", False): - ratio, basis = _get_asking_sold_ratio(db, 1) - - assert ratio == 0.82 - assert basis == "per_rooms" - - -def test_flag_off_cache_key_is_legacy_not_none() -> None: - """Flag OFF must cache under (bucket, '_legacy'), NOT (bucket, None). - - This prevents key collision with the flag-ON Step-3 'all' fallback which also - caches under (bucket, None). If the process flips tier_aware_ratio_enabled ON - without restart, the '_legacy' entry is not accidentally served as the 'all' row. - """ - from app.core.config import settings - from app.services import estimator - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - bucket = min(max(2, 0), 4) - db = _db_sequence([_FakeRow(ratio=0.77, basis="per_rooms")]) - - with patch.object(settings, "tier_aware_ratio_enabled", False): - _get_asking_sold_ratio(db, 2) - - # Flag-OFF entry must be under "_legacy", not None. - assert (bucket, "_legacy") in estimator._asking_sold_ratio_cache - assert (bucket, None) not in estimator._asking_sold_ratio_cache - - -# ── 2. Flag ON, anchor in low band -> shrunk tier ratio ────────────────────── - - -def test_flag_on_low_band_returns_shrunk_tier_ratio() -> None: - """Flag ON, anchor < t33 -> tier='low', shrunk = tier*w + all*(1-w).""" - from app.core.config import settings - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - # DB calls sequence: - # 1. bounds (t33=130000, t66=165000) - # 2. tier row (ratio=0.91, n_deals=200) - # 3. 'all' row (ratio=0.80) - db = _db_sequence( - [ - _FakeRow(t33=130_000, t66=165_000), # bounds - _FakeRow(ratio=0.91, n_deals=200), # tier row - _FakeRow(ratio=0.80), # 'all' row - ] - ) - - k = 150 - w = 200 / (200 + k) - expected_ratio = 0.91 * w + 0.80 * (1.0 - w) - - with ( - patch.object(settings, "tier_aware_ratio_enabled", True), - patch.object(settings, "tier_ratio_shrink_k", k), - ): - ratio, basis = _get_asking_sold_ratio(db, 2, anchor_ppm2=90_000.0) - - assert ratio is not None - assert abs(ratio - expected_ratio) < 1e-9 - assert basis is not None - assert basis.startswith("per_rooms_tier") - assert "low" in basis - - -# ── 3. Flag ON, no tier row -> fallback to 'all' ───────────────────────────── - - -def test_flag_on_no_tier_row_falls_back_to_all() -> None: - """Flag ON, anchor in mid band, but no tier row for (bucket,mid) -> 'all' fallback.""" - from app.core.config import settings - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - # DB calls: - # 1. bounds (t33=130000, t66=165000) - # 2. tier row -> None (cell not populated) - # 3. 'all' row (ratio=0.80) -- used as both shrink-target AND fallback - # Since tier_row is None, we skip shrink and fall through to step 3. - db = _db_sequence( - [ - _FakeRow(t33=130_000, t66=165_000), # bounds - None, # tier row missing - _FakeRow(ratio=0.80), # 'all' row (shrink target query) - _FakeRow(ratio=0.80), # 'all' row (fallback step 3) - ] - ) - - with ( - patch.object(settings, "tier_aware_ratio_enabled", True), - patch.object(settings, "tier_ratio_shrink_k", 150), - ): - ratio, basis = _get_asking_sold_ratio(db, 2, anchor_ppm2=145_000.0) - - assert ratio == 0.80 - assert basis == "per_rooms_all" - - -# ── 4. Flag ON, no 'all' row -> global fallback ─────────────────────────────── - - -def test_flag_on_no_all_row_falls_back_to_global() -> None: - """Flag ON, bounds present, tier+all rows missing -> global -1 fallback.""" - from app.core.config import settings - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - db = _db_sequence( - [ - _FakeRow(t33=130_000, t66=165_000), # bounds - None, # tier row -> None - None, # 'all' shrink-target -> None - None, # step-3 'all' fallback -> None - _FakeRow(ratio=0.79), # step-4 global (-1, all) - ] - ) - - with patch.object(settings, "tier_aware_ratio_enabled", True): - ratio, basis = _get_asking_sold_ratio(db, 2, anchor_ppm2=145_000.0) - - assert ratio == 0.79 - assert basis == "global_all" - - -# ── 5. Flag ON, no tables / empty -> (None, None), no raise ────────────────── - - -def test_flag_on_no_tables_returns_none_none() -> None: - """Flag ON, db.execute raises (tables not migrated) -> (None, None), no raise.""" - from app.core.config import settings - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - db = MagicMock() - db.execute.side_effect = RuntimeError("relation asking_to_sold_tier_bounds does not exist") - - with patch.object(settings, "tier_aware_ratio_enabled", True): - result = _get_asking_sold_ratio(db, 2, anchor_ppm2=90_000.0) - - assert result == (None, None) - # rollback must be called on error - db.rollback.assert_called() - - -# ── 6. Flag ON, no bounds row -> step-3 'all' fallback (skip tier entirely) ── - - -def test_flag_on_no_bounds_row_falls_back_to_all() -> None: - """Flag ON, bounds row None -> skip tier, go straight to 'all' step 3.""" - from app.core.config import settings - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - db = _db_sequence( - [ - None, # bounds -> None (bucket has no bound data) - _FakeRow(ratio=0.80), # step-3 'all' row - ] - ) - - with patch.object(settings, "tier_aware_ratio_enabled", True): - ratio, basis = _get_asking_sold_ratio(db, 2, anchor_ppm2=90_000.0) - - assert ratio == 0.80 - assert basis == "per_rooms_all" - - -# ── 7. Cache: (bucket, tier) key memoises per tier ─────────────────────────── - - -def test_tier_cache_key_is_bucket_tier_tuple() -> None: - """Second call with same (bucket, anchor in same tier) -> served from cache after bounds. - - Flow for second call: - - cache_key starts as (bucket, None) -- miss - - fetches bounds (1 DB query) -> determines tier_key="low" - - cache_key = (bucket, "low") -- HIT (populated by first call) - - returns cached ratio without further DB queries - Total DB queries: first call (3) + second call (1 bounds) = 4. - """ - from app.core.config import settings - from app.services.estimator import _get_asking_sold_ratio - - _clear_ratio_cache() - db = _db_sequence( - [ - # First call: bounds + tier_row + all_row - _FakeRow(t33=130_000, t66=165_000), - _FakeRow(ratio=0.91, n_deals=200), - _FakeRow(ratio=0.80), - # Second call: bounds only (tier cache hit after that) - _FakeRow(t33=130_000, t66=165_000), - ] - ) - - with ( - patch.object(settings, "tier_aware_ratio_enabled", True), - patch.object(settings, "tier_ratio_shrink_k", 150), - ): - first = _get_asking_sold_ratio(db, 2, anchor_ppm2=90_000.0) - second = _get_asking_sold_ratio(db, 2, anchor_ppm2=95_000.0) # same 'low' tier - - # Both calls return the same result (same tier). - assert first == second - # First call: 3 queries (bounds + tier_row + all_row). - # Second call: 1 query (bounds) + cache hit on (bucket, "low"). - assert db.execute.call_count == 4 diff --git a/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py b/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py index 54f10b81..3b32277a 100644 --- a/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py +++ b/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py @@ -272,27 +272,17 @@ class _FakeMappingResult: class _FakeDB: """Minimal stand-in for a SQLAlchemy Session — records execute() calls. - Execute call order (#928 extended task): + Execute call order: 1. DELETE asking_to_sold_ratios 2. REDERIVE INSERT asking_to_sold_ratios 3. COUNTERS SELECT (returns counters_row) - 4. DELETE asking_to_sold_ratios_tiered (tiered rows) - 5. DELETE asking_to_sold_tier_bounds (tier bounds) - 6. REDERIVE_BOUNDS INSERT - 7. REDERIVE_TIERED INSERT - 8. TIERED_COUNTERS SELECT (returns tiered_counters_row) """ def __init__( self, counters_row: dict[str, Any], - tiered_counters_row: dict[str, Any] | None = None, ) -> None: self._counters_row = counters_row - self._tiered_counters_row = tiered_counters_row or { - "tiered_rows_written": 0, - "tier_bounds_written": 0, - } self.executed: list[Any] = [] self.committed = False self.rolled_back = False @@ -302,9 +292,6 @@ class _FakeDB: # 3rd call is the counters SELECT (asking_to_sold_ratios). if len(self.executed) == 3: return _FakeMappingResult(self._counters_row) - # 8th call is the tiered counters SELECT. - if len(self.executed) == 8: - return _FakeMappingResult(self._tiered_counters_row) return _FakeMappingResult(None) def commit(self) -> None: @@ -317,8 +304,8 @@ class _FakeDB: def test_counter_logic_with_fake_db(monkeypatch: pytest.MonkeyPatch) -> None: """recompute_asking_to_sold_ratios maps the counters row into its return + marks done. - #928: counters now include tiered_rows_written + tier_bounds_written (5 keys total). - FakeDB has 8 execute() calls (old: 3, tiered: 5). + #2002: tiered refresh removed — counters are the 3 legacy keys, FakeDB has 3 + execute() calls (DELETE + REDERIVE INSERT + COUNTERS SELECT). """ marked: dict[str, Any] = {} monkeypatch.setattr( @@ -330,7 +317,6 @@ def test_counter_logic_with_fake_db(monkeypatch: pytest.MonkeyPatch) -> None: db = _FakeDB( counters_row={"rows_written": 4, "per_rooms_rows": 3, "used_global_fallback": 1}, - tiered_counters_row={"tiered_rows_written": 15, "tier_bounds_written": 5}, ) out = ratio_mod.recompute_asking_to_sold_ratios(db, run_id=99) # type: ignore[arg-type] @@ -338,15 +324,11 @@ def test_counter_logic_with_fake_db(monkeypatch: pytest.MonkeyPatch) -> None: "rows_written": 4, "per_rooms_rows": 3, "used_global_fallback": 1, - "tiered_rows_written": 15, - "tier_bounds_written": 5, } assert out == expected assert db.committed is True - # Eight statements: DELETE(old) + REDERIVE(old) + COUNTERS(old) - # + DELETE_TIERED_ROWS + DELETE_TIER_BOUNDS + REDERIVE_BOUNDS + REDERIVE_TIERED - # + TIERED_COUNTERS (#928 extension). - assert len(db.executed) == 8 + # Three statements: DELETE + REDERIVE INSERT + COUNTERS SELECT (legacy per-rooms). + assert len(db.executed) == 3 assert marked["run_id"] == 99 assert marked["counters"] == expected diff --git a/tradein-mvp/backend/tests/test_backtest_fixture_roundtrip.py b/tradein-mvp/backend/tests/test_backtest_fixture_roundtrip.py index f5859892..1b188f8a 100644 --- a/tradein-mvp/backend/tests/test_backtest_fixture_roundtrip.py +++ b/tradein-mvp/backend/tests/test_backtest_fixture_roundtrip.py @@ -164,7 +164,7 @@ def _build_fixture() -> dict[str, Any]: {"price_per_m2": 308_000.0, "area_m2": 82.0, "rooms": 3, "floor": 6, "total_floors": 9}, ], anchor_tier_fetched=None, - ratio_calls=[[310_000.0, [0.92, "per_rooms_tier:high"]]], + ratio_calls=[[310_000.0, [0.92, "per_rooms"]]], qi_calls=[], qis_calls=[], address="ул. Тестовая, 3", diff --git a/tradein-mvp/backend/tests/test_estimator_ratio_tier_fix.py b/tradein-mvp/backend/tests/test_estimator_ratio_tier_fix.py index 069d4bc1..478e1db2 100644 --- a/tradein-mvp/backend/tests/test_estimator_ratio_tier_fix.py +++ b/tradein-mvp/backend/tests/test_estimator_ratio_tier_fix.py @@ -163,7 +163,7 @@ def test_ratio_tier_uses_final_headline_after_anchor() -> None: def _spy(db_inner: Any, rooms: Any, anchor_ppm2: Any = None) -> tuple: captured_anchor_ppm2.append(anchor_ppm2) - return (0.78, "per_rooms_tier:high") + return (0.78, "per_rooms") async def _run() -> Any: with ( diff --git a/tradein-mvp/backend/tests/test_segment_guard_1186.py b/tradein-mvp/backend/tests/test_segment_guard_1186.py index e614f2c4..999e41c4 100644 --- a/tradein-mvp/backend/tests/test_segment_guard_1186.py +++ b/tradein-mvp/backend/tests/test_segment_guard_1186.py @@ -74,9 +74,9 @@ def test_tier_a_anchor_gated_python_side_not_sql_guard() -> None: # (a) Tier C SQL guard сохранён (поиск находит единственное вхождение — Tier C). assert _GUARD_RE.search(src), "Tier C anchor SQL lost the canonical novostroyki guard" # (b) Tier A теперь gated Python-side (#1774). - assert "estimate_sb_tier_a_allow_primary_if_secondary_present" in src, ( - "Tier A no longer references the #1774 gating flag — gated relaxation missing" - ) + assert ( + "estimate_sb_tier_a_allow_primary_if_secondary_present" in src + ), "Tier A no longer references the #1774 gating flag — gated relaxation missing" # Hardened gate: secondary_count ≥ primary_count (заменил has_secondary). assert "secondary_count" in src, "Tier A lacks secondary_count Python-side gate (#1774)" assert "primary_count" in src, "Tier A lacks primary_count comparison in hardened gate (#1774)" @@ -94,8 +94,6 @@ def test_tier_c_anchor_has_canonical_guard_not_segment_param() -> None: # ── asking_to_sold_ratio.py static checks ───────────────────────────────────── _REDERIVE_SQL_TEXT = str(ratio_mod._REDERIVE_SQL.text) -_REDERIVE_BOUNDS_TEXT = str(ratio_mod._REDERIVE_BOUNDS_SQL.text) -_REDERIVE_TIERED_TEXT = str(ratio_mod._REDERIVE_TIERED_SQL.text) def test_ask_side_cte_has_guard() -> None: @@ -114,20 +112,6 @@ def test_ask_global_cte_has_guard() -> None: ), f"Expected ≥2 guard occurrences in _REDERIVE_SQL (ask_side + ask_global), got {matches}" -def test_bounds_sql_has_guard() -> None: - """_REDERIVE_BOUNDS_SQL (tier bounds percentiles) — guard обязателен.""" - assert _GUARD_RE.search(_REDERIVE_BOUNDS_TEXT), "_REDERIVE_BOUNDS_SQL lacks novostroyki guard" - - -def test_tiered_ask_ctes_have_guard() -> None: - """_REDERIVE_TIERED_SQL: ask_tiered + ask_side_all + ask_global — минимум 3 вхождения.""" - matches = len(_GUARD_RE.findall(_REDERIVE_TIERED_TEXT)) - assert matches >= 3, ( - f"Expected ≥3 guard occurrences in _REDERIVE_TIERED_SQL " - f"(ask_tiered + ask_side_all + ask_global), got {matches}" - ) - - # ── Поведенческие тесты: _fetch_analogs (mock DB) ──────────────────────────── -- 2.45.3