From 4d93e5a1aa4e351650fefd85446d4ec28086ffe1 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 30 May 2026 19:51:18 +0300 Subject: [PATCH] =?UTF-8?q?fix(ratio):=20ppm=C2=B2=20=D0=BF=D0=BE=D1=82?= =?UTF-8?q?=D0=BE=D0=BB=D0=BE=D0=BA=20asking=E2=86=92sold=20=D0=B2=20?= =?UTF-8?q?=D0=BA=D0=BE=D0=BD=D1=84=D0=B8=D0=B3,=20default=201.2M=20(Refs?= =?UTF-8?q?=20#767)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit asking_to_sold_ratio фильтровал сделки по хардкоду price_per_m2 BETWEEN 30000 AND 600000 (4 CTE) → премиум ЕКБ >600k выпадал → ratio из global-fallback → bias выкупной цены премиума. Все 4 хардкода → BETWEEN :ppm2_min AND :ppm2_max (bind-params, не f-string). settings.asking_ratio_ppm2_max default 1_200_000 (env ASKING_RATIO_PPM2_MAX), _PPM2_MIN=30_000. Docstring обновлён. NB: точный потолок верифицировать QA против SELECT max(price_per_m2) FROM deals WHERE source='rosreestr' (worker без live-DB; 1.2M — запас, env-override без деплоя). pytest -k asking 21 pass, ruff clean. --- tradein-mvp/backend/app/core/config.py | 8 +++++ .../backend/app/tasks/asking_to_sold_ratio.py | 33 +++++++++++++------ .../tests/test_asking_to_sold_ratio.py | 23 ++++++++++--- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index d9327655..0ad00792 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -108,6 +108,14 @@ class Settings(BaseSettings): # Конфигурируется через env ESTIMATE_QUOTA_LIMIT. Default 15. estimate_quota_limit: int = 15 + # Фильтр junk-/премиум-порога для asking→sold derivation (#767). + # Нижняя граница 30 000 ₽/м² отсекает нежилые/технические сделки; менять не стоит. + # Верхняя граница — поднята с 600 000 до 1 200 000 ₽/м², чтобы покрыть ЕКБ-premium + # (>600k). Точное значение стоит сверить с `SELECT max(price_per_m2) FROM deals + # WHERE source='rosreestr'` на проде — см. QA-note в asking_to_sold_ratio.py. + # ENV: ASKING_RATIO_PPM2_MAX. + asking_ratio_ppm2_max: int = 1_200_000 + # SSRF-защита для admin scrape endpoints (#756). # Список хостов которым разрешено передавать абсолютные URL в параметрах *_url. # Относительные пути (без netloc) проходят без проверки — хост подставляется diff --git a/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py b/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py index aec1e8c3..a5ebb17f 100644 --- a/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py +++ b/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py @@ -22,8 +22,9 @@ refresh потреблял свежие ДКП-сделки того же дня SQL derivation ниже — БАЙТ-В-БАЙТ та же логика, что seed в data/sql/080_asking_to_sold_ratios.sql (deal_side / ask_side / per_bucket + deal_global / ask_global / global_row: трейлинг-12мес -окно, ppm²-полоса [30000,600000], бакет LEAST(GREATEST(rooms,0),4), порог n_deals>=30 AND -n_listings>=30 для per_rooms, global -1 строка всегда). ON CONFLICT убран — DELETE идёт первым, +окно, ppm²-полоса [_PPM2_MIN, settings.asking_ratio_ppm2_max] (default [30000,1200000]), +бакет LEAST(GREATEST(rooms,0),4), порог n_deals>=30 AND n_listings>=30 для per_rooms, +global -1 строка всегда). ON CONFLICT убран — DELETE идёт первым, конфликтов нет (повторный прогон в одной tx невозможен, refresh = re-seed по семантике). """ @@ -34,8 +35,15 @@ import logging from sqlalchemy import text from sqlalchemy.orm import Session +from app.core.config import settings from app.services import scrape_runs as runs_mod +# Нижняя граница ppm² — отсекает нежилые/технические сделки; не меняется. +_PPM2_MIN: int = 30_000 +# Верхняя граница берётся из settings.asking_ratio_ppm2_max (default 1_200_000). +# QA-note: точное значение сверить с `SELECT max(price_per_m2) FROM deals +# WHERE source='rosreestr'` на проде — ceiling должен быть > max(ppm²) premium-сделок. + logger = logging.getLogger(__name__) # ── True-mirror cleanup: drop all #648 rows before re-derivation ────────────── @@ -52,12 +60,14 @@ _DELETE_SQL = text( # ── Derivation + re-seed (БАЙТ-В-БАЙТ из 080, ON CONFLICT убран — DELETE идёт первым) ── # deal_side / ask_side / per_bucket + deal_global / ask_global / global_row: # sold_median = percentile_cont(0.5) по deals.price_per_m2 (source='rosreestr', -# ppm² ∈ [30000,600000], deal_date >= CURRENT_DATE − 12 months), +# ppm² ∈ [_PPM2_MIN, settings.asking_ratio_ppm2_max], deal_date >= CURRENT_DATE − 12 months), # бакет LEAST(GREATEST(rooms,0),4). -# ask_median = percentile_cont(0.5) по listings.price_per_m2 (is_active, та же ppm²-полоса). +# ask_median = percentile_cont(0.5) по listings.price_per_m2 +# (is_active, та же ppm²-полоса [_PPM2_MIN, asking_ratio_ppm2_max]). # per_rooms строки — только при n_deals>=30 AND n_listings>=30 AND ask>0 AND sold>0. # global -1 строка (basis='global_fallback') — всегда (если ask>0 AND sold>0). window_months=12. -# Литералы (порог/окно/полоса) — без bind-параметров, поэтому CAST(:x AS type) не нужен. +# Порог/окно — литералы; ppm²-полоса передаётся bind-параметрами :ppm2_min/:ppm2_max +# (безопасно от SQL-инъекций; CAST не нужен — psycopg v3 передаёт int напрямую). _REDERIVE_SQL = text( """ WITH @@ -70,7 +80,7 @@ _REDERIVE_SQL = text( FROM deals WHERE source = 'rosreestr' AND rooms IS NOT NULL - AND price_per_m2 BETWEEN 30000 AND 600000 + AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max AND deal_date >= CURRENT_DATE - INTERVAL '12 months' GROUP BY LEAST(GREATEST(rooms, 0), 4) ), @@ -83,7 +93,7 @@ _REDERIVE_SQL = text( FROM listings WHERE is_active AND rooms IS NOT NULL - AND price_per_m2 BETWEEN 30000 AND 600000 + AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max GROUP BY LEAST(GREATEST(rooms, 0), 4) ), -- Per-rooms строки: только бакеты с обеими сторонами, прошедшие порог 30/30 и ask>0. @@ -116,7 +126,7 @@ _REDERIVE_SQL = text( FROM deals WHERE source = 'rosreestr' AND rooms IS NOT NULL - AND price_per_m2 BETWEEN 30000 AND 600000 + AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max AND deal_date >= CURRENT_DATE - INTERVAL '12 months' ), -- ASKING медиана по ВСЕМ активным listings (без бакет-фильтра) — для global row. @@ -127,7 +137,7 @@ _REDERIVE_SQL = text( FROM listings WHERE is_active AND rooms IS NOT NULL - AND price_per_m2 BETWEEN 30000 AND 600000 + AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max ), -- Global fallback строка rooms_bucket=-1 (пишется всегда, если ask>0). global_row AS ( @@ -197,7 +207,10 @@ def recompute_asking_to_sold_ratios(db: Session, run_id: int) -> dict[str, int]: # DELETE + re-derive INSERT в одной транзакции (НЕ коммитим между ними — # таблица не должна остаться пустой, если INSERT упадёт). db.execute(_DELETE_SQL) - db.execute(_REDERIVE_SQL) + db.execute( + _REDERIVE_SQL, + {"ppm2_min": _PPM2_MIN, "ppm2_max": settings.asking_ratio_ppm2_max}, + ) row = db.execute(_COUNTERS_SQL).mappings().first() if row is not None: diff --git a/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py b/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py index 4e68adf8..00738fb9 100644 --- a/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py +++ b/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py @@ -91,7 +91,9 @@ def test_rederivation_uses_12_month_window() -> None: def test_rederivation_applies_3030_threshold_and_ppm_band() -> None: assert "d.n_deals >= 30" in _REDERIVE_SQL assert "a.n_listings >= 30" in _REDERIVE_SQL - assert "price_per_m2 BETWEEN 30000 AND 600000" in _REDERIVE_SQL + # ppm² bounds are now bind parameters (not hardcoded literals) — prevents SQL injection + # and makes the ceiling configurable via settings.asking_ratio_ppm2_max (#767). + assert "price_per_m2 BETWEEN :ppm2_min AND :ppm2_max" in _REDERIVE_SQL assert "LEAST(GREATEST(rooms, 0), 4)" in _REDERIVE_SQL assert "percentile_cont(0.5)" in _REDERIVE_SQL @@ -129,6 +131,10 @@ def test_migration_080_derivation_is_subset_of_refresh_sql() -> None: Strip -- comments + normalise whitespace, then confirm the 080 CTE body (deal_side … per_bucket UNION ALL) is present verbatim in the refresh SQL — so refresh has exact re-seed semantics on the EXECUTABLE derivation. + + ppm² bounds are compared after normalisation: the 080 seed uses literal integers + (30000 / 600000) while the refresh uses bind params (:ppm2_min / :ppm2_max) — + both are replaced with the token PPM2_BAND_PLACEHOLDER before the subset check. """ seed_sql = _MIGRATION_080.read_text("utf-8") # Extract the WITH … (up to the ON CONFLICT) from the seed. @@ -136,7 +142,16 @@ def test_migration_080_derivation_is_subset_of_refresh_sql() -> None: onconflict_idx = seed_sql.index("ON CONFLICT (rooms_bucket, district) DO UPDATE") seed_derivation = seed_sql[with_idx:onconflict_idx] - assert _strip_sql(seed_derivation) in _strip_sql(_REDERIVE_SQL) + def _normalise_ppm2(s: str) -> str: + """Replace ppm² bound tokens with a common placeholder for cross-comparison.""" + # Seed uses literal integers; refresh uses bind params. + s = re.sub(r"30000 AND 600000", "PPM2_BAND_PLACEHOLDER", s) + s = re.sub(r":ppm2_min AND :ppm2_max", "PPM2_BAND_PLACEHOLDER", s) + return s + + assert _strip_sql(_normalise_ppm2(seed_derivation)) in _strip_sql( + _normalise_ppm2(_REDERIVE_SQL) + ) # ── Counters query ──────────────────────────────────────────────────────────── @@ -279,9 +294,7 @@ def test_counter_logic_with_fake_db(monkeypatch: pytest.MonkeyPatch) -> None: ) monkeypatch.setattr(ratio_mod.runs_mod, "mark_failed", lambda *a, **k: None) - db = _FakeDB( - counters_row={"rows_written": 4, "per_rooms_rows": 3, "used_global_fallback": 1} - ) + db = _FakeDB(counters_row={"rows_written": 4, "per_rooms_rows": 3, "used_global_fallback": 1}) out = ratio_mod.recompute_asking_to_sold_ratios(db, run_id=99) # type: ignore[arg-type] assert out == {"rows_written": 4, "per_rooms_rows": 3, "used_global_fallback": 1}