From 5b83d1e2dba86d69bbf28c95e8c61e43ebbca945 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sun, 13 Sep 2026 13:34:09 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20=D0=B1=D1=8D=D0=BA=D1=82=D0=B5?= =?UTF-8?q?=D1=81=D1=82=20=D0=BE=D1=86=D0=B5=D0=BD=D1=89=D0=B8=D0=BA=D0=B0?= =?UTF-8?q?=20=D1=83=D0=BC=D0=B5=D0=B5=D1=82=20=D0=BB=D1=8E=D0=B1=D0=BE?= =?UTF-8?q?=D0=B9=20=D1=80=D0=B5=D0=B3=D0=B8=D0=BE=D0=BD,=20=D0=BD=D0=B5?= =?UTF-8?q?=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20=D0=95=D0=BA=D0=B0?= =?UTF-8?q?=D1=82=D0=B5=D1=80=D0=B8=D0=BD=D0=B1=D1=83=D1=80=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Обещание точности лендинга измерялось только на екатеринбургских ДКП: скрипт бэктеста не имел параметра региона (region_code был зашит числом 66), а сам SELECT сделок вообще не фильтровал по deals.region_code. Пока в базе жил только регион 66, это было незаметно — но теперь там же лежат 212 937 московских ДКП (69 138 за последние 12 мес.) и 113 351 подмосковных, и безусловный "unscoped" запрос молча смешал бы все три региона в одной выборке. Прогон по Москве до этой правки сделать было нечем. Добавлен CLI-флаг --region (по идиоме app/tasks/msk_raw_import.py: явный список поддерживаемых значений, отказ на неизвестном коде ДО любого запроса к БД). Источник допустимых значений — реестр REGIONS из app/services/regions.py, а не отдельный литерал: регион заводится в реестре один раз и становится доступен здесь автоматически. Дефолт оставлен 66 — поведение существующих вызовов не меняется. region_code проведён во все 4 варианта SQL выборки сделок (обычная/scattered × city/no-city) и в per-city PPM2-band lookup (_resolve_city_ppm2_band). Отдельно поправлен _predict_full_spine: ДКП-коридор (_fetch_dkp_corridor) резолвил регион только по умолчанию (66) независимо от сделки — теперь регион резолвится ПО КООРДИНАТАМ каждой сделки через regions.region_for_point, зеркаля то, как это делает сам prod-эстиматор (estimator.py ~4868-4870). Без этого фикса флаг --region был бы декоративным: SELECT сделок брал бы верный регион, но коридор аналогов всё равно считался бы по Свердловской области. Проверено: --help показывает --region {50,66,77}; --region 99 отклоняется argparse ДО подключения к БД (exit 2); ruff check/format check чисты; все 83 существующих теста backtest_* проходят без изменений. Полный прогон по Москве в этой задаче не запускался (долгий, отдельная проверка): python -m scripts.backtest_estimator --region 77 --since 2024-09-01 --sample 300 --- .../backend/scripts/backtest_estimator.py | 139 +++++++++++++++--- 1 file changed, 117 insertions(+), 22 deletions(-) diff --git a/tradein-mvp/backend/scripts/backtest_estimator.py b/tradein-mvp/backend/scripts/backtest_estimator.py index eff9093f..a6dd8f84 100644 --- a/tradein-mvp/backend/scripts/backtest_estimator.py +++ b/tradein-mvp/backend/scripts/backtest_estimator.py @@ -134,6 +134,10 @@ USAGE # oblast D: per-city validation (exact deals.city name, not a slug): python -m scripts.backtest_estimator --city "Нижний Тагил" --sample 300 + + # #3520 любой регион (choices come from app.services.regions.REGIONS): + # Москва (region_code=77, 212 937 ДКП, 69 138 за последние 12 мес.): + python -m scripts.backtest_estimator --region 77 --since 2024-09-01 --sample 300 """ from __future__ import annotations @@ -156,6 +160,15 @@ from typing import Any from sqlalchemy import text from sqlalchemy.orm import Session +from app.services.regions import REGIONS, region_for_point + +# Регион по умолчанию — Свердловская обл., прежнее (единственное до #3520) +# поведение скрипта. Список допустимых значений — РЕЕСТР REGIONS +# (app/services/regions.py), а не отдельный литерал: регион появляется в +# реестре один раз и автоматически становится доступен и здесь, и в +# msk_raw_import, и в самом эстиматоре. +DEFAULT_BACKTEST_REGION_CODE = 66 + def _import_estimator() -> tuple[Any, Any]: """Lazy import of the estimator's pure funcs (_filter_outliers, _percentile). @@ -1153,6 +1166,7 @@ _SAMPLE_SQL = text( house_type FROM deals WHERE source = 'rosreestr' + AND region_code = CAST(:region_code AS int) AND geom IS NOT NULL AND price_per_m2 BETWEEN CAST(:ppm2_min AS numeric) AND CAST(:ppm2_max AS numeric) AND rooms IS NOT NULL @@ -1204,6 +1218,7 @@ _SAMPLE_SQL_SCATTERED = text( house_type FROM deals WHERE source = 'rosreestr' + AND region_code = CAST(:region_code AS int) AND geom IS NOT NULL AND price_per_m2 BETWEEN CAST(:ppm2_min AS numeric) AND CAST(:ppm2_max AS numeric) AND rooms IS NOT NULL @@ -1233,6 +1248,7 @@ _SAMPLE_SQL_SCATTERED_CITY = text( house_type FROM deals WHERE source = 'rosreestr' + AND region_code = CAST(:region_code AS int) AND geom IS NOT NULL AND price_per_m2 BETWEEN CAST(:ppm2_min AS numeric) AND CAST(:ppm2_max AS numeric) AND rooms IS NOT NULL @@ -1271,10 +1287,9 @@ _CANDIDATES_SQL = text( # _fetch_dkp_corridor COALESCEs against (migration 178, key (region_code, city) # since migration 298 — #3051 "Москва"). Looked up only when --city is set (see # _resolve_city_ppm2_band); the default (city=None) path never issues this -# query. region_code defaults to 66 (Свердловская обл.) — this script has no -# region CLI flag yet, so every call is scoped to the oblast, matching every -# existing --city invocation (byte-identical to the pre-298 unscoped lookup, -# which only ever saw region-66 rows since the table was oblast-only before). +# query. region_code (#3520 "любой регион") is threaded from the CLI --region +# flag end-to-end now (default 66, Свердловская обл. — unchanged behaviour for +# every pre-existing invocation, which never passed --region). _CITY_PPM2_BAND_SQL = text( """ SELECT ppm2_min, ppm2_max @@ -1288,10 +1303,13 @@ _CITY_PPM2_BAND_SQL = text( def _sample_sql(city: str | None, scattered: bool = False) -> Any: """ДКП deal-sample SELECT — optionally scoped to one ``deals.city`` (oblast D). - ``city is None`` (default) returns the SAME ``_SAMPLE_SQL`` object used - before this change — literal identity, not just equal text — so the - default CLI invocation (and the frozen EKB regression gate, which never - calls this path at all) see byte-identical SQL. + ``city is None`` (default) returns the SAME ``_SAMPLE_SQL`` object every + time — literal identity, not just equal text (asserted by + ``test_sample_sql_city_none_returns_original_object``) — so callers that + cache/compare the query object see a stable reference. All four SQL + variants (unscoped/scattered × city/no-city) carry a ``region_code`` + predicate (#3520 "любой регион") bound from ``_load_sample``'s + ``region_code`` param, default 66. When ``city`` is set, an extra ``AND city = CAST(:city AS text)`` predicate scopes the sample to that ``deals.city`` value — populated oblast-wide by @@ -1321,6 +1339,7 @@ def _sample_sql(city: str | None, scattered: bool = False) -> Any: house_type FROM deals WHERE source = 'rosreestr' + AND region_code = CAST(:region_code AS int) AND geom IS NOT NULL AND price_per_m2 BETWEEN CAST(:ppm2_min AS numeric) AND CAST(:ppm2_max AS numeric) AND rooms IS NOT NULL @@ -1348,9 +1367,9 @@ def _resolve_city_ppm2_band( mirrors estimator._fetch_dkp_corridor's own COALESCE fallback) or any DB error → the globals; read-only best-effort, never raises. - ``region_code`` defaults to 66 (Свердловская обл.) — this script has no - region CLI flag yet (out of scope, #3051 sub-PR B); every existing - --city caller keeps its byte-identical lookup. + ``region_code`` defaults to 66 (Свердловская обл.) for backward-compat + direct callers; ``_load_sample`` (#3520) always passes the actual + ``--region`` value through explicitly. """ if city is None: return float(PPM2_MIN), float(PPM2_MAX) @@ -1376,22 +1395,32 @@ def _load_sample( city: str | None = None, scattered: bool = False, seed: str = "mera", + region_code: int = DEFAULT_BACKTEST_REGION_CODE, ) -> list[DealSample]: """Run the held-out ДКП deal sampling SELECT → list[DealSample]. + ``region_code`` (#3520 "любой регион", default 66 — Свердловская обл.) + scopes the sample to one ``deals.region_code`` value — REQUIRED once + ``deals`` carries more than one region (212 937 Москва + 113 351 + Московская обл. rows alongside 108 623 ЕКБ), otherwise the "unscoped" + query would silently mix regions together. Must be a key of + ``app.services.regions.REGIONS`` — validated by ``main()``/``_parse_args`` + (CLI ``choices``) so this function only ever sees a supported code. + ``city`` (oblast D, default None) scopes the sample to one ``deals.city`` value via ``_sample_sql`` and sources the PPM2 sanity band from - ``deal_city_price_bands`` for that city (``_resolve_city_ppm2_band``, - falls back to the module globals). Default None is byte-identical to the - pre-oblast-D behaviour: same SQL object, same PPM2_MIN/PPM2_MAX globals, - no extra query. + ``deal_city_price_bands`` for that (region_code, city) pair + (``_resolve_city_ppm2_band``, falls back to the module globals). Default + None is byte-identical to the pre-oblast-D behaviour: same SQL object, + same PPM2_MIN/PPM2_MAX globals, no extra query. """ if city is None: ppm2_min: float = PPM2_MIN ppm2_max: float = PPM2_MAX else: - ppm2_min, ppm2_max = _resolve_city_ppm2_band(db, city) + ppm2_min, ppm2_max = _resolve_city_ppm2_band(db, city, region_code=region_code) params: dict[str, Any] = { + "region_code": region_code, "ppm2_min": ppm2_min, "ppm2_max": ppm2_max, "since": since, @@ -1947,8 +1976,21 @@ def _predict_full_spine( # harness measures the SAME corridor prod actually computes today, not the # pre-C2 unscoped behaviour — else the backtest validates stale semantics. target_city = m._resolve_target_city(deal.address) + # #3520 "любой регион": mirror estimate_quality's OWN region_code resolution + # (estimator.py ~4868-4870, region_for_point(geo.lat, geo.lon) with a + # DEFAULT_REGION_CODE fallback for points outside every registered bbox) — + # without this, _fetch_dkp_corridor's default (region 66) would scope a + # Москва/Московская обл. deal's corridor query to the WRONG region and + # silently return no rows (or, worse, an unrelated same-named street in 66). + target_region = region_for_point(deal.lat, deal.lon) + target_region_code = target_region.code if target_region else DEFAULT_BACKTEST_REGION_CODE dkp_raw = m._fetch_dkp_corridor( - db, address=deal.address, rooms=deal.rooms, area=deal.area_m2, city=target_city + db, + address=deal.address, + rooms=deal.rooms, + area=deal.area_m2, + city=target_city, + region_code=target_region_code, ) # #1966 prod parity: same-building anchor pre-fetch is GATED exactly like # estimate_quality — no-area / no-address → ([], None) instead of an @@ -2169,6 +2211,7 @@ def run_backtest( city: str | None = None, scattered: bool = False, seed: str = "mera", + region_code: int = DEFAULT_BACKTEST_REGION_CODE, ) -> dict[str, Any]: """Drive the full read-only backtest and return a metrics dict. @@ -2184,12 +2227,29 @@ def run_backtest( out-of-sample accuracy (see _derive_room_ratios). Pass ``holdout_split=True`` to fit on even-id deals and evaluate on the odd-id half for an honest number. + ``region_code`` (#3520, default 66) scopes the deal sample to one + ``deals.region_code`` — see ``_load_sample``. + ``city`` (oblast D, default None) scopes the deal sample to one ``deals.city`` value — see ``_load_sample``. Default None is unscoped (byte-identical to the pre-oblast-D behaviour). """ - deals = _load_sample(db, sample=sample, since=since, city=city, scattered=scattered, seed=seed) - logger.info("loaded sample: %d ДКП deals (since=%s, city=%s)", len(deals), since, city) + deals = _load_sample( + db, + sample=sample, + since=since, + city=city, + scattered=scattered, + seed=seed, + region_code=region_code, + ) + logger.info( + "loaded sample: %d ДКП deals (since=%s, city=%s, region=%d)", + len(deals), + since, + city, + region_code, + ) matched_rows: list[tuple[float, float, int]] = [] matched_ids: list[int] = [] @@ -2264,6 +2324,7 @@ def run_backtest_full( city: str | None = None, scattered: bool = False, seed: str = "mera", + region_code: int = DEFAULT_BACKTEST_REGION_CODE, ) -> dict[str, Any]: """Drive the FULL-spine read-only backtest and return a metrics dict (#1966). @@ -2289,6 +2350,14 @@ def run_backtest_full( ``house_id_resolution`` coverage block (resolved / total / imv_reachable) is attached to the returned metrics. Default False → byte-identical prior output. + ``region_code`` (#3520, default 66) scopes the deal sample to one + ``deals.region_code`` — see ``_load_sample``. Independently of this flag, + ``_predict_full_spine`` ALWAYS resolves each deal's OWN region from its + lat/lon via ``regions.region_for_point`` (mirroring prod's + ``estimate_quality`` region resolution) before calling + ``_fetch_dkp_corridor``, so the ДКП corridor is scoped correctly even if a + sample somehow mixed regions. + ``city`` (oblast D, default None) scopes the deal sample to one ``deals.city`` value — see ``_load_sample``. Default None is unscoped (byte-identical to the pre-oblast-D behaviour). Independently of this flag, @@ -2296,9 +2365,21 @@ def run_backtest_full( ``_fetch_dkp_corridor`` (oblast C2 parity fix) — see its docstring. """ est = _import_estimator_full() - deals = _load_sample(db, sample=sample, since=since, city=city, scattered=scattered, seed=seed) + deals = _load_sample( + db, + sample=sample, + since=since, + city=city, + scattered=scattered, + seed=seed, + region_code=region_code, + ) logger.info( - "loaded sample: %d ДКП deals (since=%s, city=%s) [full spine]", len(deals), since, city + "loaded sample: %d ДКП deals (since=%s, city=%s, region=%d) [full spine]", + len(deals), + since, + city, + region_code, ) predictions: list[Prediction] = [] @@ -2482,6 +2563,17 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Соль для --spread scattered. Тот же seed даёт ТУ ЖЕ выборку: два " "прогона подряд отличаются только правкой, а не составом выборки.", ) + p.add_argument( + "--region", + type=int, + default=DEFAULT_BACKTEST_REGION_CODE, + choices=sorted(REGIONS), + help="#3520 'любой регион': deals.region_code to backtest against (default " + f"{DEFAULT_BACKTEST_REGION_CODE} — Свердловская обл., unchanged behaviour). " + "Supported values come from the app.services.regions.REGIONS registry — " + + ", ".join(f"{code} ({REGIONS[code].name})" for code in sorted(REGIONS)) + + ". An unsupported code is rejected by argparse before any DB call.", + ) p.add_argument( "--city", default=None, @@ -2599,9 +2691,10 @@ def main(argv: list[str] | None = None) -> int: raise SystemExit("--calibrate-segments is only supported with --engine full") logger.info( - "backtest start: engine=%s sample=%d since=%s radius=%dm " + "backtest start: engine=%s region=%d sample=%d since=%s radius=%dm " "rooms_tolerance=%d holdout_split=%s dump_fixture=%s resolve_house_id=%s city=%s", args.engine, + args.region, args.sample, args.since, args.radius, @@ -2624,6 +2717,7 @@ def main(argv: list[str] | None = None) -> int: city=args.city, scattered=(args.spread == "scattered"), seed=args.seed, + region_code=args.region, ) else: metrics = run_backtest( @@ -2636,6 +2730,7 @@ def main(argv: list[str] | None = None) -> int: city=args.city, scattered=(args.spread == "scattered"), seed=args.seed, + region_code=args.region, ) finally: db.close() -- 2.45.3