tradein-mvp/backend/scripts/backtest_estimator.py

@@ -134,6 +134,10 @@ USAGE
 
     # oblast D: per-city validation (exact deals.city name, not a slug):
     python -m scripts.backtest_estimator --city "Нижний Тагил" --sample 300
+
+    # #3520 любой регион (choices come from app.services.regions.REGIONS):
+    # Москва (region_code=77, 212 937 ДКП, 69 138 за последние 12 мес.):
+    python -m scripts.backtest_estimator --region 77 --since 2024-09-01 --sample 300
 """
 
 from __future__ import annotations
@@ -156,6 +160,15 @@ from typing import Any
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
+from app.services.regions import REGIONS, region_for_point
+
+# Регион по умолчанию — Свердловская обл., прежнее (единственное до #3520)
+# поведение скрипта. Список допустимых значений — РЕЕСТР REGIONS
+# (app/services/regions.py), а не отдельный литерал: регион появляется в
+# реестре один раз и автоматически становится доступен и здесь, и в
+# msk_raw_import, и в самом эстиматоре.
+DEFAULT_BACKTEST_REGION_CODE = 66
+
 
 def _import_estimator() -> tuple[Any, Any]:
     """Lazy import of the estimator's pure funcs (_filter_outliers, _percentile).
@@ -1153,6 +1166,7 @@ _SAMPLE_SQL = text(
         house_type
     FROM deals
     WHERE source = 'rosreestr'
+      AND region_code = CAST(:region_code AS int)
       AND geom IS NOT NULL
       AND price_per_m2 BETWEEN CAST(:ppm2_min AS numeric) AND CAST(:ppm2_max AS numeric)
       AND rooms IS NOT NULL
@@ -1204,6 +1218,7 @@ _SAMPLE_SQL_SCATTERED = text(
         house_type
     FROM deals
     WHERE source = 'rosreestr'
+      AND region_code = CAST(:region_code AS int)
       AND geom IS NOT NULL
       AND price_per_m2 BETWEEN CAST(:ppm2_min AS numeric) AND CAST(:ppm2_max AS numeric)
       AND rooms IS NOT NULL
@@ -1233,6 +1248,7 @@ _SAMPLE_SQL_SCATTERED_CITY = text(
         house_type
     FROM deals
     WHERE source = 'rosreestr'
+      AND region_code = CAST(:region_code AS int)
       AND geom IS NOT NULL
       AND price_per_m2 BETWEEN CAST(:ppm2_min AS numeric) AND CAST(:ppm2_max AS numeric)
       AND rooms IS NOT NULL
@@ -1271,10 +1287,9 @@ _CANDIDATES_SQL = text(
 # _fetch_dkp_corridor COALESCEs against (migration 178, key (region_code, city)
 # since migration 298 — #3051 "Москва"). Looked up only when --city is set (see
 # _resolve_city_ppm2_band); the default (city=None) path never issues this
-# query. region_code defaults to 66 (Свердловская обл.) — this script has no
-# region CLI flag yet, so every call is scoped to the oblast, matching every
-# existing --city invocation (byte-identical to the pre-298 unscoped lookup,
-# which only ever saw region-66 rows since the table was oblast-only before).
+# query. region_code (#3520 "любой регион") is threaded from the CLI --region
+# flag end-to-end now (default 66, Свердловская обл. — unchanged behaviour for
+# every pre-existing invocation, which never passed --region).
 _CITY_PPM2_BAND_SQL = text(
     """
     SELECT ppm2_min, ppm2_max
@@ -1288,10 +1303,13 @@ _CITY_PPM2_BAND_SQL = text(
 def _sample_sql(city: str | None, scattered: bool = False) -> Any:
     """ДКП deal-sample SELECT — optionally scoped to one ``deals.city`` (oblast D).
 
-    ``city is None`` (default) returns the SAME ``_SAMPLE_SQL`` object used
-    before this change — literal identity, not just equal text — so the
-    default CLI invocation (and the frozen EKB regression gate, which never
-    calls this path at all) see byte-identical SQL.
+    ``city is None`` (default) returns the SAME ``_SAMPLE_SQL`` object every
+    time — literal identity, not just equal text (asserted by
+    ``test_sample_sql_city_none_returns_original_object``) — so callers that
+    cache/compare the query object see a stable reference. All four SQL
+    variants (unscoped/scattered × city/no-city) carry a ``region_code``
+    predicate (#3520 "любой регион") bound from ``_load_sample``'s
+    ``region_code`` param, default 66.
 
     When ``city`` is set, an extra ``AND city = CAST(:city AS text)`` predicate
     scopes the sample to that ``deals.city`` value — populated oblast-wide by
@@ -1321,6 +1339,7 @@ def _sample_sql(city: str | None, scattered: bool = False) -> Any:
             house_type
         FROM deals
         WHERE source = 'rosreestr'
+          AND region_code = CAST(:region_code AS int)
           AND geom IS NOT NULL
           AND price_per_m2 BETWEEN CAST(:ppm2_min AS numeric) AND CAST(:ppm2_max AS numeric)
           AND rooms IS NOT NULL
@@ -1348,9 +1367,9 @@ def _resolve_city_ppm2_band(
     mirrors estimator._fetch_dkp_corridor's own COALESCE fallback) or any DB
     error → the globals; read-only best-effort, never raises.
 
-    ``region_code`` defaults to 66 (Свердловская обл.) — this script has no
-    region CLI flag yet (out of scope, #3051 sub-PR B); every existing
-    --city caller keeps its byte-identical lookup.
+    ``region_code`` defaults to 66 (Свердловская обл.) for backward-compat
+    direct callers; ``_load_sample`` (#3520) always passes the actual
+    ``--region`` value through explicitly.
     """
     if city is None:
         return float(PPM2_MIN), float(PPM2_MAX)
@@ -1376,22 +1395,32 @@ def _load_sample(
     city: str | None = None,
     scattered: bool = False,
     seed: str = "mera",
+    region_code: int = DEFAULT_BACKTEST_REGION_CODE,
 ) -> list[DealSample]:
     """Run the held-out ДКП deal sampling SELECT → list[DealSample].
 
+    ``region_code`` (#3520 "любой регион", default 66 — Свердловская обл.)
+    scopes the sample to one ``deals.region_code`` value — REQUIRED once
+    ``deals`` carries more than one region (212 937 Москва + 113 351
+    Московская обл. rows alongside 108 623 ЕКБ), otherwise the "unscoped"
+    query would silently mix regions together. Must be a key of
+    ``app.services.regions.REGIONS`` — validated by ``main()``/``_parse_args``
+    (CLI ``choices``) so this function only ever sees a supported code.
+
     ``city`` (oblast D, default None) scopes the sample to one ``deals.city``
     value via ``_sample_sql`` and sources the PPM2 sanity band from
-    ``deal_city_price_bands`` for that city (``_resolve_city_ppm2_band``,
-    falls back to the module globals). Default None is byte-identical to the
-    pre-oblast-D behaviour: same SQL object, same PPM2_MIN/PPM2_MAX globals,
-    no extra query.
+    ``deal_city_price_bands`` for that (region_code, city) pair
+    (``_resolve_city_ppm2_band``, falls back to the module globals). Default
+    None is byte-identical to the pre-oblast-D behaviour: same SQL object,
+    same PPM2_MIN/PPM2_MAX globals, no extra query.
     """
     if city is None:
         ppm2_min: float = PPM2_MIN
         ppm2_max: float = PPM2_MAX
     else:
-        ppm2_min, ppm2_max = _resolve_city_ppm2_band(db, city)
+        ppm2_min, ppm2_max = _resolve_city_ppm2_band(db, city, region_code=region_code)
     params: dict[str, Any] = {
+        "region_code": region_code,
         "ppm2_min": ppm2_min,
         "ppm2_max": ppm2_max,
         "since": since,
@@ -1947,8 +1976,21 @@ def _predict_full_spine(
     # harness measures the SAME corridor prod actually computes today, not the
     # pre-C2 unscoped behaviour — else the backtest validates stale semantics.
     target_city = m._resolve_target_city(deal.address)
+    # #3520 "любой регион": mirror estimate_quality's OWN region_code resolution
+    # (estimator.py ~4868-4870, region_for_point(geo.lat, geo.lon) with a
+    # DEFAULT_REGION_CODE fallback for points outside every registered bbox) —
+    # without this, _fetch_dkp_corridor's default (region 66) would scope a
+    # Москва/Московская обл. deal's corridor query to the WRONG region and
+    # silently return no rows (or, worse, an unrelated same-named street in 66).
+    target_region = region_for_point(deal.lat, deal.lon)
+    target_region_code = target_region.code if target_region else DEFAULT_BACKTEST_REGION_CODE
     dkp_raw = m._fetch_dkp_corridor(
-        db, address=deal.address, rooms=deal.rooms, area=deal.area_m2, city=target_city
+        db,
+        address=deal.address,
+        rooms=deal.rooms,
+        area=deal.area_m2,
+        city=target_city,
+        region_code=target_region_code,
     )
     # #1966 prod parity: same-building anchor pre-fetch is GATED exactly like
     # estimate_quality — no-area / no-address → ([], None) instead of an
@@ -2169,6 +2211,7 @@ def run_backtest(
     city: str | None = None,
     scattered: bool = False,
     seed: str = "mera",
+    region_code: int = DEFAULT_BACKTEST_REGION_CODE,
 ) -> dict[str, Any]:
     """Drive the full read-only backtest and return a metrics dict.
 
@@ -2184,12 +2227,29 @@ def run_backtest(
     out-of-sample accuracy (see _derive_room_ratios). Pass ``holdout_split=True``
     to fit on even-id deals and evaluate on the odd-id half for an honest number.
 
+    ``region_code`` (#3520, default 66) scopes the deal sample to one
+    ``deals.region_code`` — see ``_load_sample``.
+
     ``city`` (oblast D, default None) scopes the deal sample to one
     ``deals.city`` value — see ``_load_sample``. Default None is unscoped
     (byte-identical to the pre-oblast-D behaviour).
     """
-    deals = _load_sample(db, sample=sample, since=since, city=city, scattered=scattered, seed=seed)
-    logger.info("loaded sample: %d ДКП deals (since=%s, city=%s)", len(deals), since, city)
+    deals = _load_sample(
+        db,
+        sample=sample,
+        since=since,
+        city=city,
+        scattered=scattered,
+        seed=seed,
+        region_code=region_code,
+    )
+    logger.info(
+        "loaded sample: %d ДКП deals (since=%s, city=%s, region=%d)",
+        len(deals),
+        since,
+        city,
+        region_code,
+    )
 
     matched_rows: list[tuple[float, float, int]] = []
     matched_ids: list[int] = []
@@ -2264,6 +2324,7 @@ def run_backtest_full(
     city: str | None = None,
     scattered: bool = False,
     seed: str = "mera",
+    region_code: int = DEFAULT_BACKTEST_REGION_CODE,
 ) -> dict[str, Any]:
     """Drive the FULL-spine read-only backtest and return a metrics dict (#1966).
 
@@ -2289,6 +2350,14 @@ def run_backtest_full(
     ``house_id_resolution`` coverage block (resolved / total / imv_reachable) is
     attached to the returned metrics. Default False → byte-identical prior output.
 
+    ``region_code`` (#3520, default 66) scopes the deal sample to one
+    ``deals.region_code`` — see ``_load_sample``. Independently of this flag,
+    ``_predict_full_spine`` ALWAYS resolves each deal's OWN region from its
+    lat/lon via ``regions.region_for_point`` (mirroring prod's
+    ``estimate_quality`` region resolution) before calling
+    ``_fetch_dkp_corridor``, so the ДКП corridor is scoped correctly even if a
+    sample somehow mixed regions.
+
     ``city`` (oblast D, default None) scopes the deal sample to one
     ``deals.city`` value — see ``_load_sample``. Default None is unscoped
     (byte-identical to the pre-oblast-D behaviour). Independently of this flag,
@@ -2296,9 +2365,21 @@ def run_backtest_full(
     ``_fetch_dkp_corridor`` (oblast C2 parity fix) — see its docstring.
     """
     est = _import_estimator_full()
-    deals = _load_sample(db, sample=sample, since=since, city=city, scattered=scattered, seed=seed)
+    deals = _load_sample(
+        db,
+        sample=sample,
+        since=since,
+        city=city,
+        scattered=scattered,
+        seed=seed,
+        region_code=region_code,
+    )
     logger.info(
-        "loaded sample: %d ДКП deals (since=%s, city=%s) [full spine]", len(deals), since, city
+        "loaded sample: %d ДКП deals (since=%s, city=%s, region=%d) [full spine]",
+        len(deals),
+        since,
+        city,
+        region_code,
     )
 
     predictions: list[Prediction] = []
@@ -2482,6 +2563,17 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
         help="Соль для --spread scattered. Тот же seed даёт ТУ ЖЕ выборку: два "
         "прогона подряд отличаются только правкой, а не составом выборки.",
     )
+    p.add_argument(
+        "--region",
+        type=int,
+        default=DEFAULT_BACKTEST_REGION_CODE,
+        choices=sorted(REGIONS),
+        help="#3520 'любой регион': deals.region_code to backtest against (default "
+        f"{DEFAULT_BACKTEST_REGION_CODE} — Свердловская обл., unchanged behaviour). "
+        "Supported values come from the app.services.regions.REGIONS registry — "
+        + ", ".join(f"{code} ({REGIONS[code].name})" for code in sorted(REGIONS))
+        + ". An unsupported code is rejected by argparse before any DB call.",
+    )
     p.add_argument(
         "--city",
         default=None,
@@ -2599,9 +2691,10 @@ def main(argv: list[str] | None = None) -> int:
         raise SystemExit("--calibrate-segments is only supported with --engine full")
 
     logger.info(
-        "backtest start: engine=%s sample=%d since=%s radius=%dm "
+        "backtest start: engine=%s region=%d sample=%d since=%s radius=%dm "
         "rooms_tolerance=%d holdout_split=%s dump_fixture=%s resolve_house_id=%s city=%s",
         args.engine,
+        args.region,
         args.sample,
         args.since,
         args.radius,
@@ -2624,6 +2717,7 @@ def main(argv: list[str] | None = None) -> int:
                 city=args.city,
                 scattered=(args.spread == "scattered"),
                 seed=args.seed,
+                region_code=args.region,
             )
         else:
             metrics = run_backtest(
@@ -2636,6 +2730,7 @@ def main(argv: list[str] | None = None) -> int:
                 city=args.city,
                 scattered=(args.spread == "scattered"),
                 seed=args.seed,
+                region_code=args.region,
             )
     finally:
         db.close()