feat(tradein/backtest): frozen fixture + baseline + hermetic CI regression gate (#1966 PR 3/3)

Commits the gzipped 277-deal prod fixture (frozen _price_from_inputs inputs),
the frozen backtest_baseline.json (overall MAPE 18.63% / bias -1.2%; per-segment
бизнес -21.5% / элит -38.6% — the systemic expensive-segment underestimation,
now measured), and a hermetic pytest gate that replays the fixture through the
full pricing spine (ZERO DB) and asserts metrics == baseline within float tolerance.

Relative regression gate, not an absolute SLA (live coverage ~55% is data-blocked
per #1966). To change estimator behaviour: regenerate the baseline
(--from-fixture --update-baseline, no DB) and justify the per-segment deltas in the PR.
The fixture (gzipped prod inputs) re-extracts rarely; the per-change artifact is the
3 KB baseline. Large-file hook excludes the fixture (re-extracted only on ground-truth refresh).

Refs #1966

.pre-commit-config.yaml

@@ -17,6 +17,11 @@ repos:
       - id: check-toml
       - id: check-added-large-files
         args: ["--maxkb=512"]
+        # #1966: the frozen backtest regression-gate fixture is gzipped prod
+        # inputs (~3 MB). It is re-extracted rarely (only when ground-truth
+        # refreshes), so it does not bloat history per estimator change — the
+        # per-change artifact is the 3 KB backtest_baseline.json.
+        exclude: ^tradein-mvp/backend/tests/fixtures/backtest_full_fixture\.json\.gz$
       - id: check-merge-conflict
       - id: detect-private-key
 

tradein-mvp/backend/tests/fixtures/.gitattributes

@@ -0,0 +1,6 @@
+# #1966 PR 3/3 — frozen backtest regression-gate artifacts.
+# The fixture is gzipped binary: never apply CRLF/text filters (would corrupt it).
+backtest_full_fixture.json.gz binary
+# The baseline is the diff-visible artifact; keep it LF so `--update-baseline`
+# regen (which writes LF) never shows spurious line-ending churn.
+backtest_baseline.json text eol=lf

tradein-mvp/backend/tests/fixtures/backtest_baseline.json

@@ -0,0 +1,154 @@
+{
+  "calibration": {
+    "high": {
+      "coverage_pct": null,
+      "mape_pct": null,
+      "n": 0,
+      "n_covered": 0
+    },
+    "low": {
+      "coverage_pct": 55.27,
+      "mape_pct": 18.63,
+      "n": 275,
+      "n_covered": 152
+    },
+    "medium": {
+      "coverage_pct": 50.0,
+      "mape_pct": 19.18,
+      "n": 2,
+      "n_covered": 1
+    }
+  },
+  "confidence_order": [
+    "high",
+    "medium",
+    "low"
+  ],
+  "expected_sold": {
+    "overall": {
+      "mape_pct": 18.63,
+      "median_bias_pct": -1.2,
+      "n": 277,
+      "n_no_analogs": 0,
+      "p25_pct": -16.25,
+      "p75_pct": 20.26
+    },
+    "per_rooms": {
+      "0": {
+        "label": "студия",
+        "mape_pct": 27.97,
+        "median_bias_pct": 27.56,
+        "n": 37,
+        "n_no_analogs": 0,
+        "p25_pct": -8.97,
+        "p75_pct": 38.45
+      },
+      "1": {
+        "label": "1к",
+        "mape_pct": 19.55,
+        "median_bias_pct": -8.08,
+        "n": 93,
+        "n_no_analogs": 0,
+        "p25_pct": -21.27,
+        "p75_pct": 13.3
+      },
+      "2": {
+        "label": "2к",
+        "mape_pct": 16.22,
+        "median_bias_pct": -8.82,
+        "n": 74,
+        "n_no_analogs": 0,
+        "p25_pct": -21.01,
+        "p75_pct": 6.13
+      },
+      "3": {
+        "label": "3к",
+        "mape_pct": 10.52,
+        "median_bias_pct": 4.08,
+        "n": 43,
+        "n_no_analogs": 0,
+        "p25_pct": -6.54,
+        "p75_pct": 11.96
+      },
+      "4": {
+        "label": "4+",
+        "mape_pct": 23.53,
+        "median_bias_pct": 14.35,
+        "n": 30,
+        "n_no_analogs": 0,
+        "p25_pct": -0.72,
+        "p75_pct": 35.27
+      }
+    },
+    "per_segment": {
+      "бизнес": {
+        "mape_pct": 22.14,
+        "median_bias_pct": -21.49,
+        "n": 46,
+        "p25_pct": -28.22,
+        "p75_pct": -9.82
+      },
+      "комфорт": {
+        "mape_pct": 16.74,
+        "median_bias_pct": -8.05,
+        "n": 104,
+        "p25_pct": -20.63,
+        "p75_pct": 7.55
+      },
+      "премиум": {
+        "mape_pct": 59.37,
+        "median_bias_pct": -59.37,
+        "n": 1,
+        "p25_pct": -59.37,
+        "p75_pct": -59.37
+      },
+      "эконом": {
+        "mape_pct": 18.01,
+        "median_bias_pct": 17.17,
+        "n": 120,
+        "p25_pct": 1.73,
+        "p75_pct": 46.96
+      },
+      "элит": {
+        "mape_pct": 38.62,
+        "median_bias_pct": -38.62,
+        "n": 6,
+        "p25_pct": -47.98,
+        "p75_pct": -33.18
+      }
+    }
+  },
+  "headline": {
+    "ask_median_ppm2": 147545.8502510892,
+    "deal_median_ppm2": 125063.0,
+    "spread_pct": 17.98
+  },
+  "range_coverage": {
+    "overall": {
+      "coverage_pct": 55.23,
+      "n": 277,
+      "n_covered": 153
+    },
+    "per_confidence": {
+      "high": {
+        "coverage_pct": null,
+        "n": 0,
+        "n_covered": 0
+      },
+      "low": {
+        "coverage_pct": 55.27,
+        "n": 275,
+        "n_covered": 152
+      },
+      "medium": {
+        "coverage_pct": 50.0,
+        "n": 2,
+        "n_covered": 1
+      }
+    }
+  },
+  "sharpness": {
+    "median_rel_width": 0.4638,
+    "n": 277
+  }
+}

tradein-mvp/backend/tests/test_backtest_regression_gate.py

@@ -0,0 +1,84 @@
+"""Hermetic estimator regression gate (#1966 PR 3/3).
+
+Replays the committed frozen backtest fixture through the full pricing spine
+(``app.services.estimator._price_from_inputs``) with ZERO DB / network, recomputes
+the backtest metrics, and asserts they match the committed baseline. Any change to
+the spine, the metric code, or a config default that moves a metric beyond float
+jitter fails this test → regenerate the baseline deliberately:
+
+    cd tradein-mvp/backend
+    uv run python -m scripts.backtest_estimator \
+        --from-fixture tests/fixtures/backtest_full_fixture.json.gz \
+        --update-baseline tests/fixtures/backtest_baseline.json
+
+and justify the per-segment MAPE / coverage deltas in the PR. This is a RELATIVE
+regression gate, not an absolute SLA (live coverage ~55% is data-blocked, see #1966).
+
+The fixture is gzipped frozen prod inputs (opaque, rarely changes); the baseline is
+the small diff-visible artifact that surfaces accuracy movement right in the PR diff.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+from pathlib import Path
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
+
+from scripts.backtest_estimator import load_fixture, replay_fixture
+
+_FIXTURES = Path(__file__).parent / "fixtures"
+_FIXTURE_PATH = _FIXTURES / "backtest_full_fixture.json.gz"
+_BASELINE_PATH = _FIXTURES / "backtest_baseline.json"
+
+# Floats: a small relative+absolute tolerance absorbs cross-platform / Python
+# libm last-ulp jitter (the replay is otherwise deterministic). A real regression
+# moves a metric by orders of magnitude more than this, so it is still caught.
+_REL_TOL = 1e-6
+_ABS_TOL = 1e-6
+
+
+def _assert_match(path: str, expected: object, actual: object) -> None:
+    if isinstance(expected, dict):
+        assert isinstance(actual, dict), f"{path}: expected dict, got {type(actual).__name__}"
+        assert (
+            expected.keys() == actual.keys()
+        ), f"{path}: key set differs\n  expected={sorted(expected)}\n  actual=  {sorted(actual)}"
+        for k in expected:
+            _assert_match(f"{path}.{k}", expected[k], actual[k])
+    elif isinstance(expected, list):
+        assert isinstance(actual, list), f"{path}: expected list, got {type(actual).__name__}"
+        assert len(expected) == len(actual), f"{path}: list length {len(actual)} != {len(expected)}"
+        for i, (e, a) in enumerate(zip(expected, actual, strict=True)):
+            _assert_match(f"{path}[{i}]", e, a)
+    elif expected is None or isinstance(expected, bool):
+        assert actual is expected or actual == expected, f"{path}: {actual!r} != {expected!r}"
+    elif isinstance(expected, int):  # bool already handled above
+        assert actual == expected, f"{path}: int {actual!r} != {expected!r}"
+    elif isinstance(expected, float):
+        assert isinstance(actual, int | float), f"{path}: {type(actual).__name__} not numeric"
+        assert math.isclose(actual, expected, rel_tol=_REL_TOL, abs_tol=_ABS_TOL), (
+            f"{path}: {actual!r} != baseline {expected!r} (Δ={actual - expected:.3e}). "
+            f"The estimator/metrics changed — if intentional, regenerate the baseline "
+            f"(--from-fixture --update-baseline) and justify the deltas in the PR."
+        )
+    else:
+        assert actual == expected, f"{path}: {actual!r} != {expected!r}"
+
+
+def test_fixture_and_baseline_committed() -> None:
+    assert _FIXTURE_PATH.exists(), f"frozen fixture missing: {_FIXTURE_PATH}"
+    assert _BASELINE_PATH.exists(), f"frozen baseline missing: {_BASELINE_PATH}"
+
+
+def test_backtest_regression_gate() -> None:
+    fixture = load_fixture(_FIXTURE_PATH)
+    baseline = json.loads(_BASELINE_PATH.read_text(encoding="utf-8"))
+    # Round-trip the replay output through JSON before comparing: the committed
+    # baseline is JSON (string object keys), while replay_fixture returns native
+    # dicts whose per_rooms buckets are int keys (0..4). Round-tripping normalises
+    # key types to match — the same transform `--update-baseline` applies on write.
+    metrics = json.loads(json.dumps(replay_fixture(fixture), ensure_ascii=False))
+    _assert_match("metrics", baseline, metrics)