diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f085b541..3175e0db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,11 @@ repos: - id: check-toml - id: check-added-large-files args: ["--maxkb=512"] + # #1966: the frozen backtest regression-gate fixture is gzipped prod + # inputs (~3 MB). It is re-extracted rarely (only when ground-truth + # refreshes), so it does not bloat history per estimator change — the + # per-change artifact is the 3 KB backtest_baseline.json. + exclude: ^tradein-mvp/backend/tests/fixtures/backtest_full_fixture\.json\.gz$ - id: check-merge-conflict - id: detect-private-key diff --git a/tradein-mvp/backend/tests/fixtures/.gitattributes b/tradein-mvp/backend/tests/fixtures/.gitattributes new file mode 100644 index 00000000..5047b712 --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/.gitattributes @@ -0,0 +1,6 @@ +# #1966 PR 3/3 — frozen backtest regression-gate artifacts. +# The fixture is gzipped binary: never apply CRLF/text filters (would corrupt it). +backtest_full_fixture.json.gz binary +# The baseline is the diff-visible artifact; keep it LF so `--update-baseline` +# regen (which writes LF) never shows spurious line-ending churn. +backtest_baseline.json text eol=lf diff --git a/tradein-mvp/backend/tests/fixtures/backtest_baseline.json b/tradein-mvp/backend/tests/fixtures/backtest_baseline.json new file mode 100644 index 00000000..56b96a3e --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/backtest_baseline.json @@ -0,0 +1,154 @@ +{ + "calibration": { + "high": { + "coverage_pct": null, + "mape_pct": null, + "n": 0, + "n_covered": 0 + }, + "low": { + "coverage_pct": 55.27, + "mape_pct": 18.63, + "n": 275, + "n_covered": 152 + }, + "medium": { + "coverage_pct": 50.0, + "mape_pct": 19.18, + "n": 2, + "n_covered": 1 + } + }, + "confidence_order": [ + "high", + "medium", + "low" + ], + "expected_sold": { + "overall": { + "mape_pct": 18.63, + "median_bias_pct": -1.2, + "n": 277, + "n_no_analogs": 0, + "p25_pct": -16.25, + "p75_pct": 20.26 + }, + "per_rooms": { + "0": { + "label": "студия", + "mape_pct": 27.97, + "median_bias_pct": 27.56, + "n": 37, + "n_no_analogs": 0, + "p25_pct": -8.97, + "p75_pct": 38.45 + }, + "1": { + "label": "1к", + "mape_pct": 19.55, + "median_bias_pct": -8.08, + "n": 93, + "n_no_analogs": 0, + "p25_pct": -21.27, + "p75_pct": 13.3 + }, + "2": { + "label": "2к", + "mape_pct": 16.22, + "median_bias_pct": -8.82, + "n": 74, + "n_no_analogs": 0, + "p25_pct": -21.01, + "p75_pct": 6.13 + }, + "3": { + "label": "3к", + "mape_pct": 10.52, + "median_bias_pct": 4.08, + "n": 43, + "n_no_analogs": 0, + "p25_pct": -6.54, + "p75_pct": 11.96 + }, + "4": { + "label": "4+", + "mape_pct": 23.53, + "median_bias_pct": 14.35, + "n": 30, + "n_no_analogs": 0, + "p25_pct": -0.72, + "p75_pct": 35.27 + } + }, + "per_segment": { + "бизнес": { + "mape_pct": 22.14, + "median_bias_pct": -21.49, + "n": 46, + "p25_pct": -28.22, + "p75_pct": -9.82 + }, + "комфорт": { + "mape_pct": 16.74, + "median_bias_pct": -8.05, + "n": 104, + "p25_pct": -20.63, + "p75_pct": 7.55 + }, + "премиум": { + "mape_pct": 59.37, + "median_bias_pct": -59.37, + "n": 1, + "p25_pct": -59.37, + "p75_pct": -59.37 + }, + "эконом": { + "mape_pct": 18.01, + "median_bias_pct": 17.17, + "n": 120, + "p25_pct": 1.73, + "p75_pct": 46.96 + }, + "элит": { + "mape_pct": 38.62, + "median_bias_pct": -38.62, + "n": 6, + "p25_pct": -47.98, + "p75_pct": -33.18 + } + } + }, + "headline": { + "ask_median_ppm2": 147545.8502510892, + "deal_median_ppm2": 125063.0, + "spread_pct": 17.98 + }, + "range_coverage": { + "overall": { + "coverage_pct": 55.23, + "n": 277, + "n_covered": 153 + }, + "per_confidence": { + "high": { + "coverage_pct": null, + "n": 0, + "n_covered": 0 + }, + "low": { + "coverage_pct": 55.27, + "n": 275, + "n_covered": 152 + }, + "medium": { + "coverage_pct": 50.0, + "n": 2, + "n_covered": 1 + } + } + }, + "sharpness": { + "median_rel_width": 0.4638, + "n": 277 + } +} diff --git a/tradein-mvp/backend/tests/fixtures/backtest_full_fixture.json.gz b/tradein-mvp/backend/tests/fixtures/backtest_full_fixture.json.gz new file mode 100644 index 00000000..dff67155 Binary files /dev/null and b/tradein-mvp/backend/tests/fixtures/backtest_full_fixture.json.gz differ diff --git a/tradein-mvp/backend/tests/test_backtest_regression_gate.py b/tradein-mvp/backend/tests/test_backtest_regression_gate.py new file mode 100644 index 00000000..15c19299 --- /dev/null +++ b/tradein-mvp/backend/tests/test_backtest_regression_gate.py @@ -0,0 +1,84 @@ +"""Hermetic estimator regression gate (#1966 PR 3/3). + +Replays the committed frozen backtest fixture through the full pricing spine +(``app.services.estimator._price_from_inputs``) with ZERO DB / network, recomputes +the backtest metrics, and asserts they match the committed baseline. Any change to +the spine, the metric code, or a config default that moves a metric beyond float +jitter fails this test → regenerate the baseline deliberately: + + cd tradein-mvp/backend + uv run python -m scripts.backtest_estimator \ + --from-fixture tests/fixtures/backtest_full_fixture.json.gz \ + --update-baseline tests/fixtures/backtest_baseline.json + +and justify the per-segment MAPE / coverage deltas in the PR. This is a RELATIVE +regression gate, not an absolute SLA (live coverage ~55% is data-blocked, see #1966). + +The fixture is gzipped frozen prod inputs (opaque, rarely changes); the baseline is +the small diff-visible artifact that surfaces accuracy movement right in the PR diff. +""" + +from __future__ import annotations + +import json +import math +import os +from pathlib import Path + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from scripts.backtest_estimator import load_fixture, replay_fixture + +_FIXTURES = Path(__file__).parent / "fixtures" +_FIXTURE_PATH = _FIXTURES / "backtest_full_fixture.json.gz" +_BASELINE_PATH = _FIXTURES / "backtest_baseline.json" + +# Floats: a small relative+absolute tolerance absorbs cross-platform / Python +# libm last-ulp jitter (the replay is otherwise deterministic). A real regression +# moves a metric by orders of magnitude more than this, so it is still caught. +_REL_TOL = 1e-6 +_ABS_TOL = 1e-6 + + +def _assert_match(path: str, expected: object, actual: object) -> None: + if isinstance(expected, dict): + assert isinstance(actual, dict), f"{path}: expected dict, got {type(actual).__name__}" + assert ( + expected.keys() == actual.keys() + ), f"{path}: key set differs\n expected={sorted(expected)}\n actual= {sorted(actual)}" + for k in expected: + _assert_match(f"{path}.{k}", expected[k], actual[k]) + elif isinstance(expected, list): + assert isinstance(actual, list), f"{path}: expected list, got {type(actual).__name__}" + assert len(expected) == len(actual), f"{path}: list length {len(actual)} != {len(expected)}" + for i, (e, a) in enumerate(zip(expected, actual, strict=True)): + _assert_match(f"{path}[{i}]", e, a) + elif expected is None or isinstance(expected, bool): + assert actual is expected or actual == expected, f"{path}: {actual!r} != {expected!r}" + elif isinstance(expected, int): # bool already handled above + assert actual == expected, f"{path}: int {actual!r} != {expected!r}" + elif isinstance(expected, float): + assert isinstance(actual, int | float), f"{path}: {type(actual).__name__} not numeric" + assert math.isclose(actual, expected, rel_tol=_REL_TOL, abs_tol=_ABS_TOL), ( + f"{path}: {actual!r} != baseline {expected!r} (Δ={actual - expected:.3e}). " + f"The estimator/metrics changed — if intentional, regenerate the baseline " + f"(--from-fixture --update-baseline) and justify the deltas in the PR." + ) + else: + assert actual == expected, f"{path}: {actual!r} != {expected!r}" + + +def test_fixture_and_baseline_committed() -> None: + assert _FIXTURE_PATH.exists(), f"frozen fixture missing: {_FIXTURE_PATH}" + assert _BASELINE_PATH.exists(), f"frozen baseline missing: {_BASELINE_PATH}" + + +def test_backtest_regression_gate() -> None: + fixture = load_fixture(_FIXTURE_PATH) + baseline = json.loads(_BASELINE_PATH.read_text(encoding="utf-8")) + # Round-trip the replay output through JSON before comparing: the committed + # baseline is JSON (string object keys), while replay_fixture returns native + # dicts whose per_rooms buckets are int keys (0..4). Round-tripping normalises + # key types to match — the same transform `--update-baseline` applies on write. + metrics = json.loads(json.dumps(replay_fixture(fixture), ensure_ascii=False)) + _assert_match("metrics", baseline, metrics)