lekss361
ec9ed56fad
All checks were successful
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m25s
Deploy Trade-In / build-backend (push) Successful in 55s
Deploy Trade-In / deploy (push) Successful in 54s
84 lines
4.1 KiB
Python
84 lines
4.1 KiB
Python
"""Hermetic estimator regression gate (#1966 PR 3/3).
|
|
|
|
Replays the committed frozen backtest fixture through the full pricing spine
|
|
(``app.services.estimator._price_from_inputs``) with ZERO DB / network, recomputes
|
|
the backtest metrics, and asserts they match the committed baseline. Any change to
|
|
the spine, the metric code, or a config default that moves a metric beyond float
|
|
jitter fails this test → regenerate the baseline deliberately:
|
|
|
|
cd tradein-mvp/backend
|
|
uv run python -m scripts.backtest_estimator \
|
|
--from-fixture tests/fixtures/backtest_full_fixture.json.gz \
|
|
--update-baseline tests/fixtures/backtest_baseline.json
|
|
|
|
and justify the per-segment MAPE / coverage deltas in the PR. This is a RELATIVE
|
|
regression gate, not an absolute SLA (live coverage ~55% is data-blocked, see #1966).
|
|
|
|
The fixture is gzipped frozen prod inputs (opaque, rarely changes); the baseline is
|
|
the small diff-visible artifact that surfaces accuracy movement right in the PR diff.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
|
|
|
from scripts.backtest_estimator import load_fixture, replay_fixture
|
|
|
|
_FIXTURES = Path(__file__).parent / "fixtures"
|
|
_FIXTURE_PATH = _FIXTURES / "backtest_full_fixture.json.gz"
|
|
_BASELINE_PATH = _FIXTURES / "backtest_baseline.json"
|
|
|
|
# Floats: a small relative+absolute tolerance absorbs cross-platform / Python
|
|
# libm last-ulp jitter (the replay is otherwise deterministic). A real regression
|
|
# moves a metric by orders of magnitude more than this, so it is still caught.
|
|
_REL_TOL = 1e-6
|
|
_ABS_TOL = 1e-6
|
|
|
|
|
|
def _assert_match(path: str, expected: object, actual: object) -> None:
|
|
if isinstance(expected, dict):
|
|
assert isinstance(actual, dict), f"{path}: expected dict, got {type(actual).__name__}"
|
|
assert (
|
|
expected.keys() == actual.keys()
|
|
), f"{path}: key set differs\n expected={sorted(expected)}\n actual= {sorted(actual)}"
|
|
for k in expected:
|
|
_assert_match(f"{path}.{k}", expected[k], actual[k])
|
|
elif isinstance(expected, list):
|
|
assert isinstance(actual, list), f"{path}: expected list, got {type(actual).__name__}"
|
|
assert len(expected) == len(actual), f"{path}: list length {len(actual)} != {len(expected)}"
|
|
for i, (e, a) in enumerate(zip(expected, actual, strict=True)):
|
|
_assert_match(f"{path}[{i}]", e, a)
|
|
elif expected is None or isinstance(expected, bool):
|
|
assert actual is expected or actual == expected, f"{path}: {actual!r} != {expected!r}"
|
|
elif isinstance(expected, int): # bool already handled above
|
|
assert actual == expected, f"{path}: int {actual!r} != {expected!r}"
|
|
elif isinstance(expected, float):
|
|
assert isinstance(actual, int | float), f"{path}: {type(actual).__name__} not numeric"
|
|
assert math.isclose(actual, expected, rel_tol=_REL_TOL, abs_tol=_ABS_TOL), (
|
|
f"{path}: {actual!r} != baseline {expected!r} (Δ={actual - expected:.3e}). "
|
|
f"The estimator/metrics changed — if intentional, regenerate the baseline "
|
|
f"(--from-fixture --update-baseline) and justify the deltas in the PR."
|
|
)
|
|
else:
|
|
assert actual == expected, f"{path}: {actual!r} != {expected!r}"
|
|
|
|
|
|
def test_fixture_and_baseline_committed() -> None:
|
|
assert _FIXTURE_PATH.exists(), f"frozen fixture missing: {_FIXTURE_PATH}"
|
|
assert _BASELINE_PATH.exists(), f"frozen baseline missing: {_BASELINE_PATH}"
|
|
|
|
|
|
def test_backtest_regression_gate() -> None:
|
|
fixture = load_fixture(_FIXTURE_PATH)
|
|
baseline = json.loads(_BASELINE_PATH.read_text(encoding="utf-8"))
|
|
# Round-trip the replay output through JSON before comparing: the committed
|
|
# baseline is JSON (string object keys), while replay_fixture returns native
|
|
# dicts whose per_rooms buckets are int keys (0..4). Round-tripping normalises
|
|
# key types to match — the same transform `--update-baseline` applies on write.
|
|
metrics = json.loads(json.dumps(replay_fixture(fixture), ensure_ascii=False))
|
|
_assert_match("metrics", baseline, metrics)
|