diff --git a/tradein-mvp/backend/scripts/backtest_estimator.py b/tradein-mvp/backend/scripts/backtest_estimator.py index 2d7a348e..ffa1e65f 100644 --- a/tradein-mvp/backend/scripts/backtest_estimator.py +++ b/tradein-mvp/backend/scripts/backtest_estimator.py @@ -89,6 +89,7 @@ from __future__ import annotations import argparse import dataclasses +import gzip import json import logging import math @@ -1194,15 +1195,6 @@ def _sanitize_json(obj: Any) -> Any: return str(obj) -def _to_hashable(obj: Any) -> Any: - """Normalise a JSON-plain value into a hashable key (list→tuple, dict→sorted items).""" - if isinstance(obj, list | tuple): - return tuple(_to_hashable(x) for x in obj) - if isinstance(obj, dict): - return tuple(sorted((str(k), _to_hashable(v)) for k, v in obj.items())) - return obj - - def _coerce_ratio_return(ret: Any) -> tuple[Any, Any]: """Recorded ``[ratio, basis]`` → ``(ratio, basis)`` tuple (unpacked at the call site).""" return (ret[0], ret[1]) @@ -1218,30 +1210,50 @@ def _coerce_qis_return(ret: Any) -> dict[str, float]: return {str(k): v for k, v in dict(ret).items()} +def load_fixture(path: str) -> dict[str, Any]: + """Load a backtest fixture JSON — transparently handles gzip (``.gz``) files. + + The committed fixture is gzipped (~3 MB raw); a plain ``.json`` path is also + accepted for ad-hoc runs. Exported for the CI regression-gate test. + """ + if str(path).endswith(".gz"): + with gzip.open(path, "rt", encoding="utf-8") as fh: + return json.loads(fh.read()) + return json.loads(Path(path).read_text(encoding="utf-8")) + + def _make_call_stub( calls: list[Any], *, label: str, coerce: Callable[[Any], Any] ) -> Callable[[Any], Any]: - """Build an exact-match replay stub from recorded ``[arg, return]`` pairs. + """Build an ORDER-based (FIFO) replay stub from recorded ``[arg, return]`` pairs. - Each recorded entry is ``[serialized_single_arg, serialized_return]``. The - stub looks up the (normalised) positional arg and raises ``KeyError`` on an - unseen arg — that must never happen for a faithful fixture (the spine is - deterministic, so replay requests exactly the args capture recorded). - ``coerce`` maps the JSON-plain recorded return back to the live callable's + The stub returns the recorded RETURNS in invocation order and IGNORES the arg + value. Rationale: the DB-derived ratio / quarter-index is a FROZEN input — when + estimator logic later shifts e.g. ``median_ppm2``, the regression gate must + surface that as a clean metric diff vs the baseline, NOT mask it as a lookup + miss. Matching on the exact computed float arg is also not bit-stable across + platforms (libm last-ulp jitter), so a Linux-captured fixture would crash when + replayed off-Linux. The recorded arg is kept (for debuggability) but never + matched. Each callable fires ≤1×/deal today (so index 0 in practice); the FIFO + stays correct if a call site ever loops. Calling the stub MORE times than + recorded raises RuntimeError — control flow diverged from capture. + + ``coerce`` maps each JSON-plain recorded return back to the live callable's return type (tuple / dict) so unpacking at the call site behaves identically. """ - table: dict[Any, Any] = {} - for arg, ret in calls: - table[_to_hashable(_sanitize_json(arg))] = coerce(ret) + returns = [coerce(ret) for _arg, ret in calls] + idx = 0 - def _stub(arg: Any) -> Any: - key = _to_hashable(_sanitize_json(arg)) - if key not in table: - raise KeyError( - f"{label}: unseen arg {arg!r} during replay — fixture is not faithful " - f"(recorded {len(table)} call(s))" + def _stub(_arg: Any) -> Any: + nonlocal idx + if idx >= len(returns): + raise RuntimeError( + f"{label}: replay made call #{idx + 1} but fixture recorded only " + f"{len(returns)} — control flow diverged from capture" ) - return table[key] + ret = returns[idx] + idx += 1 + return ret return _stub @@ -1249,8 +1261,8 @@ def _make_call_stub( def replay_fixture(fixture: dict[str, Any]) -> dict[str, Any]: """Replay a frozen backtest fixture through the full spine — hermetic, ZERO DB. - For every captured deal record: rebuild the ``GeocodeResult``, build exact-match - stubs for the 3 DB callables from the recorded call-lists, call + For every captured deal record: rebuild the ``GeocodeResult``, build order-based + (FIFO) stubs for the 3 DB callables from the recorded call-lists, call ``_price_from_inputs`` with the frozen kwargs, and rebuild the ``Prediction`` EXACTLY as ``_predict_full_spine`` does. Runs the harness's own ``_compute_full_metrics`` over the replayed predictions plus a city-wide @@ -1847,9 +1859,9 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--from-fixture", metavar="PATH", default=None, - help="Hermetic replay: load the frozen fixture at PATH, re-run the spine " - "offline (ZERO DB) via replay_fixture, and print the metrics JSON. No DB " - "connection is opened.", + help="Hermetic replay: load the frozen fixture at PATH (gzip-transparent — " + "a .gz path is decompressed on the fly), re-run the spine offline (ZERO DB) " + "via replay_fixture, and print the metrics JSON. No DB connection is opened.", ) p.add_argument( "--update-baseline", @@ -1867,7 +1879,7 @@ def main(argv: list[str] | None = None) -> int: # ── Hermetic replay path (#1966 PR 3/3) — ZERO DB, no SessionLocal opened. ── if args.from_fixture: - fixture = json.loads(Path(args.from_fixture).read_text(encoding="utf-8")) + fixture = load_fixture(args.from_fixture) metrics = replay_fixture(fixture) print(json.dumps(metrics, ensure_ascii=False, indent=2, sort_keys=True)) if args.update_baseline: diff --git a/tradein-mvp/backend/tests/test_backtest_fixture_roundtrip.py b/tradein-mvp/backend/tests/test_backtest_fixture_roundtrip.py index 6b35d1d4..10adf67d 100644 --- a/tradein-mvp/backend/tests/test_backtest_fixture_roundtrip.py +++ b/tradein-mvp/backend/tests/test_backtest_fixture_roundtrip.py @@ -20,9 +20,14 @@ import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") +import gzip import json +import tempfile +from pathlib import Path from typing import Any +import pytest + from scripts import backtest_estimator as bt # --------------------------------------------------------------------------- # @@ -222,14 +227,54 @@ def test_replay_fixture_segments_span_multiple_bands() -> None: assert len(non_empty) >= 3 -def test_unseen_callable_arg_raises_keyerror() -> None: - # A fixture whose recorded ratio call does not match the median the spine - # actually computes must surface a clear KeyError (not a silent wrong answer). +def test_replay_is_arg_insensitive_order_based() -> None: + # Order-based (FIFO) replay returns the recorded ratio REGARDLESS of the arg + # value the spine actually computes — so a recorded arg that can never equal + # the live median (999_999.0) still replays cleanly. This is the cross-platform + # robustness contract: a Linux-captured fixture must replay off-Linux even when + # libm last-ulp jitter shifts the computed median_ppm2 by an ulp. fixture = _build_fixture() - fixture["deals"][0]["ratio_calls"] = [[999_999.0, [0.9, "per_rooms_all"]]] - try: + fixture["deals"][0]["ratio_calls"] = [[999_999.0, [0.95, "per_rooms_all"]]] + metrics = bt.replay_fixture(fixture) + # Replay succeeded (no KeyError) and still priced every deal. + assert metrics["expected_sold"]["overall"]["n"] == len(fixture["deals"]) + # The recorded ratio (0.95) was applied to deal 1 (median 100k → expected_sold + # ppm² 95k), so its signed error vs SOLD 100k is -5% — proving the recorded + # RETURN drove the result, not the (mismatched) arg. + econom = metrics["expected_sold"]["per_segment"]["эконом"] + assert econom["n"] == 1 + assert econom["median_bias_pct"] == pytest.approx(-5.0) + + +def test_replay_exhaustion_raises_runtime_error() -> None: + # If a callable's recorded list is SHORTER than the spine's call count (here: + # the spine calls ratio_resolver once but nothing is recorded), the exhaustion + # guard must raise a clear RuntimeError — control flow diverged from capture, + # NOT a silent wrong answer. + fixture = _build_fixture() + fixture["deals"][0]["ratio_calls"] = [] # spine will still request the ratio + with pytest.raises(RuntimeError) as exc: bt.replay_fixture(fixture) - except KeyError as exc: - assert "ratio_resolver" in str(exc) - else: - raise AssertionError("expected KeyError for an unrecorded ratio_resolver arg") + assert "ratio_resolver" in str(exc.value) + assert "diverged from capture" in str(exc.value) + + +def test_load_fixture_plain_and_gzip_roundtrip() -> None: + # load_fixture is gzip-transparent: a .gz path is decompressed, a plain .json + # path is read directly. Both must yield a replay-able fixture dict. + fixture = _build_fixture() + payload = json.dumps(fixture, ensure_ascii=False) + with tempfile.TemporaryDirectory() as d: + plain = Path(d) / "fx.json" + plain.write_text(payload, encoding="utf-8") + gz = Path(d) / "fx.json.gz" + with gzip.open(gz, "wt", encoding="utf-8") as fh: + fh.write(payload) + + loaded_plain = bt.load_fixture(str(plain)) + loaded_gz = bt.load_fixture(str(gz)) + assert loaded_plain == loaded_gz == fixture + # Both load paths produce identical replay metrics. + m_plain = bt.replay_fixture(loaded_plain) + m_gz = bt.replay_fixture(loaded_gz) + assert json.dumps(m_plain, sort_keys=True) == json.dumps(m_gz, sort_keys=True)