From cc66f518631dd985fc4e2576f408fb0e44181ef4 Mon Sep 17 00:00:00 2001 From: Light1YT Date: Thu, 4 Jun 2026 16:24:08 +0500 Subject: [PATCH] =?UTF-8?q?fix(backtest):=20add=20binomial=20significance?= =?UTF-8?q?=20gate=20to=20=C2=A79.6=20verdict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OOS verdict flagged a variant 'candidate to promote' on hit-rate >= 0.5+margin + lag_stable alone. On thin data this over-claims: Source A Almon-ADL scored 6/10 (0.60) lag-stable and was flagged as signal, but P(X>=6|10,0.5)~=0.377 -- a coin flip. Live ground-truth confirmed no signal (full-sample R2~=0.003, wrong sign). Add exact stdlib-only one-sided binomial _binom_sf_ge + _VERDICT_ALPHA=0.05 and require P(X>=hits|n_test,0.5) < alpha in both verdict() and cross_source_verdict() on top of the effect-size margin. hits recovered exactly as round(hit_rate*n_test) (n_test==scored invariant; no evaluator shape change). Verdict text now states n_test + the binomial p on pass and fail. Evaluator/estimator math and the read-only SELECT discipline untouched. Refs #978. --- backend/scripts/backtest_rate_sensitivity.py | 102 +++++++++++++-- .../scripts/test_backtest_rate_sensitivity.py | 123 +++++++++++++++++- 2 files changed, 212 insertions(+), 13 deletions(-) diff --git a/backend/scripts/backtest_rate_sensitivity.py b/backend/scripts/backtest_rate_sensitivity.py index 93e75371..5a61c6b0 100644 --- a/backend/scripts/backtest_rate_sensitivity.py +++ b/backend/scripts/backtest_rate_sensitivity.py @@ -188,6 +188,14 @@ _MIN_BACKTEST_MONTHS: int = 18 # tiny test window can't flip the verdict on one lucky month. _VERDICT_HITRATE_MARGIN: float = 0.05 +# One-sided significance level for the verdict's binomial gate. The hit-rate must +# be statistically distinguishable from a fair coin (p=0.5) at this α before the +# verdict promotes — i.e. P(X ≥ hits | n_test, 0.5) < _VERDICT_ALPHA. This is a +# HARD gate on top of the effect-size margin above: it prevents promoting on THIN +# data where a high hit-rate is consistent with chance (the #978 near-miss: +# 6/10 = 0.60 has one-sided p ≈ 0.377, indistinguishable from a coin flip). +_VERDICT_ALPHA: float = 0.05 + # Source B premise filter — residential квартиры, the only segment §9.6 scores # (mirrors sales_series._DEFAULT_PREMISE_KIND). _PREMISE_KIND: str = "квартира" @@ -368,6 +376,35 @@ def _round_or_none(value: float | None, digits: int) -> float | None: # --------------------------------------------------------------------------- # +def _binom_sf_ge(k: int, n: int, p: float = 0.5) -> float: + """Exact one-sided binomial survival: P(X ≥ k) for X ~ Binomial(n, p). PURE. + + Computed EXACTLY via ``math.comb`` (stdlib only — NO scipy/statsmodels, to + mirror the §9.6 engine's no-heavy-deps discipline): + ``Σ_{i=k..n} C(n, i) · p^i · (1−p)^(n−i)``. + + This is the probability the observed directional hit count (or anything more + extreme) arises by chance from a fair coin — small ⇒ the hit-rate is real + signal, not luck. It is the verdict's significance gate (see ``_VERDICT_ALPHA``). + + Guards (return 1.0 = "no evidence against the null"): + • ``n <= 0`` → 1.0 (no trials, nothing to distinguish); + • ``k`` clamped to ``[0, n]``; + • ``k <= 0`` → 1.0 (P(X ≥ 0) = 1 trivially). + """ + if n <= 0: + return 1.0 + k = max(0, min(k, n)) + if k <= 0: + return 1.0 + q = 1.0 - p + total = 0.0 + for i in range(k, n + 1): + total += math.comb(n, i) * (p**i) * (q ** (n - i)) + # Clamp tiny floating-point overshoot — a probability can't exceed 1.0. + return min(1.0, total) + + def _rate_first_diff(rate_levels: list[float | None]) -> list[float | None]: """First difference of the key_rate level series: out[t] = r_t − r_{t-1}. @@ -880,16 +917,26 @@ def verdict( ) -> dict[str, Any]: """Decide whether the EKB-wide tier shows OOS predictive value. PURE. - The engine is a promotion CANDIDATE when, on the EKB-wide tier: + The engine is a promotion CANDIDATE when, on the EKB-wide tier, ALL hold: • a gated lag was found and scored on a non-empty test window, AND • the OOS directional hit-rate beats the 0.5 coin-flip baseline by at - least ``margin``, AND + least ``margin`` (minimum EFFECT SIZE), AND + • the hit-rate is STATISTICALLY SIGNIFICANT vs a fair coin: the exact + one-sided binomial ``P(X ≥ hits | n_test, 0.5) < _VERDICT_ALPHA`` (a HARD + gate — a high rate on a tiny window is consistent with chance and must + NOT promote; this is the #978 near-miss: 6/10 = 0.60 has p ≈ 0.377), AND • the winning lag is the same on TRAIN and on the full-sample refit (lag stability — a lag that jumps between windows is not a signal). + ``hits`` (the integer directional-hit count the binomial needs) is recovered + as ``round(oos_hit_rate * n_test)``: both the hit-rate and n_test derive from + the SAME integer division in ``evaluate_oos`` (``hits / scored`` with + ``n_test == scored``), so this round-trips EXACTLY without threading a new + field through the deep-reviewed evaluator return shape. + Returns ``{"promote": bool, "reason": str, "thin_warning": str | None}``. - Honest: if the OOS test window is tiny the reason says so even when the - hit-rate happens to clear the bar. + Honest: if the OOS test window is tiny the reason says so, and significance + is now a HARD gate, not just an advisory caveat. """ if ekb.skipped is not None: return { @@ -908,6 +955,11 @@ def verdict( } beats_coin = ekb.oos_hit_rate >= 0.5 + margin + # Recover the integer hit count exactly (see docstring) and test significance. + hits = round(ekb.oos_hit_rate * ekb.n_test) + p_value = _binom_sf_ge(hits, ekb.n_test, 0.5) + significant = p_value < _VERDICT_ALPHA + thin_warning: str | None = None if ekb.n_test < min(_MIN_BACKTEST_MONTHS // 2, 6): thin_warning = ( @@ -915,11 +967,13 @@ def verdict( "confidence is weak — treat the verdict as indicative, not proof." ) - if beats_coin and ekb.lag_stable: + if beats_coin and ekb.lag_stable and significant: reason = ( f"engine has OOS predictive value (candidate to promote from " - f"advisory): EKB-wide OOS hit-rate={ekb.oos_hit_rate:.2f} > " - f"0.5+{margin:.2f} and lag stable (lag={ekb.train_lag})" + f"advisory): EKB-wide OOS hit-rate={ekb.oos_hit_rate:.2f} over " + f"n_test={ekb.n_test} > 0.5+{margin:.2f}, lag stable " + f"(lag={ekb.train_lag}), and significant " + f"(one-sided binomial p={p_value:.3f} < {_VERDICT_ALPHA:.2f})" ) return {"promote": True, "reason": reason, "thin_warning": thin_warning} @@ -928,6 +982,13 @@ def verdict( bits.append(f"hit-rate={ekb.oos_hit_rate:.2f} ≤ 0.5+{margin:.2f}") if not ekb.lag_stable: bits.append(f"lag unstable (train={ekb.train_lag}, full={ekb.full_sample_lag})") + # Significance reported whenever the effect size cleared the bar but the + # window is too thin to rule out chance — the #978 transparency requirement. + if beats_coin and not significant: + bits.append( + f"hit-rate={ekb.oos_hit_rate:.2f} over n_test={ekb.n_test} is not " + f"significant (one-sided binomial p={p_value:.2f} ≥ {_VERDICT_ALPHA:.2f})" + ) reason = "insufficient OOS signal — keep advisory (" + "; ".join(bits) + ")" return {"promote": False, "reason": reason, "thin_warning": thin_warning} @@ -1372,7 +1433,15 @@ def cross_source_verdict( label = _variant_label_for_run(run) hr = ekb.oos_hit_rate scorable = ekb.skipped is None and hr is not None and ekb.n_test >= 1 - beats = bool(scorable and hr is not None and hr >= 0.5 + margin and ekb.lag_stable) + beats_margin = bool(scorable and hr is not None and hr >= 0.5 + margin) + # Significance gate (mirrors verdict()): recover the integer hit count + # exactly from the rate × window (same integer division in evaluate_oos) + # and require the one-sided binomial p < _VERDICT_ALPHA. A high hit-rate + # on a thin window is NOT a signal — the #978 6/10 near-miss. + hits = round(hr * ekb.n_test) if scorable and hr is not None else 0 + p_value = _binom_sf_ge(hits, ekb.n_test, 0.5) if scorable else 1.0 + significant = p_value < _VERDICT_ALPHA + beats = bool(beats_margin and ekb.lag_stable and significant) thin = scorable and ekb.n_test < min(min_months // 2, 6) if beats: signal_variants.append(label) @@ -1389,6 +1458,8 @@ def cross_source_verdict( "oos_hit_rate": _round_or_none(hr, 4), "n_test": ekb.n_test, "lag_stable": ekb.lag_stable, + "binom_p": _round_or_none(p_value, 4) if scorable else None, + "significant": significant, "beats_coin": beats, "skipped": ekb.skipped, } @@ -1409,7 +1480,20 @@ def cross_source_verdict( why = r["skipped"] or "no gated lag / empty test window" lines.append(f" {r['variant']:<{label_w}} → not scorable ({why})") else: - tag = "SIGNAL > coin-flip" if r["beats_coin"] else "no signal (≤ coin-flip)" + # Spell out WHY a variant does/doesn't count as signal — including the + # binomial p so a "high rate but thin" row is transparent (#978). + if r["beats_coin"]: + tag = ( + f"SIGNAL > coin-flip (binomial p={_fmt_rate(r['binom_p'])} " + f"< {_VERDICT_ALPHA:.2f})" + ) + elif not r["significant"]: + tag = ( + f"no signal (hit-rate not significant: binomial " + f"p={_fmt_rate(r['binom_p'])} ≥ {_VERDICT_ALPHA:.2f})" + ) + else: + tag = "no signal (≤ coin-flip / lag unstable)" lines.append( f" {r['variant']:<{label_w}} → OOS_hit={_fmt_rate(r['oos_hit_rate'])} " f"(n_test={r['n_test']}, lag_stable={'yes' if r['lag_stable'] else 'no'}) " diff --git a/backend/tests/scripts/test_backtest_rate_sensitivity.py b/backend/tests/scripts/test_backtest_rate_sensitivity.py index 2fe9a3bb..570f0402 100644 --- a/backend/tests/scripts/test_backtest_rate_sensitivity.py +++ b/backend/tests/scripts/test_backtest_rate_sensitivity.py @@ -244,6 +244,46 @@ def _seasonal_units( return [base * fac[m.month] for m in months] +# --------------------------------------------------------------------------- # +# _binom_sf_ge — exact one-sided binomial survival (verdict significance gate) +# --------------------------------------------------------------------------- # + + +class TestBinomSfGe: + def test_known_values(self) -> None: + # The #978 near-miss: 6/10 heads is NOT distinguishable from a fair coin. + assert math.isclose(bt._binom_sf_ge(6, 10, 0.5), 0.376953125, abs_tol=1e-9) + # A clearly-significant tail. + assert math.isclose(bt._binom_sf_ge(9, 10, 0.5), 0.0107421875, abs_tol=1e-9) + # 5/5 perfect over a tiny window is just barely significant (p < 0.05). + assert math.isclose(bt._binom_sf_ge(5, 5, 0.5), 0.03125, abs_tol=1e-12) + + def test_k_zero_or_below_is_one(self) -> None: + # P(X ≥ 0) = 1 trivially; negative k clamps to 0 → 1.0. + assert bt._binom_sf_ge(0, 10, 0.5) == 1.0 + assert bt._binom_sf_ge(-3, 10, 0.5) == 1.0 + + def test_n_zero_returns_one(self) -> None: + # No trials → no evidence against the null → 1.0 (never promotes). + assert bt._binom_sf_ge(3, 0, 0.5) == 1.0 + assert bt._binom_sf_ge(0, 0, 0.5) == 1.0 + + def test_k_clamped_to_n(self) -> None: + # k > n clamps to n → P(X ≥ n) = p^n (only the all-success term). + assert math.isclose(bt._binom_sf_ge(20, 10, 0.5), 0.5**10, abs_tol=1e-12) + # k == n → exactly the all-success probability. + assert math.isclose(bt._binom_sf_ge(4, 4, 0.5), 0.0625, abs_tol=1e-12) + + def test_full_distribution_sums_to_one(self) -> None: + # P(X ≥ 0) over all i must be 1 for any n (sanity on the comb sum). + for n in (1, 3, 7, 12, 35): + assert math.isclose(bt._binom_sf_ge(0, n, 0.5), 1.0, abs_tol=1e-9) + + def test_non_half_p(self) -> None: + # Works for p ≠ 0.5: P(X ≥ 1 | n=2, p=0.1) = 1 − (0.9)^2 = 0.19. + assert math.isclose(bt._binom_sf_ge(1, 2, 0.1), 0.19, abs_tol=1e-12) + + # --------------------------------------------------------------------------- # # _time_ordered_split # --------------------------------------------------------------------------- # @@ -1062,10 +1102,15 @@ def _tier( class TestVerdict: - def test_promote_when_beats_coin_and_lag_stable(self) -> None: - vd = bt.verdict(_tier(oos_hit_rate=0.75, lag_stable=True)) + def test_promote_when_beats_coin_and_lag_stable_and_significant(self) -> None: + # hit-rate clears 0.5+margin, lag stable, AND a wide-enough window makes it + # statistically significant: hits=round(0.71·35)=25, P(X≥25|35)≈0.008<0.05. + vd = bt.verdict(_tier(oos_hit_rate=0.71, n_test=35, n_train=80, lag_stable=True)) assert vd["promote"] is True assert "OOS predictive value" in vd["reason"] + # The promote message exposes the significance p (#978 transparency). + assert "binomial p=" in vd["reason"] + assert "n_test=35" in vd["reason"] def test_keep_advisory_when_at_coin_flip(self) -> None: vd = bt.verdict(_tier(oos_hit_rate=0.52, lag_stable=True)) # ≤ 0.5+margin @@ -1086,12 +1131,45 @@ class TestVerdict: vd = bt.verdict(_tier(oos_hit_rate=None)) assert vd["promote"] is False - def test_thin_warning_set_for_small_test_window(self) -> None: - vd = bt.verdict(_tier(oos_hit_rate=0.9, n_test=3, lag_stable=True)) + def test_does_not_promote_six_of_ten_not_significant(self) -> None: + # REGRESSION GUARD — the exact #978 near-miss. Source A Almon-ADL scored + # oos_hit_rate=0.60 with n_test=10 (6/10) and lag_stable. The OLD rule + # (hit-rate ≥ 0.5+margin AND lag_stable) over-claimed "candidate to + # promote". But P(X≥6|10, 0.5)≈0.377 ≥ 0.05 — indistinguishable from a + # coin flip. The significance gate MUST keep it advisory. + vd = bt.verdict(_tier(oos_hit_rate=0.60, n_test=10, n_train=23, lag_stable=True)) + assert vd["promote"] is False + assert "keep advisory" in vd["reason"] + assert "not significant" in vd["reason"] + # The message names n_test and the binomial p so the WHY is transparent. + assert "n_test=10" in vd["reason"] + assert "p=0.38" in vd["reason"] + + def test_small_n_perfect_score_does_not_promote(self) -> None: + # A tiny window at 100% still can't promote: P(X≥4|4, 0.5)=0.0625 ≥ 0.05. + # Proves a perfect-but-thin run is not enough to clear significance. + vd = bt.verdict(_tier(oos_hit_rate=1.0, n_test=4, n_train=10, lag_stable=True)) + assert vd["promote"] is False + assert "not significant" in vd["reason"] + assert "n_test=4" in vd["reason"] + + def test_thin_warning_set_but_significant_still_promotes(self) -> None: + # A small window (n_test=5 < 6) sets the thin_warning, but 5/5 is the + # smallest perfect window that IS significant: P(X≥5|5, 0.5)=0.03125<0.05. + # So it promotes AND carries the thin caveat — the caveat is advisory, + # significance is the hard gate. + vd = bt.verdict(_tier(oos_hit_rate=1.0, n_test=5, n_train=13, lag_stable=True)) assert vd["promote"] is True assert vd["thin_warning"] is not None assert "small" in vd["thin_warning"] + def test_thin_window_high_rate_blocked_by_significance(self) -> None: + # The original "thin window" scenario (hit-rate=0.9, n_test=3): under the + # stricter rule it does NOT promote — hits=round(2.7)=3, P(X≥3|3)=0.125. + vd = bt.verdict(_tier(oos_hit_rate=0.9, n_test=3, lag_stable=True)) + assert vd["promote"] is False + assert "not significant" in vd["reason"] + class TestTierLift: def test_positive_lift_beats_ekb(self) -> None: @@ -1325,6 +1403,28 @@ class TestCrossSourceVerdict: assert cv["rows"][1]["deseasonalized"] is True assert cv["rows"][2]["estimator"] == bt._ESTIMATOR_ALMON + def test_six_of_ten_not_significant_no_signal(self) -> None: + # REGRESSION GUARD (#978) — the same near-miss in the cross-source path: + # a Source A row at oos_hit_rate=0.60, n_test=10, lag_stable=True must NOT + # count as signal (P(X≥6|10)≈0.377 ≥ 0.05). The gate applies in BOTH the + # per-variant verdict() and cross_source_verdict(). + runs = [ + _run( + bt._SOURCE_A, + False, + _tier(source=bt._SOURCE_A, oos_hit_rate=0.60, n_test=10, n_train=23), + ), + ] + cv = bt.cross_source_verdict(runs) + assert cv["promote_any"] is False + assert cv["signal_variants"] == [] + # The rendered line spells out the failed-significance reason + the p. + row = cv["rows"][0] + assert row["significant"] is False + assert row["beats_coin"] is False + joined = "\n".join(cv["lines"]) + assert "not significant" in joined + def test_candidate_method_recovers_signal_is_flagged(self) -> None: # raw best_lag no signal, but the Almon-ADL variant clears coin-flip+margin # (lag stable) → flagged as a variant recovering signal worth inspecting. @@ -1343,6 +1443,21 @@ class TestCrossSourceVerdict: # Conclusion offers the candidate-method reading. assert "candidate method" in cv["conclusion"] + def test_significant_wide_window_counts_as_signal(self) -> None: + # A genuinely-significant detrended variant (hit-rate=0.71 over n_test=35, + # lag stable) DOES count as signal: P(X≥25|35)≈0.008 < 0.05. + runs = [ + _run( + bt._SOURCE_B, + True, + _tier(detrended=True, oos_hit_rate=0.71, n_test=35, n_train=80), + ), + ] + cv = bt.cross_source_verdict(runs) + assert cv["promote_any"] is True + assert "B detrended" in cv["signal_variants"] + assert cv["rows"][0]["significant"] is True + # --------------------------------------------------------------------------- # # DB layer SQL SHAPE — mocked session, asserts CAST not :: and read-only