backend/scripts/backtest_rate_sensitivity.py

@@ -188,6 +188,14 @@ _MIN_BACKTEST_MONTHS: int = 18
 # tiny test window can't flip the verdict on one lucky month.
 _VERDICT_HITRATE_MARGIN: float = 0.05
 
+# One-sided significance level for the verdict's binomial gate. The hit-rate must
+# be statistically distinguishable from a fair coin (p=0.5) at this α before the
+# verdict promotes — i.e. P(X ≥ hits | n_test, 0.5) < _VERDICT_ALPHA. This is a
+# HARD gate on top of the effect-size margin above: it prevents promoting on THIN
+# data where a high hit-rate is consistent with chance (the #978 near-miss:
+# 6/10 = 0.60 has one-sided p ≈ 0.377, indistinguishable from a coin flip).
+_VERDICT_ALPHA: float = 0.05
+
 # Source B premise filter — residential квартиры, the only segment §9.6 scores
 # (mirrors sales_series._DEFAULT_PREMISE_KIND).
 _PREMISE_KIND: str = "квартира"
@@ -368,6 +376,35 @@ def _round_or_none(value: float | None, digits: int) -> float | None:
 # --------------------------------------------------------------------------- #
 
 
+def _binom_sf_ge(k: int, n: int, p: float = 0.5) -> float:
+    """Exact one-sided binomial survival: P(X ≥ k) for X ~ Binomial(n, p). PURE.
+
+    Computed EXACTLY via ``math.comb`` (stdlib only — NO scipy/statsmodels, to
+    mirror the §9.6 engine's no-heavy-deps discipline):
+    ``Σ_{i=k..n} C(n, i) · p^i · (1−p)^(n−i)``.
+
+    This is the probability the observed directional hit count (or anything more
+    extreme) arises by chance from a fair coin — small ⇒ the hit-rate is real
+    signal, not luck. It is the verdict's significance gate (see ``_VERDICT_ALPHA``).
+
+    Guards (return 1.0 = "no evidence against the null"):
+      • ``n <= 0`` → 1.0 (no trials, nothing to distinguish);
+      • ``k`` clamped to ``[0, n]``;
+      • ``k <= 0`` → 1.0 (P(X ≥ 0) = 1 trivially).
+    """
+    if n <= 0:
+        return 1.0
+    k = max(0, min(k, n))
+    if k <= 0:
+        return 1.0
+    q = 1.0 - p
+    total = 0.0
+    for i in range(k, n + 1):
+        total += math.comb(n, i) * (p**i) * (q ** (n - i))
+    # Clamp tiny floating-point overshoot — a probability can't exceed 1.0.
+    return min(1.0, total)
+
+
 def _rate_first_diff(rate_levels: list[float | None]) -> list[float | None]:
     """First difference of the key_rate level series: out[t] = r_t − r_{t-1}.
 
@@ -880,16 +917,26 @@ def verdict(
 ) -> dict[str, Any]:
     """Decide whether the EKB-wide tier shows OOS predictive value. PURE.
 
-    The engine is a promotion CANDIDATE when, on the EKB-wide tier:
+    The engine is a promotion CANDIDATE when, on the EKB-wide tier, ALL hold:
       • a gated lag was found and scored on a non-empty test window, AND
       • the OOS directional hit-rate beats the 0.5 coin-flip baseline by at
-        least ``margin``, AND
+        least ``margin`` (minimum EFFECT SIZE), AND
+      • the hit-rate is STATISTICALLY SIGNIFICANT vs a fair coin: the exact
+        one-sided binomial ``P(X ≥ hits | n_test, 0.5) < _VERDICT_ALPHA`` (a HARD
+        gate — a high rate on a tiny window is consistent with chance and must
+        NOT promote; this is the #978 near-miss: 6/10 = 0.60 has p ≈ 0.377), AND
       • the winning lag is the same on TRAIN and on the full-sample refit
         (lag stability — a lag that jumps between windows is not a signal).
 
+    ``hits`` (the integer directional-hit count the binomial needs) is recovered
+    as ``round(oos_hit_rate * n_test)``: both the hit-rate and n_test derive from
+    the SAME integer division in ``evaluate_oos`` (``hits / scored`` with
+    ``n_test == scored``), so this round-trips EXACTLY without threading a new
+    field through the deep-reviewed evaluator return shape.
+
     Returns ``{"promote": bool, "reason": str, "thin_warning": str | None}``.
-    Honest: if the OOS test window is tiny the reason says so even when the
-    hit-rate happens to clear the bar.
+    Honest: if the OOS test window is tiny the reason says so, and significance
+    is now a HARD gate, not just an advisory caveat.
     """
     if ekb.skipped is not None:
         return {
@@ -908,6 +955,11 @@ def verdict(
         }
 
     beats_coin = ekb.oos_hit_rate >= 0.5 + margin
+    # Recover the integer hit count exactly (see docstring) and test significance.
+    hits = round(ekb.oos_hit_rate * ekb.n_test)
+    p_value = _binom_sf_ge(hits, ekb.n_test, 0.5)
+    significant = p_value < _VERDICT_ALPHA
+
     thin_warning: str | None = None
     if ekb.n_test < min(_MIN_BACKTEST_MONTHS // 2, 6):
         thin_warning = (
@@ -915,11 +967,13 @@ def verdict(
             "confidence is weak — treat the verdict as indicative, not proof."
         )
 
-    if beats_coin and ekb.lag_stable:
+    if beats_coin and ekb.lag_stable and significant:
         reason = (
             f"engine has OOS predictive value (candidate to promote from "
-            f"advisory): EKB-wide OOS hit-rate={ekb.oos_hit_rate:.2f} > "
-            f"0.5+{margin:.2f} and lag stable (lag={ekb.train_lag})"
+            f"advisory): EKB-wide OOS hit-rate={ekb.oos_hit_rate:.2f} over "
+            f"n_test={ekb.n_test} > 0.5+{margin:.2f}, lag stable "
+            f"(lag={ekb.train_lag}), and significant "
+            f"(one-sided binomial p={p_value:.3f} < {_VERDICT_ALPHA:.2f})"
         )
         return {"promote": True, "reason": reason, "thin_warning": thin_warning}
 
@@ -928,6 +982,13 @@ def verdict(
         bits.append(f"hit-rate={ekb.oos_hit_rate:.2f} ≤ 0.5+{margin:.2f}")
     if not ekb.lag_stable:
         bits.append(f"lag unstable (train={ekb.train_lag}, full={ekb.full_sample_lag})")
+    # Significance reported whenever the effect size cleared the bar but the
+    # window is too thin to rule out chance — the #978 transparency requirement.
+    if beats_coin and not significant:
+        bits.append(
+            f"hit-rate={ekb.oos_hit_rate:.2f} over n_test={ekb.n_test} is not "
+            f"significant (one-sided binomial p={p_value:.2f} ≥ {_VERDICT_ALPHA:.2f})"
+        )
     reason = "insufficient OOS signal — keep advisory (" + "; ".join(bits) + ")"
     return {"promote": False, "reason": reason, "thin_warning": thin_warning}
 
@@ -1372,7 +1433,15 @@ def cross_source_verdict(
         label = _variant_label_for_run(run)
         hr = ekb.oos_hit_rate
         scorable = ekb.skipped is None and hr is not None and ekb.n_test >= 1
-        beats = bool(scorable and hr is not None and hr >= 0.5 + margin and ekb.lag_stable)
+        beats_margin = bool(scorable and hr is not None and hr >= 0.5 + margin)
+        # Significance gate (mirrors verdict()): recover the integer hit count
+        # exactly from the rate × window (same integer division in evaluate_oos)
+        # and require the one-sided binomial p < _VERDICT_ALPHA. A high hit-rate
+        # on a thin window is NOT a signal — the #978 6/10 near-miss.
+        hits = round(hr * ekb.n_test) if scorable and hr is not None else 0
+        p_value = _binom_sf_ge(hits, ekb.n_test, 0.5) if scorable else 1.0
+        significant = p_value < _VERDICT_ALPHA
+        beats = bool(beats_margin and ekb.lag_stable and significant)
         thin = scorable and ekb.n_test < min(min_months // 2, 6)
         if beats:
             signal_variants.append(label)
@@ -1389,6 +1458,8 @@ def cross_source_verdict(
                 "oos_hit_rate": _round_or_none(hr, 4),
                 "n_test": ekb.n_test,
                 "lag_stable": ekb.lag_stable,
+                "binom_p": _round_or_none(p_value, 4) if scorable else None,
+                "significant": significant,
                 "beats_coin": beats,
                 "skipped": ekb.skipped,
             }
@@ -1409,7 +1480,20 @@ def cross_source_verdict(
             why = r["skipped"] or "no gated lag / empty test window"
             lines.append(f"  {r['variant']:<{label_w}} → not scorable ({why})")
         else:
-            tag = "SIGNAL > coin-flip" if r["beats_coin"] else "no signal (≤ coin-flip)"
+            # Spell out WHY a variant does/doesn't count as signal — including the
+            # binomial p so a "high rate but thin" row is transparent (#978).
+            if r["beats_coin"]:
+                tag = (
+                    f"SIGNAL > coin-flip (binomial p={_fmt_rate(r['binom_p'])} "
+                    f"< {_VERDICT_ALPHA:.2f})"
+                )
+            elif not r["significant"]:
+                tag = (
+                    f"no signal (hit-rate not significant: binomial "
+                    f"p={_fmt_rate(r['binom_p'])} ≥ {_VERDICT_ALPHA:.2f})"
+                )
+            else:
+                tag = "no signal (≤ coin-flip / lag unstable)"
             lines.append(
                 f"  {r['variant']:<{label_w}} → OOS_hit={_fmt_rate(r['oos_hit_rate'])} "
                 f"(n_test={r['n_test']}, lag_stable={'yes' if r['lag_stable'] else 'no'}) "

backend/tests/scripts/test_backtest_rate_sensitivity.py

@@ -244,6 +244,46 @@ def _seasonal_units(
     return [base * fac[m.month] for m in months]
 
 
+# --------------------------------------------------------------------------- #
+# _binom_sf_ge — exact one-sided binomial survival (verdict significance gate)
+# --------------------------------------------------------------------------- #
+
+
+class TestBinomSfGe:
+    def test_known_values(self) -> None:
+        # The #978 near-miss: 6/10 heads is NOT distinguishable from a fair coin.
+        assert math.isclose(bt._binom_sf_ge(6, 10, 0.5), 0.376953125, abs_tol=1e-9)
+        # A clearly-significant tail.
+        assert math.isclose(bt._binom_sf_ge(9, 10, 0.5), 0.0107421875, abs_tol=1e-9)
+        # 5/5 perfect over a tiny window is just barely significant (p < 0.05).
+        assert math.isclose(bt._binom_sf_ge(5, 5, 0.5), 0.03125, abs_tol=1e-12)
+
+    def test_k_zero_or_below_is_one(self) -> None:
+        # P(X ≥ 0) = 1 trivially; negative k clamps to 0 → 1.0.
+        assert bt._binom_sf_ge(0, 10, 0.5) == 1.0
+        assert bt._binom_sf_ge(-3, 10, 0.5) == 1.0
+
+    def test_n_zero_returns_one(self) -> None:
+        # No trials → no evidence against the null → 1.0 (never promotes).
+        assert bt._binom_sf_ge(3, 0, 0.5) == 1.0
+        assert bt._binom_sf_ge(0, 0, 0.5) == 1.0
+
+    def test_k_clamped_to_n(self) -> None:
+        # k > n clamps to n → P(X ≥ n) = p^n (only the all-success term).
+        assert math.isclose(bt._binom_sf_ge(20, 10, 0.5), 0.5**10, abs_tol=1e-12)
+        # k == n → exactly the all-success probability.
+        assert math.isclose(bt._binom_sf_ge(4, 4, 0.5), 0.0625, abs_tol=1e-12)
+
+    def test_full_distribution_sums_to_one(self) -> None:
+        # P(X ≥ 0) over all i must be 1 for any n (sanity on the comb sum).
+        for n in (1, 3, 7, 12, 35):
+            assert math.isclose(bt._binom_sf_ge(0, n, 0.5), 1.0, abs_tol=1e-9)
+
+    def test_non_half_p(self) -> None:
+        # Works for p ≠ 0.5: P(X ≥ 1 | n=2, p=0.1) = 1 − (0.9)^2 = 0.19.
+        assert math.isclose(bt._binom_sf_ge(1, 2, 0.1), 0.19, abs_tol=1e-12)
+
+
 # --------------------------------------------------------------------------- #
 # _time_ordered_split
 # --------------------------------------------------------------------------- #
@@ -1062,10 +1102,15 @@ def _tier(
 
 
 class TestVerdict:
-    def test_promote_when_beats_coin_and_lag_stable(self) -> None:
-        vd = bt.verdict(_tier(oos_hit_rate=0.75, lag_stable=True))
+    def test_promote_when_beats_coin_and_lag_stable_and_significant(self) -> None:
+        # hit-rate clears 0.5+margin, lag stable, AND a wide-enough window makes it
+        # statistically significant: hits=round(0.71·35)=25, P(X≥25|35)≈0.008<0.05.
+        vd = bt.verdict(_tier(oos_hit_rate=0.71, n_test=35, n_train=80, lag_stable=True))
         assert vd["promote"] is True
         assert "OOS predictive value" in vd["reason"]
+        # The promote message exposes the significance p (#978 transparency).
+        assert "binomial p=" in vd["reason"]
+        assert "n_test=35" in vd["reason"]
 
     def test_keep_advisory_when_at_coin_flip(self) -> None:
         vd = bt.verdict(_tier(oos_hit_rate=0.52, lag_stable=True))  # ≤ 0.5+margin
@@ -1086,12 +1131,45 @@ class TestVerdict:
         vd = bt.verdict(_tier(oos_hit_rate=None))
         assert vd["promote"] is False
 
-    def test_thin_warning_set_for_small_test_window(self) -> None:
-        vd = bt.verdict(_tier(oos_hit_rate=0.9, n_test=3, lag_stable=True))
+    def test_does_not_promote_six_of_ten_not_significant(self) -> None:
+        # REGRESSION GUARD — the exact #978 near-miss. Source A Almon-ADL scored
+        # oos_hit_rate=0.60 with n_test=10 (6/10) and lag_stable. The OLD rule
+        # (hit-rate ≥ 0.5+margin AND lag_stable) over-claimed "candidate to
+        # promote". But P(X≥6|10, 0.5)≈0.377 ≥ 0.05 — indistinguishable from a
+        # coin flip. The significance gate MUST keep it advisory.
+        vd = bt.verdict(_tier(oos_hit_rate=0.60, n_test=10, n_train=23, lag_stable=True))
+        assert vd["promote"] is False
+        assert "keep advisory" in vd["reason"]
+        assert "not significant" in vd["reason"]
+        # The message names n_test and the binomial p so the WHY is transparent.
+        assert "n_test=10" in vd["reason"]
+        assert "p=0.38" in vd["reason"]
+
+    def test_small_n_perfect_score_does_not_promote(self) -> None:
+        # A tiny window at 100% still can't promote: P(X≥4|4, 0.5)=0.0625 ≥ 0.05.
+        # Proves a perfect-but-thin run is not enough to clear significance.
+        vd = bt.verdict(_tier(oos_hit_rate=1.0, n_test=4, n_train=10, lag_stable=True))
+        assert vd["promote"] is False
+        assert "not significant" in vd["reason"]
+        assert "n_test=4" in vd["reason"]
+
+    def test_thin_warning_set_but_significant_still_promotes(self) -> None:
+        # A small window (n_test=5 < 6) sets the thin_warning, but 5/5 is the
+        # smallest perfect window that IS significant: P(X≥5|5, 0.5)=0.03125<0.05.
+        # So it promotes AND carries the thin caveat — the caveat is advisory,
+        # significance is the hard gate.
+        vd = bt.verdict(_tier(oos_hit_rate=1.0, n_test=5, n_train=13, lag_stable=True))
         assert vd["promote"] is True
         assert vd["thin_warning"] is not None
         assert "small" in vd["thin_warning"]
 
+    def test_thin_window_high_rate_blocked_by_significance(self) -> None:
+        # The original "thin window" scenario (hit-rate=0.9, n_test=3): under the
+        # stricter rule it does NOT promote — hits=round(2.7)=3, P(X≥3|3)=0.125.
+        vd = bt.verdict(_tier(oos_hit_rate=0.9, n_test=3, lag_stable=True))
+        assert vd["promote"] is False
+        assert "not significant" in vd["reason"]
+
 
 class TestTierLift:
     def test_positive_lift_beats_ekb(self) -> None:
@@ -1325,6 +1403,28 @@ class TestCrossSourceVerdict:
         assert cv["rows"][1]["deseasonalized"] is True
         assert cv["rows"][2]["estimator"] == bt._ESTIMATOR_ALMON
 
+    def test_six_of_ten_not_significant_no_signal(self) -> None:
+        # REGRESSION GUARD (#978) — the same near-miss in the cross-source path:
+        # a Source A row at oos_hit_rate=0.60, n_test=10, lag_stable=True must NOT
+        # count as signal (P(X≥6|10)≈0.377 ≥ 0.05). The gate applies in BOTH the
+        # per-variant verdict() and cross_source_verdict().
+        runs = [
+            _run(
+                bt._SOURCE_A,
+                False,
+                _tier(source=bt._SOURCE_A, oos_hit_rate=0.60, n_test=10, n_train=23),
+            ),
+        ]
+        cv = bt.cross_source_verdict(runs)
+        assert cv["promote_any"] is False
+        assert cv["signal_variants"] == []
+        # The rendered line spells out the failed-significance reason + the p.
+        row = cv["rows"][0]
+        assert row["significant"] is False
+        assert row["beats_coin"] is False
+        joined = "\n".join(cv["lines"])
+        assert "not significant" in joined
+
     def test_candidate_method_recovers_signal_is_flagged(self) -> None:
         # raw best_lag no signal, but the Almon-ADL variant clears coin-flip+margin
         # (lag stable) → flagged as a variant recovering signal worth inspecting.
@@ -1343,6 +1443,21 @@ class TestCrossSourceVerdict:
         # Conclusion offers the candidate-method reading.
         assert "candidate method" in cv["conclusion"]
 
+    def test_significant_wide_window_counts_as_signal(self) -> None:
+        # A genuinely-significant detrended variant (hit-rate=0.71 over n_test=35,
+        # lag stable) DOES count as signal: P(X≥25|35)≈0.008 < 0.05.
+        runs = [
+            _run(
+                bt._SOURCE_B,
+                True,
+                _tier(detrended=True, oos_hit_rate=0.71, n_test=35, n_train=80),
+            ),
+        ]
+        cv = bt.cross_source_verdict(runs)
+        assert cv["promote_any"] is True
+        assert "B detrended" in cv["signal_variants"]
+        assert cv["rows"][0]["significant"] is True
+
 
 # --------------------------------------------------------------------------- #
 # DB layer SQL SHAPE — mocked session, asserts CAST not :: and read-only