260 lines
9.4 KiB
Python
260 lines
9.4 KiB
Python
"""Unit tests for the read-only backtest harness (issue #648).
|
|
|
|
Covers the PURE aggregation / metric helpers, factored out of the DB code so
|
|
they're testable without a live database:
|
|
- _compute_metrics — signed/abs error %, median bias, MAPE, per-rooms split
|
|
- _errors_summary — bias / MAPE / p25 / p75 of a signed-error list
|
|
- _bucketize_rooms / _rooms_label — 4+ collapse, студия labelling
|
|
|
|
No DB / network / mocks: these operate on plain lists/tuples.
|
|
|
|
NOTE: importing scripts.backtest_estimator pulls app.services.estimator →
|
|
app.core.config.Settings, which REQUIRES DATABASE_URL. Set a dummy value
|
|
BEFORE importing app modules (same pattern as tests/test_estimator_pure_units.py
|
|
and tests/test_audit_address_mismatch.py).
|
|
"""
|
|
|
|
import os
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
|
|
|
import math
|
|
|
|
import pytest
|
|
|
|
from scripts import backtest_estimator as bt
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# _bucketize_rooms / _rooms_label
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_bucketize_studio_and_negative_clamp_to_zero() -> None:
|
|
assert bt._bucketize_rooms(0) == 0
|
|
assert bt._bucketize_rooms(-3) == 0
|
|
|
|
|
|
def test_bucketize_four_plus_collapses() -> None:
|
|
assert bt._bucketize_rooms(4) == 4
|
|
assert bt._bucketize_rooms(5) == 4
|
|
assert bt._bucketize_rooms(9) == 4
|
|
|
|
|
|
def test_bucketize_passthrough_for_one_to_three() -> None:
|
|
assert bt._bucketize_rooms(1) == 1
|
|
assert bt._bucketize_rooms(2) == 2
|
|
assert bt._bucketize_rooms(3) == 3
|
|
|
|
|
|
def test_rooms_label() -> None:
|
|
assert bt._rooms_label(0) == "студия"
|
|
assert bt._rooms_label(1) == "1к"
|
|
assert bt._rooms_label(3) == "3к"
|
|
assert bt._rooms_label(4) == "4+"
|
|
assert bt._rooms_label(7) == "4+"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# _errors_summary
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_errors_summary_empty_returns_all_none() -> None:
|
|
s = bt._errors_summary([])
|
|
assert s["n"] == 0
|
|
assert s["median_bias_pct"] is None
|
|
assert s["mape_pct"] is None
|
|
assert s["p25_pct"] is None
|
|
assert s["p75_pct"] is None
|
|
|
|
|
|
def test_errors_summary_uses_median_abs_for_mape_not_mean() -> None:
|
|
# signed errors with an asymmetric outlier: median |err| (=10) differs
|
|
# sharply from the MEAN |err| (=40). The brief defines MAPE as the MEDIAN
|
|
# absolute error, so we assert the robust median is used.
|
|
signed = [10.0, 10.0, 10.0, 130.0]
|
|
s = bt._errors_summary(signed)
|
|
assert s["mape_pct"] == 10.0 # median(|10,10,10,130|) = 10, not mean 40
|
|
assert s["median_bias_pct"] == 10.0 # median([10,10,10,130]) = 10
|
|
|
|
|
|
def test_errors_summary_signed_bias_can_be_negative() -> None:
|
|
# Under-prediction → negative bias.
|
|
s = bt._errors_summary([-20.0, -10.0, -30.0])
|
|
assert s["median_bias_pct"] == -20.0
|
|
assert s["mape_pct"] == 20.0 # median of |[-20,-10,-30]| = median[10,20,30]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# _compute_metrics — signed/abs error %, bias, MAPE, per-rooms
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_compute_metrics_empty_overall_is_none_per_rooms_all_present() -> None:
|
|
m = bt._compute_metrics([])
|
|
assert m["overall"]["n"] == 0
|
|
assert m["overall"]["median_bias_pct"] is None
|
|
assert m["overall"]["mape_pct"] is None
|
|
assert m["overall"]["n_no_analogs"] == 0
|
|
# Every room bucket must still appear (with n=0) so the report renders.
|
|
assert set(m["per_rooms"].keys()) == set(bt.ROOM_BUCKETS)
|
|
for bucket in bt.ROOM_BUCKETS:
|
|
assert m["per_rooms"][bucket]["n"] == 0
|
|
assert m["per_rooms"][bucket]["median_bias_pct"] is None
|
|
assert m["per_rooms"][bucket]["label"] == bt._rooms_label(bucket)
|
|
|
|
|
|
def test_compute_metrics_known_plus_22_pct_overprediction() -> None:
|
|
# The headline finding: asking median over-predicts SOLD by ~+22%.
|
|
# pred = 1.22 * sold for every row → signed error must be exactly +22%,
|
|
# MAPE +22%, p25 == p75 == +22% (no spread).
|
|
rows = [
|
|
(122_000.0, 100_000.0, 1),
|
|
(244_000.0, 200_000.0, 2),
|
|
(366_000.0, 300_000.0, 3),
|
|
]
|
|
m = bt._compute_metrics(rows)
|
|
assert m["overall"]["n"] == 3
|
|
assert m["overall"]["median_bias_pct"] == pytest.approx(22.0)
|
|
assert m["overall"]["mape_pct"] == pytest.approx(22.0)
|
|
assert m["overall"]["p25_pct"] == pytest.approx(22.0)
|
|
assert m["overall"]["p75_pct"] == pytest.approx(22.0)
|
|
|
|
|
|
def test_compute_metrics_signed_error_formula() -> None:
|
|
# Single row, hand-computed: 100*(150k-120k)/120k = +25.0%.
|
|
m = bt._compute_metrics([(150_000.0, 120_000.0, 2)])
|
|
assert m["overall"]["median_bias_pct"] == pytest.approx(25.0)
|
|
assert m["overall"]["mape_pct"] == pytest.approx(25.0)
|
|
|
|
|
|
def test_compute_metrics_abs_error_distinct_from_signed() -> None:
|
|
# Mixed over/under: signed bias near 0 but MAPE (median |err|) is positive.
|
|
# rows: +50%, -50%, +50%, -50% → median signed in {-50,+50} band,
|
|
# median |err| = 50.
|
|
rows = [
|
|
(150_000.0, 100_000.0, 1), # +50
|
|
(50_000.0, 100_000.0, 1), # -50
|
|
(150_000.0, 100_000.0, 1), # +50
|
|
(50_000.0, 100_000.0, 1), # -50
|
|
]
|
|
m = bt._compute_metrics(rows)
|
|
assert m["overall"]["mape_pct"] == pytest.approx(50.0)
|
|
# signed median of [-50,-50,50,50] = 0.0 (mean of two middles)
|
|
assert m["overall"]["median_bias_pct"] == pytest.approx(0.0)
|
|
|
|
|
|
def test_compute_metrics_per_rooms_split_and_four_plus_collapse() -> None:
|
|
rows = [
|
|
(110_000.0, 100_000.0, 0), # студия: +10
|
|
(130_000.0, 100_000.0, 0), # студия: +30 → median bucket 0 = +20
|
|
(90_000.0, 100_000.0, 2), # 2к: -10
|
|
(200_000.0, 100_000.0, 5), # 4+ (5 collapses): +100
|
|
(300_000.0, 100_000.0, 4), # 4+ : +200 → median bucket 4 = +150
|
|
]
|
|
m = bt._compute_metrics(rows)
|
|
|
|
assert m["per_rooms"][0]["n"] == 2
|
|
assert m["per_rooms"][0]["median_bias_pct"] == pytest.approx(20.0)
|
|
assert m["per_rooms"][0]["label"] == "студия"
|
|
|
|
assert m["per_rooms"][2]["n"] == 1
|
|
assert m["per_rooms"][2]["median_bias_pct"] == pytest.approx(-10.0)
|
|
|
|
# rooms=5 and rooms=4 both land in bucket 4.
|
|
assert m["per_rooms"][4]["n"] == 2
|
|
assert m["per_rooms"][4]["median_bias_pct"] == pytest.approx(150.0)
|
|
assert m["per_rooms"][4]["label"] == "4+"
|
|
|
|
# buckets 1 and 3 had no rows.
|
|
assert m["per_rooms"][1]["n"] == 0
|
|
assert m["per_rooms"][3]["n"] == 0
|
|
|
|
# overall n counts every matched row.
|
|
assert m["overall"]["n"] == 5
|
|
|
|
|
|
def test_compute_metrics_drops_nonpositive_sold() -> None:
|
|
# sold_ppm2 <= 0 cannot be divided → row dropped, not counted, no crash.
|
|
rows = [
|
|
(120_000.0, 0.0, 1), # dropped
|
|
(120_000.0, -5.0, 2), # dropped
|
|
(122_000.0, 100_000.0, 1), # kept → +22
|
|
]
|
|
m = bt._compute_metrics(rows)
|
|
assert m["overall"]["n"] == 1
|
|
assert m["overall"]["median_bias_pct"] == pytest.approx(22.0)
|
|
|
|
|
|
def test_compute_metrics_carries_no_analog_counts() -> None:
|
|
rows = [(122_000.0, 100_000.0, 1)]
|
|
m = bt._compute_metrics(
|
|
rows,
|
|
n_no_analogs=7,
|
|
per_rooms_no_analogs={1: 4, 2: 3},
|
|
)
|
|
assert m["overall"]["n_no_analogs"] == 7
|
|
assert m["per_rooms"][1]["n_no_analogs"] == 4
|
|
assert m["per_rooms"][2]["n_no_analogs"] == 3
|
|
# bucket with no skipped deals defaults to 0.
|
|
assert m["per_rooms"][0]["n_no_analogs"] == 0
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Rendering smoke tests — table + empty render must not crash.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_render_table_runs_on_real_metrics() -> None:
|
|
m = bt._compute_metrics([(122_000.0, 100_000.0, 1)], n_no_analogs=2)
|
|
headline = {
|
|
"deal_median_ppm2": 100_000.0,
|
|
"ask_median_ppm2": 122_000.0,
|
|
"spread_pct": 22.0,
|
|
}
|
|
out = bt._render_table(m, headline)
|
|
assert "BACKTEST" in out
|
|
assert "OVERALL" in out
|
|
assert "+22.0" in out # bias rendered with sign
|
|
assert "100 000" in out # ppm2 formatted with space thousands separator
|
|
|
|
|
|
def test_render_table_handles_empty_sample() -> None:
|
|
m = bt._compute_metrics([])
|
|
headline = {"deal_median_ppm2": None, "ask_median_ppm2": None, "spread_pct": None}
|
|
out = bt._render_table(m, headline)
|
|
assert "n/a" in out # None metrics render as n/a, no crash
|
|
|
|
|
|
def test_fmt_helpers_handle_none_and_nan_safely() -> None:
|
|
assert bt._fmt_pct(None) == " n/a"
|
|
assert bt._fmt_ppm2(None) == "n/a"
|
|
# sanity: finite values format
|
|
assert "+22" in bt._fmt_pct(22.0)
|
|
assert not math.isnan(22.0)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# argparse — defaults match the brief.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_argparse_defaults() -> None:
|
|
ns = bt._parse_args([])
|
|
assert ns.sample == 300
|
|
assert ns.since == "2025-06-01"
|
|
assert ns.radius == 1000
|
|
assert ns.rooms_tolerance == 0
|
|
assert ns.json is False
|
|
|
|
|
|
def test_argparse_overrides() -> None:
|
|
ns = bt._parse_args(
|
|
["--sample", "50", "--since", "2024-01-01", "--radius", "2000",
|
|
"--rooms-tolerance", "1", "--json"]
|
|
)
|
|
assert ns.sample == 50
|
|
assert ns.since == "2024-01-01"
|
|
assert ns.radius == 2000
|
|
assert ns.rooms_tolerance == 1
|
|
assert ns.json is True
|