gendesign/backend/tests/sql/test_mv_layout_velocity_weighted_avg.py

"""
Regression test for fix #295 — mv_layout_velocity weighted AVG.

Proves that the count-weighted formula
    SUM(val * cnt) / NULLIF(SUM(cnt), 0)
produces the correct result and differs from naive AVG when zero-deal months are present.

Uses psycopg v3 (never psycopg2) with a temporary table containing known data.
All bind parameters use CAST(:x AS type) — never :x::type.
"""

import os
from decimal import Decimal

import psycopg
import pytest

# ---------------------------------------------------------------------------
# DSN + skip guard
#
# This is an INTEGRATION test — it needs a real PostgreSQL (it CREATE TEMP TABLE
# and runs the weighted-AVG SQL). On CI (Docker) Postgres is reachable; on a dev
# laptop without the SSH tunnel it is not. We:
#   1. Prefer TEST_DATABASE_URL (mirror tests/integration/conftest.py), then
#      DATABASE_URL — but normalize the SQLAlchemy dialect suffix `+psycopg`,
#      which psycopg.connect() cannot parse (other test modules set DATABASE_URL
#      to `postgresql+psycopg://...` via os.environ.setdefault → ProgrammingError).
#   2. Probe connectivity at import time; if the DB is unreachable, SKIP the whole
#      module cleanly (never ERROR). On CI the probe succeeds → tests run, so the
#      #295 weighted-AVG regression stays guarded.
# ---------------------------------------------------------------------------


def _get_dsn() -> str:
    raw = os.environ.get("TEST_DATABASE_URL") or os.environ.get(
        "DATABASE_URL",
        "postgresql://gendesign@localhost:15432/gendesign",
    )
    # SQLAlchemy dialect form `postgresql+psycopg://` → plain libpq DSN for psycopg.
    return raw.replace("+psycopg", "")


def _db_reachable() -> tuple[bool, str]:
    try:
        with psycopg.connect(_get_dsn(), connect_timeout=3):
            return True, ""
    except Exception as e:  # OperationalError / ProgrammingError / etc.
        return False, str(e)


_DB_OK, _DB_ERR = _db_reachable()
pytestmark = pytest.mark.skipif(
    not _DB_OK,
    reason=(
        "Нет доступной plain-postgres БД (TEST_DATABASE_URL/DATABASE_URL) — "
        f"интеграционный тест #295 пропущен off-CI: {_DB_ERR}"
    ),
)


@pytest.fixture()
def conn():
    """Open a psycopg v3 connection and roll back after the test."""
    with psycopg.connect(_get_dsn(), autocommit=False) as c:
        yield c
        c.rollback()


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

_CREATE_TEMP = """
CREATE TEMP TABLE _test_ocm (
    room_bucket TEXT,
    deals_total_count INTEGER,
    deals_total_avg_area_m2 NUMERIC,
    deals_total_avg_price_thousand_rub_per_m2 NUMERIC
) ON COMMIT DROP;
"""

_INSERT_ROW = """
INSERT INTO _test_ocm VALUES (
    CAST(%s AS text),
    CAST(%s AS integer),
    CAST(%s AS numeric),
    CAST(%s AS numeric)
);
"""

# Weighted aggregate query — mirrors the fixed mv_layout_velocity formula exactly.
_WEIGHTED_QUERY = """
SELECT
    room_bucket,
    SUM(deals_total_count)                                               AS total_deals,
    AVG(deals_total_avg_area_m2)::numeric(10,2)                         AS naive_avg_area,
    (SUM(deals_total_avg_area_m2 * deals_total_count)
        / NULLIF(SUM(deals_total_count), 0))::numeric(10,2)             AS weighted_avg_area,
    AVG(deals_total_avg_price_thousand_rub_per_m2)::numeric(12,2)       AS naive_avg_price,
    (SUM(deals_total_avg_price_thousand_rub_per_m2 * deals_total_count)
        / NULLIF(SUM(deals_total_count), 0))::numeric(12,2)             AS weighted_avg_price
FROM _test_ocm
GROUP BY room_bucket
ORDER BY room_bucket;
"""


# ---------------------------------------------------------------------------
# Test cases
# ---------------------------------------------------------------------------


class TestWeightedAvgFormula:
    """Unit-level verification of the count-weighted formula using a temp table."""

    def test_zero_deal_months_skew_naive_avg(self, conn: psycopg.Connection) -> None:
        """
        Scenario: studio, 3 months in window.
          Month 1: 10 deals, avg_area=25, avg_price=180
          Month 2:  0 deals, avg_area=0,  avg_price=0     ← zero row
          Month 3:  0 deals, avg_area=0,  avg_price=0     ← zero row

        Naive AVG: area = (25+0+0)/3 = 8.33  (wrong — dragged down by zeros)
        Weighted:  area = (25*10) / 10 = 25   (correct)

        This replicates the exact bug reported in #295.
        """
        with conn.cursor() as cur:
            cur.execute(_CREATE_TEMP)
            rows = [
                ("studio", 10, 25.0, 180.0),
                ("studio", 0, 0.0, 0.0),
                ("studio", 0, 0.0, 0.0),
            ]
            cur.executemany(_INSERT_ROW, rows)
            cur.execute(_WEIGHTED_QUERY)
            result = cur.fetchone()

        assert result is not None
        room, total_deals, naive_area, weighted_area, naive_price, weighted_price = result

        assert room == "studio"
        assert total_deals == 10

        # Naive AVG is wrong — zero rows drag it down
        assert naive_area == Decimal("8.33"), (
            f"Expected naive AVG=8.33 (showing the bug), got {naive_area}"
        )
        assert naive_price == Decimal("60.00"), (
            f"Expected naive price=60.00 (showing the bug), got {naive_price}"
        )

        # Weighted AVG is correct — ignores zero-deal months
        assert weighted_area == Decimal("25.00"), (
            f"Expected weighted area=25.00, got {weighted_area}"
        )
        assert weighted_price == Decimal("180.00"), (
            f"Expected weighted price=180.00, got {weighted_price}"
        )

        # The whole point: weighted != naive when zeros present
        assert weighted_area != naive_area
        assert weighted_price != naive_price

    def test_weighted_differs_from_naive_sparse_project(self, conn: psycopg.Connection) -> None:
        """
        Scenario inspired by real data: 3-room flat, project with 5 deals
        spread across 5 active months out of 15 total months.

        Mirrors the 'Vitamin-квартал на Титова' / 3-room case from prod
        where buggy=27.14m² vs correct=81.43m² (3x undercount).
        """
        with conn.cursor() as cur:
            cur.execute(_CREATE_TEMP)
            # 10 zero-deal months + 5 active months with avg_area=81
            rows = [("3", 0, 0.0, 0.0)] * 10 + [("3", 1, 81.0, 136.0)] * 5
            cur.executemany(_INSERT_ROW, rows)
            cur.execute(_WEIGHTED_QUERY)
            result = cur.fetchone()

        assert result is not None
        room, total_deals, naive_area, weighted_area, _naive_price, weighted_price = result

        assert room == "3"
        assert total_deals == 5

        # Weighted: 5 deals * 81 / 5 deals = 81.00
        assert weighted_area == Decimal("81.00")
        assert weighted_price == Decimal("136.00")

        # Naive: (10*0 + 5*81) / 15 = 27.00 — wrong
        assert naive_area == Decimal("27.00")

        # Correction factor > 2x
        ratio = float(weighted_area) / float(naive_area)
        assert ratio > 2.5, f"Expected correction factor > 2.5x for sparse project, got {ratio:.2f}"

    def test_no_zero_months_weighted_equals_naive(self, conn: psycopg.Connection) -> None:
        """
        When every month has deals, weighted and naive AVG should be equal
        (within numeric(10,2) rounding) only if counts are uniform.
        When counts differ between months, weighted still differs — and is more accurate.
        But if all counts are equal, they match exactly.
        """
        with conn.cursor() as cur:
            cur.execute(_CREATE_TEMP)
            # All months active, same deal count → weighted == naive
            rows = [
                ("1", 5, 38.0, 170.0),
                ("1", 5, 40.0, 175.0),
                ("1", 5, 36.0, 165.0),
            ]
            cur.executemany(_INSERT_ROW, rows)
            cur.execute(_WEIGHTED_QUERY)
            result = cur.fetchone()

        assert result is not None
        _, total_deals, naive_area, weighted_area, naive_price, weighted_price = result

        assert total_deals == 15
        # With equal weights (5 each), weighted == naive
        assert weighted_area == naive_area, (
            f"Equal-weight case: expected weighted={naive_area}, got {weighted_area}"
        )
        assert weighted_price == naive_price

    def test_nullif_prevents_division_by_zero(self, conn: psycopg.Connection) -> None:
        """
        When all months have 0 deals, NULLIF(SUM(cnt), 0) → NULL instead of divide-by-zero.
        The buggy AVG() also returns 0 (not NULL) for all-zero rows, which is arguably
        worse — it emits a fake value rather than NULL.
        """
        with conn.cursor() as cur:
            cur.execute(_CREATE_TEMP)
            rows = [
                ("2", 0, 0.0, 0.0),
                ("2", 0, 0.0, 0.0),
            ]
            cur.executemany(_INSERT_ROW, rows)
            cur.execute(_WEIGHTED_QUERY)
            result = cur.fetchone()

        assert result is not None
        _, total_deals, naive_area, weighted_area, _naive_price, weighted_price = result

        assert total_deals == 0
        # Weighted formula returns NULL for all-zero-deal case (correct — no real data)
        assert weighted_area is None, (
            f"Expected weighted_area=NULL for all-zero-deal case, got {weighted_area}"
        )
        assert weighted_price is None, (
            f"Expected weighted_price=NULL for all-zero-deal case, got {weighted_price}"
        )
        # Naive AVG returns 0.00 — a misleading non-NULL value
        assert naive_area == Decimal("0.00")

    def test_hand_computed_weighted_average(self, conn: psycopg.Connection) -> None:
        """
        End-to-end hand-computed check with multiple room buckets and
        mixed deal counts to verify the formula is exactly correct.

        Hand computation:
          studio: area = (30*8 + 28*12 + 27*3) / (8+12+3) = (240+336+81)/23 = 657/23 ≈ 28.57
                  price = (160*8 + 170*12 + 155*3) / 23 = (1280+2040+465)/23 = 3785/23 ≈ 164.57
          1-room: area = (38*15 + 40*5) / 20 = (570+200)/20 = 770/20 = 38.50
                  price = (175*15 + 180*5) / 20 = (2625+900)/20 = 3525/20 = 176.25
        """
        with conn.cursor() as cur:
            cur.execute(_CREATE_TEMP)
            rows = [
                ("studio", 8, 30.0, 160.0),
                ("studio", 12, 28.0, 170.0),
                ("studio", 3, 27.0, 155.0),
                ("1", 15, 38.0, 175.0),
                ("1", 5, 40.0, 180.0),
            ]
            cur.executemany(_INSERT_ROW, rows)
            cur.execute(_WEIGHTED_QUERY)
            rows_out = cur.fetchall()

        by_room = {r[0]: r for r in rows_out}

        studio = by_room["studio"]
        assert studio[3] == Decimal("28.57"), (
            f"studio weighted_area: expected 28.57, got {studio[3]}"
        )
        assert studio[5] == Decimal("164.57"), (
            f"studio weighted_price: expected 164.57, got {studio[5]}"
        )

        one_room = by_room["1"]
        assert one_room[3] == Decimal("38.50"), (
            f"1-room weighted_area: expected 38.50, got {one_room[3]}"
        )
        assert one_room[5] == Decimal("176.25"), (
            f"1-room weighted_price: expected 176.25, got {one_room[5]}"
        )