gendesign/backend/tests/services/site_finder/test_supply_layers.py
bot-backend c8e7cb1c79
All checks were successful
Deploy / build-frontend (push) Has been skipped
Deploy / changes (push) Successful in 7s
Deploy / build-backend (push) Successful in 2m8s
Deploy / build-worker (push) Successful in 3m16s
Deploy / deploy (push) Successful in 5m41s
perf(objective_lots): inline DISTINCT ON for 3 view-fullscan consumers + cache landing stats (#1953) (#2067)
2026-06-28 17:25:41 +00:00

661 lines
28 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Unit-тесты supply-layers service (#950 EPIC 6, ТЗ §9.3).
Чистые тесты (без живой БД):
• pure-арифметика (_hidden_units clamp ≥ 0, _l2_confidence по coverage,
_round_or_none) — hand-built фикстуры, включая thin / NULL / negative-clamp.
• compute_layer1/2/3 через MagicMock-сессию — форма SQL (CAST(:x AS type) не :x::type),
params, whitelisted source/confidence, graceful-on-empty (→ [], no crash) и
КРИТИЧНО L2 mapping-free (никаких objective_lots/complexes в L2-SQL).
psycopg v3 правило проверяется явно: bind-параметры — CAST(:x AS type).
"""
from __future__ import annotations
import os
from collections.abc import Iterator
from datetime import date
from unittest.mock import MagicMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from app.services.site_finder.supply_layers import (
SupplyLayerRow,
_hidden_units,
_l2_confidence,
_round_or_none,
compute_all_layers,
compute_layer1_open,
compute_layer2_hidden,
compute_layer3_future,
)
# Разрешённые значения CHECK-констрейнтов supply_layers (м.125). Любой эмит ВНЕ
# этих множеств → INSERT worker'а #950 Step5 упадёт ГРОМКО, поэтому тестим явно.
_ALLOWED_SOURCES = {"objective", "domrf_declared", "domrf_multiphase"}
_ALLOWED_CONFIDENCE = {"high", "medium", "low"}
# Резолвер admin→micro (Step 2) — патчим в L1-тестах. ТОЛЬКО L1 (open) резолвится
# (objective_lots = микро-вокабуляр); L2/L3 бьют domrf_kn_objects.district_name, а это
# УЖЕ админ-вокабуляр (м.57 spatial-join) → их district НЕ резолвится.
_RESOLVE = "app.services.site_finder.supply_layers.resolve_objective_districts"
@pytest.fixture(autouse=True)
def _patch_resolver() -> Iterator[MagicMock]:
"""По умолчанию резолвер = identity raw-микро (district→[district], None→None).
Так _mock_db остаётся валиден (резолвер не трогает db). L1-тесты, проверяющие
admin→micro-разворот, переопределяют .side_effect.
"""
with patch(_RESOLVE) as m:
m.side_effect = lambda _db, d: [d] if d is not None else None
yield m
# ── pure: _hidden_units (L2 clamp) ────────────────────────────────────────────
class TestHiddenUnits:
def test_basic(self) -> None:
# 100 в ПД 30 выставлено = 70 скрыто
assert _hidden_units(100, 30) == 70
def test_clamp_never_negative(self) -> None:
# free_flats > flat_count (рассинхрон снапшотов) → 0, НИКОГДА не негатив
assert _hidden_units(30, 100) == 0
def test_equal_is_zero(self) -> None:
assert _hidden_units(50, 50) == 0
def test_zero_free_all_hidden(self) -> None:
# ничего не выставлено → весь ПД скрыт
assert _hidden_units(80, 0) == 80
def test_none_flat_count_returns_none(self) -> None:
assert _hidden_units(None, 10) is None
def test_none_free_flats_returns_none(self) -> None:
# free_flats IS NULL → объект исключается (нет sold-колонки → не вычислить)
assert _hidden_units(100, None) is None
def test_both_none(self) -> None:
assert _hidden_units(None, None) is None
# ── pure: _l2_confidence (coverage) ───────────────────────────────────────────
class TestL2Confidence:
def test_full_coverage_medium(self) -> None:
# все объекты с free_flats → надёжный hidden, но всё равно НЕ high (это оценка ПД)
assert _l2_confidence(10, 10) == "medium"
def test_majority_coverage_medium(self) -> None:
# 7/10 = 0.7 ≥ 0.6 порог
assert _l2_confidence(10, 7) == "medium"
def test_minority_coverage_low(self) -> None:
# 3/10 = 0.3 < 0.6 → много исключённых NULL-объектов → недооценка → low
assert _l2_confidence(10, 3) == "low"
def test_no_objects_low(self) -> None:
assert _l2_confidence(0, 0) == "low"
def test_zero_with_free_flats_low(self) -> None:
assert _l2_confidence(10, 0) == "low"
def test_never_high(self) -> None:
# L2 никогда не 'high' — даже при 100% coverage
assert _l2_confidence(1000, 1000) != "high"
# ── pure: _round_or_none ──────────────────────────────────────────────────────
class TestRoundOrNone:
def test_rounds(self) -> None:
assert _round_or_none(1234.5678, 1) == 1234.6
def test_none_passthrough(self) -> None:
assert _round_or_none(None, 1) is None
# ── SupplyLayerRow.as_dict ────────────────────────────────────────────────────
class TestSupplyLayerRowAsDict:
def test_serialises_and_rounds_area(self) -> None:
row = SupplyLayerRow(
layer=2,
district_name="Автовокзал",
complex_id=None,
obj_class=None,
dev_group_name=None,
units_estimate=70,
area_estimate=1234.5678,
expected_online_date=None,
source="domrf_declared",
confidence="medium",
method="test",
snapshot_date=date(2026, 6, 3),
)
d = row.as_dict()
assert d["layer"] == 2
assert d["district_name"] == "Автовокзал"
assert d["dev_group_name"] is None
assert d["units_estimate"] == 70
assert d["area_estimate"] == 1234.6
assert d["source"] == "domrf_declared"
assert d["confidence"] == "medium"
assert d["snapshot_date"] == date(2026, 6, 3)
def test_dev_group_name_carried_for_l3(self) -> None:
# #970: L3-строка несёт непустой dev_group_name → попадает в as_dict (ключ UPSERT).
row = SupplyLayerRow(
layer=3,
district_name="Академический",
complex_id=None,
obj_class=None,
dev_group_name="Брусника",
units_estimate=900,
area_estimate=45000.0,
expected_online_date=date(2028, 1, 1),
source="domrf_multiphase",
confidence="low",
method="test",
snapshot_date=date(2026, 6, 3),
)
assert row.as_dict()["dev_group_name"] == "Брусника"
def test_none_area_survives(self) -> None:
row = SupplyLayerRow(
layer=2,
district_name="X",
complex_id=None,
obj_class=None,
dev_group_name=None,
units_estimate=0,
area_estimate=None,
expected_online_date=None,
source="domrf_declared",
confidence="low",
method="test",
snapshot_date=None,
)
assert row.as_dict()["area_estimate"] is None
# ── MagicMock-сессия helpers (зеркало test_market_metrics) ────────────────────
def _mock_db(rows: list[dict]) -> MagicMock:
"""Сессия с одним execute → .mappings().all() == rows."""
db = MagicMock()
result = MagicMock()
result.mappings.return_value.all.return_value = rows
db.execute.return_value = result
return db
def _executed_sql(db: MagicMock, call_index: int = 0) -> str:
args, _kwargs = db.execute.call_args_list[call_index]
return str(args[0])
def _executed_params(db: MagicMock, call_index: int = 0) -> dict:
args, _kwargs = db.execute.call_args_list[call_index]
return args[1]
def _assert_no_double_colon_cast(sql: str) -> None:
"""psycopg v3: запрещён :name::type (SQLAlchemy его молча роняет)."""
import re
assert re.search(r":[a-z_]+::[a-z]", sql) is None, f"double-colon cast in SQL:\n{sql}"
# ── Layer1 (open) ─────────────────────────────────────────────────────────────
class TestLayer1Open:
def test_sql_uses_cast_not_double_colon(self) -> None:
db = _mock_db([])
compute_layer1_open(db, district="Автовокзал")
sql = _executed_sql(db)
# District-фильтр L1 теперь массив резолвнутых микро (ANY), не скаляр.
assert "CAST(:districts AS text[])" in sql
_assert_no_double_colon_cast(sql)
def test_sql_dedups_physflat_inline(self) -> None:
# Перф-батч #2 (#1953-followup): L1 дедупит INLINE через DISTINCT ON по
# physflat-ключу (последний снапшот), НЕ через v_objective_lots_latest
# (view = оптимизационный барьер → full-scan 1.76M, ~6с). Анти-регресс
# #1964 сохранён: чтение СЫРОГО objective_lots БЕЗ дедупа раздуло бы
# units_available ~2.91× (мульти lot_id/физлот) — поэтому требуем DISTINCT ON.
db = _mock_db([])
compute_layer1_open(db, district="Автовокзал")
sql = _executed_sql(db)
norm = " ".join(sql.split())
assert "DISTINCT ON" in norm, "L1 должен physflat-дедупить (анти-инфляция #1964)"
for col in ("project_name", "corpus_name", "section", "floor", "lot_number"):
assert col in norm, f"physflat-ключ DISTINCT ON должен включать {col}"
assert "snapshot_date DESC" in norm, "дедуп к последнему снапшоту физлота"
assert "FROM objective_lots" in norm, "L1 читает сырой objective_lots (inline-дедуп)"
assert "v_objective_lots_latest" not in sql, "view = full-scan, должен быть inline"
def test_params(self) -> None:
db = _mock_db([])
compute_layer1_open(db, district="Автовокзал", premise_kind="квартира")
p = _executed_params(db)
# district резолвится в набор микро (identity → ['Автовокзал']) → ANY(:districts).
assert p["has_district"] is True
assert p["districts"] == ["Автовокзал"]
assert "district" not in p
assert p["premise_kind"] == "квартира"
def test_rows_built_with_whitelisted_values(self) -> None:
rows = [
{
"district": "Автовокзал",
"obj_class": "комфорт",
"units_available": 42,
"area_available": 2100.0,
},
{
"district": "Автовокзал",
"obj_class": "бизнес",
"units_available": 10,
"area_available": 700.0,
},
]
db = _mock_db(rows)
out = compute_layer1_open(db, district="Автовокзал")
assert len(out) == 2
for r in out:
assert r.layer == 1
assert r.source == "objective"
assert r.confidence == "high"
assert r.source in _ALLOWED_SOURCES
assert r.confidence in _ALLOWED_CONFIDENCE
assert r.expected_online_date is None
assert r.complex_id is None
assert r.dev_group_name is None # #970: L1 не имеет dev_group
first = out[0]
assert first.obj_class == "комфорт"
assert first.units_estimate == 42
assert first.area_estimate == 2100.0
def test_zero_units_skipped(self) -> None:
rows = [
{"district": "X", "obj_class": "комфорт", "units_available": 0, "area_available": 0}
]
db = _mock_db(rows)
assert compute_layer1_open(db, district="X") == []
def test_zero_area_becomes_none(self) -> None:
rows = [
{"district": "X", "obj_class": "комфорт", "units_available": 5, "area_available": 0}
]
db = _mock_db(rows)
out = compute_layer1_open(db, district="X")
assert out[0].area_estimate is None
def test_empty_graceful(self) -> None:
assert compute_layer1_open(_mock_db([]), district="Пусто") == []
def test_query_exception_graceful(self) -> None:
db = MagicMock()
db.execute.side_effect = RuntimeError("db down")
assert compute_layer1_open(db, district="Сбой") == []
class TestLayer1DistrictResolution:
"""Step 2: L1 (open) резолвит админ-имя → микро-набор в SQL-фильтре.
Тот же баг, что в market_metrics: админ-имя по objective_lots.district давало 0
строк открытого предложения. Резолвер разворачивает его в чистые микро.
"""
def test_admin_resolves_to_micros_in_sql_params(self, _patch_resolver: MagicMock) -> None:
_patch_resolver.side_effect = lambda _db, d: (
["Втузгородок", "ЖБИ"] if d == "Кировский" else None
)
db = _mock_db([])
compute_layer1_open(db, district="Кировский")
_patch_resolver.assert_called_once()
assert _patch_resolver.call_args.args[1] == "Кировский"
p = _executed_params(db)
# КРИТИЧНО: фильтр содержит МИКРО, а НЕ админ-имя (иначе 0 строк).
assert p["has_district"] is True
assert p["districts"] == ["Втузгородок", "ЖБИ"]
assert "Кировский" not in p["districts"]
def test_resolver_none_drops_district_filter(self, _patch_resolver: MagicMock) -> None:
_patch_resolver.side_effect = lambda _db, _d: None
db = _mock_db([])
compute_layer1_open(db, district="не определён")
p = _executed_params(db)
assert p["has_district"] is False
assert p["districts"] == []
# ── Layer2 (hidden) — THE CRUX ────────────────────────────────────────────────
class TestLayer2Hidden:
def test_sql_uses_cast_not_double_colon(self) -> None:
db = _mock_db([])
compute_layer2_hidden(db, district="Автовокзал")
sql = _executed_sql(db)
assert "CAST(:district AS text)" in sql
_assert_no_double_colon_cast(sql)
def test_mapping_free_no_objective_or_complexes_join(self) -> None:
# КРИТИЧНО (ТЗ §9.3): L2 — mapping-free. domrf↔objective маппинг (2.5%) НЕ нужен,
# т.к. flat_count и free_flats на ОДНОЙ строке domrf_kn_objects (м.124).
db = _mock_db([])
compute_layer2_hidden(db, district="Автовокзал")
sql = _executed_sql(db).lower()
assert "domrf_kn_objects" in sql
assert "objective_lots" not in sql
assert "objective_complex_mapping" not in sql
assert "complex_sources" not in sql
assert "complexes" not in sql
def test_sql_clamps_at_zero(self) -> None:
# SQL должен клампить hidden ≥ 0 (GREATEST) — защита от free_flats > flat_count.
db = _mock_db([])
compute_layer2_hidden(db, district="X")
sql = _executed_sql(db).upper()
assert "GREATEST(0" in sql
def test_sql_excludes_null_free_flats(self) -> None:
# free_flats IS NULL объекты исключены из суммы (нет sold-колонки → не вычислить).
db = _mock_db([])
compute_layer2_hidden(db, district="X")
sql = _executed_sql(db)
assert "free_flats IS NOT NULL" in sql
def test_sql_dedups_latest_snapshot(self) -> None:
# КРИТИЧНО: domrf_kn_objects = UNIQUE(obj_id, snapshot_date), накапливает снапшоты.
# Без DISTINCT ON (obj_id) + snapshot_date DESC наивный SUM умножил бы hidden на
# число снапшотов (cross-snapshot double-count). Этот guard locks-in защиту.
db = _mock_db([])
compute_layer2_hidden(db, district="X")
sql = _executed_sql(db)
assert "DISTINCT ON (o.obj_id)" in sql
assert "snapshot_date DESC" in sql
def test_params_region_ekb(self) -> None:
db = _mock_db([])
compute_layer2_hidden(db, district="Автовокзал")
p = _executed_params(db)
assert p["region_cd"] == 66
assert p["district"] == "Автовокзал"
def test_district_not_resolved_admin_vocab_preserved(self, _patch_resolver: MagicMock) -> None:
# КРИТИЧНО: L2 бьёт domrf_kn_objects.district_name (УЖЕ админ-вокабуляр, м.57
# spatial-join) → резолвер НЕ вызывается, админ-имя идёт в фильтр как есть.
# Иначе резолв в микро дал бы 0 строк (domrf не знает микро-имён).
db = _mock_db([])
compute_layer2_hidden(db, district="Кировский")
_patch_resolver.assert_not_called()
p = _executed_params(db)
assert p["district"] == "Кировский"
assert "districts" not in p
def test_hidden_aggregate_medium_confidence(self) -> None:
rows = [
{
"district_name": "Автовокзал",
"n_objects_total": 10,
"n_with_free_flats": 9, # 0.9 coverage → medium
"hidden_units": 540,
}
]
db = _mock_db(rows)
out = compute_layer2_hidden(db, district="Автовокзал")
assert len(out) == 1
row = out[0]
assert row.layer == 2
assert row.source == "domrf_declared"
assert row.confidence == "medium"
assert row.units_estimate == 540
assert row.obj_class is None # district-агрегат класс-agnostic
assert row.dev_group_name is None # #970: L2 не имеет dev_group
assert row.area_estimate is None
assert row.expected_online_date is None
assert row.source in _ALLOWED_SOURCES
assert row.confidence in _ALLOWED_CONFIDENCE
def test_low_coverage_low_confidence(self) -> None:
rows = [
{
"district_name": "X",
"n_objects_total": 10,
"n_with_free_flats": 2, # 0.2 coverage → low
"hidden_units": 100,
}
]
db = _mock_db(rows)
out = compute_layer2_hidden(db, district="X")
assert out[0].confidence == "low"
def test_no_objects_with_free_flats_skipped(self) -> None:
# район где НИ один объект не имеет free_flats → строки нет (а не units=0)
rows = [
{
"district_name": "X",
"n_objects_total": 5,
"n_with_free_flats": 0,
"hidden_units": 0,
}
]
db = _mock_db(rows)
assert compute_layer2_hidden(db, district="X") == []
def test_units_never_negative_even_if_sql_leaks(self) -> None:
# двойная защита: даже если SQL вернёт негатив, Python клампит в 0
rows = [
{
"district_name": "X",
"n_objects_total": 4,
"n_with_free_flats": 4,
"hidden_units": -50,
}
]
db = _mock_db(rows)
out = compute_layer2_hidden(db, district="X")
assert out[0].units_estimate == 0
def test_empty_graceful(self) -> None:
assert compute_layer2_hidden(_mock_db([]), district="Пусто") == []
def test_query_exception_graceful(self) -> None:
db = MagicMock()
db.execute.side_effect = RuntimeError("db down")
assert compute_layer2_hidden(db, district="Сбой") == []
# ── Layer3 (future) ───────────────────────────────────────────────────────────
class TestLayer3Future:
def test_sql_uses_cast_not_double_colon(self) -> None:
db = _mock_db([])
compute_layer3_future(db, district="Академический")
sql = _executed_sql(db)
assert "CAST(:district AS text)" in sql
assert "CAST(:horizon AS interval)" in sql
_assert_no_double_colon_cast(sql)
def test_params(self) -> None:
db = _mock_db([])
compute_layer3_future(db, district="Академический")
p = _executed_params(db)
assert p["region_cd"] == 66
assert p["district"] == "Академический"
assert p["horizon"] == "12 months"
assert p["max_free_flats"] == 5
def test_sql_dedups_latest_snapshot(self) -> None:
# Тот же anti-double-count guard, что в L2: один объект может иметь несколько
# снапшотов → DISTINCT ON (obj_id) + snapshot_date DESC обязателен до агрегации.
db = _mock_db([])
compute_layer3_future(db, district="Академический")
sql = _executed_sql(db)
assert "DISTINCT ON (o.obj_id)" in sql
assert "snapshot_date DESC" in sql
def test_future_row_low_confidence_with_date(self) -> None:
rows = [
{
"district_name": "Академический",
"dev_group_name": "Брусника",
"n_objects": 3,
"units_future": 900,
"area_future": 45000.0,
"expected_online_date": date(2028, 1, 1),
}
]
db = _mock_db(rows)
out = compute_layer3_future(db, district="Академический")
assert len(out) == 1
row = out[0]
assert row.layer == 3
assert row.source == "domrf_multiphase"
assert row.confidence == "low"
assert row.units_estimate == 900
assert row.area_estimate == 45000.0
assert row.expected_online_date == date(2028, 1, 1)
# #970: dev_group_name из SQL-строки протянут в dataclass (ключ-колонка UPSERT)
assert row.dev_group_name == "Брусника"
assert row.source in _ALLOWED_SOURCES
assert row.confidence in _ALLOWED_CONFIDENCE
def test_same_district_different_dev_groups_distinct_keys(self) -> None:
# #970 (REGRESSION GUARD): два dev_group'а ОДНОГО района должны давать РАЗНЫЕ
# логические ключи (layer, district_name, complex_id, obj_class, dev_group_name,
# source, snapshot_date), иначе UPSERT воркера схлопнет их в один (был ~95.6%
# потерь L3). complex_id/obj_class у L3 всегда NULL → раньше единственным
# различителем был dev_group_name, но его НЕ было в строке → коллапс.
rows = [
{
"district_name": "Академический",
"dev_group_name": "Брусника",
"n_objects": 5,
"units_future": 900,
"area_future": 45000.0,
"expected_online_date": date(2028, 1, 1),
},
{
"district_name": "Академический",
"dev_group_name": "Атомстройкомплекс",
"n_objects": 7,
"units_future": 1200,
"area_future": 60000.0,
"expected_online_date": date(2029, 6, 1),
},
]
db = _mock_db(rows)
out = compute_layer3_future(db, district="Академический")
assert len(out) == 2
# каждый dev_group протянут в свою строку (не только в method-текст)
assert {r.dev_group_name for r in out} == {"Брусника", "Атомстройкомплекс"}
# ключевой инвариант: UPSERT-ключи двух строк РАЗЛИЧНЫ (не схлопываются)
def _key(r: SupplyLayerRow) -> tuple:
d = r.as_dict()
return (
d["layer"],
d["district_name"],
d["complex_id"],
d["obj_class"],
d["dev_group_name"],
d["source"],
d["snapshot_date"],
)
keys = [_key(r) for r in out]
assert len(set(keys)) == 2, f"dev_group keys collapsed: {keys}"
# всё, КРОМЕ dev_group_name, у них идентично — единственный различитель = dev_group
assert keys[0][:4] == keys[1][:4] # layer/district/complex_id/obj_class совпадают
assert keys[0][4] != keys[1][4] # dev_group_name различается
def test_zero_units_and_area_become_none(self) -> None:
rows = [
{
"district_name": "X",
"dev_group_name": None,
"n_objects": 1,
"units_future": 0,
"area_future": 0,
"expected_online_date": date(2029, 6, 1),
}
]
db = _mock_db(rows)
out = compute_layer3_future(db, district="X")
assert out[0].units_estimate is None
assert out[0].area_estimate is None
# строка всё равно эмитится — есть expected_online_date (полезный future-сигнал)
assert out[0].expected_online_date == date(2029, 6, 1)
def test_empty_graceful(self) -> None:
assert compute_layer3_future(_mock_db([]), district="Пусто") == []
def test_query_exception_graceful(self) -> None:
db = MagicMock()
db.execute.side_effect = RuntimeError("db down")
assert compute_layer3_future(db, district="Сбой") == []
# ── compute_all_layers (объединённый) ─────────────────────────────────────────
class TestComputeAllLayers:
def test_combines_three_layers(self) -> None:
l1 = [
{"district": "A", "obj_class": "комфорт", "units_available": 5, "area_available": 250.0}
]
l2 = [
{"district_name": "A", "n_objects_total": 4, "n_with_free_flats": 4, "hidden_units": 80}
]
l3 = [
{
"district_name": "A",
"dev_group_name": "Dev",
"n_objects": 1,
"units_future": 100,
"area_future": 5000.0,
"expected_online_date": date(2028, 1, 1),
}
]
db = MagicMock()
results = []
for rows in (l1, l2, l3):
res = MagicMock()
res.mappings.return_value.all.return_value = rows
results.append(res)
db.execute.side_effect = results
out = compute_all_layers(db, district="A")
layers = sorted(r.layer for r in out)
assert layers == [1, 2, 3]
# все source/confidence whitelisted
assert all(r.source in _ALLOWED_SOURCES for r in out)
assert all(r.confidence in _ALLOWED_CONFIDENCE for r in out)
def test_one_layer_failing_does_not_kill_others(self) -> None:
# L1 падает, L2/L3 пустые — compute_all не бросает, возвращает []
db = MagicMock()
l1_fail = RuntimeError("l1 down")
l2_ok = MagicMock()
l2_ok.mappings.return_value.all.return_value = []
l3_ok = MagicMock()
l3_ok.mappings.return_value.all.return_value = []
db.execute.side_effect = [l1_fail, l2_ok, l3_ok]
out = compute_all_layers(db, district="A")
assert out == []