fix(estimator): low-findings аудита «Мера» — cian sanity, deal floor-edge, price_trend dedup, DEAL_MAX verify #1863

Merged
lekss361 merged 4 commits from fix/estimator-low-findings into main 2026-06-21 06:16:35 +00:00
6 changed files with 459 additions and 26 deletions

View file

@ -169,6 +169,25 @@ class Settings(BaseSettings):
# Дефолт 6 (консервативно); аудит предложил 3 — конфигурируемо.
estimate_price_trend_max_age_months: int = 6
# ── Mera-audit fix-3: cross-source dedup в price_trend ────────────────────
# Один объект на avito_imv + yandex_valuation с разными ext_item_id даёт
# double-count в house_placement_history → шум в помесячной медиане тренда.
# True (дефолт) = дедуплицировать строки перед агрегацией по ключу
# (round(area_m2,0), floor, COALESCE(last_price,start_price),
# COALESCE(last_price_date,start_price_date)), приоритет avito_imv.
# False = старое поведение без дедупа (backward-compat).
# ENV: ESTIMATE_PRICE_TREND_DEDUP_ENABLED.
estimate_price_trend_dedup_enabled: bool = True
# ── Mera-audit fix-1: Cian valuation sanity bounds ────────────────────────
# API-ответ Cian иногда возвращает garbage-значения (999_999 или 9_999_999_999).
# sale_price_rub вне [min, max] → результат отбрасывается (return None, не кэшируется).
# low_price > high_price или отрицательные значения → также сброс.
# Дефолты: 500_000 (мин. рыночная квартира ЕКБ) и 500_000_000 (500 Мабс. потолок).
# ENV: CIAN_VALUATION_MIN_RUB, CIAN_VALUATION_MAX_RUB.
cian_valuation_min_rub: float = 500_000
cian_valuation_max_rub: float = 500_000_000
# ── #audit-4: MAD-clip after similarity-weighting ─────────────────────────
# True = clip происходит ПОСЛЕ similarity-weighting (на взвешенных ppm²).
# False = clip ДО weighting (старое поведение). Дефолт True.

View file

@ -93,8 +93,15 @@ SBER_TIME_FACTOR_MAX = 1.6
# guard-bands (НЕ относительные) для ЕКБ-вторички: рынок ~100400 К/м² (ср. пороги
# _band_haircut 180/350К), премиум до ~680К. Нижняя/верхняя границы заведомо вне
# рынка — режут «4.98 М за 125 м²» = 39.7 К/м² и т.п. Этаж: ЕКБ-максимум ~52 эт.
#
# Mera-audit fix-4: верифицировано vs данных prod deals (2026-06-21):
# deals p90=172_419 ₽/м², p95=199_583 ₽/м², p99=278_681 ₽/м², p99.9=500_000 ₽/м²
# deals >800k = 13 штук (0.026%) — нерыночные outlier'ы, не легитимный премиум.
# listings p99 (is_active, <800k фильтр) = 392_795 ₽/м².
# Вывод: DEAL_MAX_PPM2=800_000 НЕ режет легитимный ЕКБ-премиум (p99.9 сделок=500k).
# Значение НЕ изменено — порог адекватен. Проверить повторно при p99 > 700_000.
DEAL_MIN_PPM2 = 50_000 # ниже = не arms-length (доля/обременение/ошибка)
DEAL_MAX_PPM2 = 800_000 # выше премиума → опечатка/коммерция
DEAL_MAX_PPM2 = 800_000 # выше премиума → опечатка/коммерция (verified p99.9=500k, 2026-06-21)
DEAL_MAX_FLOOR = 60 # выше реального максимума ЕКБ → битый этаж (напр. floor:100)
# Когорта по году постройки — типизация массовой застройки РФ.
@ -3348,23 +3355,27 @@ def _fetch_price_trend(
# #audit-3: freshness_months — фильтр по scraped_at чтобы исключить стale items
# (yandex_valuation из 2024 и т.д.). Применяется ко ВСЕМ source в этом запросе
# (avito_imv + yandex_valuation), не только к yandex_valuation. Дефолт из config.
# Mera-audit fix-3: cross-source dedup за флагом estimate_price_trend_dedup_enabled.
_fresh_months = freshness_months if freshness_months is not None else months
_dedup_enabled = settings.estimate_price_trend_dedup_enabled
try:
rows = (
db.execute(
text(
"""
SELECT to_char(
date_trunc('month',
COALESCE(last_price_date, start_price_date)),
'YYYY-MM'
) AS month,
round(
percentile_cont(0.5) WITHIN GROUP (
ORDER BY COALESCE(last_price, start_price)
/ NULLIF(area_m2, 0)
)
)::int AS ppm2
if _dedup_enabled:
# Дедупликация near-дублей: один объект на avito_imv + yandex_valuation
# c разными ext_item_id → double-count в медиане. DISTINCT ON по ключу
# (round(area_m2,0), floor, price, price_date) с приоритетом avito_imv
# (source='avito_imv' сортируется раньше через CASE).
trend_sql = """
WITH deduped AS (
SELECT DISTINCT ON (
ROUND(area_m2),
floor,
COALESCE(last_price, start_price),
COALESCE(last_price_date, start_price_date)
)
area_m2,
floor,
COALESCE(last_price, start_price) AS price,
COALESCE(last_price_date, start_price_date) AS price_date
FROM house_placement_history
WHERE house_id = CAST(:hid AS bigint)
AND area_m2 > 0
@ -3375,10 +3386,53 @@ def _fetch_price_trend(
AND scraped_at > (now()
- CAST(:fresh_months AS integer)
* CAST('1 month' AS interval))
GROUP BY 1
ORDER BY 1 ASC
"""
),
ORDER BY
ROUND(area_m2),
floor,
COALESCE(last_price, start_price),
COALESCE(last_price_date, start_price_date),
CASE source WHEN 'avito_imv' THEN 0 ELSE 1 END ASC,
scraped_at DESC
)
SELECT to_char(date_trunc('month', price_date), 'YYYY-MM') AS month,
round(
percentile_cont(0.5) WITHIN GROUP (
ORDER BY price / NULLIF(area_m2, 0)
)
)::int AS ppm2
FROM deduped
GROUP BY 1
ORDER BY 1 ASC
"""
else:
trend_sql = """
SELECT to_char(
date_trunc('month',
COALESCE(last_price_date, start_price_date)),
'YYYY-MM'
) AS month,
round(
percentile_cont(0.5) WITHIN GROUP (
ORDER BY COALESCE(last_price, start_price)
/ NULLIF(area_m2, 0)
)
)::int AS ppm2
FROM house_placement_history
WHERE house_id = CAST(:hid AS bigint)
AND area_m2 > 0
AND COALESCE(last_price, start_price) > 0
AND COALESCE(last_price_date, start_price_date) IS NOT NULL
AND COALESCE(last_price_date, start_price_date) > (CURRENT_DATE
- (CAST(:months AS integer) || ' months')::interval)
AND scraped_at > (now()
- CAST(:fresh_months AS integer)
* CAST('1 month' AS interval))
GROUP BY 1
ORDER BY 1 ASC
"""
rows = (
db.execute(
text(trend_sql),
{"hid": target_house_id, "months": months, "fresh_months": _fresh_months},
)
.mappings()
@ -4000,12 +4054,21 @@ def _fetch_analogs(
def _is_plausible_deal(
price_per_m2: float | None, floor: int | None, total_floors: int | None
price_per_m2: float | None,
floor: int | None,
total_floors: int | None,
area_m2: float | None = None,
price_rub: float | None = None,
) -> bool:
"""#699: True если ДКП-сделка правдоподобна (не выброс по ppm²/этажу).
"""#699 + Mera-audit fix-2: True если ДКП-сделка правдоподобна (не выброс).
Абсолютные guard-bands (см. DEAL_* константы). None-поля не судим (keep
нечем сравнивать). Этаж > total_floors физически невозможен drop.
нечем сравнивать). Проверки:
- price_per_m2 вне [DEAL_MIN_PPM2, DEAL_MAX_PPM2] drop
- floor < 1 или floor > DEAL_MAX_FLOOR drop (битый парсер: floor=-5/999)
- floor > total_floors физически невозможен drop
- area_m2 задана и <= 0 drop (битый парсер)
- price_rub задана и <= 0 drop (нерыночная/технческая сделка)
"""
if price_per_m2 is not None and not (DEAL_MIN_PPM2 <= price_per_m2 <= DEAL_MAX_PPM2):
return False
@ -4014,6 +4077,10 @@ def _is_plausible_deal(
return False
if total_floors is not None and total_floors > 0 and floor > total_floors:
return False
if area_m2 is not None and area_m2 <= 0:
return False
if price_rub is not None and price_rub <= 0:
return False
return True
@ -4055,13 +4122,19 @@ def _fetch_deals(
.all()
)
# #699: отсекаем ДКП-выбросы (битый этаж / нерыночный ppm²) до выдачи в
# actual_deals и expected_sold — иначе floor:100 / 39.7К-м² шумят демо.
# #699 + Mera-audit fix-2: отсекаем ДКП-выбросы (битый этаж / нерыночный ppm²
# / нулевая площадь / нулевая цена) до выдачи в actual_deals и expected_sold.
deals = [dict(r) for r in rows]
clean = [
d
for d in deals
if _is_plausible_deal(d.get("price_per_m2"), d.get("floor"), d.get("total_floors"))
if _is_plausible_deal(
d.get("price_per_m2"),
d.get("floor"),
d.get("total_floors"),
d.get("area_m2"),
d.get("price_rub"),
)
]
if len(clean) < len(deals):
logger.info("deals sanitize #699: %d%d (dropped outliers)", len(deals), len(clean))

View file

@ -216,6 +216,12 @@ async def estimate_via_cian_valuation(
# 7. Extract structured result
result = _parse_valuation_state(state)
# 7a. Sanity-check price bounds (Mera-audit fix-1).
# Битый API-ответ (999_999 < min или 9_999_999_999 > max) → не кэшируем, возвращаем None.
# Аналогично проверяем low/high bound: low>high или отрицательные → drop.
if not _is_price_sane(result):
return None
# 8. Persist to cache (24h TTL)
_save_to_cache(
db,
@ -355,6 +361,48 @@ def _parse_change_pct(value_str: str) -> float | None:
return None
def _is_price_sane(result: CianValuationResult) -> bool:
"""Mera-audit fix-1: проверка разумного диапазона цены Cian Valuation.
Возвращает False ( caller вернёт None, не кэшируем) если:
- sale_price_rub задан и вне [settings.cian_valuation_min_rub, cian_valuation_max_rub]
- sale_price_from или sale_price_to отрицательные
- sale_price_from > sale_price_to (инвертированный диапазон)
None-поля не судим: Cian может не вернуть диапазон это не ошибка.
"""
price = result.sale_price_rub
if price is not None:
min_rub = settings.cian_valuation_min_rub
max_rub = settings.cian_valuation_max_rub
if price < min_rub or price > max_rub:
logger.warning(
"Cian valuation sanity: sale_price_rub=%.0f вне [%.0f, %.0f] → drop",
price,
min_rub,
max_rub,
)
return False
low = result.sale_price_from
high = result.sale_price_to
if low is not None and low < 0:
logger.warning("Cian valuation sanity: sale_price_from=%.0f < 0 → drop", low)
return False
if high is not None and high < 0:
logger.warning("Cian valuation sanity: sale_price_to=%.0f < 0 → drop", high)
return False
if low is not None and high is not None and low > high:
logger.warning(
"Cian valuation sanity: low=%.0f > high=%.0f (инвертированный диапазон) → drop",
low,
high,
)
return False
return True
def _load_from_cache(db: Session, cache_key: str) -> CianValuationResult | None:
"""SELECT from external_valuations where cache_key + source='cian_valuation' + not expired."""
row = (

View file

@ -1,4 +1,5 @@
"""Tests for cian_valuation.py — Stage 7 Cian Valuation Calculator scraper."""
import os
# Settings требует DATABASE_URL при инициализации (fail-fast).
@ -12,6 +13,7 @@ import pytest
from app.services.scrapers.cian_valuation import (
CianValuationResult,
_is_price_sane,
_parse_change_pct,
_parse_num,
_parse_valuation_state,
@ -344,6 +346,8 @@ async def test_cache_hit_returns_cached(monkeypatch):
"chart_change_direction": "increase",
"external_house_id": 999,
"filters_hash": "xyz789",
"low_price": 6500000, # required by _load_from_cache → sale_price_from
"high_price": 7500000, # required by _load_from_cache → sale_price_to
}
db.execute.return_value.mappings.return_value.first.return_value = cached_row
@ -371,6 +375,78 @@ async def test_cache_hit_returns_cached(monkeypatch):
assert len(result.chart) == 1
# ---------------------------------------------------------------------------
# Mera-audit fix-1: _is_price_sane — bounds validation
# ---------------------------------------------------------------------------
def test_price_sane_valid_price() -> None:
"""Обычная цена ЕКБ-квартиры — проходит санитарную проверку."""
r = CianValuationResult(
sale_price_rub=7_500_000, sale_price_from=6_900_000, sale_price_to=8_100_000
)
assert _is_price_sane(r) is True
def test_price_sane_none_price() -> None:
"""sale_price_rub=None — проверка цены не судит None (graceful)."""
r = CianValuationResult(sale_price_rub=None)
assert _is_price_sane(r) is True
def test_price_sane_below_min_returns_false() -> None:
"""Цена 999_999 < дефолтный min (500_000... нет, 999_999 > 500_000).
Тест: цена ниже min=500_000 задаём 100_000 False."""
r = CianValuationResult(sale_price_rub=100_000)
assert _is_price_sane(r) is False
def test_price_sane_above_max_returns_false() -> None:
"""Цена 9_999_999_999 (> 500М) → False."""
r = CianValuationResult(sale_price_rub=9_999_999_999)
assert _is_price_sane(r) is False
def test_price_sane_at_min_boundary() -> None:
"""Цена ровно на нижней границе дефолта (500_000) → True."""
r = CianValuationResult(sale_price_rub=500_000)
assert _is_price_sane(r) is True
def test_price_sane_at_max_boundary() -> None:
"""Цена ровно на верхней границе дефолта (500_000_000) → True."""
r = CianValuationResult(sale_price_rub=500_000_000)
assert _is_price_sane(r) is True
def test_price_sane_inverted_range_returns_false() -> None:
"""low > high (инвертированный диапазон) → False."""
r = CianValuationResult(
sale_price_rub=7_000_000,
sale_price_from=8_000_000, # from > to
sale_price_to=6_000_000,
)
assert _is_price_sane(r) is False
def test_price_sane_negative_low_returns_false() -> None:
"""sale_price_from < 0 → False."""
r = CianValuationResult(sale_price_rub=7_000_000, sale_price_from=-100)
assert _is_price_sane(r) is False
def test_price_sane_negative_high_returns_false() -> None:
"""sale_price_to < 0 → False."""
r = CianValuationResult(sale_price_rub=7_000_000, sale_price_to=-1)
assert _is_price_sane(r) is False
def test_price_sane_none_bounds_ok() -> None:
"""sale_price_from/to=None при валидной цене → True (bounds не обязательны)."""
r = CianValuationResult(sale_price_rub=7_000_000, sale_price_from=None, sale_price_to=None)
assert _is_price_sane(r) is True
@pytest.mark.asyncio
async def test_estimate_use_cache_false_skips_cache(monkeypatch):
"""use_cache=False → cache lookup не вызывается, идём сразу в load_session."""

View file

@ -50,3 +50,73 @@ def test_null_fields_not_judged() -> None:
assert _is_plausible_deal(None, None, None) is True
assert _is_plausible_deal(None, 100, None) is False # floor всё равно судим
assert _is_plausible_deal(39_700, None, None) is False # ppm² всё равно судим
# ---------------------------------------------------------------------------
# Mera-audit fix-2: floor edge cases + area_m2/price_rub guards
# ---------------------------------------------------------------------------
def test_drops_floor_negative() -> None:
# floor=-5 из битого парсера → drop.
assert _is_plausible_deal(150_000, -5, 9) is False
def test_drops_floor_zero() -> None:
# floor=0 — не существует (1-indexed) → drop.
assert _is_plausible_deal(150_000, 0, 9) is False
def test_drops_floor_999() -> None:
# floor=999 из битого парсера → drop (> DEAL_MAX_FLOOR=60).
assert _is_plausible_deal(150_000, 999, None) is False
def test_keeps_floor_none_graceful() -> None:
# floor=None → допустимо, не судим.
assert _is_plausible_deal(150_000, None, None) is True
def test_keeps_floor_1_valid() -> None:
assert _is_plausible_deal(150_000, 1, 9) is True
def test_keeps_floor_at_max() -> None:
assert _is_plausible_deal(150_000, DEAL_MAX_FLOOR, None) is True
def test_drops_area_m2_zero() -> None:
# area_m2=0 из битого парсера → drop.
assert _is_plausible_deal(150_000, 5, 9, area_m2=0.0) is False
def test_drops_area_m2_negative() -> None:
# area_m2=-10 → drop.
assert _is_plausible_deal(150_000, 5, 9, area_m2=-10.0) is False
def test_keeps_area_m2_none() -> None:
# area_m2=None → не судим (graceful).
assert _is_plausible_deal(150_000, 5, 9, area_m2=None) is True
def test_keeps_area_m2_valid() -> None:
assert _is_plausible_deal(150_000, 5, 9, area_m2=55.0) is True
def test_drops_price_rub_zero() -> None:
# price_rub=0 → нерыночная/техническая сделка → drop.
assert _is_plausible_deal(150_000, 5, 9, price_rub=0.0) is False
def test_drops_price_rub_negative() -> None:
assert _is_plausible_deal(150_000, 5, 9, price_rub=-1.0) is False
def test_keeps_price_rub_none() -> None:
# price_rub=None → не судим (graceful).
assert _is_plausible_deal(150_000, 5, 9, price_rub=None) is True
def test_keeps_price_rub_valid() -> None:
assert _is_plausible_deal(150_000, 5, 9, price_rub=8_000_000.0) is True

View file

@ -0,0 +1,147 @@
"""Mera-audit fix-3: cross-source dedup в _fetch_price_trend.
Проверяет что при dedup_enabled=True один объект на avito_imv + yandex_valuation
с разными ext_item_id не double-count'ится в месячной медиане.
"""
from __future__ import annotations
import os
from typing import Any
from unittest.mock import MagicMock, patch
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
import pytest
from app.services.estimator import _fetch_price_trend
def _make_mock_db(source1_rows: list[dict], source2_rows: list[dict]) -> MagicMock:
"""Возвращает mock db: Source1 (houses_price_dynamics) → source1_rows,
Source2 (house_placement_history) source2_rows."""
mock_db = MagicMock()
def side_effect(*args: Any, **kwargs: Any) -> MagicMock:
call_text = str(args[0].text) if args else ""
result = MagicMock()
if "houses_price_dynamics" in call_text:
result.mappings.return_value.all.return_value = source1_rows
else:
result.mappings.return_value.all.return_value = source2_rows
return result
mock_db.execute.side_effect = side_effect
return mock_db
# ---------------------------------------------------------------------------
# Fix-3: dedup enabled (default True)
# ---------------------------------------------------------------------------
def test_dedup_enabled_flag_is_true_by_default() -> None:
"""estimate_price_trend_dedup_enabled дефолт True."""
from app.core.config import settings
assert settings.estimate_price_trend_dedup_enabled is True
def test_price_trend_dedup_sql_uses_distinct_on_when_enabled() -> None:
"""Когда dedup включён, SQL-запрос к house_placement_history содержит DISTINCT ON."""
mock_db = _make_mock_db([], [{"month": "2026-01", "ppm2": 150_000}])
with patch("app.services.estimator.settings") as mock_settings:
mock_settings.estimate_price_trend_dedup_enabled = True
mock_settings.estimate_price_trend_max_age_months = 6
_fetch_price_trend(mock_db, target_house_id=42, min_points=1)
calls = mock_db.execute.call_args_list
# Второй вызов — Source 2 (house_placement_history)
assert len(calls) >= 2
second_sql = str(calls[1].args[0].text) if calls[1].args else ""
assert "DISTINCT ON" in second_sql.upper() or "distinct on" in second_sql.lower()
def test_price_trend_dedup_disabled_no_distinct_on() -> None:
"""Когда dedup выключен, SQL НЕ содержит DISTINCT ON (legacy path)."""
mock_db = _make_mock_db([], [{"month": "2026-01", "ppm2": 150_000}])
with patch("app.services.estimator.settings") as mock_settings:
mock_settings.estimate_price_trend_dedup_enabled = False
mock_settings.estimate_price_trend_max_age_months = 6
_fetch_price_trend(mock_db, target_house_id=42, min_points=1)
calls = mock_db.execute.call_args_list
assert len(calls) >= 2
second_sql = str(calls[1].args[0].text) if calls[1].args else ""
assert "distinct on" not in second_sql.lower()
def test_price_trend_returns_result_with_dedup_enabled() -> None:
"""Функция корректно возвращает точки тренда при dedup_enabled=True."""
rows = [
{"month": "2025-11", "ppm2": 148_000},
{"month": "2025-12", "ppm2": 152_000},
{"month": "2026-01", "ppm2": 155_000},
]
mock_db = _make_mock_db([], rows)
with patch("app.services.estimator.settings") as mock_settings:
mock_settings.estimate_price_trend_dedup_enabled = True
mock_settings.estimate_price_trend_max_age_months = 6
result = _fetch_price_trend(mock_db, target_house_id=42, min_points=3)
assert result is not None
assert len(result) == 3
assert result[0]["month"] == "2025-11"
assert result[0]["ppm2"] == 148_000
def test_price_trend_dedup_disabled_returns_same_points() -> None:
"""Флаг OFF не ломает возврат данных (backward-compat)."""
rows = [
{"month": "2025-11", "ppm2": 148_000},
{"month": "2025-12", "ppm2": 152_000},
{"month": "2026-01", "ppm2": 155_000},
]
mock_db = _make_mock_db([], rows)
with patch("app.services.estimator.settings") as mock_settings:
mock_settings.estimate_price_trend_dedup_enabled = False
mock_settings.estimate_price_trend_max_age_months = 6
result = _fetch_price_trend(mock_db, target_house_id=42, min_points=3)
assert result is not None
assert len(result) == 3
def test_price_trend_source1_preferred_over_dedup_path() -> None:
"""Если Source1 (houses_price_dynamics) вернул ≥ min_points → Source2 не вызывается."""
source1_rows = [
{"month": "2025-11", "ppm2": 140_000},
{"month": "2025-12", "ppm2": 145_000},
{"month": "2026-01", "ppm2": 150_000},
]
mock_db = _make_mock_db(source1_rows, [])
with patch("app.services.estimator.settings") as mock_settings:
mock_settings.estimate_price_trend_dedup_enabled = True
mock_settings.estimate_price_trend_max_age_months = 6
result = _fetch_price_trend(mock_db, target_house_id=42, min_points=3)
# Source1 вернул 3 точки — Source2 не должен вызываться
assert result is not None
assert len(result) == 3
calls = mock_db.execute.call_args_list
# Ровно один вызов (Source1)
assert len(calls) == 1
if __name__ == "__main__": # pragma: no cover
raise SystemExit(pytest.main([__file__, "-q"]))