Merge pull request 'fix(tradein): guard listing_segment во всех оценочных читателях listings — novostroyki вне comps (#1186)' (#1192) from fix/tradein-segment-guards into main
All checks were successful
Deploy Trade-In / changes (push) Successful in 6s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 32s
Deploy Trade-In / build-backend (push) Successful in 1m2s
Deploy Trade-In / deploy (push) Successful in 40s

Reviewed-on: #1192
This commit is contained in:
lekss361 2026-06-12 09:06:28 +00:00
commit d64a57cfcd
5 changed files with 327 additions and 30 deletions

View file

@ -1428,15 +1428,15 @@ def _fetch_anchor_comps(
lon: float | None,
rooms: int | None,
area: float | None,
listing_segment: str | None = None,
) -> tuple[list[dict[str, Any]], str | None]:
"""Тированный набор комплов для same-building якоря. Стоп на 1-м тире с ≥ min_comps.
Tier A SAME BUILDING: normalized street + base house no (+ литера если есть).
RELAXED rooms (без фильтра), БЕЗ area±15%. Не группируем по house_id_fk
один дом дробится на несколько fk (Хохрякова 48 = 7085/9878/12797).
Tier C micro-radius 500m (ST_DWithin) + same listing_segment + rooms match
+ area±25%. (Tier B «тот же ЖК» skip: complex_id/cian_zhk_url ненадёжны.)
Tier C micro-radius 500m (ST_DWithin) + вторичка-канон guard (#1186): NULL = legacy
вторичка + rooms match + area±25%. (Tier B «тот же ЖК» skip: complex_id/cian_zhk_url
ненадёжны.)
Tier D фолбэк: None tier (caller остаётся на радиусном median-пути).
Excludes lots без price_per_m2. is_active=true. Best-effort: ([], None) на ошибке.
@ -1470,6 +1470,8 @@ def _fetch_anchor_comps(
AND price_per_m2 > 0
AND lower(translate(address, 'ёЁ', 'ее')) LIKE :street_like
AND lower(translate(address, 'ёЁ', 'ее')) ~ :house_re
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
"""
),
{
@ -1513,11 +1515,8 @@ def _fetch_anchor_comps(
AND price_per_m2 > 0
AND rooms = CAST(:rooms AS integer)
AND area_m2 BETWEEN :area_min AND :area_max
AND (
CAST(:segment AS text) IS NULL
OR listing_segment IS NULL
OR listing_segment = CAST(:segment AS text)
)
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
AND geom IS NOT NULL
AND ST_DWithin(
geom::geography,
@ -1530,7 +1529,6 @@ def _fetch_anchor_comps(
"rooms": rooms,
"area_min": area * 0.75,
"area_max": area * 1.25,
"segment": listing_segment,
"lon": lon,
"lat": lat,
},
@ -2154,7 +2152,7 @@ async def estimate_quality(
# Якорь из комплов ТОГО ЖЕ ДОМА (Tier A) / micro-radius (Tier C). Когда он
# сработал — он ЗАМЕНЯЕТ радиусную медиану для median_price/ppm²/range (premium
# больше не размывается массовой застройкой). За флагом; OFF ⇒ точно старое
# поведение. listing_segment для Tier C берём из самого частого среди аналогов.
# поведение. Сегмент в Tier C фиксирован guard'ом (#1186): вторичка + NULL-legacy.
anchor_tier: str | None = None
# #694: комплы того же дома, на которых РЕАЛЬНО построен headline-якорь.
# Заполняется только когда anchor мутировал median (ниже) — тогда UI-аналоги
@ -2173,13 +2171,6 @@ async def estimate_quality(
and payload.area_m2
and (payload.address or (geo is not None and geo.full_address))
):
seg_counts: dict[str, int] = {}
for lot in listings_clean:
seg = lot.get("listing_segment")
if seg:
seg_counts[seg] = seg_counts.get(seg, 0) + 1
target_segment = max(seg_counts, key=seg_counts.get) if seg_counts else None
comps, anchor_tier = _fetch_anchor_comps(
db,
# payload.address (сырой ввод) ПЕРВИЧЕН: на проде геокод часто
@ -2192,7 +2183,6 @@ async def estimate_quality(
lon=geo.lon,
rooms=payload.rooms,
area=payload.area_m2,
listing_segment=target_segment,
)
anchor = _compute_same_building_anchor(
comps,
@ -3246,10 +3236,11 @@ _COMMON_WHERE = """
OR year_built BETWEEN CAST(:cohort_year_min AS integer)
AND CAST(:cohort_year_max AS integer)
)
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
-- Исключаем новостройки из comp-пула вторички: девелоперский прайс искажает
-- медиану /м². NULL сегмент пропускаем (rosreestr/avito/yandex без сегмента
-- это вторичка или неклассифицированный объект).
AND (listing_segment IS NULL OR listing_segment <> 'novostroyki')
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
"""
# Note: Tier W has its own inline copy of the cohort clause (PR #519 line
# ~1280). Не удалять — Tier W не использует _COMMON_WHERE из-за inline
@ -3598,9 +3589,9 @@ def _fetch_analogs(
OR year_built BETWEEN CAST(:cohort_year_min AS integer)
AND CAST(:cohort_year_max AS integer)
)
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
-- Tier W: исключаем новостройки из comp-пула (sync с _COMMON_WHERE).
-- NULL сегмент пропускаем rosreestr/avito/yandex без сегмента вторичка.
AND (listing_segment IS NULL OR listing_segment <> 'novostroyki')
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
-- 2026-05-23: Avito coords теперь real (PR #487 убрал jitter после
-- C-5 audit). Listings с NULL coords отфильтруются через ST_DWithin
-- (geom IS NULL не matches). geocode-missing-listings backfill

View file

@ -393,13 +393,16 @@ async def pull_sber_indices(
# Task B: best-effort reconciliation to our scrape-median asking ppm²
try:
our_median = db.execute(
text("""
SELECT percentile_cont(0.5)
WITHIN GROUP (ORDER BY price_per_m2) AS m
FROM listings
WHERE is_active = true
AND price_per_m2 BETWEEN 50000 AND 800000
""")
text(
# novostroyki guard (#1186): NULL = legacy вторичка до м.011
"SELECT percentile_cont(0.5)"
" WITHIN GROUP (ORDER BY price_per_m2) AS m"
" FROM listings"
" WHERE is_active = true"
" AND price_per_m2 BETWEEN 50000 AND 800000"
" AND (listing_segment IS NULL"
" OR listing_segment = 'vtorichka')"
)
).scalar()
if our_median is not None and latest_val:
sber_latest = latest_val

View file

@ -94,6 +94,8 @@ _REDERIVE_SQL = text(
WHERE is_active
AND rooms IS NOT NULL
AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
GROUP BY LEAST(GREATEST(rooms, 0), 4)
),
-- Per-rooms строки: только бакеты с обеими сторонами, прошедшие порог 30/30 и ask>0.
@ -138,6 +140,8 @@ _REDERIVE_SQL = text(
WHERE is_active
AND rooms IS NOT NULL
AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
),
-- Global fallback строка rooms_bucket=-1 (пишется всегда, если ask>0).
global_row AS (
@ -215,6 +219,8 @@ _REDERIVE_BOUNDS_SQL = text(
WHERE is_active
AND rooms IS NOT NULL
AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
GROUP BY LEAST(GREATEST(rooms, 0), 4)
HAVING COUNT(*) > 0
)
@ -259,6 +265,8 @@ _REDERIVE_TIERED_SQL = text(
WHERE l.is_active
AND l.rooms IS NOT NULL
AND l.price_per_m2 BETWEEN :ppm2_min AND :ppm2_max
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
AND (l.listing_segment IS NULL OR l.listing_segment = 'vtorichka')
),
deal_side_tier AS (
SELECT
@ -320,6 +328,8 @@ _REDERIVE_TIERED_SQL = text(
WHERE is_active
AND rooms IS NOT NULL
AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
GROUP BY LEAST(GREATEST(rooms, 0), 4)
),
all_rows AS (
@ -361,6 +371,8 @@ _REDERIVE_TIERED_SQL = text(
WHERE is_active
AND rooms IS NOT NULL
AND price_per_m2 BETWEEN :ppm2_min AND :ppm2_max
-- novostroyki guard (#1186): NULL = legacy вторичка до м.011
AND (listing_segment IS NULL OR listing_segment = 'vtorichka')
),
global_row AS (
SELECT

View file

@ -135,6 +135,9 @@ def test_migration_080_derivation_is_subset_of_refresh_sql() -> None:
ppm² bounds are compared after normalisation: the 080 seed uses literal integers
(30000 / 600000) while the refresh uses bind params (:ppm2_min / :ppm2_max)
both are replaced with the token PPM2_BAND_PLACEHOLDER before the subset check.
#1186: the refresh now adds the novostroyki guard predicate to each ask_* CTE;
it is normalised away here so the 080 seed (no guard) still matches.
"""
seed_sql = _MIGRATION_080.read_text("utf-8")
# Extract the WITH … (up to the ON CONFLICT) from the seed.
@ -149,8 +152,17 @@ def test_migration_080_derivation_is_subset_of_refresh_sql() -> None:
s = re.sub(r":ppm2_min AND :ppm2_max", "PPM2_BAND_PLACEHOLDER", s)
return s
assert _strip_sql(_normalise_ppm2(seed_derivation)) in _strip_sql(
_normalise_ppm2(_REDERIVE_SQL)
def _drop_segment_guard(s: str) -> str:
"""Remove the #1186 novostroyki guard predicate (absent in the 080 seed)."""
return re.sub(
r"AND\s*\((?:\w+\.)?listing_segment\s+IS\s+NULL\s+"
r"OR\s+(?:\w+\.)?listing_segment\s*=\s*'vtorichka'\)",
"",
s,
)
assert _strip_sql(_normalise_ppm2(_drop_segment_guard(seed_derivation))) in _strip_sql(
_normalise_ppm2(_drop_segment_guard(_REDERIVE_SQL))
)

View file

@ -0,0 +1,279 @@
"""#1186 — query-side guard: novostroyki строки не попадают в comp-пул вторички.
Тесты статические (SQL-string inspection) + поведенческие (_fetch_analogs mock).
Каноничный предикат: (listing_segment IS NULL OR listing_segment = 'vtorichka')
NULL = legacy вторичка до миграции 011 отбрасывать нельзя.
"""
from __future__ import annotations
import inspect
import os
import re
from typing import Any
from unittest.mock import MagicMock
# Settings требует DATABASE_URL при инициализации (fail-fast, C-3).
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services import estimator as est_mod
from app.tasks import asking_to_sold_ratio as ratio_mod
# ── Canonical guard predicate ─────────────────────────────────────────────────
_GUARD = "(listing_segment IS NULL OR listing_segment = 'vtorichka')"
_GUARD_RE = re.compile(
r"\((?:\w+\.)?listing_segment\s+IS\s+NULL\s+OR\s+(?:\w+\.)?listing_segment\s*=\s*'vtorichka'\)",
re.IGNORECASE,
)
def _norm(s: str) -> str:
"""Drop -- comments и склей пробелы для устойчивого поиска."""
no_comments = re.sub(r"--[^\n]*", "", s)
return re.sub(r"\s+", " ", no_comments).strip()
# ── estimator.py static checks ────────────────────────────────────────────────
def test_common_where_has_canonical_guard() -> None:
"""_COMMON_WHERE используется Tier S и Tier H — должен содержать канон-предикат."""
assert _GUARD_RE.search(
est_mod._COMMON_WHERE
), "_COMMON_WHERE lacks novostroyki guard — Tier S/H comp set contaminated"
def test_common_where_no_old_neq_form() -> None:
"""Старый вариант listing_segment <> 'novostroyki' запрещён — не охватывает иные сегменты."""
assert "listing_segment <> 'novostroyki'" not in est_mod._COMMON_WHERE
assert "listing_segment != 'novostroyki'" not in est_mod._COMMON_WHERE
def test_tier_w_inline_has_canonical_guard() -> None:
"""Tier W использует inline SQL, а не _COMMON_WHERE — нужен собственный guard."""
# Получаем исходник функции _fetch_analogs.
src = inspect.getsource(est_mod._fetch_analogs)
# Tier W — inline блок (не через _COMMON_WHERE).
assert _GUARD_RE.search(src), "_fetch_analogs Tier W inline SQL lacks novostroyki guard"
# Старой формы быть не должно.
assert "listing_segment <> 'novostroyki'" not in src
assert "listing_segment != 'novostroyki'" not in src
def test_tier_a_anchor_has_canonical_guard() -> None:
"""Tier A anchor (same building) — ранее не имел segment-guard (#1186 fix)."""
src = inspect.getsource(est_mod._fetch_anchor_comps)
assert _GUARD_RE.search(src), "_fetch_anchor_comps Tier A SQL lacks novostroyki guard"
def test_tier_c_anchor_has_canonical_guard_not_segment_param() -> None:
"""Tier C anchor — канон-предикат, не CAST(:segment AS text) (тот вариант отбрасывал NULL)."""
src = inspect.getsource(est_mod._fetch_anchor_comps)
# Canonical guard present.
assert _GUARD_RE.search(src)
# Старый параметрический вариант убран.
assert "listing_segment = CAST(:segment AS text)" not in src
# ── asking_to_sold_ratio.py static checks ─────────────────────────────────────
_REDERIVE_SQL_TEXT = str(ratio_mod._REDERIVE_SQL.text)
_REDERIVE_BOUNDS_TEXT = str(ratio_mod._REDERIVE_BOUNDS_SQL.text)
_REDERIVE_TIERED_TEXT = str(ratio_mod._REDERIVE_TIERED_SQL.text)
def test_ask_side_cte_has_guard() -> None:
"""ask_side CTE в _REDERIVE_SQL (per-rooms asking медиана) — guard обязателен."""
assert _GUARD_RE.search(
_REDERIVE_SQL_TEXT
), "ask_side CTE in _REDERIVE_SQL lacks novostroyki guard"
def test_ask_global_cte_has_guard() -> None:
"""ask_global CTE в _REDERIVE_SQL (global fallback asking медиана) — guard обязателен."""
# _REDERIVE_SQL содержит два `ask_global`-блока; ищем оба через count.
matches = len(_GUARD_RE.findall(_REDERIVE_SQL_TEXT))
assert (
matches >= 2
), f"Expected ≥2 guard occurrences in _REDERIVE_SQL (ask_side + ask_global), got {matches}"
def test_bounds_sql_has_guard() -> None:
"""_REDERIVE_BOUNDS_SQL (tier bounds percentiles) — guard обязателен."""
assert _GUARD_RE.search(_REDERIVE_BOUNDS_TEXT), "_REDERIVE_BOUNDS_SQL lacks novostroyki guard"
def test_tiered_ask_ctes_have_guard() -> None:
"""_REDERIVE_TIERED_SQL: ask_tiered + ask_side_all + ask_global — минимум 3 вхождения."""
matches = len(_GUARD_RE.findall(_REDERIVE_TIERED_TEXT))
assert matches >= 3, (
f"Expected ≥3 guard occurrences in _REDERIVE_TIERED_SQL "
f"(ask_tiered + ask_side_all + ask_global), got {matches}"
)
# ── Поведенческие тесты: _fetch_analogs (mock DB) ────────────────────────────
def _make_listing(
*,
source: str = "avito",
address: str = "ул. Ленина, 1",
distance_m: float = 100.0,
relevance_score: float = 0.1,
price_rub: float = 5_000_000.0,
area_m2: float = 40.0,
rooms: int = 2,
listing_segment: str | None = None,
) -> dict[str, Any]:
"""Минимальный listing-dict, имитирующий вывод DB mapping."""
return {
"source": source,
"source_url": f"https://{source}.ru/offer/1",
"address": address,
"lat": 56.838,
"lon": 60.595,
"rooms": rooms,
"area_m2": area_m2,
"floor": 3,
"total_floors": 9,
"price_rub": price_rub,
"price_per_m2": price_rub / area_m2,
"listing_date": None,
"days_on_market": 15,
"photo_urls": [],
"scraped_at": None,
"distance_m": distance_m,
"relevance_score": relevance_score,
"building_cadastral_number": None,
"listing_segment": listing_segment,
}
def _make_db_mock(rows: list[dict[str, Any]]) -> MagicMock:
"""Session mock: db.execute().mappings().all() возвращает rows."""
db = MagicMock()
db.execute.return_value.mappings.return_value.all.return_value = rows
return db
def test_fetch_analogs_sql_guard_present_novostroyki_excluded() -> None:
"""SQL guard корректно записан: listing_segment='novostroyki' не должен присутствовать
в WHERE-предикате (guard зарыт в SQL, а не в Python post-filter).
Тест проверяет СТАТИКУ SQL внутри _fetch_analogs (Tier W path), а не runtime фильтрацию
runtime-фильтрацию сделала бы только интеграционная БД. Убеждаемся, что кто-то в будущем
не переписал SQL, убрав guard.
"""
src = inspect.getsource(est_mod._fetch_analogs)
# Guard должен присутствовать хотя бы один раз в теле функции.
assert _GUARD_RE.search(
src
), "_fetch_analogs SQL no longer contains novostroyki guard — guard was removed!"
def test_null_segment_listing_not_excluded_by_guard() -> None:
"""listing_segment=NULL (legacy вторичка до м.011) должны ПОПАСТЬ в comps.
Поведенческий тест: mock возвращает строку с segment=None, _fetch_analogs
должен включить её в результат (_stratify_candidates не отфильтровывает NULL).
"""
from app.services.estimator import _fetch_analogs
null_seg_listing = _make_listing(listing_segment=None, distance_m=50.0, relevance_score=0.05)
db = _make_db_mock([null_seg_listing])
result, _fallback, _tier = _fetch_analogs(
db, lat=56.838, lon=60.595, rooms=2, area=40.0, radius_m=1000
)
# NULL-segment listing должен присутствовать в результате (не отброшен Python-стороной).
assert any(
r.get("source") == "avito" for r in result
), "NULL-segment listing was unexpectedly excluded from comp set"
def test_vtorichka_segment_listing_included() -> None:
"""listing_segment='vtorichka' — явный вторичный сегмент — должен попасть в comps."""
from app.services.estimator import _fetch_analogs
vtor_listing = _make_listing(listing_segment="vtorichka", distance_m=80.0)
db = _make_db_mock([vtor_listing])
result, _, _ = _fetch_analogs(db, lat=56.838, lon=60.595, rooms=2, area=40.0, radius_m=1000)
assert len(result) >= 1, "vtorichka-segment listing was unexpectedly excluded"
# ── Негативные assert'ы: запрещённые формы guard'а во всём estimator'е ────────
def _estimator_full_src() -> str:
"""Полный исходник модуля estimator (для статической проверки всех SQL-блоков)."""
return inspect.getsource(est_mod)
def test_no_bare_vtorichka_eq_without_null_guard() -> None:
"""Запрет голого `listing_segment = 'vtorichka'` без IS NULL OR части.
Такой вариант отбросит NULL (legacy вторичку до миграции 011) баг охвата.
"""
src = _estimator_full_src()
# Ищем вхождения = 'vtorichka' которым НЕ предшествует IS NULL OR на той же строке.
bad_matches = [
line
for line in src.splitlines()
if "= 'vtorichka'" in line and "IS NULL OR" not in line and "--" not in line.lstrip()[:3]
]
assert not bad_matches, (
"Found bare `= 'vtorichka'` without IS NULL OR guard in estimator.py:\n"
+ "\n".join(bad_matches)
)
def test_no_neq_novostroyki_form_in_estimator() -> None:
"""Запрет формы `listing_segment <> 'novostroyki'` / `!= 'novostroyki'` в estimator.
Эта форма пропускает прочие сегменты (коммерция и т.п.) и не охватывает NULL.
Канон: IS NULL OR = 'vtorichka'.
"""
src = _estimator_full_src()
assert (
"<> 'novostroyki'" not in src
), "estimator.py contains deprecated `<> 'novostroyki'` form — use canonical guard"
assert (
"!= 'novostroyki'" not in src
), "estimator.py contains deprecated `!= 'novostroyki'` form — use canonical guard"
def test_anchor_comps_no_dead_listing_segment_param() -> None:
"""_fetch_anchor_comps не должна принимать listing_segment как параметр.
После #1186 guard хардкожен в SQL — параметр убран, чтобы caller не мог
случайно передать 'novostroyki' или другой сегмент и обойти guard.
"""
import inspect as _inspect
sig = _inspect.signature(est_mod._fetch_anchor_comps)
assert "listing_segment" not in sig.parameters, (
"_fetch_anchor_comps still has `listing_segment` parameter — "
"it was removed in #1186 review fixup; guard is hardcoded in SQL"
)
def test_fetch_anchor_comps_tier_c_canonical_guard_not_parametric() -> None:
"""Tier C в _fetch_anchor_comps использует канон-guard, а не параметрическое сравнение.
Параметрическая форма `listing_segment = CAST(:segment AS text)` исключала NULL
и давала caller'у возможность выбрать произвольный сегмент (включая novostroyki).
"""
src = inspect.getsource(est_mod._fetch_anchor_comps)
assert "CAST(:segment AS text)" not in src, (
"_fetch_anchor_comps Tier C still uses parametric segment filter — "
"must be hardcoded canonical guard"
)
# И канон-guard на месте (Tier C-блок).
assert _GUARD_RE.search(
src
), "_fetch_anchor_comps lacks canonical guard after removing parametric form"