1586 lines
66 KiB
Python
1586 lines
66 KiB
Python
"""Trade-In Estimator — реальное SQL aggregation поверх listings + deals.
|
||
|
||
Заменяет старый _mock_estimate() из api/v1/trade_in.py.
|
||
|
||
Алгоритм:
|
||
1. Geocode address → (lat, lon)
|
||
2. SELECT listings с фильтрами:
|
||
- PostGIS ST_DWithin (geom, point, 1000m) — радиус поиска
|
||
- source ≠ avito (у Avito фейковые anchor-jitter координаты — не гео-аналог)
|
||
- rooms = target_rooms (точное совпадение)
|
||
- area_m2 BETWEEN target × 0.85 AND target × 1.15
|
||
- scraped_at > NOW() - 14 days (свежие)
|
||
- is_active = true
|
||
3. Tukey outlier filter (1.5 × IQR rule)
|
||
4. Median / Q1 / Q3 / count → confidence
|
||
5. То же для deals (period = 12 mo).
|
||
6. Сохранить в trade_in_estimates + вернуть AggregatedEstimate
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import math
|
||
import re
|
||
from datetime import UTC, datetime, timedelta
|
||
from typing import Any
|
||
from uuid import uuid4
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.schemas.trade_in import AggregatedEstimate, AnalogLot, TradeInEstimateInput
|
||
from app.services.geocoder import GeocodeResult, geocode
|
||
from app.services.house_metadata import get_house_metadata
|
||
from app.services.scrapers.avito_imv import (
|
||
IMVAddressNotFoundError,
|
||
IMVAuthError,
|
||
IMVEvaluation,
|
||
IMVTransientError,
|
||
compute_imv_cache_key,
|
||
evaluate_via_imv,
|
||
save_imv_evaluation,
|
||
)
|
||
from app.services.scrapers.cian_valuation import (
|
||
CianValuationResult,
|
||
estimate_via_cian_valuation,
|
||
)
|
||
from app.services.scrapers.yandex_valuation import (
|
||
YandexValuationResult,
|
||
YandexValuationScraper,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ── Constants ────────────────────────────────────────────────────────────────
|
||
DEFAULT_RADIUS_M = 1000 # ПО ВСТРЕЧЕ ПТИЦЫ: «локация не дальше 800-1000 м»
|
||
FALLBACK_RADIUS_M = 2000
|
||
AREA_TOLERANCE = 0.15 # ±15% площади
|
||
MAX_ANALOGS_PER_ADDRESS = 5 # анти-bias: не больше 5 лотов с одного адреса
|
||
MIN_ANALOGS_PER_SOURCE = 5 # гарантированный минимум на live source
|
||
LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней
|
||
DEALS_PERIOD_MONTHS = 12 # сделки за последний год
|
||
|
||
# Когорта по году постройки — типизация массовой застройки РФ.
|
||
# Используется как hard-filter в Tier 0 _fetch_analogs (PR 9, 2026-05-24).
|
||
# Если target_year не задан — cohort = None → фильтр отключён, Tier 0 пропускается.
|
||
COHORTS = (
|
||
# (cohort_name, year_min_inclusive, year_max_inclusive)
|
||
('khrushchev', 1955, 1969), # Хрущёвки 5-эт
|
||
('brezhnev', 1970, 1989), # Брежневка кирпич/панель 9–12-эт
|
||
('late_soviet', 1985, 1999), # Поздний СССР (overlap с brezhnev intentional)
|
||
('2000s', 2000, 2010), # Ранние новостройки
|
||
('modern', 2011, 2100), # Современные ЖК
|
||
)
|
||
# Минимум аналогов чтобы остаться на Tier 0 (с cohort); ниже — fallback на Tier A.
|
||
MIN_ANALOGS_TIER_0 = 5
|
||
|
||
|
||
def _target_cohort_range(year_built: int | None) -> tuple[int, int] | None:
|
||
"""Maps a target year to its cohort year range [min, max] inclusive.
|
||
|
||
Returns None if year_built is None — caller will skip cohort filter.
|
||
Picks the FIRST matching cohort (so 1988 → 'brezhnev', not 'late_soviet').
|
||
"""
|
||
if year_built is None:
|
||
return None
|
||
for _name, ymin, ymax in COHORTS:
|
||
if ymin <= year_built <= ymax:
|
||
return (ymin, ymax)
|
||
# Out-of-range год (например, 1900 или 2050) — cohort фильтр не применяем,
|
||
# лучше показать что есть в радиусе, чем 0 результатов.
|
||
return None
|
||
|
||
|
||
# Поправочные коэффициенты на состояние ремонта. Аналоги в выборке — микс
|
||
# состояний (≈ "стандартный/косметический"), коэффициент сдвигает медиану под
|
||
# конкретный ремонт целевой квартиры. Встреча Птицы: ремонт влияет на цену.
|
||
_IMV_HOUSE_TYPE_MAP: dict[str | None, str | None] = {
|
||
"panel": "panel",
|
||
"brick": "brick",
|
||
"monolith": "monolith",
|
||
"monolith_brick": "monolith_brick",
|
||
"monolithic": "monolith",
|
||
"block": "block",
|
||
"wood": "wood",
|
||
None: None,
|
||
}
|
||
_IMV_REPAIR_MAP: dict[str | None, str | None] = {
|
||
"needs_repair": "required",
|
||
"standard": "cosmetic",
|
||
"good": "euro",
|
||
"excellent": "designer",
|
||
None: None,
|
||
}
|
||
|
||
_REPAIR_COEF: dict[str, float] = {
|
||
"needs_repair": 0.92, # требует ремонта — ниже рынка
|
||
"standard": 0.98,
|
||
"good": 1.03,
|
||
"excellent": 1.08, # евроремонт — выше рынка
|
||
}
|
||
_REPAIR_LABEL: dict[str | None, str] = {
|
||
"needs_repair": "требует ремонта",
|
||
"standard": "стандартный ремонт",
|
||
"good": "хороший ремонт",
|
||
"excellent": "евроремонт",
|
||
}
|
||
|
||
|
||
def _repair_coefficient(repair_state: str | None) -> float:
|
||
"""Множитель к медиане по состоянию ремонта. None → 1.0 (без поправки)."""
|
||
if not repair_state:
|
||
return 1.0
|
||
return _REPAIR_COEF.get(repair_state, 1.0)
|
||
|
||
|
||
# ── Avito IMV cache lookup (Stage 3) ────────────────────────────────────────
|
||
IMV_CACHE_TTL_HOURS = 24
|
||
|
||
# Префиксы в адресе, которые Avito-геокодер не распознаёт (не жилые назначения).
|
||
# Пример: "Склад, ул. Заводская, д. 44-а" → "ул. Заводская, д. 44-а"
|
||
_NOISE_PREFIX_RE = re.compile(
|
||
r"(Склад|Гараж|Подсобка|Нежилое|Помещение|Цех),\s*",
|
||
flags=re.IGNORECASE,
|
||
)
|
||
|
||
YANDEX_VALUATION_CACHE_TTL_HOURS = 24
|
||
YANDEX_VALUATION_DEFAULT_CATEGORY = "APARTMENT"
|
||
YANDEX_VALUATION_DEFAULT_TYPE = "SELL"
|
||
|
||
|
||
async def _get_or_fetch_imv_cached(
|
||
db: Session,
|
||
*,
|
||
address: str,
|
||
rooms: int,
|
||
area_m2: float,
|
||
floor: int,
|
||
floor_at_home: int,
|
||
house_type: str,
|
||
renovation_type: str,
|
||
has_balcony: bool,
|
||
has_loggia: bool,
|
||
estimate_id_for_link: Any = None,
|
||
) -> IMVEvaluation | None:
|
||
"""Cached IMV lookup. TTL 24h по cache_key (sha256 of address + params).
|
||
|
||
1. compute cache_key
|
||
2. SELECT из avito_imv_evaluations WHERE cache_key = :ck AND fetched_at > NOW() - 24h
|
||
3. Если hit → возвращаем reconstructed IMVEvaluation
|
||
4. Cache miss → call evaluate_via_imv, save_imv_evaluation, return
|
||
|
||
Graceful: на любой error возвращаем None (estimator продолжает без IMV).
|
||
"""
|
||
try:
|
||
cache_key = compute_imv_cache_key(
|
||
address, rooms, area_m2, floor, floor_at_home,
|
||
house_type, renovation_type, has_balcony, has_loggia,
|
||
)
|
||
|
||
existing = db.execute(
|
||
text(
|
||
"""
|
||
SELECT id, cache_key, address, rooms, area_m2, floor, floor_at_home,
|
||
house_type, renovation_type, has_balcony, has_loggia,
|
||
lat, lon, geo_hash, avito_address_id, avito_location_id,
|
||
avito_metro_id, avito_district_id,
|
||
recommended_price, lower_price, higher_price, market_count,
|
||
raw_response, fetched_at
|
||
FROM avito_imv_evaluations
|
||
WHERE cache_key = :ck
|
||
AND fetched_at > NOW() - (:ttl_hours || ' hours')::interval
|
||
ORDER BY fetched_at DESC
|
||
LIMIT 1
|
||
"""
|
||
),
|
||
{"ck": cache_key, "ttl_hours": IMV_CACHE_TTL_HOURS},
|
||
).mappings().first()
|
||
|
||
if existing is not None:
|
||
logger.info(
|
||
"imv: cache HIT key=%s recommended=%d",
|
||
cache_key[:8], existing["recommended_price"],
|
||
)
|
||
from app.services.scrapers.avito_imv import IMVGeo
|
||
return IMVEvaluation(
|
||
cache_key=existing["cache_key"],
|
||
address=existing["address"],
|
||
rooms=existing["rooms"],
|
||
area_m2=float(existing["area_m2"]),
|
||
floor=existing["floor"],
|
||
floor_at_home=existing["floor_at_home"],
|
||
house_type=existing["house_type"],
|
||
renovation_type=existing["renovation_type"],
|
||
has_balcony=existing["has_balcony"],
|
||
has_loggia=existing["has_loggia"],
|
||
geo=IMVGeo(
|
||
geo_hash=existing["geo_hash"] or "",
|
||
lat=existing["lat"],
|
||
lon=existing["lon"],
|
||
avito_address_id=existing["avito_address_id"],
|
||
avito_location_id=existing["avito_location_id"],
|
||
avito_metro_id=existing["avito_metro_id"],
|
||
avito_district_id=existing["avito_district_id"],
|
||
),
|
||
recommended_price=existing["recommended_price"],
|
||
lower_price=existing["lower_price"],
|
||
higher_price=existing["higher_price"],
|
||
market_count=existing["market_count"],
|
||
raw_response=existing.get("raw_response"),
|
||
)
|
||
|
||
# Cache miss — fresh fetch
|
||
logger.info("imv: cache MISS key=%s — fetching fresh", cache_key[:8])
|
||
result = await evaluate_via_imv(
|
||
address=address, rooms=rooms, area_m2=area_m2,
|
||
floor=floor, floor_at_home=floor_at_home,
|
||
house_type=house_type, renovation_type=renovation_type,
|
||
has_balcony=has_balcony, has_loggia=has_loggia,
|
||
)
|
||
save_imv_evaluation(db, result, estimate_id=estimate_id_for_link)
|
||
logger.info(
|
||
"imv: fresh recommended=%d range=(%d, %d) count=%d",
|
||
result.recommended_price, result.lower_price, result.higher_price,
|
||
result.market_count or 0,
|
||
)
|
||
return result
|
||
|
||
except IMVAddressNotFoundError as e:
|
||
logger.warning("imv: address not found in Avito geocoder: %s", e)
|
||
# Retry once with noise prefixes stripped (e.g. "Склад, ул. X" → "ул. X")
|
||
cleaned = _NOISE_PREFIX_RE.sub("", address)
|
||
if cleaned != address:
|
||
logger.info(
|
||
"imv: retry with cleaned address %r → %r",
|
||
address[:60],
|
||
cleaned[:60],
|
||
)
|
||
try:
|
||
result = await evaluate_via_imv(
|
||
address=cleaned,
|
||
rooms=rooms,
|
||
area_m2=area_m2,
|
||
floor=floor,
|
||
floor_at_home=floor_at_home,
|
||
house_type=house_type,
|
||
renovation_type=renovation_type,
|
||
has_balcony=has_balcony,
|
||
has_loggia=has_loggia,
|
||
)
|
||
save_imv_evaluation(db, result, estimate_id=estimate_id_for_link)
|
||
logger.info(
|
||
"imv: retry OK recommended=%d range=(%d, %d) count=%d",
|
||
result.recommended_price,
|
||
result.lower_price,
|
||
result.higher_price,
|
||
result.market_count or 0,
|
||
)
|
||
return result
|
||
except IMVAddressNotFoundError:
|
||
logger.warning("imv: cleaned address also not found — giving up")
|
||
except Exception as retry_exc:
|
||
logger.warning("imv: retry failed: %s", retry_exc)
|
||
return None
|
||
except IMVAuthError as e:
|
||
logger.error(
|
||
"imv: auth/quota error — manual action required: %s", e,
|
||
)
|
||
return None
|
||
except IMVTransientError as e:
|
||
logger.warning("imv: transient error, skipping retry in estimator context: %s", e)
|
||
return None
|
||
except Exception as e:
|
||
logger.warning("imv: fetch failed — estimator продолжает без IMV: %s", e)
|
||
return None
|
||
|
||
|
||
# ── Yandex Valuation cache lookup (Stage 8) ─────────────────────────────────
|
||
|
||
def _yandex_valuation_cache_key(
|
||
address: str, offer_category: str, offer_type: str
|
||
) -> str:
|
||
"""SHA256 cache key for Yandex Valuation lookups."""
|
||
payload = f"{address}|{offer_category}|{offer_type}"
|
||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||
|
||
|
||
async def _get_or_fetch_yandex_valuation_cached(
|
||
db: Session,
|
||
*,
|
||
address: str,
|
||
offer_category: str = YANDEX_VALUATION_DEFAULT_CATEGORY,
|
||
offer_type: str = YANDEX_VALUATION_DEFAULT_TYPE,
|
||
) -> YandexValuationResult | None:
|
||
"""Cached Yandex Valuation lookup. TTL 24h via external_valuations table.
|
||
|
||
Returns None on any error / cache miss + fetch failure — caller continues
|
||
without Yandex enrichment (graceful degradation).
|
||
"""
|
||
cache_key = _yandex_valuation_cache_key(address, offer_category, offer_type)
|
||
|
||
# Cache lookup
|
||
try:
|
||
cached = db.execute(
|
||
text(
|
||
"""
|
||
SELECT raw_payload, fetched_at
|
||
FROM external_valuations
|
||
WHERE source = 'yandex_valuation'
|
||
AND cache_key = :ck
|
||
AND expires_at > NOW()
|
||
ORDER BY fetched_at DESC
|
||
LIMIT 1
|
||
"""
|
||
),
|
||
{"ck": cache_key},
|
||
).mappings().first()
|
||
except Exception as e:
|
||
logger.warning("yandex_valuation: cache lookup failed: %s", e)
|
||
cached = None
|
||
|
||
if cached is not None and cached.get("raw_payload"):
|
||
try:
|
||
payload_dict = (
|
||
cached["raw_payload"]
|
||
if isinstance(cached["raw_payload"], dict)
|
||
else json.loads(cached["raw_payload"])
|
||
)
|
||
logger.info(
|
||
"yandex_valuation: cache HIT key=%s items=%d",
|
||
cache_key[:8],
|
||
len(payload_dict.get("history_items", [])),
|
||
)
|
||
return YandexValuationResult.model_validate(payload_dict)
|
||
except Exception as e:
|
||
logger.warning("yandex_valuation: cache deserialize failed — refetching: %s", e)
|
||
|
||
# Fresh fetch
|
||
try:
|
||
async with YandexValuationScraper() as scraper:
|
||
result = await scraper.fetch_house_history(
|
||
address=address,
|
||
offer_category=offer_category,
|
||
offer_type=offer_type,
|
||
)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"yandex_valuation: fetch failed — estimator продолжает без Yandex: %s", e
|
||
)
|
||
return None
|
||
|
||
if result is None:
|
||
logger.info("yandex_valuation: empty result for address=%s", address[:60])
|
||
return None
|
||
|
||
# Save to cache (UPSERT on (source, cache_key))
|
||
try:
|
||
db.execute(
|
||
text(
|
||
"""
|
||
INSERT INTO external_valuations (
|
||
source, cache_key, address,
|
||
raw_payload,
|
||
fetched_at, expires_at
|
||
) VALUES (
|
||
'yandex_valuation', :ck, :addr,
|
||
CAST(:payload AS jsonb),
|
||
NOW(), NOW() + (:ttl_hours || ' hours')::interval
|
||
)
|
||
ON CONFLICT (source, cache_key) DO UPDATE
|
||
SET raw_payload = EXCLUDED.raw_payload,
|
||
fetched_at = NOW(),
|
||
expires_at = NOW() + (:ttl_hours || ' hours')::interval
|
||
"""
|
||
),
|
||
{
|
||
"ck": cache_key,
|
||
"addr": address,
|
||
"payload": json.dumps(result.model_dump(mode="json"), ensure_ascii=False),
|
||
"ttl_hours": YANDEX_VALUATION_CACHE_TTL_HOURS,
|
||
},
|
||
)
|
||
db.commit()
|
||
logger.info(
|
||
"yandex_valuation: fresh fetch saved key=%s items=%d",
|
||
cache_key[:8],
|
||
len(result.history_items),
|
||
)
|
||
except Exception as e:
|
||
logger.warning("yandex_valuation: cache save failed (continuing): %s", e)
|
||
db.rollback()
|
||
|
||
return result
|
||
|
||
|
||
def _save_yandex_history_items(
|
||
db: Session,
|
||
result: YandexValuationResult,
|
||
) -> int:
|
||
"""Persist history items to house_placement_history. Returns saved count.
|
||
|
||
house_id stays NULL — estimator doesn't compute target_house_id yet.
|
||
Idempotent via UNIQUE (source, ext_item_id); we synthesize ext_item_id from
|
||
(address|date|area|floor) hash since Yandex history items don't carry an
|
||
explicit ID.
|
||
|
||
Batch semantics (closes finding #5 from 2026-05-24 audit): all items
|
||
inserted under a single try/except; on any failure the whole batch is
|
||
rolled back as a unit (prior per-item rollback() destroyed the parent tx).
|
||
"""
|
||
if not result.history_items:
|
||
return 0
|
||
|
||
rows = []
|
||
for item in result.history_items:
|
||
# Synthesize stable ext_item_id (no native ID in valuation page)
|
||
ext_seed = (
|
||
f"{result.address}|{item.publish_date}|{item.area_m2}|{item.floor}|"
|
||
f"{item.start_price}|{item.last_price}"
|
||
)
|
||
ext_item_id = hashlib.sha256(ext_seed.encode("utf-8")).hexdigest()[:32]
|
||
rows.append({
|
||
"ext_id": ext_item_id,
|
||
"rooms": item.rooms,
|
||
"area": item.area_m2,
|
||
"floor": item.floor,
|
||
"start_price": item.start_price,
|
||
"last_price": item.last_price,
|
||
"publish_date": item.publish_date,
|
||
"exposure": item.exposure_days,
|
||
"raw": json.dumps(item.model_dump(mode="json"), ensure_ascii=False),
|
||
})
|
||
|
||
sql = text(
|
||
"""
|
||
INSERT INTO house_placement_history (
|
||
source, ext_item_id,
|
||
rooms, area_m2, floor,
|
||
start_price, start_price_date,
|
||
last_price, last_price_date,
|
||
exposure_days,
|
||
raw_payload
|
||
) VALUES (
|
||
'yandex_valuation', :ext_id,
|
||
:rooms, :area, :floor,
|
||
:start_price, :publish_date,
|
||
:last_price, :publish_date,
|
||
:exposure,
|
||
CAST(:raw AS jsonb)
|
||
)
|
||
ON CONFLICT (source, ext_item_id) DO NOTHING
|
||
"""
|
||
)
|
||
|
||
try:
|
||
for row in rows:
|
||
db.execute(sql, row)
|
||
db.commit()
|
||
return len(rows)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"yandex_valuation: failed to save history batch (%d items): %s",
|
||
len(rows), e,
|
||
)
|
||
db.rollback()
|
||
return 0
|
||
|
||
|
||
# ── Public ───────────────────────────────────────────────────────────────────
|
||
async def estimate_quality(
|
||
payload: TradeInEstimateInput, db: Session
|
||
) -> AggregatedEstimate:
|
||
"""Главная функция — оценка квартиры по реальным данным.
|
||
|
||
NOTE 2026-05-24: rosreestr_deals temporarily NOT included in actual_deals
|
||
output (they contain ДДУ first-market data that skews secondary-market
|
||
median). See Decision_TradeIn_DataQuality_8PR_Roadmap.
|
||
|
||
Returns:
|
||
AggregatedEstimate с estimate_id, медианой, диапазоном, аналогами.
|
||
"""
|
||
# 1. Geocode
|
||
geo: GeocodeResult | None = None
|
||
if payload.address:
|
||
geo = await geocode(payload.address, db)
|
||
|
||
if geo is None:
|
||
# Без координат не можем искать через PostGIS. Возвращаем low confidence.
|
||
logger.warning("geocode failed for %s — returning low-confidence estimate", payload.address)
|
||
return _empty_estimate(payload, db, reason="address_not_geocoded")
|
||
|
||
# 2. #392: обогащаем год / тип дома из картографии (OSM Overpass), если
|
||
# пользователь их не указал — это улучшает house-match аналогов (#6).
|
||
# Best-effort: при недоступности OSM target_* остаются None.
|
||
target_year = payload.year_built
|
||
target_house_type = payload.house_type
|
||
if target_year is None or target_house_type is None:
|
||
house_meta = await get_house_metadata(geo.lat, geo.lon, db)
|
||
if house_meta is not None:
|
||
if target_year is None:
|
||
target_year = house_meta.year_built
|
||
if target_house_type is None:
|
||
target_house_type = house_meta.house_type
|
||
|
||
# 3. Four-tier fallback (PR 9 — added Tier 0 with cohort filter):
|
||
# 0) 1km + ±15% area + cohort match (year_built — если задан)
|
||
# a) 1km + ±15% area (без cohort — drop fallback)
|
||
# b) 2km + ±15% area (fallback_used = True)
|
||
# c) 2km + ±25% area (fallback_used = True, area_widened = True)
|
||
cohort_range = _target_cohort_range(target_year)
|
||
|
||
if cohort_range is not None:
|
||
cy_min, cy_max = cohort_range
|
||
listings_tier0, _, analog_tier = _fetch_analogs(
|
||
db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2,
|
||
radius_m=DEFAULT_RADIUS_M,
|
||
full_address=geo.full_address,
|
||
year_built=target_year, house_type=target_house_type,
|
||
total_floors=payload.total_floors,
|
||
cohort_year_min=cy_min, cohort_year_max=cy_max,
|
||
)
|
||
else:
|
||
listings_tier0 = []
|
||
analog_tier = 'W'
|
||
|
||
if len(listings_tier0) >= MIN_ANALOGS_TIER_0:
|
||
listings = listings_tier0
|
||
fallback_used = False
|
||
else:
|
||
# Tier 0 пуст/мал — graceful fallback на Tier A без cohort
|
||
listings, fallback_used, analog_tier = _fetch_analogs(
|
||
db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2,
|
||
radius_m=DEFAULT_RADIUS_M,
|
||
full_address=geo.full_address,
|
||
year_built=target_year, house_type=target_house_type,
|
||
total_floors=payload.total_floors,
|
||
)
|
||
area_widened = False
|
||
|
||
if len(listings) < 5:
|
||
listings_wide, _, analog_tier_wide = _fetch_analogs(
|
||
db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2,
|
||
radius_m=FALLBACK_RADIUS_M,
|
||
full_address=geo.full_address,
|
||
year_built=target_year, house_type=target_house_type,
|
||
total_floors=payload.total_floors,
|
||
)
|
||
if len(listings_wide) > len(listings):
|
||
listings = listings_wide
|
||
fallback_used = True
|
||
analog_tier = analog_tier_wide
|
||
|
||
# Tier C: если даже на 2км мало — расширяем area tolerance до ±25%
|
||
# (актуально для отдалённых районов / новостроек с нестандартной планировкой)
|
||
if len(listings) < 3:
|
||
listings_widearea, _, analog_tier_wa = _fetch_analogs(
|
||
db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2,
|
||
radius_m=FALLBACK_RADIUS_M, area_tolerance=0.25,
|
||
full_address=geo.full_address,
|
||
year_built=target_year, house_type=target_house_type,
|
||
total_floors=payload.total_floors,
|
||
)
|
||
if len(listings_widearea) > len(listings):
|
||
listings = listings_widearea
|
||
fallback_used = True
|
||
area_widened = True
|
||
analog_tier = analog_tier_wa
|
||
|
||
# 3. Outlier filter
|
||
listings_clean = _filter_outliers(listings)
|
||
|
||
# 4. Aggregation
|
||
if listings_clean:
|
||
prices_ppm2 = sorted(lot["price_per_m2"] for lot in listings_clean if lot["price_per_m2"])
|
||
median_ppm2 = _percentile(prices_ppm2, 0.5)
|
||
q1_ppm2 = _percentile(prices_ppm2, 0.25)
|
||
q3_ppm2 = _percentile(prices_ppm2, 0.75)
|
||
median_price = int(median_ppm2 * payload.area_m2)
|
||
range_low = int(q1_ppm2 * payload.area_m2)
|
||
range_high = int(q3_ppm2 * payload.area_m2)
|
||
n_analogs = len(listings_clean)
|
||
else:
|
||
median_ppm2 = 0
|
||
median_price = 0
|
||
range_low = 0
|
||
range_high = 0
|
||
n_analogs = 0
|
||
|
||
# 4b. Поправка на состояние ремонта (встреча Птицы: ремонт влияет на цену).
|
||
# Аналоги — микс состояний; коэффициент сдвигает оценку под ремонт клиента.
|
||
repair_coef = _repair_coefficient(payload.repair_state)
|
||
repair_note = ""
|
||
if listings_clean and repair_coef != 1.0:
|
||
median_price = int(median_price * repair_coef)
|
||
range_low = int(range_low * repair_coef)
|
||
range_high = int(range_high * repair_coef)
|
||
median_ppm2 = median_ppm2 * repair_coef
|
||
pct = round((repair_coef - 1.0) * 100)
|
||
repair_note = (
|
||
f" Цена скорректирована на состояние ремонта "
|
||
f"({_REPAIR_LABEL.get(payload.repair_state, '')} {pct:+d}%)."
|
||
)
|
||
|
||
confidence, explanation = _compute_confidence(
|
||
n_analogs, median_ppm2, q1_ppm2 if listings_clean else 0,
|
||
q3_ppm2 if listings_clean else 0, fallback_used, area_widened,
|
||
listings=listings_clean,
|
||
)
|
||
|
||
# Tier note — информируем пользователя о качестве house-match
|
||
tier_note = ""
|
||
if analog_tier == "S":
|
||
tier_note = " (аналоги из того же дома)"
|
||
elif analog_tier == "H":
|
||
tf_str = f"{payload.total_floors}-эт." if payload.total_floors else ""
|
||
yr_str = f"{target_year}±15 г." if target_year else ""
|
||
parts_str = ", ".join(p for p in [yr_str, tf_str] if p)
|
||
tier_note = f" (аналоги из домов того же класса: {parts_str})" if parts_str else ""
|
||
else:
|
||
tier_note = " (нет аналогов в том же доме/классе — расширили поиск)"
|
||
|
||
explanation = (explanation or "") + tier_note + repair_note
|
||
|
||
# ── Stage 3: Avito IMV evaluation as 5-th source (on-demand cached) ──
|
||
imv_eval: IMVEvaluation | None = None
|
||
imv_house_type = _IMV_HOUSE_TYPE_MAP.get(target_house_type)
|
||
imv_renovation = _IMV_REPAIR_MAP.get(payload.repair_state)
|
||
# IMV требует: address, rooms, area, floor, floor_at_home, house_type, renovation_type.
|
||
# Если payload не содержит required fields — skip IMV (graceful).
|
||
if (
|
||
geo is not None
|
||
and geo.full_address
|
||
and payload.rooms is not None
|
||
and payload.area_m2
|
||
and payload.floor is not None
|
||
and payload.total_floors is not None
|
||
and imv_house_type is not None
|
||
and imv_renovation is not None
|
||
):
|
||
imv_eval = await _get_or_fetch_imv_cached(
|
||
db,
|
||
address=geo.full_address,
|
||
rooms=payload.rooms,
|
||
area_m2=payload.area_m2,
|
||
floor=payload.floor,
|
||
floor_at_home=payload.total_floors,
|
||
house_type=imv_house_type,
|
||
renovation_type=imv_renovation,
|
||
has_balcony=bool(payload.has_balcony),
|
||
has_loggia=False, # payload не разделяет балкон/лоджия → дефолт False
|
||
)
|
||
|
||
# Include IMV в sources_used если получили
|
||
sources_used_pre = sorted({lot.get("source") for lot in listings_clean if lot.get("source")})
|
||
if imv_eval is not None:
|
||
sources_used_pre = sorted(set(sources_used_pre) | {"avito_imv"})
|
||
|
||
# ── Stage 8: Yandex Valuation as on-demand source (anonymous, cached 24h) ──
|
||
yandex_val: YandexValuationResult | None = None
|
||
if geo is not None and geo.full_address:
|
||
yandex_val = await _get_or_fetch_yandex_valuation_cached(
|
||
db, address=geo.full_address,
|
||
)
|
||
if yandex_val is not None:
|
||
sources_used_pre = sorted(set(sources_used_pre) | {"yandex_valuation"})
|
||
saved_hist = _save_yandex_history_items(db, yandex_val)
|
||
logger.info(
|
||
"yandex_valuation: history items processed=%d saved=%d"
|
||
" (house_id=NULL — matching deferred)",
|
||
len(yandex_val.history_items), saved_hist,
|
||
)
|
||
|
||
# ── Stage 9: Cian Valuation as 7th source (on-demand, 24h cached, graceful if no cookies) ──
|
||
cian_val: CianValuationResult | None = None
|
||
if (
|
||
geo is not None
|
||
and geo.full_address
|
||
and payload.rooms is not None
|
||
and payload.area_m2
|
||
and payload.floor is not None
|
||
and payload.total_floors is not None
|
||
):
|
||
try:
|
||
cian_val = await estimate_via_cian_valuation(
|
||
db,
|
||
address=geo.full_address,
|
||
total_area=payload.area_m2,
|
||
rooms_count=payload.rooms,
|
||
floor=payload.floor,
|
||
total_floors=payload.total_floors,
|
||
repair_type="cosmetic",
|
||
deal_type="sale",
|
||
use_cache=True,
|
||
)
|
||
if cian_val is not None and cian_val.sale_price_rub:
|
||
sources_used_pre = sorted(set(sources_used_pre) | {"cian_valuation"})
|
||
logger.info(
|
||
"cian_valuation: price=%s accuracy=%s house_id=%s",
|
||
cian_val.sale_price_rub,
|
||
cian_val.sale_accuracy,
|
||
cian_val.external_house_id,
|
||
)
|
||
except Exception as exc:
|
||
logger.warning("cian_valuation: lookup failed (graceful): %s", exc)
|
||
|
||
# 5. Deals — отключены 2026-05-24 (см. Decision_TradeIn_DataQuality_8PR_Roadmap).
|
||
# rosreestr_deals содержит ДДУ первичного рынка (177-200 К/м²), что искажает
|
||
# оценку вторички (~114 К/м²). Возвращаемся к этому когда отделим вторичные
|
||
# сделки от ДДУ в источнике.
|
||
deals: list[dict[str, Any]] = []
|
||
|
||
# 6. Сохраняем в trade_in_estimates
|
||
estimate_id = uuid4()
|
||
now = datetime.now(tz=UTC)
|
||
expires_at = now + timedelta(hours=24)
|
||
|
||
analogs_lots = [_listing_to_analog(lot) for lot in listings_clean[:10]]
|
||
deals_lots = [_deal_to_analog(d) for d in deals[:10]]
|
||
freshness_pre = _compute_freshness_minutes(listings_clean)
|
||
db.execute(
|
||
text(
|
||
"""
|
||
INSERT INTO trade_in_estimates (
|
||
id, address, lat, lon,
|
||
area_m2, rooms, floor, total_floors,
|
||
year_built, house_type, repair_state, has_balcony,
|
||
ownership_type, has_mortgage, client_name, client_phone,
|
||
median_price, range_low, range_high, median_price_per_m2,
|
||
confidence, confidence_explanation, n_analogs,
|
||
analogs, actual_deals,
|
||
sources_used, data_freshness_minutes,
|
||
expires_at
|
||
) VALUES (
|
||
CAST(:id AS uuid),
|
||
:address, :lat, :lon,
|
||
:area, :rooms, :floor, :total_floors,
|
||
:year_built, :house_type, :repair_state, :has_balcony,
|
||
:ownership_type, :has_mortgage, :client_name, :client_phone,
|
||
:median_price, :range_low, :range_high, :median_ppm2,
|
||
:confidence, :explanation, :n_analogs,
|
||
CAST(:analogs_json AS jsonb),
|
||
CAST(:deals_json AS jsonb),
|
||
CAST(:sources_json AS jsonb),
|
||
:freshness,
|
||
:expires_at
|
||
)
|
||
"""
|
||
),
|
||
{
|
||
"id": str(estimate_id),
|
||
"address": geo.full_address,
|
||
"lat": geo.lat,
|
||
"lon": geo.lon,
|
||
"area": payload.area_m2,
|
||
"rooms": payload.rooms,
|
||
"floor": payload.floor,
|
||
"total_floors": payload.total_floors,
|
||
"year_built": target_year,
|
||
"house_type": target_house_type,
|
||
"repair_state": payload.repair_state,
|
||
"has_balcony": payload.has_balcony,
|
||
"ownership_type": payload.ownership_type,
|
||
"has_mortgage": payload.has_mortgage,
|
||
"client_name": payload.client_name,
|
||
"client_phone": payload.client_phone,
|
||
"median_price": median_price,
|
||
"range_low": range_low,
|
||
"range_high": range_high,
|
||
"median_ppm2": int(median_ppm2),
|
||
"confidence": confidence,
|
||
"explanation": explanation,
|
||
"n_analogs": n_analogs,
|
||
"analogs_json": json.dumps(
|
||
[a.model_dump(mode="json") for a in analogs_lots], ensure_ascii=False
|
||
),
|
||
"deals_json": json.dumps(
|
||
[a.model_dump(mode="json") for a in deals_lots], ensure_ascii=False
|
||
),
|
||
"sources_json": json.dumps(sources_used_pre, ensure_ascii=False),
|
||
"freshness": freshness_pre,
|
||
"expires_at": expires_at,
|
||
},
|
||
)
|
||
|
||
# Link saved IMV evaluation к этому estimate_id атомарно с основным INSERT
|
||
# (closes finding #4 from 2026-05-24 audit — prior code committed estimate first,
|
||
# then UPDATEd IMV in a separate tx, racing against concurrent estimators
|
||
# sharing the same cache_key).
|
||
if imv_eval is not None:
|
||
db.execute(
|
||
text(
|
||
"""
|
||
UPDATE avito_imv_evaluations
|
||
SET estimate_id = CAST(:estimate_id AS uuid)
|
||
WHERE cache_key = :cache_key
|
||
AND (estimate_id IS NULL OR estimate_id = CAST(:estimate_id AS uuid))
|
||
"""
|
||
),
|
||
{"estimate_id": str(estimate_id), "cache_key": imv_eval.cache_key},
|
||
)
|
||
|
||
db.commit()
|
||
|
||
logger.info(
|
||
"estimate: id=%s addr=%s rooms=%d area=%.1f → median=%d (n=%d, conf=%s)%s%s",
|
||
estimate_id,
|
||
geo.full_address[:60],
|
||
payload.rooms,
|
||
payload.area_m2,
|
||
median_price,
|
||
n_analogs,
|
||
confidence,
|
||
f" imv={imv_eval.recommended_price}" if imv_eval else "",
|
||
f" cian={cian_val.sale_price_rub}" if cian_val and cian_val.sale_price_rub else "",
|
||
)
|
||
|
||
sources_used = sorted({lot.source for lot in analogs_lots if lot.source})
|
||
if imv_eval is not None:
|
||
sources_used = sorted(set(sources_used) | {"avito_imv"})
|
||
if yandex_val is not None:
|
||
sources_used = sorted(set(sources_used) | {"yandex_valuation"})
|
||
if cian_val is not None and cian_val.sale_price_rub:
|
||
sources_used = sorted(set(sources_used) | {"cian_valuation"})
|
||
freshness_min = _compute_freshness_minutes(listings_clean)
|
||
|
||
return AggregatedEstimate(
|
||
estimate_id=estimate_id,
|
||
median_price_rub=median_price,
|
||
range_low_rub=range_low,
|
||
range_high_rub=range_high,
|
||
median_price_per_m2=int(median_ppm2),
|
||
confidence=confidence,
|
||
confidence_explanation=explanation,
|
||
n_analogs=n_analogs,
|
||
period_months=DEALS_PERIOD_MONTHS,
|
||
analogs=analogs_lots,
|
||
actual_deals=deals_lots,
|
||
expires_at=expires_at,
|
||
target_address=geo.full_address,
|
||
target_lat=geo.lat,
|
||
target_lon=geo.lon,
|
||
sources_used=sources_used,
|
||
data_freshness_minutes=freshness_min,
|
||
est_days_on_market=_estimate_days_on_market(listings_clean, deals),
|
||
area_m2=payload.area_m2,
|
||
rooms=payload.rooms,
|
||
floor=payload.floor,
|
||
total_floors=payload.total_floors,
|
||
year_built=target_year,
|
||
house_type=target_house_type,
|
||
repair_state=payload.repair_state,
|
||
has_balcony=payload.has_balcony,
|
||
)
|
||
|
||
|
||
def _estimate_days_on_market(
|
||
listings: list[dict[str, Any]], deals: list[dict[str, Any]]
|
||
) -> int | None:
|
||
"""Прогноз срока продажи — медиана days_on_market по аналогам/сделкам.
|
||
|
||
Возвращает None если ни у одного аналога нет данных о сроке экспозиции
|
||
(наши парсеры не всегда его отдают — честно показываем «нет данных»).
|
||
"""
|
||
values = [
|
||
int(lot["days_on_market"])
|
||
for lot in (*listings, *deals)
|
||
if lot.get("days_on_market") and int(lot["days_on_market"]) > 0
|
||
]
|
||
if len(values) < 3:
|
||
return None
|
||
values.sort()
|
||
return values[len(values) // 2]
|
||
|
||
|
||
def _compute_freshness_minutes(lots: list[dict[str, Any]]) -> int | None:
|
||
"""Минут с последнего парсинга — для UI «обновлено N мин назад»."""
|
||
if not lots:
|
||
return None
|
||
from datetime import datetime as _dt
|
||
|
||
now = _dt.now(tz=UTC)
|
||
scraped = [lot.get("scraped_at") or lot.get("listing_date") for lot in lots]
|
||
scraped_dt: list[datetime] = []
|
||
for s in scraped:
|
||
if s is None:
|
||
continue
|
||
# listings rows из mappings — scraped_at это datetime, не date
|
||
if hasattr(s, "tzinfo"):
|
||
scraped_dt.append(s if s.tzinfo else s.replace(tzinfo=UTC))
|
||
if not scraped_dt:
|
||
return None
|
||
return int((now - max(scraped_dt)).total_seconds() / 60)
|
||
|
||
|
||
# ── Internals ────────────────────────────────────────────────────────────────
|
||
|
||
# Compiled regexes for _extract_short_addr — module-level for performance.
|
||
|
||
# Strips leading admin prefixes: «Россия», «Свердловская область», «г. Екатеринбург» etc.
|
||
_ADMIN_PREFIX_RE = re.compile(
|
||
r"^(?:"
|
||
r"\s*(?:Россия|РФ|Российская\s+Федерация)\s*,?\s*|"
|
||
r"\s*[А-Яа-яёЁ][А-Яа-яёЁ\s-]+(?:\s+(?:обл(?:асть)?|р-н|район|округ|край|республика))\.?\s*,?\s*|"
|
||
r"\s*(?:г(?:ород)?|гор)\.?\s*[А-Яа-яёЁ][А-Яа-яёЁ\s-]+\s*,?\s*|"
|
||
r"\s*[А-Я][а-яё-]+(?:\s+[А-Я][а-яё-]+)?\s*,?\s*"
|
||
r")+",
|
||
flags=re.UNICODE,
|
||
)
|
||
|
||
# Recognizes start of a street keyword.
|
||
_STREET_START_RE = re.compile(
|
||
r"(?:ул\.|улица|пр\.|пр-т|проспект|пер\.|переулок|"
|
||
r"б-р|бульвар|ш\.|шоссе|наб\.|набережная|проезд|тракт|пл\.|площадь|"
|
||
r"мкр\.?|микрорайон)\s+",
|
||
flags=re.IGNORECASE | re.UNICODE,
|
||
)
|
||
|
||
# Drops trailing apartment / office / corpus noise from the end.
|
||
_TRAILING_NOISE_RE = re.compile(
|
||
r"\s*,\s*(?:кв\.?\s*\d+|корп\.?\s*\w+|оф\.?\s*\d+|пом\.?\s*\d+|подъезд\s*\d+).*$",
|
||
flags=re.IGNORECASE | re.UNICODE,
|
||
)
|
||
|
||
|
||
def _extract_short_addr(full_address: str | None) -> str | None:
|
||
"""Извлекает «улица + номер дома» из полного адреса для поиска в том же доме.
|
||
|
||
Примеры:
|
||
"Свердловская область, г. Екатеринбург, ул. Заводская, д. 44-а" → "ул. Заводская, д. 44-а"
|
||
"Россия, Екатеринбург, ул. Малышева, 1" → "ул. Малышева, 1"
|
||
"РФ, Свердловская обл., Екатеринбург, ул. Ленина, 5, кв. 12" → "ул. Ленина, 5"
|
||
"г. Екатеринбург, проспект Ленина, 50" → "проспект Ленина, 50"
|
||
"Екатеринбург, ул. Крауля, 48/2" → "ул. Крауля, 48/2"
|
||
|
||
Алгоритм:
|
||
1. Отрезаем trailing кв./корп./оф. noise.
|
||
2. Ищем первый street-keyword токен (ул./пр./пер. и т.д.) — возвращаем с него.
|
||
3. Fallback: агрессивно strip admin-prefix regex, вернуть остаток.
|
||
4. None если строка пустая или нечего возвращать.
|
||
"""
|
||
if not full_address:
|
||
return None
|
||
s = full_address.strip()
|
||
s = _TRAILING_NOISE_RE.sub("", s)
|
||
|
||
# Find first street-keyword position and return from there.
|
||
m = _STREET_START_RE.search(s)
|
||
if m:
|
||
return s[m.start():].strip(" ,.")
|
||
|
||
# Fallback: strip known admin prefixes, return whatever remains.
|
||
s = _ADMIN_PREFIX_RE.sub("", s)
|
||
return s.strip(" ,.") or None
|
||
|
||
|
||
def _stratify_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
"""Стратифицированная выборка Approach B — гарантирует MIN_ANALOGS_PER_SOURCE слотов.
|
||
|
||
Candidates должны быть уже отсортированы по relevance_score (ASC).
|
||
"""
|
||
guaranteed: list[dict[str, Any]] = []
|
||
guaranteed_ids: set[int] = set()
|
||
by_source: dict[str, list[dict[str, Any]]] = {}
|
||
for row in candidates:
|
||
src = row.get("source") or "unknown"
|
||
by_source.setdefault(src, []).append(row)
|
||
|
||
for _src, src_rows in by_source.items():
|
||
quota = min(len(src_rows), MIN_ANALOGS_PER_SOURCE)
|
||
for row in src_rows[:quota]:
|
||
if id(row) not in guaranteed_ids:
|
||
guaranteed.append(row)
|
||
guaranteed_ids.add(id(row))
|
||
|
||
remaining_slots = 50 - len(guaranteed)
|
||
remainder: list[dict[str, Any]] = []
|
||
if remaining_slots > 0:
|
||
for row in candidates:
|
||
if id(row) not in guaranteed_ids:
|
||
remainder.append(row)
|
||
if len(remainder) >= remaining_slots:
|
||
break
|
||
|
||
result = guaranteed + remainder
|
||
result.sort(key=lambda r: r.get("relevance_score") or 0.0)
|
||
return result[:50]
|
||
|
||
|
||
_ANALOG_SELECT_COLS = """
|
||
source, source_url, address, lat, lon,
|
||
rooms, area_m2, floor, total_floors,
|
||
price_rub, price_per_m2,
|
||
listing_date, days_on_market, photo_urls,
|
||
scraped_at
|
||
"""
|
||
|
||
_COMMON_WHERE = """
|
||
AND rooms = :rooms
|
||
AND area_m2 BETWEEN :area_min AND :area_max
|
||
AND is_active = true
|
||
AND scraped_at > NOW() - (:fresh_days || ' days')::interval
|
||
AND price_rub > 0
|
||
"""
|
||
|
||
|
||
def _fetch_analogs(
|
||
db: Session, *, lat: float, lon: float, rooms: int, area: float, radius_m: int,
|
||
full_address: str | None = None,
|
||
area_tolerance: float = AREA_TOLERANCE,
|
||
year_built: int | None = None, house_type: str | None = None,
|
||
total_floors: int | None = None,
|
||
cohort_year_min: int | None = None, # NEW: lower bound year_built inclusive
|
||
cohort_year_max: int | None = None, # NEW: upper bound year_built inclusive
|
||
# TODO: когда listings получит колонку house_id_fk — добавить ext_house_id JOIN для Tier S.
|
||
) -> tuple[list[dict[str, Any]], bool, str]:
|
||
"""SELECT аналогов — трёхуровневый house-match (S → H → W).
|
||
|
||
**Tier S (same building):** address ILIKE prefix-match по short_addr.
|
||
Если ≥3 результатов → возвращаем только их; tier='S'.
|
||
|
||
**Tier H (same class):** PostGIS + rooms + area + year ±15 + total_floors ±30%.
|
||
Если ≥5 результатов → возвращаем; tier='H'.
|
||
Пропускается если year_built или total_floors неизвестны.
|
||
|
||
**Tier W (wide / current):** текущая логика без year/floors WHERE фильтра.
|
||
tier='W'.
|
||
|
||
House-match relevance_score используется для сортировки в Tier H и W.
|
||
|
||
Стратифицированная выборка (Approach B):
|
||
1. SQL вытягивает до 300 кандидатов с per-address row_number (cap MAX_ANALOGS_PER_ADDRESS).
|
||
2. Python гарантирует MIN_ANALOGS_PER_SOURCE слотов каждому live source.
|
||
3. Оставшиеся слоты заполняются из остальных кандидатов по relevance.
|
||
4. Итоговый список отсортирован по relevance, LIMIT 50.
|
||
|
||
Когортный фильтр (PR 9): если переданы cohort_year_min/max — добавляется
|
||
hard-filter WHERE year_built BETWEEN min AND max OR year_built IS NULL.
|
||
NULL допускается чтобы не отсеивать листинги с неизвестным годом
|
||
(типично для Avito anonymous-address объявлений).
|
||
|
||
Returns:
|
||
(list_of_listings_as_dicts, fallback_radius_used_flag, tier)
|
||
tier: 'S' | 'H' | 'W'
|
||
"""
|
||
area_min = area * (1 - area_tolerance)
|
||
area_max = area * (1 + area_tolerance)
|
||
base_params: dict[str, Any] = {
|
||
"rooms": rooms,
|
||
"area_min": area_min,
|
||
"area_max": area_max,
|
||
"fresh_days": LISTINGS_FRESH_DAYS,
|
||
"max_per_addr": MAX_ANALOGS_PER_ADDRESS,
|
||
}
|
||
|
||
# ── Tier S: same building ─────────────────────────────────────────────────
|
||
short_addr = _extract_short_addr(full_address)
|
||
|
||
if short_addr:
|
||
tier_s_params = {
|
||
**base_params,
|
||
"short_addr_prefix": short_addr + "%",
|
||
}
|
||
|
||
tier_s_rows = db.execute(
|
||
text(
|
||
f"""
|
||
WITH base AS (
|
||
SELECT
|
||
{_ANALOG_SELECT_COLS},
|
||
0.0 AS distance_m,
|
||
0.0 AS relevance_score,
|
||
row_number() OVER (PARTITION BY address ORDER BY scraped_at DESC) AS rn_addr
|
||
FROM listings
|
||
WHERE address ILIKE :short_addr_prefix
|
||
{_COMMON_WHERE}
|
||
)
|
||
SELECT
|
||
source, source_url, address, lat, lon,
|
||
rooms, area_m2, floor, total_floors,
|
||
price_rub, price_per_m2,
|
||
listing_date, days_on_market, photo_urls,
|
||
scraped_at, distance_m, relevance_score
|
||
FROM base
|
||
WHERE rn_addr <= :max_per_addr
|
||
ORDER BY scraped_at DESC
|
||
LIMIT 300
|
||
"""
|
||
),
|
||
tier_s_params,
|
||
).mappings().all()
|
||
|
||
tier_s = [dict(r) for r in tier_s_rows]
|
||
if len(tier_s) >= 3:
|
||
logger.info(
|
||
"analogs tier=S addr_prefix=%r → %d results",
|
||
short_addr,
|
||
len(tier_s),
|
||
)
|
||
return _stratify_candidates(tier_s), radius_m > DEFAULT_RADIUS_M, "S"
|
||
|
||
# ── Tier H: same class (year ±15, total_floors ±30%) ─────────────────────
|
||
if year_built is not None and total_floors is not None:
|
||
year_min = year_built - 15
|
||
year_max = year_built + 15
|
||
tf_min = math.floor(total_floors * 0.7)
|
||
tf_max = math.ceil(total_floors * 1.3)
|
||
|
||
tier_h_rows = db.execute(
|
||
text(
|
||
f"""
|
||
WITH base AS (
|
||
SELECT
|
||
{_ANALOG_SELECT_COLS},
|
||
ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography)
|
||
AS distance_m,
|
||
(
|
||
ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography)
|
||
/ 1000.0
|
||
+ CASE
|
||
WHEN year_built IS NOT NULL
|
||
THEN abs(year_built - CAST(:target_year AS integer)) / 12.0
|
||
ELSE 0
|
||
END
|
||
+ CASE
|
||
WHEN CAST(:target_house_type AS text) IS NOT NULL
|
||
AND house_type IS NOT NULL
|
||
AND house_type <> CAST(:target_house_type AS text)
|
||
THEN 1.5
|
||
ELSE 0
|
||
END
|
||
) AS relevance_score,
|
||
row_number() OVER (
|
||
PARTITION BY address
|
||
ORDER BY (
|
||
ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography)
|
||
/ 1000.0
|
||
+ CASE
|
||
WHEN year_built IS NOT NULL
|
||
THEN abs(year_built - CAST(:target_year AS integer)) / 12.0
|
||
ELSE 0
|
||
END
|
||
+ CASE
|
||
WHEN CAST(:target_house_type AS text) IS NOT NULL
|
||
AND house_type IS NOT NULL
|
||
AND house_type <> CAST(:target_house_type AS text)
|
||
THEN 1.5
|
||
ELSE 0
|
||
END
|
||
)
|
||
) AS rn_addr
|
||
FROM listings
|
||
WHERE ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius)
|
||
{_COMMON_WHERE}
|
||
AND total_floors BETWEEN CAST(:tf_min AS integer)
|
||
AND CAST(:tf_max AS integer)
|
||
AND year_built BETWEEN CAST(:year_min AS integer)
|
||
AND CAST(:year_max AS integer)
|
||
)
|
||
SELECT
|
||
source, source_url, address, lat, lon,
|
||
rooms, area_m2, floor, total_floors,
|
||
price_rub, price_per_m2,
|
||
listing_date, days_on_market, photo_urls,
|
||
scraped_at, distance_m, relevance_score
|
||
FROM base
|
||
WHERE rn_addr <= :max_per_addr
|
||
ORDER BY relevance_score
|
||
LIMIT 300
|
||
"""
|
||
),
|
||
{
|
||
**base_params,
|
||
"lat": lat,
|
||
"lon": lon,
|
||
"radius": radius_m,
|
||
"target_year": year_built,
|
||
"target_house_type": house_type,
|
||
"tf_min": tf_min,
|
||
"tf_max": tf_max,
|
||
"year_min": year_min,
|
||
"year_max": year_max,
|
||
},
|
||
).mappings().all()
|
||
|
||
tier_h = [dict(r) for r in tier_h_rows]
|
||
if len(tier_h) >= 5:
|
||
logger.info(
|
||
"analogs tier=H year=%d±15 tf=%d-%d → %d results",
|
||
year_built, tf_min, tf_max, len(tier_h),
|
||
)
|
||
return _stratify_candidates(tier_h), radius_m > DEFAULT_RADIUS_M, "H"
|
||
|
||
logger.info(
|
||
"analogs tier=H year=%d±15 tf=%d-%d → only %d (fallthrough to W)",
|
||
year_built, tf_min, tf_max, len(tier_h),
|
||
)
|
||
|
||
# ── Tier W: wide (current logic, year/floors only in relevance sort) ──────
|
||
tier_w_rows = db.execute(
|
||
text(
|
||
"""
|
||
WITH base AS (
|
||
SELECT
|
||
source, source_url, address, lat, lon,
|
||
rooms, area_m2, floor, total_floors,
|
||
price_rub, price_per_m2,
|
||
listing_date, days_on_market, photo_urls,
|
||
scraped_at,
|
||
ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography)
|
||
AS distance_m,
|
||
(
|
||
ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography)
|
||
/ 1000.0
|
||
-- CAST обязателен: target_year / target_house_type приходят NULL
|
||
-- без типа → PostgreSQL "could not determine data type of parameter"
|
||
-- (AmbiguousParameter). Явный тип снимает неоднозначность.
|
||
+ CASE
|
||
WHEN CAST(:target_year AS integer) IS NOT NULL
|
||
AND year_built IS NOT NULL
|
||
THEN abs(year_built - CAST(:target_year AS integer)) / 12.0
|
||
ELSE 0
|
||
END
|
||
+ CASE
|
||
WHEN CAST(:target_house_type AS text) IS NOT NULL
|
||
AND house_type IS NOT NULL
|
||
AND house_type <> CAST(:target_house_type AS text)
|
||
THEN 1.5
|
||
ELSE 0
|
||
END
|
||
) AS relevance_score,
|
||
row_number() OVER (
|
||
PARTITION BY address
|
||
ORDER BY (
|
||
ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography)
|
||
/ 1000.0
|
||
+ CASE
|
||
WHEN CAST(:target_year AS integer) IS NOT NULL
|
||
AND year_built IS NOT NULL
|
||
THEN abs(year_built - CAST(:target_year AS integer)) / 12.0
|
||
ELSE 0
|
||
END
|
||
+ CASE
|
||
WHEN CAST(:target_house_type AS text) IS NOT NULL
|
||
AND house_type IS NOT NULL
|
||
AND house_type <> CAST(:target_house_type AS text)
|
||
THEN 1.5
|
||
ELSE 0
|
||
END
|
||
)
|
||
) AS rn_addr
|
||
FROM listings
|
||
WHERE ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius)
|
||
AND rooms = :rooms
|
||
AND area_m2 BETWEEN :area_min AND :area_max
|
||
AND is_active = true
|
||
AND scraped_at > NOW() - (:fresh_days || ' days')::interval
|
||
AND price_rub > 0
|
||
-- Когортный фильтр (PR 9): отсеивает разные эпохи застройки
|
||
-- (хрущёвка vs новостройка). Если cohort_year_min IS NULL —
|
||
-- фильтр прозрачен. CAST обязателен — psycopg3 prepared statement
|
||
-- не выводит тип $N при IS NULL в predicate (см. PR #518 fix).
|
||
AND (
|
||
CAST(:cohort_year_min AS integer) IS NULL
|
||
OR year_built IS NULL
|
||
OR year_built BETWEEN CAST(:cohort_year_min AS integer)
|
||
AND CAST(:cohort_year_max AS integer)
|
||
)
|
||
-- 2026-05-23: Avito coords теперь real (PR #487 убрал jitter после
|
||
-- C-5 audit). Listings с NULL coords отфильтруются через ST_DWithin
|
||
-- (geom IS NULL → не matches). geocode-missing-listings backfill
|
||
-- подтягивает координаты для address-only Avito листингов.
|
||
)
|
||
SELECT
|
||
source, source_url, address, lat, lon,
|
||
rooms, area_m2, floor, total_floors,
|
||
price_rub, price_per_m2,
|
||
listing_date, days_on_market, photo_urls,
|
||
scraped_at,
|
||
distance_m,
|
||
relevance_score
|
||
FROM base
|
||
WHERE rn_addr <= :max_per_addr
|
||
ORDER BY relevance_score
|
||
LIMIT 300
|
||
"""
|
||
),
|
||
{
|
||
"lat": lat,
|
||
"lon": lon,
|
||
"radius": radius_m,
|
||
"rooms": rooms,
|
||
"area_min": area_min,
|
||
"area_max": area_max,
|
||
"fresh_days": LISTINGS_FRESH_DAYS,
|
||
"target_year": year_built,
|
||
"target_house_type": house_type,
|
||
"max_per_addr": MAX_ANALOGS_PER_ADDRESS,
|
||
"cohort_year_min": cohort_year_min, # NEW
|
||
"cohort_year_max": cohort_year_max, # NEW
|
||
},
|
||
).mappings().all()
|
||
|
||
candidates: list[dict[str, Any]] = [dict(r) for r in tier_w_rows]
|
||
logger.info("analogs tier=W radius=%dm → %d candidates", radius_m, len(candidates))
|
||
return _stratify_candidates(candidates), radius_m > DEFAULT_RADIUS_M, "W"
|
||
|
||
|
||
def _fetch_deals(
|
||
db: Session, *, lat: float, lon: float, rooms: int, area: float, radius_m: int
|
||
) -> list[dict[str, Any]]:
|
||
rows = db.execute(
|
||
text(
|
||
"""
|
||
SELECT
|
||
source, address, lat, lon,
|
||
rooms, area_m2, floor, total_floors,
|
||
price_rub, price_per_m2,
|
||
deal_date, days_on_market,
|
||
ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) AS distance_m
|
||
FROM deals
|
||
WHERE ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius)
|
||
AND rooms = :rooms
|
||
AND area_m2 BETWEEN :area_min AND :area_max
|
||
AND deal_date > NOW() - (:months || ' months')::interval
|
||
AND price_rub > 0
|
||
ORDER BY deal_date DESC
|
||
LIMIT 30
|
||
"""
|
||
),
|
||
{
|
||
"lat": lat,
|
||
"lon": lon,
|
||
"radius": radius_m,
|
||
"rooms": rooms,
|
||
"area_min": area * (1 - AREA_TOLERANCE),
|
||
"area_max": area * (1 + AREA_TOLERANCE),
|
||
"months": DEALS_PERIOD_MONTHS,
|
||
},
|
||
).mappings().all()
|
||
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def _filter_outliers(lots: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
"""Tukey IQR rule: исключаем точки вне [Q1 - 1.5×IQR, Q3 + 1.5×IQR]."""
|
||
if len(lots) < 5:
|
||
return lots # на маленькой выборке нечего фильтровать
|
||
|
||
prices = sorted(lot["price_per_m2"] for lot in lots if lot.get("price_per_m2"))
|
||
if len(prices) < 4:
|
||
return lots
|
||
|
||
q1 = _percentile(prices, 0.25)
|
||
q3 = _percentile(prices, 0.75)
|
||
iqr = q3 - q1
|
||
low = q1 - 1.5 * iqr
|
||
high = q3 + 1.5 * iqr
|
||
|
||
clean = [lot for lot in lots if low <= lot.get("price_per_m2", 0) <= high]
|
||
if len(clean) < len(lots):
|
||
logger.info("outlier filter: %d → %d (Q1=%d Q3=%d)", len(lots), len(clean), q1, q3)
|
||
return clean
|
||
|
||
|
||
def _percentile(sorted_values: list[float], p: float) -> float:
|
||
"""Linear interpolation percentile (не округляем — оставляем float)."""
|
||
if not sorted_values:
|
||
return 0.0
|
||
if len(sorted_values) == 1:
|
||
return float(sorted_values[0])
|
||
n = len(sorted_values)
|
||
rank = p * (n - 1)
|
||
lo = int(rank)
|
||
hi = min(lo + 1, n - 1)
|
||
frac = rank - lo
|
||
return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * frac
|
||
|
||
|
||
def _compute_confidence(
|
||
n_analogs: int,
|
||
median_ppm2: float,
|
||
q1: float,
|
||
q3: float,
|
||
fallback_radius_used: bool,
|
||
area_widened: bool = False,
|
||
listings: list[dict] | None = None,
|
||
) -> tuple[str, str]:
|
||
"""Confidence + explanation string.
|
||
|
||
Уровень определяется по количеству уникальных адресов, а не по raw n_analogs.
|
||
Это защищает от overstated confidence когда много лотов из одного здания
|
||
(например, MIN_ANALOGS_PER_SOURCE=5 + same-building bias).
|
||
|
||
high — unique_addr ≥ 7 AND IQR/median < 0.15
|
||
medium — unique_addr ≥ 4 OR (unique_addr ≥ 2 AND IQR/median < 0.25)
|
||
low — иначе
|
||
|
||
Downgrade на один уровень если avg_lots_per_addr > 2.5 (concentration bias).
|
||
"""
|
||
if median_ppm2 == 0:
|
||
return "low", "Не найдено аналогов — попробуйте уточнить адрес или расширить параметры."
|
||
|
||
# Вычисляем метрики уникальных адресов
|
||
if listings:
|
||
unique_addrs = {
|
||
(lot.get("address") or "").strip().lower() for lot in listings if lot.get("address")
|
||
}
|
||
unique_addr_count = len(unique_addrs)
|
||
avg_lots_per_addr = n_analogs / max(unique_addr_count, 1)
|
||
else:
|
||
unique_addr_count = n_analogs # fallback: считаем каждый лот уникальным
|
||
avg_lots_per_addr = 1.0
|
||
|
||
iqr = q3 - q1
|
||
iqr_pct = iqr / median_ppm2 if median_ppm2 > 0 else 1.0
|
||
notes = []
|
||
if fallback_radius_used:
|
||
notes.append("расширили радиус до 2 км")
|
||
if area_widened:
|
||
notes.append("расширили допуск по площади до ±25%")
|
||
fallback_note = f" ({', '.join(notes)} из-за нехватки данных)" if notes else ""
|
||
|
||
# Базовый уровень по уникальным адресам
|
||
if unique_addr_count >= 7 and iqr_pct < 0.15:
|
||
base = "high"
|
||
elif unique_addr_count >= 4:
|
||
base = "medium"
|
||
elif unique_addr_count >= 2 and iqr_pct < 0.25:
|
||
base = "medium"
|
||
else:
|
||
base = "low"
|
||
|
||
# Downgrade на один шаг если слишком много лотов сконцентрировано на малом числе адресов
|
||
if avg_lots_per_addr > 2.5 and base != "low":
|
||
downgrade_map = {"high": "medium", "medium": "low"}
|
||
downgraded = downgrade_map[base]
|
||
explanation = (
|
||
f"Найдено {n_analogs} аналогов из {unique_addr_count} разных адресов, "
|
||
f"разброс цены ±{int(iqr_pct * 100 / 2)}% от медианы{fallback_note}. "
|
||
f"Снижена точность (≥2.5 лотов на адрес — возможен bias)."
|
||
)
|
||
return downgraded, explanation
|
||
|
||
explanation = (
|
||
f"Найдено {n_analogs} аналогов из {unique_addr_count} разных адресов, "
|
||
f"разброс цены ±{int(iqr_pct * 100 / 2)}% от медианы{fallback_note}."
|
||
)
|
||
return base, explanation
|
||
|
||
|
||
def _listing_to_analog(row: dict[str, Any]) -> AnalogLot:
|
||
return AnalogLot(
|
||
address=row.get("address") or "",
|
||
area_m2=float(row.get("area_m2") or 0),
|
||
rooms=int(row.get("rooms") or 0),
|
||
floor=row.get("floor"),
|
||
total_floors=row.get("total_floors"),
|
||
price_rub=int(row["price_rub"]),
|
||
price_per_m2=int(row.get("price_per_m2") or 0),
|
||
listing_date=row.get("listing_date"),
|
||
days_on_market=row.get("days_on_market"),
|
||
photo_url=(row["photo_urls"] or [None])[0] if row.get("photo_urls") else None,
|
||
source=row.get("source"),
|
||
source_url=row.get("source_url"),
|
||
distance_m=int(row["distance_m"]) if row.get("distance_m") is not None else None,
|
||
)
|
||
|
||
|
||
def _deal_to_analog(row: dict[str, Any]) -> AnalogLot:
|
||
"""deals не имеют photo_url — упрощённо."""
|
||
return AnalogLot(
|
||
address=row.get("address") or "",
|
||
area_m2=float(row.get("area_m2") or 0),
|
||
rooms=int(row.get("rooms") or 0),
|
||
floor=row.get("floor"),
|
||
total_floors=row.get("total_floors"),
|
||
price_rub=int(row["price_rub"]),
|
||
price_per_m2=int(row.get("price_per_m2") or 0),
|
||
listing_date=row.get("deal_date"),
|
||
days_on_market=row.get("days_on_market"),
|
||
photo_url=None,
|
||
source=row.get("source"),
|
||
source_url=None, # rosreestr сделки без публичной ссылки
|
||
distance_m=int(row["distance_m"]) if row.get("distance_m") is not None else None,
|
||
)
|
||
|
||
|
||
def _empty_estimate(
|
||
payload: TradeInEstimateInput, db: Session, *, reason: str
|
||
) -> AggregatedEstimate:
|
||
"""Fallback когда нет данных для оценки.
|
||
|
||
Сохраняет запись в БД (confidence='low', пустые analogs/deals), чтобы GET /estimate/{id}
|
||
не возвращал 404. C-4 security audit.
|
||
"""
|
||
estimate_id = uuid4()
|
||
now = datetime.now(tz=UTC)
|
||
expires_at = now + timedelta(hours=24)
|
||
|
||
db.execute(
|
||
text(
|
||
"""
|
||
INSERT INTO trade_in_estimates (
|
||
id, address,
|
||
area_m2, rooms, floor, total_floors,
|
||
year_built, house_type, repair_state, has_balcony,
|
||
ownership_type, has_mortgage, client_name, client_phone,
|
||
median_price, range_low, range_high, median_price_per_m2,
|
||
confidence, confidence_explanation, n_analogs,
|
||
analogs, actual_deals,
|
||
sources_used,
|
||
expires_at
|
||
) VALUES (
|
||
CAST(:id AS uuid), :address,
|
||
:area, :rooms, :floor, :total_floors,
|
||
:year_built, :house_type, :repair_state, :has_balcony,
|
||
:ownership_type, :has_mortgage, :client_name, :client_phone,
|
||
0, 0, 0, 0,
|
||
'low', :explanation, 0,
|
||
'[]'::jsonb, '[]'::jsonb,
|
||
'[]'::jsonb,
|
||
:expires_at
|
||
)
|
||
"""
|
||
),
|
||
{
|
||
"id": str(estimate_id),
|
||
"address": payload.address,
|
||
"area": payload.area_m2,
|
||
"rooms": payload.rooms,
|
||
"floor": payload.floor,
|
||
"total_floors": payload.total_floors,
|
||
"year_built": payload.year_built,
|
||
"house_type": payload.house_type,
|
||
"repair_state": payload.repair_state,
|
||
"has_balcony": payload.has_balcony,
|
||
"ownership_type": payload.ownership_type,
|
||
"has_mortgage": payload.has_mortgage,
|
||
"client_name": payload.client_name,
|
||
"client_phone": payload.client_phone,
|
||
"explanation": reason,
|
||
"expires_at": expires_at,
|
||
},
|
||
)
|
||
db.commit()
|
||
logger.info(
|
||
"empty_estimate: id=%s reason=%s addr=%s", estimate_id, reason, payload.address[:60]
|
||
)
|
||
|
||
return AggregatedEstimate(
|
||
estimate_id=estimate_id,
|
||
median_price_rub=0,
|
||
range_low_rub=0,
|
||
range_high_rub=0,
|
||
median_price_per_m2=0,
|
||
confidence="low",
|
||
confidence_explanation=reason,
|
||
n_analogs=0,
|
||
period_months=DEALS_PERIOD_MONTHS,
|
||
analogs=[],
|
||
actual_deals=[],
|
||
expires_at=expires_at,
|
||
)
|