"""Trade-In Estimator — реальное SQL aggregation поверх listings + deals. Заменяет старый _mock_estimate() из api/v1/trade_in.py. Алгоритм: 1. Geocode address → (lat, lon) 2. SELECT listings с фильтрами: - PostGIS ST_DWithin (geom, point, 1000m) — радиус поиска - source ≠ avito (у Avito фейковые anchor-jitter координаты — не гео-аналог) - rooms = target_rooms (точное совпадение) - area_m2 BETWEEN target × 0.85 AND target × 1.15 - scraped_at > NOW() - 14 days (свежие) - is_active = true 3. Tukey outlier filter (1.5 × IQR rule) 4. Median / Q1 / Q3 / count → confidence 5. То же для deals (period = 12 mo). 6. Сохранить в trade_in_estimates + вернуть AggregatedEstimate """ from __future__ import annotations import asyncio import hashlib import json import logging import math import re import time from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from typing import Any from uuid import uuid4 from sqlalchemy import text from sqlalchemy.orm import Session from app.core.config import settings from app.schemas.trade_in import ( AggregatedEstimate, AnalogLot, AvitoImvSummary, CianValuationSummary, DkpCorridor, PriceTrendPoint, TradeInEstimateInput, ) from app.services.dadata import DadataAddressResult from app.services.dadata import clean_address as dadata_clean_address from app.services.geocoder import GeocodeResult, geocode from app.services.house_metadata import get_house_metadata from app.services.matching.houses import match_house_readonly, match_or_create_house from app.services.scrapers.avito_imv import ( IMVAddressNotFoundError, IMVAuthError, IMVEvaluation, IMVTransientError, compute_imv_cache_key, evaluate_via_imv, save_imv_evaluation, ) from app.services.scrapers.cian_valuation import ( CianValuationResult, estimate_via_cian_valuation, ) from app.services.scrapers.yandex_valuation import ( YandexValuationResult, YandexValuationScraper, ) logger = logging.getLogger(__name__) # ── Constants ──────────────────────────────────────────────────────────────── DEFAULT_RADIUS_M = 1000 # ПО ВСТРЕЧЕ ПТИЦЫ: «локация не дальше 800-1000 м» FALLBACK_RADIUS_M = 2000 AREA_TOLERANCE = 0.15 # ±15% площади MAX_ANALOGS_PER_ADDRESS = 5 # анти-bias: не больше 5 лотов с одного адреса MIN_ANALOGS_PER_SOURCE = 5 # гарантированный минимум на live source LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней DEALS_PERIOD_MONTHS = 12 # сделки за последний год # #794: СберИндекс time-adjustment of frozen Rosreestr ДКП deals. # Rosreestr deals freeze ~2026-01; the sber monthly index re-bases a stale deal's ppm² # to the latest available month. Region fixed to Свердловская обл. (tradein MVP = ЕКБ). SBER_TIME_ADJUST_REGION = "Свердловская область" # Coefficient series preference: hedonic (quality-adjusted, cleanest) → deals (fallback). SBER_COEFF_DASHBOARDS = ("residential_real_estate_prices", "real_estate_deals") SBER_TIME_FACTOR_MIN = 0.7 # clamp guards against bad/sparse index months SBER_TIME_FACTOR_MAX = 1.6 # #699: санитизация ДКП-выбросов (Росреестр `deals`). В сырых сделках встречаются # нерыночные/битые записи — доли, сделки с обременением, опечатки этажа/площади — # которые шумят actual_deals (display) и dkp_corridor/expected_sold. Абсолютные # guard-bands (НЕ относительные) для ЕКБ-вторички: рынок ~100–400 К/м² (ср. пороги # _band_haircut 180/350К), премиум до ~680К. Нижняя/верхняя границы заведомо вне # рынка — режут «4.98 М за 125 м²» = 39.7 К/м² и т.п. Этаж: ЕКБ-максимум ~52 эт. # # Mera-audit fix-4: верифицировано vs данных prod deals (2026-06-21): # deals p90=172_419 ₽/м², p95=199_583 ₽/м², p99=278_681 ₽/м², p99.9=500_000 ₽/м² # deals >800k = 13 штук (0.026%) — нерыночные outlier'ы, не легитимный премиум. # listings p99 (is_active, <800k фильтр) = 392_795 ₽/м². # Вывод: DEAL_MAX_PPM2=800_000 НЕ режет легитимный ЕКБ-премиум (p99.9 сделок=500k). # Значение НЕ изменено — порог адекватен. Проверить повторно при p99 > 700_000. DEAL_MIN_PPM2 = 50_000 # ниже = не arms-length (доля/обременение/ошибка) DEAL_MAX_PPM2 = 800_000 # выше премиума → опечатка/коммерция (verified p99.9=500k, 2026-06-21) DEAL_MAX_FLOOR = 60 # выше реального максимума ЕКБ → битый этаж (напр. floor:100) # Когорта по году постройки — типизация массовой застройки РФ. # Используется как hard-filter в Tier 0 _fetch_analogs (PR 9, 2026-05-24). # Если target_year не задан — cohort = None → фильтр отключён, Tier 0 пропускается. COHORTS = ( # (cohort_name, year_min_inclusive, year_max_inclusive) ("khrushchev", 1955, 1969), # Хрущёвки 5-эт ("brezhnev", 1970, 1989), # Брежневка кирпич/панель 9–12-эт ( "late_soviet", 1990, 1999, ), # Поздний СССР (no overlap; first-match would never pick old range) ("2000s", 2000, 2010), # Ранние новостройки ("modern", 2011, 2100), # Современные ЖК ) # Минимум аналогов чтобы остаться на Tier 0 (с cohort); ниже — fallback на Tier A. MIN_ANALOGS_TIER_0 = 5 def _target_cohort_range(year_built: int | None) -> tuple[int, int] | None: """Maps a target year to its cohort year range [min, max] inclusive. Returns None if year_built is None — caller will skip cohort filter. Picks the FIRST matching cohort (so 1988 → 'brezhnev', not 'late_soviet'). """ if year_built is None: return None for _name, ymin, ymax in COHORTS: if ymin <= year_built <= ymax: return (ymin, ymax) # Out-of-range год (например, 1900 или 2050) — cohort фильтр не применяем, # лучше показать что есть в радиусе, чем 0 результатов. return None # Маппинг наших house_type → словарь Avito-IMV (внешний source). НЕ путать с # _REPAIR_COEF (heuristic-множитель ниже). _IMV_HOUSE_TYPE_MAP: dict[str | None, str | None] = { "panel": "panel", "brick": "brick", "monolith": "monolith", "monolith_brick": "monolith_brick", "monolithic": "monolith", "block": "block", "wood": "wood", None: None, } _IMV_REPAIR_MAP: dict[str | None, str | None] = { "needs_repair": "required", "standard": "cosmetic", "good": "euro", "excellent": "designer", None: None, } # Множители к медиане по состоянию ремонта. Аналоги в выборке — микс состояний; # коэффициент сдвигает оценку под ремонт целевой квартиры (встреча Птицы: ремонт # влияет на цену). # # WARNING: tunable МАРКЕТ-ЭВРИСТИКА, НЕ data-derived (issue #7). Вывести из данных пока # нельзя: listings.repair_state покрыт только ~2% (coverage вырастет после #621 backfill), # а медианы по нему confounded by area (немонотонны). Baseline = standard = 1.00 (no-op: # было 0.98, срезало каждую «стандартную» оценку на 2% — пофикшено). Пересмотреть при # coverage > 20% и наборе достаточной выборки по каждому bucket-у (#7). # После #621: repair_state нормализован → needs_repair/standard/good/excellent на инgesте. _REPAIR_COEF: dict[str, float] = { "needs_repair": 0.94, # требует ремонта — ниже рынка "standard": 1.00, # baseline "good": 1.05, "excellent": 1.10, # евроремонт — выше рынка } _REPAIR_LABEL: dict[str | None, str] = { "needs_repair": "требует ремонта", "standard": "стандартный ремонт", "good": "хороший ремонт", "excellent": "евроремонт", } def _repair_coefficient(repair_state: str | None) -> float: """Множитель к медиане по состоянию ремонта. None → 1.0 (без поправки).""" if not repair_state: return 1.0 return _REPAIR_COEF.get(repair_state, 1.0) # ── Asking→sold correction ratio lookup (#648 Stage 3) ────────────────────── # Таблица asking_to_sold_ratios (migration 080) хранит per-rooms коэффициент # ratio = median(SOLD ppm²) / median(ASKING ppm²) (~0.72–0.93). Estimator # домножает ASKING-медиану на этот ratio, получая параллельную expected_sold # цену (релевантную для выкупа). Headline asking-медиана НЕ меняется. # # Кэш: tiny in-process dict {bucket: (ratio, basis, fetched_monotonic)} с TTL. # Ratio дрейфует медленно (refresh-задача раз в сутки, Stage 4), поэтому 300с # TTL более чем достаточно и снимает по SELECT'у с каждой оценки. Single-worker # uvicorn/scheduler — GIL делает dict-доступ atomic enough (без явного lock). _ASKING_SOLD_RATIO_CACHE_TTL_S = 300.0 # Cache key: (bucket, tier_or_None) — tier=None for flag-OFF / no-anchor path (#928). _asking_sold_ratio_cache: dict[tuple[int, str | None], tuple[float | None, str | None, float]] = {} def _get_asking_sold_ratio( db: Session, rooms: int | None, anchor_ppm2: float | None = None, ) -> tuple[float | None, str | None]: """Возвращает (ratio, basis) asking→sold для бакета комнат. bucket = min(max(rooms or 0, 0), 4). Flag-OFF (settings.tier_aware_ratio_enabled=False) или anchor_ppm2=None: Запрос к asking_to_sold_ratios (старая таблица) — байт-в-байт как до #928. Fallback: per-rooms строка → global -1. Flag-ON и anchor_ppm2 задан (#928 ppm2-tier path): 1. Читаем t33/t66 из asking_to_sold_tier_bounds для (bucket, ''). Нет bounds-строки → fallback к 'all' строке (шаг 3). 2. tier = 'low'/'mid'/'high' по anchor_ppm2 vs t33/t66. Читаем asking_to_sold_ratios_tiered (bucket,'',tier) → tier_ratio, n_deals. Shrink: w = n_deals/(n_deals + settings.tier_ratio_shrink_k). Также читаем (bucket,'','all') как shrink-target. shrunk = tier_ratio * w + all_ratio * (1 - w). basis = f'per_rooms_tier:{tier}'. 3. Fallback (bucket,'','all') → basis='per_rooms_all'. 4. Fallback (-1,'','all') → basis='global_all'. 5. Ничего → (None, None). Таблицы нет / любая ошибка → (None, None), НЕ raise (graceful). Кэшируется на ключ (bucket, tier|None) с TTL _ASKING_SOLD_RATIO_CACHE_TTL_S. """ bucket = min(max(rooms or 0, 0), 4) # Determine tier key for cache: # flag-OFF or no anchor_ppm2 → use None (legacy path, reads old table). tier_key: str | None = None use_tier_path = settings.tier_aware_ratio_enabled and anchor_ppm2 is not None cache_key: tuple[int, str | None] = (bucket, tier_key) # may be updated below if not use_tier_path: # ── Flag-OFF path: byte-identical behavior (legacy asking_to_sold_ratios) ── # Use a distinct key "_legacy" to avoid collision with the flag-ON Step-3 'all' # fallback which also caches under (bucket, None). Without this, a process that # flips tier_aware_ratio_enabled ON without restart could serve the old-table ratio # from the flag-OFF entry for up to the TTL — wrong table. cache_key = (bucket, "_legacy") cached = _asking_sold_ratio_cache.get(cache_key) if cached is not None: ratio, basis, fetched = cached if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: return ratio, basis ratio: float | None = None basis: str | None = None try: row = db.execute( text( """ SELECT ratio, basis FROM asking_to_sold_ratios WHERE rooms_bucket = CAST(:b AS int) AND district = '' """ ), {"b": bucket}, ).fetchone() if row is None: # Бакет тонкий (n<30 при seed'е) или отсутствует → global fallback (-1). row = db.execute( text( """ SELECT ratio, basis FROM asking_to_sold_ratios WHERE rooms_bucket = -1 AND district = '' """ ), ).fetchone() if row is not None and row.ratio is not None: ratio = float(row.ratio) basis = row.basis except Exception as exc: # Таблицы может не быть на свежей/старой БД (миграция 080 не применена), # либо транзакция в сбойном состоянии — graceful: без sold-коррекции. # ОБЯЗАТЕЛЬНО rollback: неудачный SELECT помечает транзакцию # InFailedSqlTransaction, и без отката следующий statement упал бы → 500. logger.debug("asking_to_sold_ratio lookup skipped (graceful): %s", exc) try: db.rollback() except Exception: pass ratio, basis = None, None _asking_sold_ratio_cache[cache_key] = (ratio, basis, time.monotonic()) return ratio, basis # ── Flag-ON path: ppm2-tier lookup (#928) ─────────────────────────────────── ratio: float | None = None basis: str | None = None try: # Step 1: read bounds for this bucket. bounds_row = db.execute( text( """ SELECT t33, t66 FROM asking_to_sold_tier_bounds WHERE rooms_bucket = CAST(:b AS int) AND district = '' """ ), {"b": bucket}, ).fetchone() if bounds_row is not None: t33 = float(bounds_row.t33) t66 = float(bounds_row.t66) tier_key = "low" if anchor_ppm2 < t33 else ("mid" if anchor_ppm2 < t66 else "high") cache_key = (bucket, tier_key) # Check cache for this (bucket, tier) key. cached = _asking_sold_ratio_cache.get(cache_key) if cached is not None: c_ratio, c_basis, fetched = cached if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: return c_ratio, c_basis # Step 2: look up tier row + 'all' row for shrink. tier_row = db.execute( text( """ SELECT ratio, n_deals FROM asking_to_sold_ratios_tiered WHERE rooms_bucket = CAST(:b AS int) AND district = '' AND ppm2_tier = CAST(:tier AS text) """ ), {"b": bucket, "tier": tier_key}, ).fetchone() all_row = db.execute( text( """ SELECT ratio FROM asking_to_sold_ratios_tiered WHERE rooms_bucket = CAST(:b AS int) AND district = '' AND ppm2_tier = 'all' """ ), {"b": bucket}, ).fetchone() if tier_row is not None and all_row is not None: tier_ratio = float(tier_row.ratio) n_deals = int(tier_row.n_deals) all_ratio = float(all_row.ratio) k = settings.tier_ratio_shrink_k w = n_deals / (n_deals + k) ratio = tier_ratio * w + all_ratio * (1.0 - w) basis = f"per_rooms_tier:{tier_key}" _asking_sold_ratio_cache[cache_key] = (ratio, basis, time.monotonic()) return ratio, basis # Step 3: fallback to 'all' for this bucket. cache_key = (bucket, None) cached = _asking_sold_ratio_cache.get(cache_key) if cached is not None: c_ratio, c_basis, fetched = cached if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: return c_ratio, c_basis all_row = db.execute( text( """ SELECT ratio FROM asking_to_sold_ratios_tiered WHERE rooms_bucket = CAST(:b AS int) AND district = '' AND ppm2_tier = 'all' """ ), {"b": bucket}, ).fetchone() if all_row is not None: ratio = float(all_row.ratio) basis = "per_rooms_all" _asking_sold_ratio_cache[cache_key] = (ratio, basis, time.monotonic()) return ratio, basis # Step 4: global fallback (-1, 'all'). global_key: tuple[int, str | None] = (-1, None) cached = _asking_sold_ratio_cache.get(global_key) if cached is not None: c_ratio, c_basis, fetched = cached if (time.monotonic() - fetched) < _ASKING_SOLD_RATIO_CACHE_TTL_S: return c_ratio, c_basis global_row = db.execute( text( """ SELECT ratio FROM asking_to_sold_ratios_tiered WHERE rooms_bucket = -1 AND district = '' AND ppm2_tier = 'all' """ ), ).fetchone() if global_row is not None: ratio = float(global_row.ratio) basis = "global_all" _asking_sold_ratio_cache[global_key] = (ratio, basis, time.monotonic()) return ratio, basis except Exception as exc: # Таблицы нет / ошибка → graceful (migration 098 ещё не применена). logger.debug("asking_to_sold_ratio tiered lookup skipped (graceful): %s", exc) try: db.rollback() except Exception: pass # Step 5: nothing found. return None, None # ── Avito IMV cache lookup (Stage 3) ──────────────────────────────────────── IMV_CACHE_TTL_HOURS = 24 # Префиксы в адресе, которые Avito-геокодер не распознаёт (не жилые назначения). # Пример: "Склад, ул. Заводская, д. 44-а" → "ул. Заводская, д. 44-а" _NOISE_PREFIX_RE = re.compile( r"(Склад|Гараж|Подсобка|Нежилое|Помещение|Цех),\s*", flags=re.IGNORECASE, ) YANDEX_VALUATION_CACHE_TTL_HOURS = 24 YANDEX_VALUATION_DEFAULT_CATEGORY = "APARTMENT" YANDEX_VALUATION_DEFAULT_TYPE = "SELL" async def _get_or_fetch_imv_cached( db: Session, *, address: str, rooms: int, area_m2: float, floor: int, floor_at_home: int, house_type: str, renovation_type: str, has_balcony: bool, has_loggia: bool, estimate_id_for_link: Any = None, ) -> IMVEvaluation | None: """Cached IMV lookup. TTL 24h по cache_key (sha256 of address + params). 1. compute cache_key 2. SELECT из avito_imv_evaluations WHERE cache_key = :ck AND fetched_at > NOW() - 24h 3. Если hit → возвращаем reconstructed IMVEvaluation 4. Cache miss → call evaluate_via_imv, save_imv_evaluation, return Graceful: на любой error возвращаем None (estimator продолжает без IMV). """ try: cache_key = compute_imv_cache_key( address, rooms, area_m2, floor, floor_at_home, house_type, renovation_type, has_balcony, has_loggia, ) existing = ( db.execute( text( """ SELECT id, cache_key, address, rooms, area_m2, floor, floor_at_home, house_type, renovation_type, has_balcony, has_loggia, lat, lon, geo_hash, avito_address_id, avito_location_id, avito_metro_id, avito_district_id, recommended_price, lower_price, higher_price, market_count, raw_response, fetched_at FROM avito_imv_evaluations WHERE cache_key = :ck AND fetched_at > NOW() - (:ttl_hours || ' hours')::interval ORDER BY fetched_at DESC LIMIT 1 """ ), {"ck": cache_key, "ttl_hours": IMV_CACHE_TTL_HOURS}, ) .mappings() .first() ) if existing is not None: logger.info( "imv: cache HIT key=%s recommended=%d", cache_key[:8], existing["recommended_price"], ) from app.services.scrapers.avito_imv import IMVGeo return IMVEvaluation( cache_key=existing["cache_key"], address=existing["address"], rooms=existing["rooms"], area_m2=float(existing["area_m2"]), floor=existing["floor"], floor_at_home=existing["floor_at_home"], house_type=existing["house_type"], renovation_type=existing["renovation_type"], has_balcony=existing["has_balcony"], has_loggia=existing["has_loggia"], geo=IMVGeo( geo_hash=existing["geo_hash"] or "", lat=existing["lat"], lon=existing["lon"], avito_address_id=existing["avito_address_id"], avito_location_id=existing["avito_location_id"], avito_metro_id=existing["avito_metro_id"], avito_district_id=existing["avito_district_id"], ), recommended_price=existing["recommended_price"], lower_price=existing["lower_price"], higher_price=existing["higher_price"], market_count=existing["market_count"], raw_response=existing.get("raw_response"), ) # Cache miss — fresh fetch logger.info("imv: cache MISS key=%s — fetching fresh", cache_key[:8]) result = await evaluate_via_imv( address=address, rooms=rooms, area_m2=area_m2, floor=floor, floor_at_home=floor_at_home, house_type=house_type, renovation_type=renovation_type, has_balcony=has_balcony, has_loggia=has_loggia, ) save_imv_evaluation(db, result, estimate_id=estimate_id_for_link) logger.info( "imv: fresh recommended=%d range=(%d, %d) count=%d", result.recommended_price, result.lower_price, result.higher_price, result.market_count or 0, ) return result except IMVAddressNotFoundError as e: logger.warning("imv: address not found in Avito geocoder: %s", e) # Retry once with noise prefixes stripped (e.g. "Склад, ул. X" → "ул. X") cleaned = _NOISE_PREFIX_RE.sub("", address) if cleaned != address: logger.info( "imv: retry with cleaned address %r → %r", address[:60], cleaned[:60], ) try: result = await evaluate_via_imv( address=cleaned, rooms=rooms, area_m2=area_m2, floor=floor, floor_at_home=floor_at_home, house_type=house_type, renovation_type=renovation_type, has_balcony=has_balcony, has_loggia=has_loggia, ) save_imv_evaluation(db, result, estimate_id=estimate_id_for_link) logger.info( "imv: retry OK recommended=%d range=(%d, %d) count=%d", result.recommended_price, result.lower_price, result.higher_price, result.market_count or 0, ) return result except IMVAddressNotFoundError: logger.warning("imv: cleaned address also not found — giving up") except Exception as retry_exc: logger.warning("imv: retry failed: %s", retry_exc) return None except IMVAuthError as e: logger.error( "imv: auth/quota error — manual action required: %s", e, ) return None except IMVTransientError as e: logger.warning("imv: transient error, skipping retry in estimator context: %s", e) return None except Exception as e: logger.warning("imv: fetch failed — estimator продолжает без IMV: %s", e) return None # ── Yandex Valuation cache lookup (Stage 8) ───────────────────────────────── def _yandex_valuation_cache_key(address: str, offer_category: str, offer_type: str) -> str: """SHA256 cache key for Yandex Valuation lookups.""" payload = f"{address}|{offer_category}|{offer_type}" return hashlib.sha256(payload.encode("utf-8")).hexdigest() async def _get_or_fetch_yandex_valuation_cached( db: Session, *, address: str, offer_category: str = YANDEX_VALUATION_DEFAULT_CATEGORY, offer_type: str = YANDEX_VALUATION_DEFAULT_TYPE, ) -> YandexValuationResult | None: """Cached Yandex Valuation lookup. TTL 24h via external_valuations table. Returns None on any error / cache miss + fetch failure — caller continues without Yandex enrichment (graceful degradation). """ cache_key = _yandex_valuation_cache_key(address, offer_category, offer_type) # Cache lookup try: cached = ( db.execute( text( """ SELECT raw_payload, fetched_at FROM external_valuations WHERE source = 'yandex_valuation' AND cache_key = :ck AND expires_at > NOW() ORDER BY fetched_at DESC LIMIT 1 """ ), {"ck": cache_key}, ) .mappings() .first() ) except Exception as e: logger.warning("yandex_valuation: cache lookup failed: %s", e) cached = None if cached is not None and cached.get("raw_payload"): try: payload_dict = ( cached["raw_payload"] if isinstance(cached["raw_payload"], dict) else json.loads(cached["raw_payload"]) ) logger.info( "yandex_valuation: cache HIT key=%s items=%d", cache_key[:8], len(payload_dict.get("history_items", [])), ) return YandexValuationResult.model_validate(payload_dict) except Exception as e: logger.warning("yandex_valuation: cache deserialize failed — refetching: %s", e) # Fresh fetch try: async with YandexValuationScraper() as scraper: result = await scraper.fetch_house_history( address=address, offer_category=offer_category, offer_type=offer_type, ) except Exception as e: logger.warning("yandex_valuation: fetch failed — estimator продолжает без Yandex: %s", e) return None if result is None: logger.info("yandex_valuation: empty result for address=%s", address[:60]) return None # Save to cache (UPSERT on (source, cache_key)) try: db.execute( text( """ INSERT INTO external_valuations ( source, cache_key, address, raw_payload, fetched_at, expires_at ) VALUES ( 'yandex_valuation', :ck, :addr, CAST(:payload AS jsonb), NOW(), NOW() + (:ttl_hours || ' hours')::interval ) ON CONFLICT (source, cache_key) DO UPDATE SET raw_payload = EXCLUDED.raw_payload, fetched_at = NOW(), expires_at = NOW() + (:ttl_hours || ' hours')::interval """ ), { "ck": cache_key, "addr": address, "payload": json.dumps(result.model_dump(mode="json"), ensure_ascii=False), "ttl_hours": YANDEX_VALUATION_CACHE_TTL_HOURS, }, ) db.commit() logger.info( "yandex_valuation: fresh fetch saved key=%s items=%d", cache_key[:8], len(result.history_items), ) except Exception as e: logger.warning("yandex_valuation: cache save failed (continuing): %s", e) db.rollback() return result def _save_yandex_history_items( db: Session, result: YandexValuationResult, ) -> int: """Persist history items to house_placement_history. Returns saved count. Resolves house_id ONCE per result via match_or_create_house() using the valuation page's address + meta (year_built/total_floors). All items from the same page share that house_id. Confidence pipeline: method_confidence <- match_or_create_house (1.0 cadastr/source, 0.9 fp, 0.7 geo, 1.0 new) final_confidence = method_confidence Idempotent via UNIQUE (source, ext_item_id); ext_item_id synthesized from (address|publish_date|area|floor|prices) hash. Batch semantics: single try/except; on any failure the batch rolls back. """ if not result.history_items: return 0 # Resolve house ONCE per page. Synthetic ext_id = sha256(address)[:16] # — stable across re-runs, distinguishes pages for different addresses. address_seed = (result.address or "").strip().lower() house_ext_id = ( hashlib.sha256(address_seed.encode("utf-8")).hexdigest()[:16] if address_seed else "unknown" ) try: house_id, method_confidence, method = match_or_create_house( db, ext_source="yandex_valuation", ext_id=house_ext_id, address=result.address, year_built=result.house.year_built, ) except Exception as e: logger.warning( "yandex_valuation: house resolution failed for address=%r: %s" " — saving with house_id=NULL", result.address, e, ) db.rollback() house_id = None method_confidence = 0.0 method = "fail" logger.info( "yandex_valuation: house resolved house_id=%s method=%s confidence=%.2f addr=%r", house_id, method, method_confidence, result.address, ) rows = [] skipped_area = 0 for item in result.history_items: # Фильтруем items с нулевой/отрицательной/отсутствующей площадью — битый парс # («0,5 м²» и пр.) сохраняет мусор в house_placement_history и искажает # price_trend. Estimator защищён NULLIF, но грязь копится → чистим на входе. if item.area_m2 is None or item.area_m2 <= 0: skipped_area += 1 continue ext_seed = ( f"{result.address}|{item.publish_date}|{item.area_m2}|{item.floor}|" f"{item.start_price}|{item.last_price}" ) ext_item_id = hashlib.sha256(ext_seed.encode("utf-8")).hexdigest()[:32] rows.append( { "ext_id": ext_item_id, "house_id": house_id, "rooms": item.rooms, "area": item.area_m2, "floor": item.floor, "total_floors": result.house.total_floors, "start_price": item.start_price, "last_price": item.last_price, "publish_date": item.publish_date, "removed_date": item.removed_date, "exposure": item.exposure_days, "confidence": float(method_confidence), "notes": f"match_method={method}" if method != "fail" else None, "raw": json.dumps(item.model_dump(mode="json"), ensure_ascii=False), } ) if skipped_area > 0: logger.info( "yandex_valuation: skipped %d/%d history items with area_m2 <= 0 or None" " (addr=%r)", skipped_area, len(result.history_items), result.address, ) sql = text( """ INSERT INTO house_placement_history ( source, ext_item_id, house_id, rooms, area_m2, floor, total_floors, start_price, start_price_date, last_price, last_price_date, removed_date, exposure_days, source_confidence, notes, raw_payload ) VALUES ( 'yandex_valuation', :ext_id, :house_id, :rooms, :area, :floor, :total_floors, :start_price, :publish_date, :last_price, :publish_date, :removed_date, :exposure, :confidence, :notes, CAST(:raw AS jsonb) ) ON CONFLICT (source, ext_item_id) DO NOTHING """ ) try: if rows: db.execute(sql, rows) db.commit() return len(rows) except Exception as e: logger.warning( "yandex_valuation: failed to save history batch (%d items): %s", len(rows), e, ) db.rollback() return 0 # ── #651: IMV / Yandex blend (killer accuracy fix) ───────────────────────────── def _fetch_house_imv_anchor( db: Session, *, target_house_id: int | None, rooms: int | None, area: float | None, ) -> dict[str, Any] | None: """Достаёт РЕАЛЬНУЮ Avito IMV-оценку target-дома из `house_imv_evaluations`. В отличие от `avito_imv_evaluations` (keyed estimate_id — пустая, on-demand скрейп), `house_imv_evaluations` популирована (~2951 домов, fresh) и keyed по house_id. Резолвим строку: WHERE house_id = target_house_id, предпочитаем запись с ближайшими rooms+area (минимизируем |Δrooms|*10 + |Δarea%|), иначе самую свежую (fetched_at DESC). Best-effort: None при любой ошибке / отсутствии house_id / пустой таблице — estimator продолжает на гео-tier'ах (no regress). Returns dict {recommended_price, lower_price, higher_price, market_count, rooms, area_m2} или None. """ if target_house_id is None: return None try: row = ( db.execute( text( """ SELECT recommended_price, lower_price, higher_price, market_count, rooms, area_m2 FROM house_imv_evaluations WHERE house_id = CAST(:hid AS bigint) AND recommended_price > 0 -- Band-guard: строка пригодна как anchor только при правдоподобном -- совпадении rooms/area. Иначе studio-only IMV запись «прилипала» -- к 3-комн. target'у (ORDER BY всё равно вернёт LIMIT 1) и при -- anchor > median×1.15 раздувала blend. NULL target/row → не -- режем (graceful, нет данных для сравнения). AND ( CAST(:rooms AS integer) IS NULL OR rooms IS NULL OR abs(rooms - CAST(:rooms AS integer)) <= 1 ) AND ( CAST(:area AS double precision) IS NULL OR area_m2 IS NULL OR area_m2 <= 0 OR area_m2 BETWEEN CAST(:area AS double precision) * 0.7 AND CAST(:area AS double precision) * 1.3 ) ORDER BY -- ближе по комнатам и площади → меньше score; NULL target → 0 (CASE WHEN CAST(:rooms AS integer) IS NOT NULL AND rooms IS NOT NULL THEN abs(rooms - CAST(:rooms AS integer)) * 10 ELSE 0 END) + (CASE WHEN CAST(:area AS double precision) IS NOT NULL AND area_m2 IS NOT NULL AND area_m2 > 0 THEN abs(area_m2 - CAST(:area AS double precision)) / area_m2 * 100 ELSE 0 END) ASC, fetched_at DESC LIMIT 1 """ ), {"hid": target_house_id, "rooms": rooms, "area": area}, ) .mappings() .first() ) except Exception as exc: # pragma: no cover — defensive logger.warning("house_imv anchor lookup failed (graceful): %s", exc) return None return dict(row) if row is not None else None def _apply_imv_blend( *, median_price: int, range_high: int, median_ppm2: float, area: float, anchor_total: int | None, anchor_higher: int | None, weight: float, threshold: float, ) -> tuple[int, int, float, bool, int | None]: """Чистая (testable без БД) blend-трансформация для #651. Если есть надёжный якорь A (`anchor_total`, ПОЛНАЯ цена за квартиру) и он выше median_price × threshold (сигнал занижения) → поднимаем медиану до blend = median*(1-w) + A*w и расширяем верх диапазона до max(range_high, anchor_higher или A). ОДНОНАПРАВЛЕННО: только повышаем (баг — занижение). Если A ниже медианы — медиану НЕ трогаем, но диапазон можем расширить, чтобы включить A (информативность). Null-guard: при anchor_total=None — no-op. Returns (new_median_price, new_range_high, new_median_ppm2, blended, anchor_used_total). """ if anchor_total is None or anchor_total <= 0 or median_price <= 0 or area <= 0: return median_price, range_high, median_ppm2, False, None blended = False new_median = median_price new_ppm2 = median_ppm2 if anchor_total > median_price * threshold: w = max(0.0, min(1.0, weight)) new_median = round(median_price * (1.0 - w) + anchor_total * w) new_ppm2 = new_median / area blended = True # Расширяем верх диапазона: предпочитаем верхнюю границу IMV-коридора, иначе сам # якорь. Только вверх (никогда не сужаем). range_top_candidate = anchor_higher if (anchor_higher and anchor_higher > 0) else anchor_total new_range_high = max(range_high, range_top_candidate, new_median) return new_median, new_range_high, new_ppm2, blended, anchor_total # ── #764: per-cadastral-quarter price index correction ─────────────────────── def _quarter_from_cadastre(cad_num: str | None) -> str | None: """Извлечь кадастровый номер квартала из кадастрового номера дома/квартиры. Формат: AA:BB:CCCCCC или AA:BB:CCCCCCC (6 или 7 цифр в третьей части). Возвращаем первые три двоеточие-разделённых компонента. Пример: "66:41:0204016:350" → "66:41:0204016". При отсутствии или некорректном формате → None. """ if not cad_num: return None parts = cad_num.split(":") if len(parts) < 3: return None quarter = ":".join(parts[:3]) # Проверяем что третья часть — числовая (квартал, не мусор) if not parts[2].isdigit(): return None return quarter def _lookup_quarter_index( db: Session, *, quarter_cad_number: str, min_n_deals: int, ) -> tuple[float, int] | None: """Поиск price_index для кадастрового квартала в FDW-таблице quarter_price_index. Возвращает (price_index, n_deals) или None при отсутствии строки / n_deals < min_n_deals / любой FDW-ошибке (graceful — backward-compatible). Использует CAST(:q AS varchar) — psycopg v3 convention. """ try: row = ( db.execute( text( """ SELECT price_index, n_deals FROM quarter_price_index WHERE quarter_cad_number = CAST(:q AS varchar) AND n_deals >= CAST(:min_n AS bigint) LIMIT 1 """ ), {"q": quarter_cad_number, "min_n": min_n_deals}, ) .mappings() .first() ) except Exception as exc: logger.warning("quarter_price_index FDW lookup failed (graceful, no-op): %s", exc) return None if row is None: return None return float(row["price_index"]), int(row["n_deals"]) def _lookup_quarter_indexes( db: Session, *, quarter_cad_numbers: list[str], min_n_deals: int, ) -> dict[str, float]: """Батч-поиск price_index для списка кадастровых кварталов (одним SQL-запросом). Возвращает {quarter_cad_number: price_index} только для кварталов, у которых n_deals >= min_n_deals. Кварталы без записи или с n_deals < порога — не попадают в словарь. При любой FDW-ошибке → {} (graceful, avg_analog_index остаётся 1.0). """ if not quarter_cad_numbers: return {} distinct = list(dict.fromkeys(quarter_cad_numbers)) # сохраняем порядок, убираем дубли try: rows = ( db.execute( text( """ SELECT quarter_cad_number, price_index FROM quarter_price_index WHERE quarter_cad_number = ANY(CAST(:quarters AS varchar[])) AND n_deals >= CAST(:min_n AS bigint) """ ), {"quarters": distinct, "min_n": min_n_deals}, ) .mappings() .all() ) except Exception as exc: logger.warning("quarter_price_index batch FDW lookup failed (graceful, no-op): %s", exc) return {} return {str(row["quarter_cad_number"]): float(row["price_index"]) for row in rows} def _apply_quarter_index( *, base_median_ppm2: float, base_median_price: int, base_range_low: int, base_range_high: int, target_index: float, avg_analog_index: float, min_factor: float = 0.6, max_factor: float = 1.8, ) -> tuple[float, int, int, int, float]: """Чистая (testable без БД) gap-correction квартального индекса (#764). Корректирует ТОЛЬКО разрыв между квартальным уровнем целевого объекта и усреднённым квартальным уровнем аналогов: factor = target_index / avg_analog_index adjusted_ppm2 = base_median_ppm2 × factor Все ценовые выходы масштабируются одним и тем же factor → median/range остаются геометрически консистентными. min_factor / max_factor — sanity-clamp (#859): belt-and-suspenders против патологичных FDW-данных. Калибруются через settings и передаются из вызывающего кода, чтобы хелпер оставался чистым (без импорта settings). Когда clamp меняет raw factor — логируется (см. caller). Returns (adjusted_ppm2, adjusted_median_price, adjusted_range_low, adjusted_range_high, factor). """ raw_factor = target_index / avg_analog_index factor = max(min_factor, min(max_factor, raw_factor)) if raw_factor != factor: logger.info( "quarter_index: factor clamped raw=%.4f → %.4f (bounds [%.2f, %.2f])" " target_index=%.3f avg_analog_index=%.3f", raw_factor, factor, min_factor, max_factor, target_index, avg_analog_index, ) adjusted_ppm2 = base_median_ppm2 * factor adjusted_median_price = round(base_median_price * factor) adjusted_range_low = round(base_range_low * factor) adjusted_range_high = round(base_range_high * factor) return adjusted_ppm2, adjusted_median_price, adjusted_range_low, adjusted_range_high, factor def _load_sber_index_series(db: Session, *, region: str) -> dict[date, float]: """#794: monthly {period_month: index_value} for region from sber_price_index. Tries SBER_COEFF_DASHBOARDS in order; returns first non-empty series. {} on any error. #audit-5a: если latest месяц серии старее sber_index_max_age_days → warning. """ for dash in SBER_COEFF_DASHBOARDS: try: rows = ( db.execute( text(""" SELECT period_month, index_value_rub_m2 FROM sber_price_index WHERE city = CAST(:region AS text) AND dashboard = CAST(:dash AS text) ORDER BY period_month """), {"region": region, "dash": dash}, ) .mappings() .all() ) except Exception as exc: logger.warning("sber_price_index lookup failed for %s (graceful): %s", dash, exc) continue if rows: series = {r["period_month"]: float(r["index_value_rub_m2"]) for r in rows} if not series: continue # #audit-5a: data-age guard — предупреждаем о stale СберИндексе. latest = max(series) today = datetime.now(tz=UTC).date() age_days = (today - latest).days if age_days > settings.sber_index_max_age_days: logger.warning( "sber_index stale #audit-5a: latest=%s age=%d days" " (> sber_index_max_age_days=%d) region=%s dash=%s" " — time-adjustment may be outdated", latest.isoformat(), age_days, settings.sber_index_max_age_days, region, dash, ) return series return {} def _sber_time_factor(series: dict[date, float], deal_month: date) -> float: """#794: factor = idx[latest] / idx[deal_month], clamped. 1.0 when no data / recent deal. series: {first-of-month date -> index value}. deal_month: first-of-month of the deal. If deal_month absent, use the nearest available month <= deal_month; if none, nearest overall. If deal_month >= latest available month -> 1.0 (no extrapolation of recent deals). """ if not series: return 1.0 latest = max(series) if deal_month >= latest: return 1.0 base = series.get(deal_month) if base is None: earlier = [m for m in series if m <= deal_month] if earlier: base = series[max(earlier)] else: base = series[min(series)] # deal older than series start → use earliest if not base or base <= 0: return 1.0 factor = series[latest] / base return max(SBER_TIME_FACTOR_MIN, min(SBER_TIME_FACTOR_MAX, factor)) def _fetch_dkp_corridor( db: Session, *, address: str | None, rooms: int | None, area: float | None, period_months: int = DEALS_PERIOD_MONTHS, area_tolerance: float = AREA_TOLERANCE, ) -> dict[str, Any] | None: """#652: коридор ₽/м² по реальным ДКП-сделкам Росреестра для target. Reuse паттерна street-deals (api/v1/trade_in.py): извлекаем улицу из адреса, фильтруем `deals` (source='rosreestr', та же rooms, площадь ±tolerance, окно period_months) и нормализуем per-m². Возвращаем low/median/high ₽/м². ADVISORY — caller не клампит, только сурфейсит + опциональная пометка. Best-effort: None при отсутствии улицы / сделок / любой ошибке. """ if not address or rooms is None or not area: return None street_name = extract_street_name(address) if not street_name: return None area_min = area * (1.0 - area_tolerance) area_max = area * (1.0 + area_tolerance) try: rows = ( db.execute( text( """ SELECT price_per_m2, deal_date FROM deals WHERE source = 'rosreestr' AND address ILIKE :street_pattern AND address ~* :street_regex AND rooms = CAST(:rooms AS integer) AND area_m2 BETWEEN :area_min AND :area_max AND deal_date > NOW() - (CAST(:period_months AS integer) || ' months')::interval AND price_per_m2 > 0 -- #699: режем нерыночные ppm²-выбросы из коридора expected_sold AND price_per_m2 BETWEEN :ppm_min AND :ppm_max """ ), { "street_pattern": "%" + street_name + "%", "street_regex": r"\m" + street_name + r"\M", "rooms": rooms, "area_min": area_min, "area_max": area_max, "period_months": period_months, "ppm_min": DEAL_MIN_PPM2, "ppm_max": DEAL_MAX_PPM2, }, ) .mappings() .all() ) except Exception as exc: # pragma: no cover — defensive logger.warning("dkp_corridor lookup failed (graceful): %s", exc) return None # #794: apply СберИндекс time-adjustment to re-base stale Rosreestr ДКП ppm² # to the latest available index month. Graceful: factor=1.0 when table is empty. series = _load_sber_index_series(db, region=SBER_TIME_ADJUST_REGION) adjusted: list[float] = [] factors_applied: list[float] = [] for r in rows: ppm2 = r["price_per_m2"] if not ppm2: continue dd = r.get("deal_date") factor = 1.0 if series and dd is not None: deal_month = date(dd.year, dd.month, 1) factor = _sber_time_factor(series, deal_month) adjusted.append(float(ppm2) * factor) factors_applied.append(factor) ppm2_values = sorted(adjusted) if not ppm2_values: return None if series and factors_applied: logger.info( "dkp_corridor #794 time-adjust: n=%d factor min=%.3f max=%.3f region=%s", len(factors_applied), min(factors_applied), max(factors_applied), SBER_TIME_ADJUST_REGION, ) # #1520: используем P10/P90 вместо абсолютных min/max, чтобы коридор был # устойчив к выбросам (один нерыночный ДКП не сдвигает границу). # При маленьких выборках (n < 10) _percentile с линейной интерполяцией # плавно сжимается к первому/последнему элементу — специального fallback # не нужно (n=3: P10 ≈ индекс 0.2, P90 ≈ индекс 2.8 → оба внутри диапазона). return { "count": len(ppm2_values), "low_ppm2": int(_percentile(ppm2_values, 0.10)), "median_ppm2": int(_percentile(ppm2_values, 0.5)), "high_ppm2": int(_percentile(ppm2_values, 0.90)), "period_months": period_months, } # ── #651/#652 v2: same-building anchor (validated, 55 golden cases) ───────────── # # Радиусная медиана ₽/м² системно занижает премиум/видовые квартиры — она мешает # топовый дом с массовой застройкой рядом. v2 строит PRIMARY якорь из комплов ТОГО # ЖЕ ДОМА (Tier A), similarity-weighted по площади+комнатам, с premium-uplift и # hard guardrail. Industry-grounded (Fannie Mae «same-project comps preferred» + # inverse-adjustment weighting + FSD-as-confidence). Полностью за флагом. # Street-alias map: ЕКБ-специфичные расхождения между golden/source-адресами и БД. # Golden «Ткачёва 13» = БД «Ткачей 13» — без алиаса 0 комплов для 5 business-кейсов # (Clever Park). Ключи/значения уже ё→е-нормализованы и lowercase. Расширяемо. # ВАЖНО: применяется к ЦЕЛЬНОМУ street_core (после strip type-words), не к токену. _STREET_ALIAS_MAP: dict[str, str] = { "ткачева": "ткачей", # «ул. Ткачёва» (родит. падеж) ↔ БД «ул. Ткачей» } # Street-type токены: канонизируем (drop type-слово, оставляем имя). Реальные prod- # адреса ставят тип ДО или ПОСЛЕ имени («улица Ткачей» И «Олимпийская наб.») — поэтому # дропаем тип ОТКУДА УГОДНО в строке, а не только ведущий keyword. Лемматизированы по # точкам/окончаниям: матчим точное слово ИЛИ «.»-сокращение. _STREET_TYPE_TOKENS: frozenset[str] = frozenset( { "улица", "ул", "у", "переулок", "пер", "проспект", "пр", "прт", "пркт", "пр-кт", "пр-т", "проезд", "бульвар", "бр", "б-р", "набережная", "наб", "шоссе", "ш", "площадь", "пл", "тракт", "аллея", "тупик", } ) # Административные токены-маркеры: всё, что относится к городу/району/мкр — НЕ часть # имени дома. Дропаем токен-маркер ВМЕСТЕ со следующим за ним словом-значением # («р-н Октябрьский», «мкр Парковый», «г Екатеринбург»). Города ЕКБ-агломерации тоже # чистим (стоят как ведущий токен «Екатеринбург,»/«Первоуральск,»). _ADMIN_MARKER_TOKENS: frozenset[str] = frozenset( {"рн", "р-н", "район", "мкр", "микрорайон", "г", "гор", "город", "обл", "область"} ) _CITY_TOKENS: frozenset[str] = frozenset( {"екатеринбург", "первоуральск", "березовский", "верхняя", "пышма", "среднеуральск", "россия"} ) # Литера корпуса, прилипшая к номеру: «16а», «204г», «57а» → base + letter (рус/лат). _HOUSE_LETTER_RE = re.compile(r"^(?P\d+)\s*(?P[а-яa-z])?$", flags=re.UNICODE) # Разбивка на токены: слова/числа, отбрасывая пунктуацию (',', '.', '·', '/', '-'). _TOKEN_SPLIT_RE = re.compile(r"[\s,;·]+", flags=re.UNICODE) def _normalize_building_key( address: str | None, ) -> tuple[str | None, int | None, str | None]: """Нормализует адрес в robust-ключ «того же дома»: (street_core, base_no, letter). Инвариантен к prod-форматам ЕКБ-вторички: - ё→е, lowercase. - отрезает город / «р-н …» / «мкр …» / «г. …» / «· …»-хвост (district-suffix). - дропает street-type слова ОТКУДА УГОДНО (улица/ул./наб./набережная/пр./… — тип может стоять ДО или ПОСЛЕ имени: «улица Ткачей» И «Олимпийская наб.»); остаток alpha-токенов = street_core («бориса ельцина», «сакко и ванцетти»). - base_no = первый числовой токен ПОСЛЕ имени улицы, толерантно к «Ткачей, 13» = «Ткачей,13» = «Ткачей,д. 13» = «Ткачей дом 13». - letter = прилипшая литера корпуса («16а»→'а', «204г»→'г'); «/N» и «кN» (корпус) схлопываются к base (тот же дом). Литеры — РАЗНЫЕ дома (204г ≠ 204д). - street_core прогоняется через _STREET_ALIAS_MAP (ткачева→ткачей). Returns (street_core, base_no, letter) — любой элемент None если не извлёкся. Best-effort: при пустом адресе → (None, None, None). """ if not address: return None, None, None norm = address.replace("ё", "е").replace("Ё", "Е").lower() # Отрезаем «· …»-хвост (Avito-формат «… · р-н Центр»): district всегда после «·». norm = norm.split("·")[0] # «д.»/«дом» перед номером → пробел (чтобы числовой токен встал отдельно). norm = re.sub(r"\b(?:д\.?|дом)\s*(?=\d)", " ", norm, flags=re.UNICODE) raw = [t for t in _TOKEN_SPLIT_RE.split(norm) if t] # 1. Вычищаем admin-маркеры (+следующее за ними значение) и города. cleaned: list[str] = [] skip_next = False for tok in raw: if skip_next: skip_next = False continue bare = tok.rstrip(".") if bare in _ADMIN_MARKER_TOKENS: skip_next = True # дропаем и сам маркер, и следующее слово-значение continue if bare in _CITY_TOKENS: continue cleaned.append(tok) # 2. House-токен = ПОСЛЕДНИЙ токен, начинающийся с цифры. Берём последний (а не # первый), чтобы числа ВНУТРИ имени улицы («8 Марта», «1905 года») не съелись # как номер дома — настоящий номер всегда в хвосте, за именем. Хвост за номером # (корпус «/N», «кN») игнорируем; всё остальное — токены улицы. base_no: int | None = None letter: str | None = None house_idx: int | None = None for i, tok in enumerate(cleaned): if tok[0].isdigit(): head = re.split(r"[/\\]", tok, maxsplit=1)[0] # «4/2» → «4»; корпус отброшен head = re.split(r"к\d", head, maxsplit=1)[0] # «105к1» → «105» m = _HOUSE_LETTER_RE.match(head) if m: base_no = int(m.group("num")) letter = m.group("letter") or None house_idx = i # 3. street_core = токены до номера дома, минус type-слова (улица/наб./пр./…) и # минус сам house-токен. Числовые префиксы имени («8 марта») сохраняем. street_tokens = [ tok for i, tok in enumerate(cleaned) if i != house_idx and tok.rstrip(".") not in _STREET_TYPE_TOKENS ] street_core = " ".join(street_tokens).strip() or None if street_core: street_core = _STREET_ALIAS_MAP.get(street_core, street_core) return street_core, base_no, letter def _anchor_comp_from_row(r: Any) -> dict[str, Any]: """Строит comp-dict из строки SQL same-building/micro-radius (#694). Несёт 5 числовых полей для _compute_same_building_anchor (price_per_m2/area_m2/rooms/floor/total_floors) + display-поля listings (address/source/source_url/price_rub/listing_date/days_on_market/photo_urls/ lat/lon), чтобы UI-аналоги отражали ИМЕННО комплы, на которых построен якорь, а не радиусные (cheaper/empty). Display-ключи best-effort: SELECT их тянет, но helper устойчив к их отсутствию (тестовые моки могут давать только числа). """ return { "price_per_m2": int(r["price_per_m2"]), "area_m2": float(r["area_m2"]) if r.get("area_m2") is not None else None, "rooms": int(r["rooms"]) if r.get("rooms") is not None else None, "floor": int(r["floor"]) if r.get("floor") is not None else None, "total_floors": int(r["total_floors"]) if r.get("total_floors") is not None else None, "address": r.get("address"), "source": r.get("source"), "source_url": r.get("source_url"), "price_rub": int(r["price_rub"]) if r.get("price_rub") is not None else None, "listing_date": r.get("listing_date"), "days_on_market": r.get("days_on_market"), "photo_urls": r.get("photo_urls"), "lat": float(r["lat"]) if r.get("lat") is not None else None, "lon": float(r["lon"]) if r.get("lon") is not None else None, } def _fetch_anchor_comps( db: Session, *, address: str | None, target_house_id: int | None, lat: float | None, lon: float | None, rooms: int | None, area: float | None, ) -> tuple[list[dict[str, Any]], str | None]: """Тированный набор комплов для same-building якоря. Стоп на 1-м тире с ≥ min_comps. Tier A — SAME BUILDING: normalized street + base house no (+ литера если есть). RELAXED rooms (без фильтра), БЕЗ area±15%. Не группируем по house_id_fk — один дом дробится на несколько fk (Хохрякова 48 = 7085/9878/12797). Tier C — micro-radius ≤500m (ST_DWithin) + вторичка-канон guard (#1186): NULL = legacy вторичка + rooms match + area±25%. (Tier B «тот же ЖК» — skip: complex_id/cian_zhk_url ненадёжны.) Tier D — фолбэк: None tier (caller остаётся на радиусном median-пути). Excludes lots без price_per_m2. is_active=true. Best-effort: ([], None) на ошибке. Returns (comps, tier) где tier ∈ {'A','C', None}. comps — list dict с ключами price_per_m2 (int>0), area_m2 (float|None), rooms (int|None), floor (int|None), total_floors (int|None) — последние два для floor-веса (#680-WB). """ min_comps = settings.estimate_sb_min_comps # ── Tier A: same building ──────────────────────────────────────────────── street, base_no, letter = _normalize_building_key(address) if street and base_no is not None: # Numeric-boundary regex: дом 204 не матчит 2040/1204; литера при наличии # обязательна (204г ≠ 204д). Корпус «/N» допускаем (тот же дом). ё→е в SQL # для symmetry с нормализатором. psycopg v3: bind через :param, оператор ~. if letter: house_re = rf"(^|[^0-9]){base_no}\s*{letter}([^а-яёa-z0-9/]|/|$)" else: house_re = rf"(^|[^0-9]){base_no}([^а-яёa-z0-9/]|/|$)" try: rows = ( db.execute( text( """ SELECT price_per_m2, area_m2, rooms, floor, total_floors, address, source, source_url, price_rub, listing_date, days_on_market, photo_urls, lat, lon, listing_segment, source_id FROM listings WHERE is_active = true AND price_per_m2 > 0 AND lower(translate(address, 'ёЁ', 'ее')) LIKE :street_like AND lower(translate(address, 'ёЁ', 'ее')) ~ :house_re """ ), { "street_like": "%" + street + "%", "house_re": house_re, }, ) .mappings() .all() ) except Exception as exc: # pragma: no cover — defensive logger.warning("anchor Tier A lookup failed (graceful): %s", exc) try: db.rollback() except Exception: pass rows = [] # #1774: gated relaxation of the #1186 guard, Tier A ONLY. В сданном доме # cian тегирует переуступки/перепродажи собственниками как 'novostroyki'; # впускаем novostroyki только если вторичных (vtorichka/NULL) НЕ МЕНЬШЕ, чем # первичных — иначе MAD-clip может выкинуть редкую вторичку и заякорить на # ценах застройщика. Чисто-первичный / primary-dominated дом → гард #1186. raw_rows = list(rows) secondary_count = sum( 1 for r in raw_rows if r.get("listing_segment") in (None, "vtorichka") ) primary_count = len(raw_rows) - secondary_count include_primary = ( settings.estimate_sb_tier_a_allow_primary_if_secondary_present and secondary_count >= 1 and secondary_count >= primary_count ) if include_primary: kept = raw_rows # mixed/delivered house: include novostroyki resales else: kept = [r for r in raw_rows if r.get("listing_segment") in (None, "vtorichka")] # Dedup дублей одного объявления (#1774: source_id=330047129 → 2 active-строки, # одна с house_id_fk, одна NULL). Ключ — (source, source_id) PRIMARY (codebase # canon: скрейперы дедупят на source_id; source_url может отличаться trailing # slash/query-param). Fallback source_url, затем атрибуты. Namespaced-теги # ('id'/'url') защищают от value-collision между id и url. deduped: dict[Any, dict[str, Any]] = {} for r in kept: sid = r.get("source_id") url = r.get("source_url") if sid is not None: key: Any = (r.get("source"), "id", sid) elif url is not None: key = (r.get("source"), "url", url) else: key = ( r.get("source"), r.get("address"), r.get("area_m2"), r.get("price_per_m2"), ) if key not in deduped: deduped[key] = r deduped_rows = list(deduped.values()) comps = [_anchor_comp_from_row(r) for r in deduped_rows if r["price_per_m2"]] if len(comps) >= min_comps: primary_n = sum(1 for r in deduped_rows if r.get("listing_segment") == "novostroyki") logger.info( "anchor tier=A street=%r base=%s letter=%s → %d comps (primary=%d)", street, base_no, letter, len(comps), primary_n, ) return comps, "A" # ── Tier C: micro-radius ≤500m + same segment + rooms + area±25% ───────── if lat is not None and lon is not None and rooms is not None and area: try: rows = ( db.execute( text( """ SELECT price_per_m2, area_m2, rooms, floor, total_floors, address, source, source_url, price_rub, listing_date, days_on_market, photo_urls, lat, lon FROM listings WHERE is_active = true AND price_per_m2 > 0 AND rooms = CAST(:rooms AS integer) AND area_m2 BETWEEN :area_min AND :area_max -- novostroyki guard (#1186): NULL = legacy вторичка до м.011 AND (listing_segment IS NULL OR listing_segment = 'vtorichka') AND geom IS NOT NULL AND ST_DWithin( geom::geography, ST_MakePoint(:lon, :lat)::geography, 500 ) """ ), { "rooms": rooms, "area_min": area * 0.75, "area_max": area * 1.25, "lon": lon, "lat": lat, }, ) .mappings() .all() ) except Exception as exc: # pragma: no cover — defensive logger.warning("anchor Tier C lookup failed (graceful): %s", exc) try: db.rollback() except Exception: pass rows = [] comps = [_anchor_comp_from_row(r) for r in rows if r["price_per_m2"]] if len(comps) >= min_comps: logger.info("anchor tier=C micro-radius → %d comps", len(comps)) return comps, "C" # Tier D — caller fallback (радиусный median-путь без anchor). return [], None def _band_haircut(anchor_ppm2: float) -> float: """asking→sold haircut, banded по ppm² (class-label в prod пуст — band на цену). Премиум (высокий ppm²) торгуется плотнее → меньше скидка; эконом — больше. Пороги ЕКБ-вторички: ≥350k → 4%; 180-350k → 5%; <180k → 7%. Дефолт из config. """ if anchor_ppm2 >= 350_000: return 0.04 if anchor_ppm2 >= 180_000: return settings.asking_to_sold_haircut # 5% mid return 0.07 def _mad_clip(values: list[float], k: float) -> list[int]: """MAD-клип: возвращает индексы элементов values, не являющихся выбросами. Выброс: |v − median| > k × MAD, где MAD = median(|v − median|). Чистая функция без side-effect'ов — возвращает список индексов выживших (не сами значения, чтобы caller мог фильтровать list[dict] по ним). При MAD == 0 (все значения одинаковы) — все элементы проходят (ни один не считается выбросом в вырожденном случае). Ожидает непустой список; caller гарантирует len(values) >= 1. """ sorted_v = sorted(values) median = _percentile(sorted_v, 0.5) deviations = sorted([abs(v - median) for v in values]) mad = _percentile(deviations, 0.5) if mad == 0.0: # Все значения идентичны — нечего отсекать. return list(range(len(values))) threshold = k * mad return [i for i, v in enumerate(values) if abs(v - median) <= threshold] def _compute_same_building_anchor( comps: list[dict[str, Any]], *, area_target: float, rooms_target: int | None, tier: str, sigma: float, rooms_boost: float, floor_target: int | None = None, total_floors_target: int | None = None, floor_sigma: float = 0.0, min_comps: int = 4, mad_k: float = 3.5, ) -> dict[str, Any] | None: """Чистая (testable без БД) свёртка комплов в anchor-оценку. 1. similarity-weighted mean ppm²: w_i = exp(−(ln(area_i/area_target))²/(2σ²)) × (rooms_boost если rooms_i==rooms_target) × FLOOR-вес. area_i=None → area-вес 1.0 (нейтрально). FLOOR-вес (#680-WB) — Gaussian по ОТНОСИТЕЛЬНОЙ вертикальной позиции pos=floor/total_floors: exp(−(pos_i−pos_t)²/(2σ_f²)), σ_f=floor_sigma. Прижимает якорь к комплам с похожим этажом (1-й/последний и видовые этажи реально отличаются по цене). floor_sigma=0 ИЛИ нет floor у target/компла → вес 1.0 (выключено / нейтрально — no regress). 2. PREMIUM uplift (class-free): target — топ-юнит ДОМА (area ≥ p66 площадей комплов) И rooms_target ≥ медианы комнат комплов И tier == 'A' → берём weighted ~p70 верхний квантиль ppm² (консервативно, только вверх). Условие по комнатам (#680-WB) не даёт мелкокомнатному юниту во флагман-доме унаследовать headline-премию флагмана (3к/153 в доме с 4к-флагманом ≠ цена флагмана). 3. haircut asking→sold (banded по anchor ppm²): anchor_sold = anchor×(1−haircut). 4. FSD = 0.07 + 0.25·CV(comp ppm²) + tier_penalty + n_penalty; range полуширина = k·fsd. confidence-банд по fsd. Returns dict {anchor_ppm2, anchor_sold_ppm2, fsd, confidence, n, cv, comp_min_ppm2, used_uplift, haircut} или None если комплов нет. """ if not comps: return None # Строим параллельные списки comps/ppm2 с гарантированным соответствием индексов. priced_pairs = [(c, float(c["price_per_m2"])) for c in comps if c.get("price_per_m2")] if not priced_pairs: return None # #755 param-3: MAD-clip — отсекаем выбросы по price_per_m2 до агрегации. # Если после клипа выживает < min_comps — якорь НЕ срабатывает (→ None → fallback). raw_ppm2 = [p for _, p in priced_pairs] # #1795 шаг 2: ужесточаем MAD-clip на малых выборках. При n < small_n_threshold # дефолтный mad_k=3.5 слишком мягок (n=7 не срезает элитные хвосты на право- # скошенном премиум-распределении → similarity-weighted mean тянется вверх). # Эффективный k выбирается ВНУТРИ функции до _mad_clip (сигнатура не меняется). # mad_k_small >= mad_k → no-op (старое поведение). # ВАЖНО: Tier A (комплы ТОГО ЖЕ дома) EXEMPT — внутридомовой спред (этаж/вид) # легитимен, агрессивный clip там съел бы реальные топ-юниты и обрушил бы якорь # < min_comps → fallback на заниженную радиусную медиану. Tier C/прочие — clip. effective_mad_k = mad_k if ( tier != "A" and settings.estimate_sb_mad_k_small_n < mad_k and len(raw_ppm2) < settings.estimate_sb_small_n_threshold ): effective_mad_k = settings.estimate_sb_mad_k_small_n surviving_idx = _mad_clip(raw_ppm2, effective_mad_k) if len(surviving_idx) < min_comps: logger.info( "anchor MAD-clip: %d comps → %d survived (< min_comps=%d) → fallback", len(priced_pairs), len(surviving_idx), min_comps, ) return None if len(surviving_idx) < len(priced_pairs): logger.info( "anchor MAD-clip: %d comps → %d after k=%.1f×MAD clip", len(priced_pairs), len(surviving_idx), effective_mad_k, ) priced_pairs = [priced_pairs[i] for i in surviving_idx] comps = [c for c, _ in priced_pairs] ppm2 = [p for _, p in priced_pairs] n = len(ppm2) # target relative vertical position (None → floor-вес отключён/нейтрален). target_pos: float | None = None if floor_sigma > 0 and floor_target and total_floors_target and total_floors_target > 0: target_pos = floor_target / total_floors_target # #audit-4: sigma > 0 guard — при sigma=0 Gaussian exp(-x²/0) → div/0/NaN. # area_sigma=0 (отключено) → нейтральный вес 1.0; floor_sigma=0 уже гейтится выше. safe_area_sigma2 = 2.0 * sigma * sigma if sigma > 0 else 0.0 safe_floor_sigma2 = 2.0 * floor_sigma * floor_sigma if floor_sigma > 0 else 0.0 # 1. similarity weights (area × rooms × floor-position) weights: list[float] = [] for c in comps: a = c.get("area_m2") if a and area_target > 0 and safe_area_sigma2 > 0: area_w = math.exp(-((math.log(a / area_target)) ** 2) / safe_area_sigma2) else: area_w = 1.0 # площадь неизвестна или sigma=0 → нейтральный area-вес rooms_match = rooms_target is not None and c.get("rooms") == rooms_target rooms_w = rooms_boost if rooms_match else 1.0 floor_w = 1.0 if target_pos is not None and safe_floor_sigma2 > 0: cf = c.get("floor") ctf = c.get("total_floors") if cf and ctf and ctf > 0: comp_pos = cf / ctf floor_w = math.exp(-((comp_pos - target_pos) ** 2) / safe_floor_sigma2) # компл без floor/total_floors → нейтральный floor-вес 1.0 weights.append(area_w * rooms_w * floor_w) wsum = sum(weights) if wsum > 0: anchor = sum(w * p for w, p in zip(weights, ppm2, strict=True)) / wsum else: anchor = _percentile(sorted(ppm2), 0.5) # #audit-4: MAD-clip ПОСЛЕ similarity-weighting (за флагом estimate_sb_clip_after_weight). # Видовые/топ-юниты с высоким ppm² могут быть выкинуты сырым clip'ом ДО weighting — # они легитимны. После weighting (anchor = weighted mean) ищем выбросы # относительно self-consistent взвешенного пространства. # Используем те же effective_mad_k; если после clip < min_comps → anchor=None. if settings.estimate_sb_clip_after_weight and n >= 2: post_clip_idx = _mad_clip(ppm2, effective_mad_k) if len(post_clip_idx) < min_comps: logger.info( "anchor post-weight MAD-clip #audit-4: %d comps → %d survived" " (< min_comps=%d) → fallback", n, len(post_clip_idx), min_comps, ) return None if len(post_clip_idx) < n: logger.info( "anchor post-weight MAD-clip #audit-4: %d comps → %d after k=%.1f×MAD", n, len(post_clip_idx), effective_mad_k, ) comps = [comps[i] for i in post_clip_idx] ppm2 = [ppm2[i] for i in post_clip_idx] weights = [weights[i] for i in post_clip_idx] n = len(ppm2) wsum = sum(weights) if wsum > 0: anchor = sum(w * p for w, p in zip(weights, ppm2, strict=True)) / wsum else: anchor = _percentile(sorted(ppm2), 0.5) # 2. premium uplift — топ-юнит дома (площадь ≥ p66 И комнаты ≥ медианы) И Tier A # → weighted p70. Условие по комнатам отсекает мелкие юниты во флагман-домах. used_uplift = False areas = [c.get("area_m2") for c in comps if c.get("area_m2")] comp_rooms = [c.get("rooms") for c in comps if c.get("rooms") is not None] if tier == "A" and areas and area_target > 0: p66_area = _percentile(sorted(areas), 0.66) rooms_ok = True if rooms_target is not None and comp_rooms: median_rooms = _percentile(sorted(comp_rooms), 0.5) rooms_ok = rooms_target >= median_rooms if area_target >= p66_area and rooms_ok: p70 = _percentile(sorted(ppm2), 0.70) if p70 > anchor: anchor = p70 used_uplift = True # 3. asking→sold haircut (banded) haircut = _band_haircut(anchor) anchor_sold = anchor * (1.0 - haircut) # 4. FSD-диапазон (tight). CV = std/mean comp ppm². mean_ppm2 = sum(ppm2) / n if mean_ppm2 > 0 and n >= 2: var = sum((p - mean_ppm2) ** 2 for p in ppm2) / n cv = math.sqrt(var) / mean_ppm2 else: cv = 0.0 tier_penalty = {"A": 0.0, "C": 0.05}.get(tier, 0.09) n_penalty = 0.05 if n < 3 else (0.02 if n < 5 else 0.0) fsd = 0.07 + 0.25 * cv + tier_penalty + n_penalty if fsd <= 0.13: confidence = "high" elif fsd <= 0.20: confidence = "medium" else: confidence = "low" # #755 param-2: confidence cap — при n < 5 комплах anchor не может быть "high" # даже если FSD укладывается в 0.13 (мало данных — самоуверенный headline опасен). if n < 5 and confidence == "high": confidence = "medium" return { "anchor_ppm2": anchor, "anchor_sold_ppm2": anchor_sold, "fsd": fsd, "confidence": confidence, "n": n, "cv": cv, "comp_min_ppm2": min(ppm2), "comp_max_ppm2": max(ppm2), "used_uplift": used_uplift, "haircut": haircut, } # ── #693 prod-fix: coarse-geocode detector (DaData-independent) ───────────── # Дом всегда оканчивается номером (1-3 цифры, опц. литера корпуса). Centroid # города/региона его НЕ содержит. Прод-сигнал грубости геокода, работающий БЕЗ # DaData: на проде DaData может быть off (token не сконфигурирован) → dadata.qc_geo # всегда None → старый гейт #707 (условный на qc_geo>=2) НЕ срабатывал НИКОГДА, # даже для региона/города. Кэш-агностичен — смотрит на geo.full_address, каким бы # провайдером/кэшем он ни был получен. # # Граница (? bool: """True если геокод разрешился лишь до centroid'а НП/города/региона (без дома). Два сигнала (OR): 1. provider confidence == 'locality' — явный centroid-маркер (на будущее: текущие провайдеры его не эмитят, но enum/кэш-колонка допускают, и так геокодер можно улучшить позже без правки гейта). 2. в geo.full_address нет house-number токена (1-3 цифры) → геокодер не дошёл до дома, вернул центр НП/города/региона. Гарантирует ZERO ложных downgrade на реальных адресах: у любого реального дома есть номер → токен матчится → не coarse. """ if geo.confidence == "locality": return True return _HOUSE_NUMBER_RE.search(geo.full_address or "") is None # ── Time-budget guard (#654) ──────────────────────────────────────────────── async def _with_budget(coro: Any, budget_s: float, *, label: str) -> Any: """Await `coro` under an asyncio.wait_for() time budget. On timeout the coroutine is cancelled and we return None — mapping a slow upstream onto the SAME graceful "None" path these enrichments already take on network error, so a single slow source degrades the estimate instead of blowing the gateway read timeout (#654: opaque Caddy 502/504). budget_s <= 0 disables the guard (await directly) — escape hatch via config. """ if budget_s is None or budget_s <= 0: return await coro try: return await asyncio.wait_for(coro, timeout=budget_s) except TimeoutError: # asyncio.TimeoutError is an alias of builtin TimeoutError (py3.11+). logger.warning("%s exceeded %.1fs budget — degrading to None (#654)", label, budget_s) return None # ── PricingResult dataclass (pure, no I/O) ────────────────────────────────── @dataclass class PricingResult: """Return type of _price_from_inputs — все переменные, нужные estimate_quality после блока.""" median_ppm2: float median_price: int range_low: int range_high: int n_analogs: int confidence: str explanation: str anchor_tier: str | None anchor_comps_used: list[dict] avito_imv_summary: AvitoImvSummary | None dkp_corridor: DkpCorridor | None expected_sold_per_m2: int | None expected_sold_price: int | None expected_sold_range_low: int | None expected_sold_range_high: int | None asking_to_sold_ratio: float | None ratio_basis: str | None sources_used_pre: list[str] listings_clean: list[dict] def _price_from_inputs( *, listings: list[dict], area_m2: float, rooms: int | None, repair_state: str | None, floor: int | None, total_floors: int | None, target_year: int | None, analog_tier: str, fallback_used: bool, area_widened: bool, anchor_comps: list[dict], anchor_tier_fetched: str | None, dkp_raw: dict | None, imv_anchor: dict | None, imv_eval: IMVEvaluation | None, yandex_val_present: bool, cian_val_present: bool, ratio_resolver: Callable[[float | None], tuple[float | None, str | None]], quarter_index_lookup: Callable[[str], tuple[float, int] | None], quarter_indexes_lookup: Callable[[list[str]], dict[str, float]], target_house_cadnum: str | None, dadata_coarse: bool, geo: GeocodeResult, dadata_qc_geo: int | None, ) -> PricingResult: """Deterministic pricing orchestration — pure, synchronous, zero I/O. Extracted from estimate_quality (#1966) to enable offline backtesting and direct unit testing. All DB lookups are injected via callables or pre-fetched values; behavior is byte-identical to the original block. """ # 3. Outlier filter listings_clean = _filter_outliers(listings) # 4. Aggregation if listings_clean: prices_ppm2 = sorted(lot["price_per_m2"] for lot in listings_clean if lot["price_per_m2"]) median_ppm2 = _percentile(prices_ppm2, 0.5) q1_ppm2 = _percentile(prices_ppm2, 0.25) q3_ppm2 = _percentile(prices_ppm2, 0.75) median_price = int(median_ppm2 * area_m2) range_low = int(q1_ppm2 * area_m2) range_high = int(q3_ppm2 * area_m2) # #2: n_analogs считается по prices_ppm2, а не len(listings_clean). n_analogs = len(prices_ppm2) else: median_ppm2 = 0.0 q1_ppm2 = 0.0 q3_ppm2 = 0.0 median_price = 0 range_low = 0 range_high = 0 n_analogs = 0 # 4b. Repair coefficient repair_coef = _repair_coefficient(repair_state) repair_note = "" if listings_clean and repair_coef != 1.0: median_price = int(median_price * repair_coef) range_low = int(range_low * repair_coef) range_high = int(range_high * repair_coef) median_ppm2 = median_ppm2 * repair_coef pct = round((repair_coef - 1.0) * 100) repair_note = ( f" Цена скорректирована на состояние ремонта " f"({_REPAIR_LABEL.get(repair_state, '')} {pct:+d}%)." ) # Build sources_used_pre from listings + external sources sources_used_pre = sorted({lot.get("source") for lot in listings_clean if lot.get("source")}) if imv_eval is not None: sources_used_pre = sorted(set(sources_used_pre) | {"avito_imv"}) if yandex_val_present: sources_used_pre = sorted(set(sources_used_pre) | {"yandex_valuation"}) if cian_val_present: sources_used_pre = sorted(set(sources_used_pre) | {"cian_valuation"}) # 4c. asking→sold variables — initialized; actual computation is after all mutations. asking_to_sold_ratio: float | None = None ratio_basis: str | None = None expected_sold_per_m2: int | None = None expected_sold_price: int | None = None expected_sold_range_low: int | None = None expected_sold_range_high: int | None = None confidence, explanation = _compute_confidence( n_analogs, median_ppm2, q1_ppm2 if listings_clean else 0, q3_ppm2 if listings_clean else 0, fallback_used, area_widened, listings=listings_clean, ) # Tier note — информируем пользователя о качестве house-match tier_note = "" if analog_tier == "S": tier_note = " (аналоги из того же дома)" elif analog_tier == "H": tf_str = f"{total_floors}-эт." if total_floors is not None else "" yr_str = f"{target_year}±15 г." if target_year else "" parts_str = ", ".join(p for p in [yr_str, tf_str] if p) tier_note = f" (аналоги из домов того же класса: {parts_str})" if parts_str else "" else: tier_note = " (нет аналогов в том же доме/классе — расширили поиск)" explanation = (explanation or "") + tier_note + repair_note # ── #651/#652 v2: same-building anchor ─────────────────────────────────── anchor_tier: str | None = anchor_tier_fetched anchor_comps_used: list[dict] = [] anchor = _compute_same_building_anchor( anchor_comps, area_target=area_m2, rooms_target=rooms, tier=anchor_tier or "", sigma=settings.estimate_sb_area_sigma, rooms_boost=settings.estimate_sb_rooms_match_boost, floor_target=floor, total_floors_target=total_floors, floor_sigma=settings.estimate_sb_floor_sigma, min_comps=settings.estimate_sb_min_comps, mad_k=settings.estimate_sb_mad_k, ) # #1795 шаг 3: гейт Tier C. if ( anchor is not None and anchor_tier == "C" and settings.estimate_anchor_tier_c_corridor_mult > 0 ): if dkp_raw is not None and dkp_raw.get("high_ppm2", 0) > 0: corridor_high_for_gate = float(dkp_raw["high_ppm2"]) else: corridor_high_for_gate = (median_ppm2 / repair_coef) * 1.3 if repair_coef else 0.0 gate_threshold = corridor_high_for_gate * settings.estimate_anchor_tier_c_corridor_mult if gate_threshold > 0 and anchor["anchor_ppm2"] > gate_threshold: logger.info( "sb_anchor Tier C gate #1795: anchor_ppm2=%d > corridor_high×%.1f=%d" " → keep radius median (anchor suppressed)", int(anchor["anchor_ppm2"]), settings.estimate_anchor_tier_c_corridor_mult, int(gate_threshold), ) anchor = None # #audit-1: low-confidence gate. if anchor is not None and settings.estimate_sb_low_conf_gate_enabled: gate_low = anchor["confidence"] == "low" gate_thin = ( anchor["n"] < settings.estimate_sb_gate_min_n and anchor["fsd"] > settings.estimate_sb_gate_max_fsd ) if gate_low or gate_thin: logger.info( "sb_anchor low-conf gate #audit-1: tier=%s n=%d fsd=%.3f conf=%s" " → suppressed (gate_low=%s gate_thin=%s) → radius fallback", anchor_tier, anchor["n"], anchor["fsd"], anchor["confidence"], gate_low, gate_thin, ) anchor = None anchor_tier = None if anchor is not None: # #694: якорь мутирует headline — UI-аналоги должны отражать ЭТИ комплы. anchor_comps_used = anchor_comps est_ppm2 = anchor["anchor_ppm2"] # PREMIUM GUARDRAIL (hard). floor_ppm2 = anchor["comp_min_ppm2"] * (1.0 - settings.estimate_sb_guardrail_tol) if est_ppm2 < floor_ppm2: est_ppm2 = floor_ppm2 new_ppm2 = est_ppm2 * repair_coef point = int(new_ppm2 * area_m2) # FSD-диапазон. half = settings.estimate_fsd_k * anchor["fsd"] new_range_low = int(point * max(0.0, 1.0 - half)) new_range_high = int(point * (1.0 + half)) # Спред комплов. spread_low = int(anchor["comp_min_ppm2"] * repair_coef * area_m2) spread_high = int(anchor["comp_max_ppm2"] * repair_coef * area_m2) new_range_low = min(new_range_low, spread_low, point) new_range_high = max(new_range_high, spread_high, point) logger.info( "sb_anchor: tier=%s n=%d radius_median_ppm2=%d → anchor_asking_ppm2=%d" " (uplift=%s haircut=%.2f) point %d → %d", anchor_tier, anchor["n"], int(median_ppm2), int(est_ppm2), anchor["used_uplift"], anchor["haircut"], median_price, point, ) median_ppm2 = new_ppm2 median_price = point range_low = new_range_low range_high = new_range_high confidence = anchor["confidence"] tier_label = "того же дома" if anchor_tier == "A" else "ближайшего окружения (≤500 м)" # #695: explanation описывает ИМЕННО якорные комплы. explanation = ( f"Оценка построена по {anchor['n']} аналогам из {tier_label}" f"{' (топ-уровень в доме)' if anchor['used_uplift'] else ''}." ) + repair_note # #695 (QA fixup): n_analogs по anchor-популяции. n_analogs = anchor["n"] # #1871 P1: ghost-anchor guard. if not listings_clean and confidence != "low": logger.warning( "estimator #1871 ghost-anchor guard: confidence %s→low " "(anchor_n=%s, radius_analogs=0)", confidence, anchor["n"], ) confidence = "low" explanation += ( " Оценка опирается только на аналоги из того же дома — " "сопоставимых предложений поблизости не найдено, поэтому " "точность ориентировочная." ) # #1871 P2: split-дома wide-corridor disclosure. if ( settings.estimate_wide_corridor_disclosure_enabled and anchor_tier == "A" and median_price > 0 ): corridor_pct = (range_high - range_low) / median_price if corridor_pct > settings.estimate_wide_corridor_threshold: confidence = _downgrade_confidence(confidence) explanation = (explanation or "") + ( " Очень широкий ценовой диапазон по дому (вероятно, дом " "разбит на секции разной этажности или разнородный фонд) — " "оценка ориентировочная, уточните по конкретной секции." ) # ── #651: IMV / Yandex blend (Tier D only, anchor_tier is None) ────────── imv_anchor_present: bool = False avito_imv_summary: AvitoImvSummary | None = None if ( anchor_tier is None and settings.estimate_imv_blend_enabled and listings_clean and median_price > 0 ): anchor_total: int | None = None anchor_higher: int | None = None anchor_label: str | None = None if imv_anchor is not None and imv_anchor.get("recommended_price"): anchor_total = int(imv_anchor["recommended_price"]) anchor_higher = ( int(imv_anchor["higher_price"]) if imv_anchor.get("higher_price") else None ) anchor_label = "оценке Avito IMV" _imv_mc = int(imv_anchor["market_count"]) if imv_anchor.get("market_count") else None avito_imv_summary = AvitoImvSummary( recommended_price=anchor_total, lower_price=( int(imv_anchor["lower_price"]) if imv_anchor.get("lower_price") else None ), higher_price=anchor_higher, market_count=_imv_mc, thin_market=( _imv_mc is not None and _imv_mc < settings.avito_imv_thin_market_threshold ), ) elif imv_eval is not None and imv_eval.recommended_price: anchor_total = int(imv_eval.recommended_price) anchor_higher = int(imv_eval.higher_price) if imv_eval.higher_price else None anchor_label = "оценке Avito IMV" avito_imv_summary = AvitoImvSummary( recommended_price=anchor_total, lower_price=(int(imv_eval.lower_price) if imv_eval.lower_price else None), higher_price=anchor_higher, market_count=imv_eval.market_count, thin_market=( imv_eval.market_count is not None and imv_eval.market_count < settings.avito_imv_thin_market_threshold ), ) # #audit-5b: thin-market warning. if avito_imv_summary is not None and avito_imv_summary.thin_market: logger.warning( "avito_imv thin_market #audit-5b: market_count=%s" " (< avito_imv_thin_market_threshold=%d) — IMV reliability low", avito_imv_summary.market_count, settings.avito_imv_thin_market_threshold, ) if anchor_total is not None: imv_anchor_present = True new_median, new_range_high, new_ppm2, blended, anchor_used = _apply_imv_blend( median_price=median_price, range_high=range_high, median_ppm2=median_ppm2, area=area_m2, anchor_total=anchor_total, anchor_higher=anchor_higher, weight=settings.estimate_imv_blend_weight, threshold=settings.estimate_imv_blend_threshold, ) if blended: logger.info( "imv_blend: median %d → %d (anchor=%d w=%.2f) range_high %d → %d", median_price, new_median, anchor_used, settings.estimate_imv_blend_weight, range_high, new_range_high, ) median_price = new_median median_ppm2 = new_ppm2 explanation = (explanation or "") + ( f" Оценка скорректирована по {anchor_label} " f"({anchor_used / 1_000_000:.1f} млн ₽)." ) sources_used_pre = sorted(set(sources_used_pre) | {"avito_imv"}) # Диапазон расширяем даже если медиану не двигали. range_high = new_range_high # Display-only IMV summary when headline built by same-building anchor. if anchor_tier is not None and avito_imv_summary is None: if imv_anchor is not None and imv_anchor.get("recommended_price"): _disp_mc = int(imv_anchor["market_count"]) if imv_anchor.get("market_count") else None avito_imv_summary = AvitoImvSummary( recommended_price=int(imv_anchor["recommended_price"]), lower_price=( int(imv_anchor["lower_price"]) if imv_anchor.get("lower_price") else None ), higher_price=( int(imv_anchor["higher_price"]) if imv_anchor.get("higher_price") else None ), market_count=_disp_mc, thin_market=( _disp_mc is not None and _disp_mc < settings.avito_imv_thin_market_threshold ), ) # ── #764: per-cadastral-quarter price index gap-correction ─────────────── if ( settings.estimate_quarter_index_enabled and anchor_tier is None # Guard-1a and not imv_anchor_present # Guard-1b and median_price > 0 and area_m2 ): target_quarter: str | None = _quarter_from_cadastre(target_house_cadnum) if target_quarter is None: for lot in listings_clean: cq = _quarter_from_cadastre(lot.get("building_cadastral_number")) if cq is not None: target_quarter = cq break if target_quarter is not None: qindex_result = quarter_index_lookup(target_quarter) if qindex_result is not None: target_qi, target_n_deals = qindex_result # Bimodal/nominal guard (Guard-4). if ( target_qi > settings.estimate_quarter_index_max_for_small_n and target_n_deals < settings.estimate_quarter_index_small_n_threshold ): logger.info( "quarter_index: bimodal guard triggered " "(index=%.3f n=%d < %d) for %s — no-op", target_qi, target_n_deals, settings.estimate_quarter_index_small_n_threshold, target_quarter, ) else: lot_quarters_for_guard2: list[str] = [] analog_quarters: list[tuple[str, float]] = [] for lot in listings_clean: lq = _quarter_from_cadastre(lot.get("building_cadastral_number")) if lq is None: continue lot_quarters_for_guard2.append(lq) lp = lot.get("price_per_m2") if lp: analog_quarters.append((lq, float(lp))) # Guard-2: same-quarter ratio. same_quarter_count = sum( 1 for lq in lot_quarters_for_guard2 if lq == target_quarter ) # #1385: знаменатель — только классифицируемые аналоги. same_quarter_ratio = ( same_quarter_count / len(lot_quarters_for_guard2) if lot_quarters_for_guard2 else 0.0 ) if same_quarter_ratio > settings.estimate_quarter_match_skip_ratio: logger.info( "quarter_index: Guard-2 skip (same-quarter ratio=%.2f > %.2f)" " for %s", same_quarter_ratio, settings.estimate_quarter_match_skip_ratio, target_quarter, ) else: distinct_analog_quarters = list( dict.fromkeys(lq for lq, _lp in analog_quarters) ) analog_index_map = quarter_indexes_lookup(distinct_analog_quarters) weighted_sum = 0.0 weight_total = 0.0 for lq, lp in analog_quarters: lot_qi = analog_index_map.get(lq) if lot_qi is None: continue weighted_sum += lp * lot_qi weight_total += lp avg_analog_index = weighted_sum / weight_total if weight_total > 0 else 1.0 ( median_ppm2, median_price, range_low, range_high, qi_factor, ) = _apply_quarter_index( base_median_ppm2=median_ppm2, base_median_price=median_price, base_range_low=range_low, base_range_high=range_high, target_index=target_qi, avg_analog_index=avg_analog_index, min_factor=settings.estimate_quarter_index_factor_min, max_factor=settings.estimate_quarter_index_factor_max, ) analogs_with_qi = sum( 1 for lq, _lp in analog_quarters if lq in analog_index_map ) logger.info( "quarter_index: applied target=%s target_qi=%.3f" " avg_analog_qi=%.3f factor=%.3f" " (same_quarter_ratio=%.2f analogs_with_qi=%d)", target_quarter, target_qi, avg_analog_index, qi_factor, same_quarter_ratio, analogs_with_qi, ) explanation = (explanation or "") + ( f" Учтена локация квартала" f" (индекс цен квартала ×{qi_factor:.2f})." ) sources_used_pre = sorted(set(sources_used_pre) | {"quarter_index"}) # ── #1795 шаг 1: soft-кламп headline к коридору ДКП-сделок ────────────── slack = settings.estimate_corridor_clamp_slack if dkp_raw is not None and median_ppm2 > 0: old_ppm2 = median_ppm2 median_ppm2, median_price, range_low, range_high, clamped = _apply_corridor_clamp( median_ppm2=median_ppm2, median_price=median_price, range_low=range_low, range_high=range_high, corridor_high_ppm2=dkp_raw["high_ppm2"], corridor_count=dkp_raw["count"], anchor_tier=anchor_tier, slack=slack, min_n=settings.estimate_corridor_clamp_min_n, enabled=settings.estimate_corridor_clamp_enabled, ) if clamped: logger.info( "corridor clamp #1795: headline %d → %d ₽/м² (corridor_high=%d ×(1+%.2f)," " count=%d, tier=%s)", int(old_ppm2), int(median_ppm2), dkp_raw["high_ppm2"], slack, dkp_raw["count"], anchor_tier, ) explanation = (explanation or "") + ( " Оценка ограничена коридором реальных сделок Росреестра по улице." ) # ── Radius-path нижний floor от DKP-коридора ───────────────────────────── if ( settings.estimate_radius_floor_enabled and anchor_tier is None and dkp_raw is not None and dkp_raw.get("low_ppm2", 0) > 0 and median_ppm2 > 0 and dkp_raw.get("count", 0) >= settings.estimate_corridor_clamp_min_n ): radius_floor_ppm2 = float(dkp_raw["low_ppm2"]) * settings.estimate_radius_floor_factor if median_ppm2 < radius_floor_ppm2: floor_factor = radius_floor_ppm2 / median_ppm2 logger.info( "radius_floor: median_ppm2=%d < dkp_low=%d × factor=%.2f = floor=%d" " → lifting (factor=%.3f)", int(median_ppm2), dkp_raw["low_ppm2"], settings.estimate_radius_floor_factor, int(radius_floor_ppm2), floor_factor, ) median_ppm2 = radius_floor_ppm2 median_price = round(median_price * floor_factor) range_low = round(range_low * floor_factor) range_high = round(range_high * floor_factor) explanation = (explanation or "") + ( " Оценка поднята до нижней границы коридора реальных сделок Росреестра." ) # 4c (cont.). expected_sold AFTER all headline mutations. if median_ppm2 > 0: asking_to_sold_ratio, ratio_basis = ratio_resolver(median_ppm2) if asking_to_sold_ratio is None: ratio_basis = None if asking_to_sold_ratio is not None and median_price > 0: effective_ratio = asking_to_sold_ratio if settings.estimate_expected_sold_le_asking and effective_ratio > 1.0: logger.info( "expected_sold ratio clamped %.3f->1.0 (rooms=%s)", effective_ratio, rooms, ) effective_ratio = 1.0 expected_sold_per_m2 = round(median_ppm2 * effective_ratio) expected_sold_price = round(median_price * effective_ratio) if settings.estimate_calibrated_pi_enabled and expected_sold_price: # #1966: калиброванный ~80% prediction interval вокруг ТОЧКИ expected_sold. # Эмпирически отношение actual_sold_ppm2 / expected_sold_per_m2 по 2366 # прод-сделкам: p10=0.649, p90=1.392 → band [×low_mult, ×high_mult] даёт # ~80% honest coverage. Старый IQR-производный band (asking-IQR × ratio) # покрывал лишь ~55% реальных продаж при заявленном «диапазоне оценки». # Множители low<1= 3: if median_ppm2 > dkp_raw["high_ppm2"] * (1.0 + slack): explanation = (explanation or "") + ( " Оценка выше коридора реальных сделок Росреестра по улице." ) elif median_ppm2 < dkp_raw["low_ppm2"] * (1.0 - slack): explanation = (explanation or "") + ( " Оценка ниже коридора реальных сделок Росреестра по улице." ) # ── #693: coarse-geo downgrade ─────────────────────────────────────────── geo_coarse = _geocode_is_coarse(geo) if (dadata_coarse or geo_coarse) and median_price > 0: logger.info( "coarse-geo gate #693: dadata_coarse=%s geo_coarse=%s anchor_tier=%s " "median=%s geo.provider=%s geo.confidence=%s geo.full_address=%r geo=(%.5f,%.5f)", dadata_coarse, geo_coarse, anchor_tier, median_price, geo.provider, geo.confidence, geo.full_address, geo.lat, geo.lon, ) if (dadata_coarse or geo_coarse) and anchor_tier != "A" and median_price > 0: if dadata_coarse: _coarse_label = {2: "населённого пункта", 3: "города", 4: "региона"}.get( dadata_qc_geo, "населённого пункта" ) else: _coarse_label = "населённого пункта" confidence = "low" explanation = (explanation or "") + ( f" Адрес определён лишь до уровня {_coarse_label} — точные координаты " "дома найти не удалось, поэтому оценка ориентировочная (аналоги взяты " "по широкой окрестности)." ) return PricingResult( median_ppm2=median_ppm2, median_price=median_price, range_low=range_low, range_high=range_high, n_analogs=n_analogs, confidence=confidence, explanation=explanation, anchor_tier=anchor_tier, anchor_comps_used=anchor_comps_used, avito_imv_summary=avito_imv_summary, dkp_corridor=dkp_corridor, expected_sold_per_m2=expected_sold_per_m2, expected_sold_price=expected_sold_price, expected_sold_range_low=expected_sold_range_low, expected_sold_range_high=expected_sold_range_high, asking_to_sold_ratio=asking_to_sold_ratio, ratio_basis=ratio_basis, sources_used_pre=sources_used_pre, listings_clean=listings_clean, ) # ── Public ─────────────────────────────────────────────────────────────────── async def estimate_quality( payload: TradeInEstimateInput, db: Session, created_by: str | None = None ) -> AggregatedEstimate: """Главная функция — оценка квартиры по реальным данным. PR M / #564 Phase 3: rosreestr_deals **included** в actual_deals output. Stale NOTE 2026-05-24 (про ДДУ contamination) устарел — importer `import-rosreestr.sh` после PR-A 2026-05-24 фильтрует doc_type='ДКП', ДДУ первички исключены. Deals идут в `actual_deals` JSONB поле AggregatedEstimate с tier classification (T0_per_house / T1_per_street) — frontend может разделять confidence в UI. Returns: AggregatedEstimate с estimate_id, медианой, диапазоном, аналогами. """ # 1. Geocode (#654: time-budgeted — Yandex/Nominatim retry chain can stack # multiple network round-trips + 1s Nominatim rate-limit sleeps). geo: GeocodeResult | None = None # Variant A: trust client-provided coords (resolved by autocomplete/map) when present # and inside the EKB bbox — skips the geocode() chain that fails on DaData-format # addresses with the Yandex key dead. Out-of-bbox / partial → ignore, geocode normally. if ( payload.lat is not None and payload.lon is not None and 60.40 <= payload.lon <= 60.85 and 56.65 <= payload.lat <= 56.95 ): geo = GeocodeResult( lat=payload.lat, lon=payload.lon, full_address=payload.address, provider="cache", confidence="exact", ) logger.info( "estimate: using client coords (%.5f, %.5f) — skipping geocode", payload.lat, payload.lon, ) if geo is None and payload.address: geo = await _with_budget( geocode(payload.address, db), settings.estimate_geocode_budget_s, label="geocode", ) if geo is None: # Без координат не можем искать через PostGIS. Возвращаем low confidence. logger.warning("geocode failed for %s — returning low-confidence estimate", payload.address) return _empty_estimate(payload, db, reason="address_not_geocoded", created_by=created_by) # 1b. DaData enrichment (PR Q1) — on-demand cleanup для target адреса. # Best-effort: graceful None при отсутствии credentials / quota / fail. # Дополняет geocode результатом kadastr_num + canonical form + nearest metro. dadata: DadataAddressResult | None = None try: dadata = await dadata_clean_address(payload.address) except Exception as exc: # pragma: no cover — defensive logger.warning("dadata: unexpected error (graceful): %s", exc) # 1c. #6 House-match: резолвим target в КАНОНИЧЕСКИЙ house_id (read-only, без # создания записи). Это даёт детерминированный Tier S «тот же дом» через # listings.house_id_fk (99% покрытие), точнее хрупкого address-string match. # cadastr от DaData → cadastr_exact tier заработает по мере backfill houses. # Best-effort: None при любой ошибке, estimator продолжает на гео-tier'ах. target_house_id: int | None = None try: match = match_house_readonly( db, address=(dadata.canonical_address if dadata else None) or geo.full_address, lat=geo.lat, lon=geo.lon, cadastral_number=(dadata.house_cadnum if dadata else None), ) if match is not None: target_house_id = match[0] logger.info( "estimate target → house_id=%s via %s (conf=%.2f)", match[0], match[2], match[1] ) except Exception as exc: # pragma: no cover — defensive logger.warning("target house match failed (graceful): %s", exc) # 2. #392: обогащаем год / тип дома из картографии (OSM Overpass), если # пользователь их не указал — это улучшает house-match аналогов (#6). # Best-effort: при недоступности OSM target_* остаются None. # #654: time-budgeted — Overpass httpx timeout 15s сам по себе близок к # gateway-таймауту; деградируем в None при превышении budget. target_year = payload.year_built target_house_type = payload.house_type if target_year is None or target_house_type is None: house_meta = await _with_budget( get_house_metadata(geo.lat, geo.lon, db), settings.estimate_house_meta_timeout_s, label="house_metadata(overpass)", ) if house_meta is not None: if target_year is None: target_year = house_meta.year_built if target_house_type is None: target_house_type = house_meta.house_type # 3. Four-tier fallback (PR 9 — added Tier 0 with cohort filter): # 0) 1km + ±15% area + cohort match (year_built — если задан) # a) 1km + ±15% area (без cohort — drop fallback) # b) 2km + ±15% area (fallback_used = True) # c) 2km + ±25% area (fallback_used = True, area_widened = True) cohort_range = _target_cohort_range(target_year) if cohort_range is not None: cy_min, cy_max = cohort_range listings_tier0, _, analog_tier = _fetch_analogs( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, radius_m=DEFAULT_RADIUS_M, full_address=geo.full_address, target_house_id=target_house_id, year_built=target_year, house_type=target_house_type, total_floors=payload.total_floors, cohort_year_min=cy_min, cohort_year_max=cy_max, ) else: listings_tier0 = [] analog_tier = "W" if len(listings_tier0) >= MIN_ANALOGS_TIER_0: listings = listings_tier0 fallback_used = False else: # Tier 0 пуст/мал — graceful fallback на Tier A без cohort listings, fallback_used, analog_tier = _fetch_analogs( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, radius_m=DEFAULT_RADIUS_M, full_address=geo.full_address, target_house_id=target_house_id, year_built=target_year, house_type=target_house_type, total_floors=payload.total_floors, ) area_widened = False if len(listings) < 5: listings_wide, _, analog_tier_wide = _fetch_analogs( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, radius_m=FALLBACK_RADIUS_M, full_address=geo.full_address, target_house_id=target_house_id, year_built=target_year, house_type=target_house_type, total_floors=payload.total_floors, ) if len(listings_wide) > len(listings): listings = listings_wide fallback_used = True analog_tier = analog_tier_wide # Tier C: если даже на 2км мало — расширяем area tolerance до ±25% # (актуально для отдалённых районов / новостроек с нестандартной планировкой) if len(listings) < 3: listings_widearea, _, analog_tier_wa = _fetch_analogs( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, radius_m=FALLBACK_RADIUS_M, area_tolerance=0.25, full_address=geo.full_address, target_house_id=target_house_id, year_built=target_year, house_type=target_house_type, total_floors=payload.total_floors, ) if len(listings_widearea) > len(listings): listings = listings_widearea fallback_used = True area_widened = True analog_tier = analog_tier_wa # ── PRE-FETCH: dkp_raw (hoisted before _price_from_inputs) ────────────── # #1795: ДКП-коридор фетчим ДО вызова _price_from_inputs, чтобы # corridor_high был доступен для Tier C-гейта и soft-клампа headline. dkp_raw = _fetch_dkp_corridor( db, address=geo.full_address, rooms=payload.rooms, area=payload.area_m2, ) # ── Stage 3: Avito IMV evaluation as 5-th source (on-demand cached) ── imv_eval: IMVEvaluation | None = None imv_house_type = _IMV_HOUSE_TYPE_MAP.get(target_house_type) imv_renovation = _IMV_REPAIR_MAP.get(payload.repair_state) if ( geo is not None and geo.full_address and payload.rooms is not None and payload.area_m2 and payload.floor is not None and payload.total_floors is not None and imv_house_type is not None and imv_renovation is not None ): imv_eval = await _get_or_fetch_imv_cached( db, address=geo.full_address, rooms=payload.rooms, area_m2=payload.area_m2, floor=payload.floor, floor_at_home=payload.total_floors, house_type=imv_house_type, renovation_type=imv_renovation, has_balcony=bool(payload.has_balcony), has_loggia=False, ) # ── Stage 8: Yandex Valuation as on-demand source (anonymous, cached 24h) ── yandex_val: YandexValuationResult | None = None if geo is not None and geo.full_address: yandex_val = await _with_budget( _get_or_fetch_yandex_valuation_cached(db, address=geo.full_address), settings.estimate_yandex_valuation_timeout_s, label="yandex_valuation", ) if yandex_val is not None: saved_hist = _save_yandex_history_items(db, yandex_val) logger.info( "yandex_valuation: history items processed=%d saved=%d" " (house_id=NULL — matching deferred)", len(yandex_val.history_items), saved_hist, ) # ── Stage 9: Cian Valuation as 7th source (on-demand, 24h cached) ────── cian_val: CianValuationResult | None = None if ( geo is not None and geo.full_address and payload.rooms is not None and payload.area_m2 and payload.floor is not None and payload.total_floors is not None ): try: cian_val = await _with_budget( estimate_via_cian_valuation( db, address=geo.full_address, total_area=payload.area_m2, rooms_count=payload.rooms, floor=payload.floor, total_floors=payload.total_floors, repair_type="cosmetic", deal_type="sale", use_cache=True, ), settings.estimate_cian_valuation_timeout_s, label="cian_valuation", ) if cian_val is not None and cian_val.sale_price_rub: logger.info( "cian_valuation: price=%s accuracy=%s house_id=%s", cian_val.sale_price_rub, cian_val.sale_accuracy, cian_val.external_house_id, ) except Exception as exc: logger.warning("cian_valuation: lookup failed (graceful): %s", exc) # ── Pre-fetch: same-building anchor comps ───────────────────────────────── # Guard mirrors the original in-block guard exactly; when false → ([], None). _anchor_comps_pre: list[dict] _anchor_tier_pre: str | None if ( settings.estimate_same_building_anchor_enabled and payload.area_m2 and (payload.address or (geo is not None and geo.full_address)) ): _anchor_comps_pre, _anchor_tier_pre = _fetch_anchor_comps( db, address=payload.address or geo.full_address, target_house_id=target_house_id, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, ) else: _anchor_comps_pre, _anchor_tier_pre = [], None # ── Pre-fetch: house IMV anchor ONCE (used for both blend and display) ─── imv_anchor_data = _fetch_house_imv_anchor( db, target_house_id=target_house_id, rooms=payload.rooms, area=payload.area_m2, ) # ── Coarse-geo signals ──────────────────────────────────────────────────── dadata_coarse = dadata is not None and dadata.qc_geo is not None and dadata.qc_geo >= 2 # ── DB-callable wrappers injected into pure pricing ─────────────────────── def _ratio_resolver( appm2: float | None, ) -> tuple[float | None, str | None]: return _get_asking_sold_ratio(db, payload.rooms, anchor_ppm2=appm2) def _qi_lookup(q: str) -> tuple[float, int] | None: return _lookup_quarter_index( db, quarter_cad_number=q, min_n_deals=settings.estimate_quarter_index_min_n_deals, ) def _qis_lookup(qs: list[str]) -> dict[str, float]: return _lookup_quarter_indexes( db, quarter_cad_numbers=qs, min_n_deals=settings.estimate_quarter_index_min_n_deals, ) # ── Deterministic pricing orchestration ────────────────────────────────── pr = _price_from_inputs( listings=listings, area_m2=payload.area_m2, rooms=payload.rooms, repair_state=payload.repair_state, floor=payload.floor, total_floors=payload.total_floors, target_year=target_year, analog_tier=analog_tier, fallback_used=fallback_used, area_widened=area_widened, anchor_comps=_anchor_comps_pre, anchor_tier_fetched=_anchor_tier_pre, dkp_raw=dkp_raw, imv_anchor=imv_anchor_data, imv_eval=imv_eval, yandex_val_present=yandex_val is not None, cian_val_present=cian_val is not None and bool(cian_val.sale_price_rub), ratio_resolver=_ratio_resolver, quarter_index_lookup=_qi_lookup, quarter_indexes_lookup=_qis_lookup, target_house_cadnum=dadata.house_cadnum if dadata else None, dadata_coarse=dadata_coarse, geo=geo, dadata_qc_geo=dadata.qc_geo if dadata else None, ) # Unpack pricing result median_price = pr.median_price median_ppm2 = pr.median_ppm2 range_low = pr.range_low range_high = pr.range_high n_analogs = pr.n_analogs confidence = pr.confidence explanation = pr.explanation anchor_tier = pr.anchor_tier anchor_comps_used = pr.anchor_comps_used avito_imv_summary = pr.avito_imv_summary dkp_corridor = pr.dkp_corridor expected_sold_per_m2 = pr.expected_sold_per_m2 expected_sold_price = pr.expected_sold_price expected_sold_range_low = pr.expected_sold_range_low expected_sold_range_high = pr.expected_sold_range_high asking_to_sold_ratio = pr.asking_to_sold_ratio ratio_basis = pr.ratio_basis sources_used_pre = pr.sources_used_pre listings_clean = pr.listings_clean # 5. Deals — ДКП-only sales (вторичка) из rosreestr_deals. # Importer фильтрует doc_type='ДКП' (PR-A 2026-05-24), ДДУ застройщиков # исключены — больше не скёюят median вторички ~110-120 К/м². deals = _fetch_deals( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, radius_m=DEFAULT_RADIUS_M, ) # 6. Сохраняем в trade_in_estimates estimate_id = uuid4() now = datetime.now(tz=UTC) expires_at = now + timedelta(hours=24) # #694: когда same-building якорь сработал, headline построен на комплах того # же дома (anchor_comps_used) — показываем ИХ, а не радиусные listings_clean # (другие/дешевле/пусто → premium headline «не подтверждён» аналогами). if anchor_tier is not None and anchor_comps_used: analogs_lots = [_anchor_comp_to_analog(c) for c in anchor_comps_used[:10]] # #1519: при сработавшем якоре метаданные (freshness/last_scraped_at/ # days_on_market) считаем по ПОКАЗАННЫМ комплам, а не по радиусной выборке — # иначе «обновлено N мин назад»/дата парсинга/срок продажи относятся к другому # набору (или = None при пустом listings_clean, хотя у комплов данные есть). metadata_lots = anchor_comps_used else: analogs_lots = [_listing_to_analog(lot) for lot in listings_clean[:10]] metadata_lots = listings_clean deals_lots = [_deal_to_analog(d) for d in deals[:10]] freshness_pre = _compute_freshness_minutes(metadata_lots) # DaData enrichment (PR Q1) — заполняется только если service отработал. # При DaData = None все колонки идут в DB как NULL (graceful). dadata_metro_json = ( json.dumps(dadata.metro, ensure_ascii=False) if dadata is not None and dadata.metro else None ) db.execute( text( """ INSERT INTO trade_in_estimates ( id, address, lat, lon, area_m2, rooms, floor, total_floors, year_built, house_type, repair_state, has_balcony, ownership_type, has_mortgage, client_name, client_phone, median_price, range_low, range_high, median_price_per_m2, confidence, confidence_explanation, n_analogs, analogs, actual_deals, sources_used, data_freshness_minutes, canonical_address, house_cadnum, house_fias_id, dadata_qc_geo, dadata_qc_house, dadata_metro, expected_sold_price, expected_sold_range_low, expected_sold_range_high, expected_sold_per_m2, asking_to_sold_ratio, ratio_basis, created_by, expires_at ) VALUES ( CAST(:id AS uuid), :address, :lat, :lon, :area, :rooms, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, :ownership_type, :has_mortgage, :client_name, :client_phone, :median_price, :range_low, :range_high, :median_ppm2, :confidence, :explanation, :n_analogs, CAST(:analogs_json AS jsonb), CAST(:deals_json AS jsonb), CAST(:sources_json AS jsonb), :freshness, :canonical_address, :house_cadnum, :house_fias_id, :dadata_qc_geo, :dadata_qc_house, CAST(:dadata_metro_json AS jsonb), :expected_sold_price, :expected_sold_range_low, :expected_sold_range_high, :expected_sold_per_m2, :asking_to_sold_ratio, :ratio_basis, :created_by, :expires_at ) """ ), { "id": str(estimate_id), "address": geo.full_address, "lat": geo.lat, "lon": geo.lon, "area": payload.area_m2, "rooms": payload.rooms, "floor": payload.floor, "total_floors": payload.total_floors, "year_built": target_year, "house_type": target_house_type, "repair_state": payload.repair_state, "has_balcony": payload.has_balcony, "ownership_type": payload.ownership_type, "has_mortgage": payload.has_mortgage, "client_name": payload.client_name, "client_phone": payload.client_phone, "median_price": median_price, "range_low": range_low, "range_high": range_high, "median_ppm2": int(median_ppm2), "confidence": confidence, "explanation": explanation, "n_analogs": n_analogs, "analogs_json": json.dumps( [a.model_dump(mode="json") for a in analogs_lots], ensure_ascii=False ), "deals_json": json.dumps( [a.model_dump(mode="json") for a in deals_lots], ensure_ascii=False ), "sources_json": json.dumps(sources_used_pre, ensure_ascii=False), "freshness": freshness_pre, "canonical_address": dadata.canonical_address if dadata else None, "house_cadnum": dadata.house_cadnum if dadata else None, "house_fias_id": dadata.house_fias_id if dadata else None, "dadata_qc_geo": dadata.qc_geo if dadata else None, "dadata_qc_house": dadata.qc_house if dadata else None, "dadata_metro_json": dadata_metro_json, "expected_sold_price": expected_sold_price, "expected_sold_range_low": expected_sold_range_low, "expected_sold_range_high": expected_sold_range_high, "expected_sold_per_m2": expected_sold_per_m2, "asking_to_sold_ratio": asking_to_sold_ratio, "ratio_basis": ratio_basis, "created_by": created_by, "expires_at": expires_at, }, ) # Link saved IMV evaluation к этому estimate_id атомарно с основным INSERT # (closes finding #4 from 2026-05-24 audit — prior code committed estimate first, # then UPDATEd IMV in a separate tx, racing against concurrent estimators # sharing the same cache_key). if imv_eval is not None: db.execute( text( """ UPDATE avito_imv_evaluations SET estimate_id = CAST(:estimate_id AS uuid) WHERE cache_key = :cache_key AND (estimate_id IS NULL OR estimate_id = CAST(:estimate_id AS uuid)) """ ), {"estimate_id": str(estimate_id), "cache_key": imv_eval.cache_key}, ) db.commit() logger.info( "estimate: id=%s addr=%s rooms=%d area=%.1f → median=%d (n=%d, conf=%s)%s%s", estimate_id, geo.full_address[:60], payload.rooms, payload.area_m2, median_price, n_analogs, confidence, f" imv={imv_eval.recommended_price}" if imv_eval else "", f" cian={cian_val.sale_price_rub}" if cian_val and cian_val.sale_price_rub else "", ) sources_used = sorted({lot.source for lot in analogs_lots if lot.source}) if imv_eval is not None: sources_used = sorted(set(sources_used) | {"avito_imv"}) if yandex_val is not None: sources_used = sorted(set(sources_used) | {"yandex_valuation"}) if cian_val is not None and cian_val.sale_price_rub: sources_used = sorted(set(sources_used) | {"cian_valuation"}) freshness_min = _compute_freshness_minutes(metadata_lots) last_scraped_at = _compute_last_scraped_at(metadata_lots) # Месячный ₽/м² тренд целевого дома (web TREND chart) — best-effort, None если нет данных. # #audit-3: передаём freshness_months из настроек — исключаем устаревшие items. price_trend_raw = _fetch_price_trend( db, target_house_id=target_house_id, freshness_months=settings.estimate_price_trend_max_age_months, ) price_trend = ( [PriceTrendPoint(month=p["month"], ppm2=p["ppm2"]) for p in price_trend_raw] if price_trend_raw else None ) # #audit-2: структурный analog_tier — стабильный enum для фронта. # anchor-путь: anchor_tier "A" → "same_building", "C" → "micro_radius". # radius-путь: analog_tier "W" → "city", остальные → "district". # None только если нет аналогов (median=0, insufficient_data=True). if median_price > 0: if anchor_tier == "A": api_analog_tier: str | None = "same_building" elif anchor_tier == "C": api_analog_tier = "micro_radius" elif analog_tier == "W": api_analog_tier = "city" else: api_analog_tier = "district" else: api_analog_tier = None # #1871 P1.2 — defensive invariant guard перед сборкой ответа: n_analogs == 0 # не может сосуществовать с confidence != 'low' (ghost-anchor). Mainline уже # честен — это belt-and-suspenders на будущие external-valuation/rehydrate пути. # За флагом estimate_confidence_floor_no_analogs (дефолт True). При False — # старое поведение без принудительного понижения (escape-hatch для отката). if settings.estimate_confidence_floor_no_analogs: confidence, explanation = _enforce_zero_analog_low( confidence, n_analogs, explanation, median_price=median_price, sources_used=sources_used, ) return AggregatedEstimate( estimate_id=estimate_id, median_price_rub=median_price, range_low_rub=range_low, range_high_rub=range_high, median_price_per_m2=int(median_ppm2), confidence=confidence, confidence_explanation=explanation, n_analogs=n_analogs, period_months=DEALS_PERIOD_MONTHS, analogs=analogs_lots, actual_deals=deals_lots, expires_at=expires_at, target_address=geo.full_address, target_lat=geo.lat, target_lon=geo.lon, sources_used=sources_used, data_freshness_minutes=freshness_min, last_scraped_at=last_scraped_at, price_trend=price_trend, est_days_on_market=_estimate_days_on_market(metadata_lots, deals), cian_valuation=( CianValuationSummary( sale_price_rub=int(cian_val.sale_price_rub) if cian_val.sale_price_rub else None, rent_price_rub=int(cian_val.rent_price_rub) if cian_val.rent_price_rub else None, chart=[ { "date": p.get("month_date") or p.get("date") or "", "price": p["price"], } for p in (cian_val.chart or []) if p.get("price") is not None ], chart_change_pct=cian_val.chart_change_pct, chart_change_direction=( cian_val.chart_change_direction if cian_val.chart_change_direction in {"increase", "decrease", "neutral"} else None ), ) if cian_val is not None else None ), avito_imv=avito_imv_summary, dkp_corridor=dkp_corridor, expected_sold_price_rub=expected_sold_price, expected_sold_range_low_rub=expected_sold_range_low, expected_sold_range_high_rub=expected_sold_range_high, expected_sold_per_m2=expected_sold_per_m2, asking_to_sold_ratio=asking_to_sold_ratio, ratio_basis=ratio_basis, area_m2=payload.area_m2, rooms=payload.rooms, floor=payload.floor, total_floors=payload.total_floors, year_built=target_year, house_type=target_house_type, repair_state=payload.repair_state, has_balcony=payload.has_balcony, canonical_address=dadata.canonical_address if dadata else None, house_cadnum=dadata.house_cadnum if dadata else None, house_fias_id=dadata.house_fias_id if dadata else None, metro_nearest=(dadata.metro if dadata and dadata.metro else []), address_precision=_qc_geo_to_precision(dadata.qc_geo if dadata else None), analog_tier=api_analog_tier, # type: ignore[arg-type] ) def _qc_geo_to_precision(qc_geo: int | None) -> str | None: # DaData qc_geo: 0=exact(house), 1=street, 2=settlement, 3=city, 4=region, 5=unknown if qc_geo is None: return None if qc_geo == 0: return "house" if qc_geo == 1: return "street" return "approximate" def _estimate_days_on_market( listings: list[dict[str, Any]], deals: list[dict[str, Any]] ) -> int | None: """Прогноз срока продажи — медиана days_on_market по аналогам/сделкам. Возвращает None если ни у одного аналога нет данных о сроке экспозиции (наши парсеры не всегда его отдают — честно показываем «нет данных»). """ values = [ int(lot["days_on_market"]) for lot in (*listings, *deals) if lot.get("days_on_market") and int(lot["days_on_market"]) > 0 ] if len(values) < 3: return None values.sort() return values[len(values) // 2] def _compute_freshness_minutes(lots: list[dict[str, Any]]) -> int | None: """Минут с последнего парсинга — для UI «обновлено N мин назад».""" if not lots: return None from datetime import datetime as _dt now = _dt.now(tz=UTC) scraped = [lot.get("scraped_at") or lot.get("listing_date") for lot in lots] scraped_dt: list[datetime] = [] for s in scraped: if s is None: continue # listings rows из mappings — scraped_at это datetime, не date if hasattr(s, "tzinfo"): scraped_dt.append(s if s.tzinfo else s.replace(tzinfo=UTC)) if not scraped_dt: return None return int((now - max(scraped_dt)).total_seconds() / 60) def _compute_last_scraped_at(lots: list[dict[str, Any]]) -> datetime | None: """Абсолютный timestamp самого свежего парсинга среди аналогов (для UI). Дополняет _compute_freshness_minutes (относительные минуты): отдаёт точную дату/время, чтобы фронт мог отрендерить «обновлено DD.MM HH:MM». None если ни у одного лота нет scraped_at/listing_date с tzinfo (graceful).""" if not lots: return None scraped = [lot.get("scraped_at") or lot.get("listing_date") for lot in lots] scraped_dt: list[datetime] = [] for s in scraped: if s is None: continue if hasattr(s, "tzinfo"): scraped_dt.append(s if s.tzinfo else s.replace(tzinfo=UTC)) return max(scraped_dt) if scraped_dt else None def _fetch_price_trend( db: Session, *, target_house_id: int | None, months: int = 24, min_points: int = 3, freshness_months: int | None = None, ) -> list[dict[str, Any]] | None: """Месячный ₽/м² тренд для целевого дома (web TREND chart) — best-effort. Предпочитает `houses_price_dynamics` (house_id, month_date, price_per_sqm) — готовая помесячная серия. В prod эта таблица пока ПУСТА, поэтому fallback — агрегация `house_placement_history` по месяцам (медиана COALESCE(last_price, start_price)/area_m2, дата = COALESCE(last_price_date, start_price_date)). Возвращает список [{month: 'YYYY-MM', ppm2: int}, ...] (≤ `months` точек, ASC по месяцу) или None если house_id не задан / точек < `min_points` / любая ошибка (graceful — фронт скрывает chart, без регрессий). """ if target_house_id is None: return None # ── Source 1 (preferred): houses_price_dynamics ────────────────────────── try: rows = ( db.execute( text( """ SELECT to_char(month_date, 'YYYY-MM') AS month, round( percentile_cont(0.5) WITHIN GROUP (ORDER BY price_per_sqm) )::int AS ppm2 FROM houses_price_dynamics WHERE house_id = CAST(:hid AS bigint) AND price_per_sqm > 0 AND month_date > (CURRENT_DATE - (CAST(:months AS integer) || ' months')::interval) GROUP BY month_date ORDER BY month_date ASC """ ), {"hid": target_house_id, "months": months}, ) .mappings() .all() ) except Exception as exc: # pragma: no cover — defensive logger.warning("price_trend houses_price_dynamics lookup failed (graceful): %s", exc) try: db.rollback() except Exception: pass rows = [] trend = [{"month": r["month"], "ppm2": int(r["ppm2"])} for r in rows if r["ppm2"]] if len(trend) >= min_points: logger.info( "price_trend house_id=%s source=houses_price_dynamics → %d points", target_house_id, len(trend), ) return trend # ── Source 2 (fallback): aggregate house_placement_history by month ────── # #audit-3: freshness_months — фильтр по scraped_at чтобы исключить стale items # (yandex_valuation из 2024 и т.д.). Применяется ко ВСЕМ source в этом запросе # (avito_imv + yandex_valuation), не только к yandex_valuation. Дефолт из config. # Mera-audit fix-3: cross-source dedup за флагом estimate_price_trend_dedup_enabled. _fresh_months = freshness_months if freshness_months is not None else months _dedup_enabled = settings.estimate_price_trend_dedup_enabled try: if _dedup_enabled: # Дедупликация near-дублей: один объект на avito_imv + yandex_valuation # c разными ext_item_id → double-count в медиане. DISTINCT ON по ключу # (round(area_m2,0), floor, price, price_date) с приоритетом avito_imv # (source='avito_imv' сортируется раньше через CASE). trend_sql = """ WITH deduped AS ( SELECT DISTINCT ON ( ROUND(area_m2), floor, COALESCE(last_price, start_price), COALESCE(last_price_date, start_price_date) ) area_m2, floor, COALESCE(last_price, start_price) AS price, COALESCE(last_price_date, start_price_date) AS price_date FROM house_placement_history WHERE house_id = CAST(:hid AS bigint) AND area_m2 > 0 AND COALESCE(last_price, start_price) > 0 AND COALESCE(last_price_date, start_price_date) IS NOT NULL AND COALESCE(last_price_date, start_price_date) > (CURRENT_DATE - (CAST(:months AS integer) || ' months')::interval) AND scraped_at > (now() - CAST(:fresh_months AS integer) * CAST('1 month' AS interval)) ORDER BY ROUND(area_m2), floor, COALESCE(last_price, start_price), COALESCE(last_price_date, start_price_date), CASE source WHEN 'avito_imv' THEN 0 ELSE 1 END ASC, scraped_at DESC ) SELECT to_char(date_trunc('month', price_date), 'YYYY-MM') AS month, round( percentile_cont(0.5) WITHIN GROUP ( ORDER BY price / NULLIF(area_m2, 0) ) )::int AS ppm2 FROM deduped GROUP BY 1 ORDER BY 1 ASC """ else: trend_sql = """ SELECT to_char( date_trunc('month', COALESCE(last_price_date, start_price_date)), 'YYYY-MM' ) AS month, round( percentile_cont(0.5) WITHIN GROUP ( ORDER BY COALESCE(last_price, start_price) / NULLIF(area_m2, 0) ) )::int AS ppm2 FROM house_placement_history WHERE house_id = CAST(:hid AS bigint) AND area_m2 > 0 AND COALESCE(last_price, start_price) > 0 AND COALESCE(last_price_date, start_price_date) IS NOT NULL AND COALESCE(last_price_date, start_price_date) > (CURRENT_DATE - (CAST(:months AS integer) || ' months')::interval) AND scraped_at > (now() - CAST(:fresh_months AS integer) * CAST('1 month' AS interval)) GROUP BY 1 ORDER BY 1 ASC """ rows = ( db.execute( text(trend_sql), {"hid": target_house_id, "months": months, "fresh_months": _fresh_months}, ) .mappings() .all() ) except Exception as exc: # pragma: no cover — defensive logger.warning("price_trend house_placement_history lookup failed (graceful): %s", exc) try: db.rollback() except Exception: pass rows = [] trend = [{"month": r["month"], "ppm2": int(r["ppm2"])} for r in rows if r["ppm2"]] if len(trend) >= min_points: logger.info( "price_trend house_id=%s source=house_placement_history → %d points", target_house_id, len(trend), ) return trend logger.info( "price_trend house_id=%s → only %d points (<%d) → None", target_house_id, len(trend), min_points, ) return None # ── Internals ──────────────────────────────────────────────────────────────── # Compiled regexes for _extract_short_addr — module-level for performance. # Strips leading admin prefixes: «Россия», «Свердловская область», «г. Екатеринбург» etc. _ADMIN_PREFIX_RE = re.compile( r"^(?:" r"\s*(?:Россия|РФ|Российская\s+Федерация)\s*,?\s*|" r"\s*[А-Яа-яёЁ][А-Яа-яёЁ\s-]+(?:\s+(?:обл(?:асть)?|р-н|район|округ|край|республика))\.?\s*,?\s*|" r"\s*(?:г(?:ород)?|гор)\.?\s*[А-Яа-яёЁ][А-Яа-яёЁ\s-]+\s*,?\s*|" r"\s*[А-Я][а-яё-]+(?:\s+[А-Я][а-яё-]+)?\s*,?\s*" r")+", flags=re.UNICODE, ) # Recognizes start of a street keyword. _STREET_START_RE = re.compile( r"(?:ул\.|улица|пр\.|пр-т|проспект|пер\.|переулок|" r"б-р|бульвар|ш\.|шоссе|наб\.|набережная|проезд|тракт|пл\.|площадь|" r"мкр\.?|микрорайон)\s+", flags=re.IGNORECASE | re.UNICODE, ) # Drops trailing apartment / office / corpus noise from the end. _TRAILING_NOISE_RE = re.compile( r"\s*,\s*(?:кв\.?\s*\d+|корп\.?\s*\w+|оф\.?\s*\d+|пом\.?\s*\d+|подъезд\s*\d+).*$", flags=re.IGNORECASE | re.UNICODE, ) def _extract_short_addr(full_address: str | None) -> str | None: """Извлекает «улица + номер дома» из полного адреса для поиска в том же доме. Примеры: "Свердловская область, г. Екатеринбург, ул. Заводская, д. 44-а" → "ул. Заводская, д. 44-а" "Россия, Екатеринбург, ул. Малышева, 1" → "ул. Малышева, 1" "РФ, Свердловская обл., Екатеринбург, ул. Ленина, 5, кв. 12" → "ул. Ленина, 5" "г. Екатеринбург, проспект Ленина, 50" → "проспект Ленина, 50" "Екатеринбург, ул. Крауля, 48/2" → "ул. Крауля, 48/2" Алгоритм: 1. Отрезаем trailing кв./корп./оф. noise. 2. Ищем первый street-keyword токен (ул./пр./пер. и т.д.) — возвращаем с него. 3. Fallback: агрессивно strip admin-prefix regex, вернуть остаток. 4. None если строка пустая или нечего возвращать. """ if not full_address: return None s = full_address.strip() s = _TRAILING_NOISE_RE.sub("", s) # Find first street-keyword position and return from there. m = _STREET_START_RE.search(s) if m: return s[m.start() :].strip(" ,.") # Fallback: strip known admin prefixes, return whatever remains. s = _ADMIN_PREFIX_RE.sub("", s) return s.strip(" ,.") or None # Ищет keyword типа улицы (ул./улица/пр./проспект/...) в адресе. # Работает для FORWARD и REVERSE форматов Nominatim. _STREET_KW_RE = re.compile( r"(? str | None: """Извлекает чистое имя улицы из адреса в FORWARD или REVERSE формате. Примеры: "Екатеринбург, ул. Космонавтов, 50" → "Космонавтов" "80, улица 8 Марта, Артек, ..., Россия" → "8 Марта" "проспект Ленина 50" → "Ленина" "Россия, Екатеринбург, ул. Малышева, 1" → "Малышева" "ул. Большая Конюшенная, 25" → "Большая Конюшенная" "" → None Алгоритм: 1. Ищем street-keyword (ул/улица/пр/проспект/...) — case-insensitive. 2. После keyword берём 1-3 слова до запятой или номера дома. 3. Если keyword не нашёлся — пытаемся первый capitalized токен с поиском до запятой или номера (fallback для адресов без keyword'а). Returns None если ничего не извлеклось. """ if not full_address or not full_address.strip(): return None s = full_address.strip() # 1. Keyword-based extraction (работает для обоих форматов: forward и reverse) m = _STREET_KW_RE.search(s) if m: rest = s[m.end() :].lstrip() nm = _STREET_NAME_RE.match(rest) if nm: return nm.group(1).strip() # 2. Fallback: нет keyword — пробуем первый capitalized токен # Используется для "Большая Конюшенная, 25" без "ул." nm = _STREET_NAME_RE.match(s) if nm: candidate = nm.group(1).strip() # Отсеиваем очевидные административные слова bad = {"Россия", "Москва", "Санкт-Петербург", "область", "район", "округ", "край"} if not any(b in candidate for b in bad): return candidate return None def _stratify_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: """Стратифицированная выборка Approach B — гарантирует MIN_ANALOGS_PER_SOURCE слотов. Candidates должны быть уже отсортированы по relevance_score (ASC). """ guaranteed: list[dict[str, Any]] = [] guaranteed_ids: set[int] = set() by_source: dict[str, list[dict[str, Any]]] = {} for row in candidates: src = row.get("source") or "unknown" by_source.setdefault(src, []).append(row) for _src, src_rows in by_source.items(): quota = min(len(src_rows), MIN_ANALOGS_PER_SOURCE) for row in src_rows[:quota]: if id(row) not in guaranteed_ids: guaranteed.append(row) guaranteed_ids.add(id(row)) remaining_slots = 50 - len(guaranteed) remainder: list[dict[str, Any]] = [] if remaining_slots > 0: for row in candidates: if id(row) not in guaranteed_ids: remainder.append(row) if len(remainder) >= remaining_slots: break result = guaranteed + remainder result.sort(key=lambda r: r.get("relevance_score") or 0.0) return result[:50] _ANALOG_SELECT_COLS = """ source, source_url, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, scraped_at, building_cadastral_number """ _COMMON_WHERE = """ AND rooms = :rooms AND area_m2 BETWEEN :area_min AND :area_max AND is_active = true AND scraped_at > NOW() - (:fresh_days || ' days')::interval AND price_rub > 0 -- Когортный фильтр (PR 10): применяется к Tier S и Tier H через _COMMON_WHERE. -- Tier W имеет свою копию этого блока в inline SQL. Если cohort_year_min IS NULL — -- фильтр прозрачен. CAST обязателен — psycopg3 prepared statement не выводит -- тип $N при IS NULL в predicate (см. PR #518 fix). AND ( CAST(:cohort_year_min AS integer) IS NULL OR year_built IS NULL OR year_built BETWEEN CAST(:cohort_year_min AS integer) AND CAST(:cohort_year_max AS integer) ) -- novostroyki guard (#1186): NULL = legacy вторичка до м.011 -- Исключаем новостройки из comp-пула вторички: девелоперский прайс искажает -- медиану ₽/м². NULL сегмент пропускаем (rosreestr/avito/yandex без сегмента — -- это вторичка или неклассифицированный объект). AND (listing_segment IS NULL OR listing_segment = 'vtorichka') """ # Note: Tier W has its own inline copy of the cohort clause (PR #519 line # ~1280). Не удалять — Tier W не использует _COMMON_WHERE из-за inline # relevance_score CASE expressions. Both code paths must stay in sync. # ── #1871 P2: radius-tier (source, source_id) dedup ────────────────────────── # Окно rn_dup в base-CTE каждого radius-тира: (source, source_id)-дубли делят # один address и переживают per-address rn_addr cap → раздувают n_analogs. # Anchor-путь дедупит по (source, source_id) (:1556); radius — нет. Ключ: # source + (id:source_id | url:source_url | ctid:ctid) namespaced-тегом (защита # от value-collision между id и url, симметрично anchor dedup). freshest по # scraped_at. ВСЕ тиры читают FROM listings (физическая таблица, без JOIN) → # source_id / source_url / ctid доступны в base-CTE даже без явного SELECT. # psycopg3: source_id::text / ctid::text — это COLUMN::type (не bind-параметр), # работает; запрещён только :name::type на bind-параметрах (их тут нет). _RN_DUP_WINDOW = """ row_number() OVER ( PARTITION BY source, CASE WHEN source_id IS NOT NULL THEN 'id:' || source_id::text WHEN source_url IS NOT NULL THEN 'url:' || source_url ELSE 'ctid:' || ctid::text END ORDER BY scraped_at DESC ) AS rn_dup""" def _fetch_analogs( db: Session, *, lat: float, lon: float, rooms: int, area: float, radius_m: int, full_address: str | None = None, area_tolerance: float = AREA_TOLERANCE, year_built: int | None = None, house_type: str | None = None, total_floors: int | None = None, cohort_year_min: int | None = None, # NEW: lower bound year_built inclusive cohort_year_max: int | None = None, # NEW: upper bound year_built inclusive target_house_id: int | None = None, # #6: canonical house for same-building Tier S ) -> tuple[list[dict[str, Any]], bool, str]: """SELECT аналогов — трёхуровневый house-match (S → H → W). **Tier S (same building):** сначала канонический match по house_id_fk (если задан target_house_id) — детерминированно «тот же дом»; fallback — address ILIKE prefix-match по short_addr. Если ≥3 результатов → tier='S'. **Tier H (same class):** PostGIS + rooms + area + year ±15 + total_floors ±30%. Если ≥5 результатов → возвращаем; tier='H'. Пропускается если year_built или total_floors неизвестны. **Tier W (wide / current):** текущая логика без year/floors WHERE фильтра. tier='W'. House-match relevance_score используется для сортировки в Tier H и W. Стратифицированная выборка (Approach B): 1. SQL вытягивает до 300 кандидатов с per-address row_number (cap MAX_ANALOGS_PER_ADDRESS). 2. Python гарантирует MIN_ANALOGS_PER_SOURCE слотов каждому live source. 3. Оставшиеся слоты заполняются из остальных кандидатов по relevance. 4. Итоговый список отсортирован по relevance, LIMIT 50. Когортный фильтр (PR 9): если переданы cohort_year_min/max — добавляется hard-filter WHERE year_built BETWEEN min AND max OR year_built IS NULL. NULL допускается чтобы не отсеивать листинги с неизвестным годом (типично для Avito anonymous-address объявлений). Returns: (list_of_listings_as_dicts, fallback_radius_used_flag, tier) tier: 'S' | 'H' | 'W' """ area_min = area * (1 - area_tolerance) area_max = area * (1 + area_tolerance) # #1871 P2: (source, source_id) dedup в radius-тирах. rn_dup-окно всегда в SQL # (безвредно без фильтра); статический фрагмент управляет только применением # `AND rn_dup = 1` в outer WHERE. Это SQL-литерал (static), НЕ data — psycopg3 # bind-параметры не задействованы, инъекции нет. dup_filter = "AND rn_dup = 1" if settings.estimate_radius_dedup_enabled else "" base_params: dict[str, Any] = { "rooms": rooms, "area_min": area_min, "area_max": area_max, "fresh_days": LISTINGS_FRESH_DAYS, "max_per_addr": MAX_ANALOGS_PER_ADDRESS, "cohort_year_min": cohort_year_min, "cohort_year_max": cohort_year_max, } # ── Tier S (canonical): same building via house_id_fk ───────────────────── # #6: детерминированный «тот же дом» через канонический house-граф. 99% # listings слинкованы (listings.house_id_fk → houses.id), что несравнимо # надёжнее address-string match'а между разнородными источниками # (Avito/Cian/Yandex форматируют адрес по-разному). Если target_house_id # неизвестен или результатов <3 — падаем на address-prefix Tier S ниже. if target_house_id is not None: tier_sc_params = {**base_params, "target_house_id": target_house_id} tier_sc_rows = ( db.execute( text( f""" WITH base AS ( SELECT {_ANALOG_SELECT_COLS}, 0.0 AS distance_m, 0.0 AS relevance_score, row_number() OVER ( PARTITION BY address ORDER BY scraped_at DESC ) AS rn_addr, {_RN_DUP_WINDOW} FROM listings WHERE house_id_fk = CAST(:target_house_id AS bigint) {_COMMON_WHERE} ) SELECT source, source_url, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, scraped_at, distance_m, relevance_score, building_cadastral_number FROM base WHERE rn_addr <= :max_per_addr {dup_filter} ORDER BY scraped_at DESC LIMIT 300 """ ), tier_sc_params, ) .mappings() .all() ) tier_sc = [dict(r) for r in tier_sc_rows] if len(tier_sc) >= 3: logger.info( "analogs tier=S(canonical) house_id=%s → %d results", target_house_id, len(tier_sc), ) return _stratify_candidates(tier_sc), radius_m > DEFAULT_RADIUS_M, "S" # ── Tier S (fallback): same building via address prefix ─────────────────── short_addr = _extract_short_addr(full_address) if short_addr: tier_s_params = { **base_params, "short_addr_prefix": short_addr + "%", } tier_s_rows = ( db.execute( text( f""" WITH base AS ( SELECT {_ANALOG_SELECT_COLS}, 0.0 AS distance_m, 0.0 AS relevance_score, row_number() OVER ( PARTITION BY address ORDER BY scraped_at DESC ) AS rn_addr, {_RN_DUP_WINDOW} FROM listings WHERE address ILIKE :short_addr_prefix {_COMMON_WHERE} ) SELECT source, source_url, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, scraped_at, distance_m, relevance_score, building_cadastral_number FROM base WHERE rn_addr <= :max_per_addr {dup_filter} ORDER BY scraped_at DESC LIMIT 300 """ ), tier_s_params, ) .mappings() .all() ) tier_s = [dict(r) for r in tier_s_rows] if len(tier_s) >= 3: logger.info( "analogs tier=S addr_prefix=%r → %d results", short_addr, len(tier_s), ) return _stratify_candidates(tier_s), radius_m > DEFAULT_RADIUS_M, "S" # ── Tier H: same class (year ±15, total_floors ±30%) ───────────────────── if year_built is not None and total_floors is not None: year_min = year_built - 15 year_max = year_built + 15 tf_min = math.floor(total_floors * 0.7) tf_max = math.ceil(total_floors * 1.3) tier_h_rows = ( db.execute( text( f""" WITH base AS ( SELECT {_ANALOG_SELECT_COLS}, id, ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) AS distance_m, ( ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) / 1000.0 + CASE WHEN year_built IS NOT NULL THEN abs(year_built - CAST(:target_year AS integer)) / 12.0 ELSE 0 END + CASE WHEN CAST(:target_house_type AS text) IS NOT NULL AND house_type IS NOT NULL AND house_type <> CAST(:target_house_type AS text) THEN 1.5 ELSE 0 END ) AS relevance_score, row_number() OVER ( PARTITION BY address ORDER BY ( ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) / 1000.0 + CASE WHEN year_built IS NOT NULL THEN abs(year_built - CAST(:target_year AS integer)) / 12.0 ELSE 0 END + CASE WHEN CAST(:target_house_type AS text) IS NOT NULL AND house_type IS NOT NULL AND house_type <> CAST(:target_house_type AS text) THEN 1.5 ELSE 0 END ) ) AS rn_addr, {_RN_DUP_WINDOW} FROM listings WHERE ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius) AND (geo_precision IS DISTINCT FROM 'city') {_COMMON_WHERE} AND total_floors BETWEEN CAST(:tf_min AS integer) AND CAST(:tf_max AS integer) AND year_built BETWEEN CAST(:year_min AS integer) AND CAST(:year_max AS integer) ) SELECT source, source_url, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, scraped_at, distance_m, relevance_score, building_cadastral_number FROM base WHERE rn_addr <= :max_per_addr {dup_filter} -- Детерминированный тай-брейкер: при равном relevance_score (один -- distance-bucket + year delta + house_type → идентичный score) -- Postgres возвращал бы строки в произвольном порядке, и тот же -- /analyze давал бы разные комп-сэты между запусками. distance_m -- ASC → scraped_at DESC (свежее) → id ASC (PK, гарантирует total -- order) делают выборку воспроизводимой. ORDER BY relevance_score ASC, distance_m ASC, scraped_at DESC NULLS LAST, id ASC LIMIT 300 """ ), { **base_params, "lat": lat, "lon": lon, "radius": radius_m, "target_year": year_built, "target_house_type": house_type, "tf_min": tf_min, "tf_max": tf_max, "year_min": year_min, "year_max": year_max, }, ) .mappings() .all() ) tier_h = [dict(r) for r in tier_h_rows] if len(tier_h) >= 5: logger.info( "analogs tier=H year=%d±15 tf=%d-%d → %d results", year_built, tf_min, tf_max, len(tier_h), ) return _stratify_candidates(tier_h), radius_m > DEFAULT_RADIUS_M, "H" logger.info( "analogs tier=H year=%d±15 tf=%d-%d → only %d (fallthrough to W)", year_built, tf_min, tf_max, len(tier_h), ) # ── Tier W: wide (current logic, year/floors only in relevance sort) ────── tier_w_rows = ( db.execute( text( f""" WITH base AS ( SELECT source, source_url, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, scraped_at, building_cadastral_number, id, ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) AS distance_m, ( ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) / 1000.0 -- CAST обязателен: target_year / target_house_type приходят NULL -- без типа → PostgreSQL "could not determine data type of parameter" -- (AmbiguousParameter). Явный тип снимает неоднозначность. + CASE WHEN CAST(:target_year AS integer) IS NOT NULL AND year_built IS NOT NULL THEN abs(year_built - CAST(:target_year AS integer)) / 12.0 ELSE 0 END + CASE WHEN CAST(:target_house_type AS text) IS NOT NULL AND house_type IS NOT NULL AND house_type <> CAST(:target_house_type AS text) THEN 1.5 ELSE 0 END ) AS relevance_score, row_number() OVER ( PARTITION BY address ORDER BY ( ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) / 1000.0 + CASE WHEN CAST(:target_year AS integer) IS NOT NULL AND year_built IS NOT NULL THEN abs(year_built - CAST(:target_year AS integer)) / 12.0 ELSE 0 END + CASE WHEN CAST(:target_house_type AS text) IS NOT NULL AND house_type IS NOT NULL AND house_type <> CAST(:target_house_type AS text) THEN 1.5 ELSE 0 END ) ) AS rn_addr, {_RN_DUP_WINDOW} FROM listings WHERE ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius) AND (geo_precision IS DISTINCT FROM 'city') AND rooms = :rooms AND area_m2 BETWEEN :area_min AND :area_max AND is_active = true AND scraped_at > NOW() - (:fresh_days || ' days')::interval AND price_rub > 0 -- Когортный фильтр (PR 9): отсеивает разные эпохи застройки -- (хрущёвка vs новостройка). Если cohort_year_min IS NULL — -- фильтр прозрачен. CAST обязателен — psycopg3 prepared statement -- не выводит тип $N при IS NULL в predicate (см. PR #518 fix). AND ( CAST(:cohort_year_min AS integer) IS NULL OR year_built IS NULL OR year_built BETWEEN CAST(:cohort_year_min AS integer) AND CAST(:cohort_year_max AS integer) ) -- novostroyki guard (#1186): NULL = legacy вторичка до м.011 -- Tier W: исключаем новостройки из comp-пула (sync с _COMMON_WHERE). AND (listing_segment IS NULL OR listing_segment = 'vtorichka') -- 2026-05-23: Avito coords теперь real (PR #487 убрал jitter после -- C-5 audit). Listings с NULL coords отфильтруются через ST_DWithin -- (geom IS NULL → не matches). geocode-missing-listings backfill -- подтягивает координаты для address-only Avito листингов. -- #769 Part E: geo_precision='city' исключает city-centroid листинги -- из radius-аналогов (centroid загрязнял comp set при ST_DWithin). -- IS DISTINCT FROM 'city' пропускает NULL (неизвестная точность — -- консервативно: листинг участвует в аналогах, не удаляем без причины). ) SELECT source, source_url, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, scraped_at, building_cadastral_number, distance_m, relevance_score FROM base WHERE rn_addr <= :max_per_addr {dup_filter} -- Детерминированный тай-брейкер: симметрично Tier H. При равном -- relevance_score Postgres возвращал бы строки в произвольном порядке -- → недетерминированная оценка. distance_m ASC → scraped_at DESC -- (свежее) → id ASC (PK, total order) делают выборку воспроизводимой. ORDER BY relevance_score ASC, distance_m ASC, scraped_at DESC NULLS LAST, id ASC LIMIT 300 """ ), { "lat": lat, "lon": lon, "radius": radius_m, "rooms": rooms, "area_min": area_min, "area_max": area_max, "fresh_days": LISTINGS_FRESH_DAYS, "target_year": year_built, "target_house_type": house_type, "max_per_addr": MAX_ANALOGS_PER_ADDRESS, "cohort_year_min": cohort_year_min, # NEW "cohort_year_max": cohort_year_max, # NEW }, ) .mappings() .all() ) candidates: list[dict[str, Any]] = [dict(r) for r in tier_w_rows] logger.info("analogs tier=W radius=%dm → %d candidates", radius_m, len(candidates)) return _stratify_candidates(candidates), radius_m > DEFAULT_RADIUS_M, "W" def _is_plausible_deal( price_per_m2: float | None, floor: int | None, total_floors: int | None, area_m2: float | None = None, price_rub: float | None = None, ) -> bool: """#699 + Mera-audit fix-2: True если ДКП-сделка правдоподобна (не выброс). Абсолютные guard-bands (см. DEAL_* константы). None-поля не судим (keep — нечем сравнивать). Проверки: - price_per_m2 вне [DEAL_MIN_PPM2, DEAL_MAX_PPM2] → drop - floor < 1 или floor > DEAL_MAX_FLOOR → drop (битый парсер: floor=-5/999) - floor > total_floors физически невозможен → drop - area_m2 задана и <= 0 → drop (битый парсер) - price_rub задана и <= 0 → drop (нерыночная/технческая сделка) """ if price_per_m2 is not None and not (DEAL_MIN_PPM2 <= price_per_m2 <= DEAL_MAX_PPM2): return False if floor is not None: if floor < 1 or floor > DEAL_MAX_FLOOR: return False if total_floors is not None and total_floors > 0 and floor > total_floors: return False if area_m2 is not None and area_m2 <= 0: return False if price_rub is not None and price_rub <= 0: return False return True def _fetch_deals( db: Session, *, lat: float, lon: float, rooms: int, area: float, radius_m: int ) -> list[dict[str, Any]]: rows = ( db.execute( text( """ SELECT source, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, deal_date, days_on_market, cadastral_number, ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) AS distance_m FROM deals WHERE ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius) AND rooms = :rooms AND area_m2 BETWEEN :area_min AND :area_max AND deal_date > NOW() - (:months || ' months')::interval AND price_rub > 0 ORDER BY deal_date DESC LIMIT 30 """ ), { "lat": lat, "lon": lon, "radius": radius_m, "rooms": rooms, "area_min": area * (1 - AREA_TOLERANCE), "area_max": area * (1 + AREA_TOLERANCE), "months": DEALS_PERIOD_MONTHS, }, ) .mappings() .all() ) # #699 + Mera-audit fix-2: отсекаем ДКП-выбросы (битый этаж / нерыночный ppm² # / нулевая площадь / нулевая цена) до выдачи в actual_deals и expected_sold. deals = [dict(r) for r in rows] clean = [ d for d in deals if _is_plausible_deal( d.get("price_per_m2"), d.get("floor"), d.get("total_floors"), d.get("area_m2"), d.get("price_rub"), ) ] if len(clean) < len(deals): logger.info("deals sanitize #699: %d → %d (dropped outliers)", len(deals), len(clean)) return clean def _apply_corridor_clamp( *, median_ppm2: float, median_price: int, range_low: int, range_high: int, corridor_high_ppm2: int, corridor_count: int, anchor_tier: str | None, slack: float, min_n: int, enabled: bool, ) -> tuple[float, int, int, int, bool]: """#1795 шаг 1: soft-кламп headline к коридору реальных ДКП-сделок Росреестра. Чистая (testable без БД) функция. Прижимает headline к cap = corridor_high_ppm2 × (1 + slack), когда ВСЕ условия: - median_ppm2 > cap (headline выше потолка коридора + slack), - corridor_count >= min_n (достаточно сделок для доверия коридору), - anchor_tier != "A" (Tier A = реальные комплы ТОГО ЖЕ дома → EXEMPT), - enabled (флаг estimate_corridor_clamp_enabled). Тогда price/range_low/range_high пересчитываются ПРОПОРЦИОНАЛЬНО (множитель = cap / median_ppm2), чтобы asking↔sold↔range остались консистентны. Возвращает (median_ppm2, median_price, range_low, range_high, clamped: bool). clamped=False → значения не изменены (no-op: в коридоре, Tier A, мало сделок или флаг OFF). """ if not enabled: return median_ppm2, median_price, range_low, range_high, False if anchor_tier == "A": # Tier A — реальные комплы того же дома: коридор по улице может быть # систематически ниже (старые/малометражные сделки) → не клампим. return median_ppm2, median_price, range_low, range_high, False if corridor_count < min_n or corridor_high_ppm2 <= 0 or median_ppm2 <= 0: return median_ppm2, median_price, range_low, range_high, False cap = corridor_high_ppm2 * (1.0 + slack) if median_ppm2 <= cap: # В коридоре (с учётом slack) — no-op (эконом/комфорт сюда не попадают). return median_ppm2, median_price, range_low, range_high, False factor = cap / median_ppm2 new_ppm2 = cap new_price = round(median_price * factor) new_low = round(range_low * factor) new_high = round(range_high * factor) return new_ppm2, new_price, new_low, new_high, True def _filter_outliers(lots: list[dict[str, Any]]) -> list[dict[str, Any]]: """Tukey IQR rule: исключаем точки вне [Q1 - k×IQR, Q3 + k×IQR]. #1795 шаг 4: на малых выборках (n < estimate_outlier_small_n_threshold) ужесточаем k с 1.5 до estimate_outlier_tukey_k_small — несколько элитных листингов на тонкой выборке премиум-дома иначе остаются в IQR-границах и тянут радиусную медиану вверх. threshold=0 → выключено (старое поведение). """ if len(lots) < 5: return lots # на маленькой выборке нечего фильтровать prices = sorted(lot["price_per_m2"] for lot in lots if lot.get("price_per_m2")) if len(prices) < 4: return lots tukey_k = 1.5 if ( settings.estimate_outlier_tukey_k_small < 1.5 and len(prices) < settings.estimate_outlier_small_n_threshold ): tukey_k = settings.estimate_outlier_tukey_k_small q1 = _percentile(prices, 0.25) q3 = _percentile(prices, 0.75) iqr = q3 - q1 low = q1 - tukey_k * iqr high = q3 + tukey_k * iqr # None-safe: listings.price_per_m2 is nullable, и lot.get(..., 0) вернёт None # (а не дефолт 0) когда ключ ПРИСУТСТВУЕТ со значением None → low <= None <= high # бросает TypeError в Python 3. Лоты без цены судить как outlier нечем — оставляем их. clean = [] for lot in lots: ppm2 = lot.get("price_per_m2") if ppm2 is None: clean.append(lot) # нечего сравнивать — keep continue if low <= ppm2 <= high: clean.append(lot) # priced лот внутри Tukey-границ — keep # else: priced outlier за пределами границ — drop if len(clean) < len(lots): logger.info("outlier filter: %d → %d (Q1=%d Q3=%d)", len(lots), len(clean), q1, q3) return clean def _percentile(sorted_values: list[float], p: float) -> float: """Linear interpolation percentile (не округляем — оставляем float).""" if not sorted_values: return 0.0 if len(sorted_values) == 1: return float(sorted_values[0]) n = len(sorted_values) rank = p * (n - 1) lo = int(rank) hi = min(lo + 1, n - 1) frac = rank - lo return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * frac def _enforce_zero_analog_low( confidence: str, n_analogs: int, explanation: str | None, *, median_price: int | float | None = None, sources_used: list[str] | None = None, ) -> tuple[str, str]: """#1871 P1.2 — defensive invariant: n_analogs == 0 МОЖЕТ означать только 'low'. Гард ловит любой будущий путь (external-valuation blend, rehydrate, миграция), который наполнит median/sources_used без единого реального аналога ("ghost-anchor"). Текущий mainline уже честен (_compute_confidence → 'low' при median=0, #755 cap) — это belt-and-suspenders, не load-bearing. Не вшивает fake median: реальное число от внешнего сервиса остаётся, мы лишь понижаем confidence + добавляем caveat. Возвращает (confidence, explanation): при n_analogs==0 и confidence!='low' — форсит 'low' + caveat-suffix; иначе значения проходят без изменений. """ if n_analogs == 0 and confidence != "low": logger.warning( "ghost_anchor_guard #1871: forcing confidence 'low' (was %s, " "median=%s, sources=%s)", confidence, median_price, sources_used, ) explanation_suffix = ( " Оценка построена без сопоставимых аналогов рядом " "(использованы только внешние оценочные сервисы) — точность " "ориентировочная, рекомендуем дополнительную проверку." ) return "low", (explanation or "") + explanation_suffix return confidence, explanation or "" def _downgrade_confidence(confidence: str) -> str: """#1871 P2 — понизить confidence на одну ступень: high→medium→low→low. Зеркалит inline downgrade-map в _compute_confidence (concentration bias) и coarse-geo gate. Неизвестные значения возвращаются как есть (defensive). """ return {"high": "medium", "medium": "low", "low": "low"}.get(confidence, confidence) def _compute_confidence( n_analogs: int, median_ppm2: float, q1: float, q3: float, fallback_radius_used: bool, area_widened: bool = False, listings: list[dict] | None = None, ) -> tuple[str, str]: """Confidence + explanation string. Уровень определяется по количеству уникальных адресов, а не по raw n_analogs. Это защищает от overstated confidence когда много лотов из одного здания (например, MIN_ANALOGS_PER_SOURCE=5 + same-building bias). high — unique_addr ≥ 7 AND IQR/median < 0.15 medium — unique_addr ≥ 4 OR (unique_addr ≥ 2 AND IQR/median < 0.25) low — иначе Downgrade на один уровень если avg_lots_per_addr > 2.5 (concentration bias). """ if median_ppm2 == 0: return "low", "Не найдено аналогов — попробуйте уточнить адрес или расширить параметры." # Вычисляем метрики уникальных адресов if listings: unique_addrs = { (lot.get("address") or "").strip().lower() for lot in listings if lot.get("address") } unique_addr_count = len(unique_addrs) avg_lots_per_addr = n_analogs / max(unique_addr_count, 1) else: unique_addr_count = n_analogs # fallback: считаем каждый лот уникальным avg_lots_per_addr = 1.0 iqr = q3 - q1 iqr_pct = iqr / median_ppm2 if median_ppm2 > 0 else 1.0 notes = [] if fallback_radius_used: notes.append("расширили радиус до 2 км") if area_widened: notes.append("расширили допуск по площади до ±25%") fallback_note = f" ({', '.join(notes)} из-за нехватки данных)" if notes else "" # Базовый уровень по уникальным адресам if unique_addr_count >= 7 and iqr_pct < 0.15: base = "high" elif unique_addr_count >= 4: base = "medium" elif unique_addr_count >= 2 and iqr_pct < 0.25: base = "medium" else: base = "low" # Downgrade на один шаг если слишком много лотов сконцентрировано на малом числе адресов if avg_lots_per_addr > 2.5 and base != "low": downgrade_map = {"high": "medium", "medium": "low"} downgraded = downgrade_map[base] explanation = ( f"Найдено {n_analogs} аналогов из {unique_addr_count} разных адресов, " f"разброс цены ±{int(iqr_pct * 100 / 2)}% от медианы{fallback_note}. " f"Снижена точность (≥2.5 лотов на адрес — возможен bias)." ) return downgraded, explanation explanation = ( f"Найдено {n_analogs} аналогов из {unique_addr_count} разных адресов, " f"разброс цены ±{int(iqr_pct * 100 / 2)}% от медианы{fallback_note}." ) return base, explanation def _listing_to_analog(row: dict[str, Any]) -> AnalogLot: return AnalogLot( address=row.get("address") or "", area_m2=float(row.get("area_m2") or 0), rooms=int(row.get("rooms") or 0), floor=row.get("floor"), total_floors=row.get("total_floors"), price_rub=int(row["price_rub"]), price_per_m2=int(row.get("price_per_m2") or 0), listing_date=row.get("listing_date"), days_on_market=row.get("days_on_market"), photo_url=(row["photo_urls"] or [None])[0] if row.get("photo_urls") else None, source=row.get("source"), source_url=row.get("source_url"), distance_m=int(row["distance_m"]) if row.get("distance_m") is not None else None, lat=float(row["lat"]) if row.get("lat") is not None else None, lon=float(row["lon"]) if row.get("lon") is not None else None, ) def _anchor_comp_to_analog(c: dict[str, Any]) -> AnalogLot: """Same-building/micro-radius comp → AnalogLot (#694). Когда якорь сработал, headline построен на этих комплах — показываем их в UI. ROBUST к отсутствию display-полей: реальные комплы несут address/source/price_rub (SELECT их тянет), но тестовые моки и legacy-строки могут давать только числа (price_per_m2/area_m2/rooms). price_rub тогда вычисляем ppm²×area (fallback на ppm² если area 0/None — без zero-result/crash). distance_m не значим для same-building → None, если явно не передан. """ ppm2 = int(c.get("price_per_m2") or 0) area = float(c.get("area_m2") or 0) price_rub_raw = c.get("price_rub") if price_rub_raw is not None: price_rub = int(price_rub_raw) elif area > 0: price_rub = round(ppm2 * area) else: # area неизвестна — деградируем до ppm² (никогда 0/crash при ppm²>0). price_rub = ppm2 photo_urls = c.get("photo_urls") return AnalogLot( address=c.get("address") or "", area_m2=area, rooms=int(c.get("rooms") or 0), floor=c.get("floor"), total_floors=c.get("total_floors"), price_rub=price_rub, price_per_m2=ppm2, listing_date=c.get("listing_date"), days_on_market=c.get("days_on_market"), photo_url=(photo_urls or [None])[0] if photo_urls else None, source=c.get("source"), source_url=c.get("source_url"), distance_m=int(c["distance_m"]) if c.get("distance_m") is not None else None, lat=float(c["lat"]) if c.get("lat") is not None else None, lon=float(c["lon"]) if c.get("lon") is not None else None, ) def _deal_to_analog(row: dict[str, Any]) -> AnalogLot: """deals не имеют photo_url — упрощённо. Tier classification (PR M / #564 Phase 3): T0_per_house — kadastr_num exact match (НЕ доступно в open dataset Росреестра) T1_per_street — street-level only (default для всех ДКП open dataset) """ kad = row.get("cadastral_number") # Per-house tier требует kadastr_num типа "66:41:0204016:10" (с участком). # Street-only patterns: "66:41:0000000:0" или NULL → T1. tier = "T0_per_house" if kad and not kad.endswith(":0") else "T1_per_street" return AnalogLot( address=row.get("address") or "", area_m2=float(row.get("area_m2") or 0), rooms=int(row.get("rooms") or 0), floor=row.get("floor"), total_floors=row.get("total_floors"), price_rub=int(row["price_rub"]), price_per_m2=int(row.get("price_per_m2") or 0), listing_date=row.get("deal_date"), days_on_market=row.get("days_on_market"), photo_url=None, source=row.get("source"), source_url=None, # rosreestr сделки без публичной ссылки distance_m=int(row["distance_m"]) if row.get("distance_m") is not None else None, lat=float(row["lat"]) if row.get("lat") is not None else None, lon=float(row["lon"]) if row.get("lon") is not None else None, tier=tier, ) def _empty_estimate( payload: TradeInEstimateInput, db: Session, *, reason: str, created_by: str | None = None ) -> AggregatedEstimate: """Fallback когда нет данных для оценки. Сохраняет запись в БД (confidence='low', пустые analogs/deals), чтобы GET /estimate/{id} не возвращал 404. C-4 security audit. """ estimate_id = uuid4() now = datetime.now(tz=UTC) expires_at = now + timedelta(hours=24) db.execute( text( """ INSERT INTO trade_in_estimates ( id, address, area_m2, rooms, floor, total_floors, year_built, house_type, repair_state, has_balcony, ownership_type, has_mortgage, client_name, client_phone, median_price, range_low, range_high, median_price_per_m2, confidence, confidence_explanation, n_analogs, analogs, actual_deals, sources_used, created_by, expires_at ) VALUES ( CAST(:id AS uuid), :address, :area, :rooms, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, :ownership_type, :has_mortgage, :client_name, :client_phone, 0, 0, 0, 0, 'low', :explanation, 0, '[]'::jsonb, '[]'::jsonb, '[]'::jsonb, :created_by, :expires_at ) """ ), { "id": str(estimate_id), "address": payload.address, "area": payload.area_m2, "rooms": payload.rooms, "floor": payload.floor, "total_floors": payload.total_floors, "year_built": payload.year_built, "house_type": payload.house_type, "repair_state": payload.repair_state, "has_balcony": payload.has_balcony, "ownership_type": payload.ownership_type, "has_mortgage": payload.has_mortgage, "client_name": payload.client_name, "client_phone": payload.client_phone, "explanation": reason, "created_by": created_by, "expires_at": expires_at, }, ) db.commit() logger.info( "empty_estimate: id=%s reason=%s addr=%s", estimate_id, reason, payload.address[:60] ) return AggregatedEstimate( estimate_id=estimate_id, median_price_rub=0, range_low_rub=0, range_high_rub=0, median_price_per_m2=0, confidence="low", confidence_explanation=reason, n_analogs=0, period_months=DEALS_PERIOD_MONTHS, analogs=[], actual_deals=[], expires_at=expires_at, cian_valuation=None, # Адрес не геокодирован (DaData не отрабатывала) → точность неизвестна. address_precision=None, analog_tier=None, # нет данных при empty estimate )