"""Trade-In Estimator — реальное SQL aggregation поверх listings + deals. Заменяет старый _mock_estimate() из api/v1/trade_in.py. Алгоритм: 1. Geocode address → (lat, lon) 2. SELECT listings с фильтрами: - PostGIS ST_DWithin (geom, point, 1000m) — радиус поиска - source ≠ avito (у Avito фейковые anchor-jitter координаты — не гео-аналог) - rooms = target_rooms (точное совпадение) - area_m2 BETWEEN target × 0.85 AND target × 1.15 - scraped_at > NOW() - 14 days (свежие) - is_active = true 3. Tukey outlier filter (1.5 × IQR rule) 4. Median / Q1 / Q3 / count → confidence 5. То же для deals (period = 12 mo). 6. Сохранить в trade_in_estimates + вернуть AggregatedEstimate """ from __future__ import annotations import hashlib import json import logging import re from datetime import UTC, datetime, timedelta from typing import Any from uuid import uuid4 from sqlalchemy import text from sqlalchemy.orm import Session from app.schemas.trade_in import AggregatedEstimate, AnalogLot, TradeInEstimateInput from app.services.geocoder import GeocodeResult, geocode from app.services.house_metadata import get_house_metadata from app.services.scrapers.avito_imv import ( IMVAddressNotFoundError, IMVAuthError, IMVEvaluation, IMVTransientError, compute_imv_cache_key, evaluate_via_imv, save_imv_evaluation, ) from app.services.scrapers.cian_valuation import ( CianValuationResult, estimate_via_cian_valuation, ) from app.services.scrapers.yandex_valuation import ( YandexValuationResult, YandexValuationScraper, ) logger = logging.getLogger(__name__) # ── Constants ──────────────────────────────────────────────────────────────── DEFAULT_RADIUS_M = 1000 # ПО ВСТРЕЧЕ ПТИЦЫ: «локация не дальше 800-1000 м» FALLBACK_RADIUS_M = 2000 AREA_TOLERANCE = 0.15 # ±15% площади MAX_ANALOGS_PER_ADDRESS = 5 # анти-bias: не больше 5 лотов с одного адреса MIN_ANALOGS_PER_SOURCE = 5 # гарантированный минимум на live source LISTINGS_FRESH_DAYS = 14 # объявления не старше 14 дней DEALS_PERIOD_MONTHS = 12 # сделки за последний год # Поправочные коэффициенты на состояние ремонта. Аналоги в выборке — микс # состояний (≈ "стандартный/косметический"), коэффициент сдвигает медиану под # конкретный ремонт целевой квартиры. Встреча Птицы: ремонт влияет на цену. _IMV_HOUSE_TYPE_MAP: dict[str | None, str | None] = { "panel": "panel", "brick": "brick", "monolith": "monolith", "monolith_brick": "monolith_brick", "monolithic": "monolith", "block": "block", "wood": "wood", None: None, } _IMV_REPAIR_MAP: dict[str | None, str | None] = { "needs_repair": "required", "standard": "cosmetic", "good": "euro", "excellent": "designer", None: None, } _REPAIR_COEF: dict[str, float] = { "needs_repair": 0.92, # требует ремонта — ниже рынка "standard": 0.98, "good": 1.03, "excellent": 1.08, # евроремонт — выше рынка } _REPAIR_LABEL: dict[str | None, str] = { "needs_repair": "требует ремонта", "standard": "стандартный ремонт", "good": "хороший ремонт", "excellent": "евроремонт", } def _repair_coefficient(repair_state: str | None) -> float: """Множитель к медиане по состоянию ремонта. None → 1.0 (без поправки).""" if not repair_state: return 1.0 return _REPAIR_COEF.get(repair_state, 1.0) # ── Avito IMV cache lookup (Stage 3) ──────────────────────────────────────── IMV_CACHE_TTL_HOURS = 24 # Префиксы в адресе, которые Avito-геокодер не распознаёт (не жилые назначения). # Пример: "Склад, ул. Заводская, д. 44-а" → "ул. Заводская, д. 44-а" _NOISE_PREFIX_RE = re.compile( r"(Склад|Гараж|Подсобка|Нежилое|Помещение|Цех),\s*", flags=re.IGNORECASE, ) YANDEX_VALUATION_CACHE_TTL_HOURS = 24 YANDEX_VALUATION_DEFAULT_CATEGORY = "APARTMENT" YANDEX_VALUATION_DEFAULT_TYPE = "SELL" async def _get_or_fetch_imv_cached( db: Session, *, address: str, rooms: int, area_m2: float, floor: int, floor_at_home: int, house_type: str, renovation_type: str, has_balcony: bool, has_loggia: bool, estimate_id_for_link: Any = None, ) -> IMVEvaluation | None: """Cached IMV lookup. TTL 24h по cache_key (sha256 of address + params). 1. compute cache_key 2. SELECT из avito_imv_evaluations WHERE cache_key = :ck AND fetched_at > NOW() - 24h 3. Если hit → возвращаем reconstructed IMVEvaluation 4. Cache miss → call evaluate_via_imv, save_imv_evaluation, return Graceful: на любой error возвращаем None (estimator продолжает без IMV). """ try: cache_key = compute_imv_cache_key( address, rooms, area_m2, floor, floor_at_home, house_type, renovation_type, has_balcony, has_loggia, ) existing = db.execute( text( """ SELECT id, cache_key, address, rooms, area_m2, floor, floor_at_home, house_type, renovation_type, has_balcony, has_loggia, lat, lon, geo_hash, avito_address_id, avito_location_id, avito_metro_id, avito_district_id, recommended_price, lower_price, higher_price, market_count, raw_response, fetched_at FROM avito_imv_evaluations WHERE cache_key = :ck AND fetched_at > NOW() - (:ttl_hours || ' hours')::interval ORDER BY fetched_at DESC LIMIT 1 """ ), {"ck": cache_key, "ttl_hours": IMV_CACHE_TTL_HOURS}, ).mappings().first() if existing is not None: logger.info( "imv: cache HIT key=%s recommended=%d", cache_key[:8], existing["recommended_price"], ) from app.services.scrapers.avito_imv import IMVGeo return IMVEvaluation( cache_key=existing["cache_key"], address=existing["address"], rooms=existing["rooms"], area_m2=float(existing["area_m2"]), floor=existing["floor"], floor_at_home=existing["floor_at_home"], house_type=existing["house_type"], renovation_type=existing["renovation_type"], has_balcony=existing["has_balcony"], has_loggia=existing["has_loggia"], geo=IMVGeo( geo_hash=existing["geo_hash"] or "", lat=existing["lat"], lon=existing["lon"], avito_address_id=existing["avito_address_id"], avito_location_id=existing["avito_location_id"], avito_metro_id=existing["avito_metro_id"], avito_district_id=existing["avito_district_id"], ), recommended_price=existing["recommended_price"], lower_price=existing["lower_price"], higher_price=existing["higher_price"], market_count=existing["market_count"], raw_response=existing.get("raw_response"), ) # Cache miss — fresh fetch logger.info("imv: cache MISS key=%s — fetching fresh", cache_key[:8]) result = await evaluate_via_imv( address=address, rooms=rooms, area_m2=area_m2, floor=floor, floor_at_home=floor_at_home, house_type=house_type, renovation_type=renovation_type, has_balcony=has_balcony, has_loggia=has_loggia, ) save_imv_evaluation(db, result, estimate_id=estimate_id_for_link) logger.info( "imv: fresh recommended=%d range=(%d, %d) count=%d", result.recommended_price, result.lower_price, result.higher_price, result.market_count or 0, ) return result except IMVAddressNotFoundError as e: logger.warning("imv: address not found in Avito geocoder: %s", e) # Retry once with noise prefixes stripped (e.g. "Склад, ул. X" → "ул. X") cleaned = _NOISE_PREFIX_RE.sub("", address) if cleaned != address: logger.info( "imv: retry with cleaned address %r → %r", address[:60], cleaned[:60], ) try: result = await evaluate_via_imv( address=cleaned, rooms=rooms, area_m2=area_m2, floor=floor, floor_at_home=floor_at_home, house_type=house_type, renovation_type=renovation_type, has_balcony=has_balcony, has_loggia=has_loggia, ) save_imv_evaluation(db, result, estimate_id=estimate_id_for_link) logger.info( "imv: retry OK recommended=%d range=(%d, %d) count=%d", result.recommended_price, result.lower_price, result.higher_price, result.market_count or 0, ) return result except IMVAddressNotFoundError: logger.warning("imv: cleaned address also not found — giving up") except Exception as retry_exc: logger.warning("imv: retry failed: %s", retry_exc) return None except IMVAuthError as e: logger.error( "imv: auth/quota error — manual action required: %s", e, ) return None except IMVTransientError as e: logger.warning("imv: transient error, skipping retry in estimator context: %s", e) return None except Exception as e: logger.warning("imv: fetch failed — estimator продолжает без IMV: %s", e) return None # ── Yandex Valuation cache lookup (Stage 8) ───────────────────────────────── def _yandex_valuation_cache_key( address: str, offer_category: str, offer_type: str ) -> str: """SHA256 cache key for Yandex Valuation lookups.""" payload = f"{address}|{offer_category}|{offer_type}" return hashlib.sha256(payload.encode("utf-8")).hexdigest() async def _get_or_fetch_yandex_valuation_cached( db: Session, *, address: str, offer_category: str = YANDEX_VALUATION_DEFAULT_CATEGORY, offer_type: str = YANDEX_VALUATION_DEFAULT_TYPE, ) -> YandexValuationResult | None: """Cached Yandex Valuation lookup. TTL 24h via external_valuations table. Returns None on any error / cache miss + fetch failure — caller continues without Yandex enrichment (graceful degradation). """ cache_key = _yandex_valuation_cache_key(address, offer_category, offer_type) # Cache lookup try: cached = db.execute( text( """ SELECT raw_payload, fetched_at FROM external_valuations WHERE source = 'yandex_valuation' AND cache_key = :ck AND expires_at > NOW() ORDER BY fetched_at DESC LIMIT 1 """ ), {"ck": cache_key}, ).mappings().first() except Exception as e: logger.warning("yandex_valuation: cache lookup failed: %s", e) cached = None if cached is not None and cached.get("raw_payload"): try: payload_dict = ( cached["raw_payload"] if isinstance(cached["raw_payload"], dict) else json.loads(cached["raw_payload"]) ) logger.info( "yandex_valuation: cache HIT key=%s items=%d", cache_key[:8], len(payload_dict.get("history_items", [])), ) return YandexValuationResult.model_validate(payload_dict) except Exception as e: logger.warning("yandex_valuation: cache deserialize failed — refetching: %s", e) # Fresh fetch try: async with YandexValuationScraper() as scraper: result = await scraper.fetch_house_history( address=address, offer_category=offer_category, offer_type=offer_type, ) except Exception as e: logger.warning( "yandex_valuation: fetch failed — estimator продолжает без Yandex: %s", e ) return None if result is None: logger.info("yandex_valuation: empty result for address=%s", address[:60]) return None # Save to cache (UPSERT on (source, cache_key)) try: db.execute( text( """ INSERT INTO external_valuations ( source, cache_key, address, raw_payload, fetched_at, expires_at ) VALUES ( 'yandex_valuation', :ck, :addr, CAST(:payload AS jsonb), NOW(), NOW() + (:ttl_hours || ' hours')::interval ) ON CONFLICT (source, cache_key) DO UPDATE SET raw_payload = EXCLUDED.raw_payload, fetched_at = NOW(), expires_at = NOW() + (:ttl_hours || ' hours')::interval """ ), { "ck": cache_key, "addr": address, "payload": json.dumps(result.model_dump(mode="json"), ensure_ascii=False), "ttl_hours": YANDEX_VALUATION_CACHE_TTL_HOURS, }, ) db.commit() logger.info( "yandex_valuation: fresh fetch saved key=%s items=%d", cache_key[:8], len(result.history_items), ) except Exception as e: logger.warning("yandex_valuation: cache save failed (continuing): %s", e) db.rollback() return result def _save_yandex_history_items( db: Session, result: YandexValuationResult, ) -> int: """Persist history items to house_placement_history. Returns saved count. house_id stays NULL — estimator doesn't compute target_house_id yet. Idempotent via UNIQUE (source, ext_item_id); we synthesize ext_item_id from (address|date|area|floor) hash since Yandex history items don't carry an explicit ID. Batch semantics (closes finding #5 from 2026-05-24 audit): all items inserted under a single try/except; on any failure the whole batch is rolled back as a unit (prior per-item rollback() destroyed the parent tx). """ if not result.history_items: return 0 rows = [] for item in result.history_items: # Synthesize stable ext_item_id (no native ID in valuation page) ext_seed = ( f"{result.address}|{item.publish_date}|{item.area_m2}|{item.floor}|" f"{item.start_price}|{item.last_price}" ) ext_item_id = hashlib.sha256(ext_seed.encode("utf-8")).hexdigest()[:32] rows.append({ "ext_id": ext_item_id, "rooms": item.rooms, "area": item.area_m2, "floor": item.floor, "start_price": item.start_price, "last_price": item.last_price, "publish_date": item.publish_date, "exposure": item.exposure_days, "raw": json.dumps(item.model_dump(mode="json"), ensure_ascii=False), }) sql = text( """ INSERT INTO house_placement_history ( source, ext_item_id, rooms, area_m2, floor, start_price, start_price_date, last_price, last_price_date, exposure_days, raw_payload ) VALUES ( 'yandex_valuation', :ext_id, :rooms, :area, :floor, :start_price, :publish_date, :last_price, :publish_date, :exposure, CAST(:raw AS jsonb) ) ON CONFLICT (source, ext_item_id) DO NOTHING """ ) try: for row in rows: db.execute(sql, row) db.commit() return len(rows) except Exception as e: logger.warning( "yandex_valuation: failed to save history batch (%d items): %s", len(rows), e, ) db.rollback() return 0 # ── Public ─────────────────────────────────────────────────────────────────── async def estimate_quality( payload: TradeInEstimateInput, db: Session ) -> AggregatedEstimate: """Главная функция — оценка квартиры по реальным данным. NOTE 2026-05-24: rosreestr_deals temporarily NOT included in actual_deals output (they contain ДДУ first-market data that skews secondary-market median). See Decision_TradeIn_DataQuality_8PR_Roadmap. Returns: AggregatedEstimate с estimate_id, медианой, диапазоном, аналогами. """ # 1. Geocode geo: GeocodeResult | None = None if payload.address: geo = await geocode(payload.address, db) if geo is None: # Без координат не можем искать через PostGIS. Возвращаем low confidence. logger.warning("geocode failed for %s — returning low-confidence estimate", payload.address) return _empty_estimate(payload, db, reason="address_not_geocoded") # 2. #392: обогащаем год / тип дома из картографии (OSM Overpass), если # пользователь их не указал — это улучшает house-match аналогов (#6). # Best-effort: при недоступности OSM target_* остаются None. target_year = payload.year_built target_house_type = payload.house_type if target_year is None or target_house_type is None: house_meta = await get_house_metadata(geo.lat, geo.lon, db) if house_meta is not None: if target_year is None: target_year = house_meta.year_built if target_house_type is None: target_house_type = house_meta.house_type # 3. Three-tier fallback: # a) 1km + ±15% area # b) 2km + ±15% area (fallback_used = True) # c) 2km + ±25% area (fallback_used = True, area_widened = True) listings, fallback_used = _fetch_analogs( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, radius_m=DEFAULT_RADIUS_M, year_built=target_year, house_type=target_house_type, ) area_widened = False if len(listings) < 5: listings_wide, _ = _fetch_analogs( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, radius_m=FALLBACK_RADIUS_M, year_built=target_year, house_type=target_house_type, ) if len(listings_wide) > len(listings): listings = listings_wide fallback_used = True # Tier C: если даже на 2км мало — расширяем area tolerance до ±25% # (актуально для отдалённых районов / новостроек с нестандартной планировкой) if len(listings) < 3: listings_widearea, _ = _fetch_analogs( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, radius_m=FALLBACK_RADIUS_M, area_tolerance=0.25, year_built=target_year, house_type=target_house_type, ) if len(listings_widearea) > len(listings): listings = listings_widearea fallback_used = True area_widened = True # 3. Outlier filter listings_clean = _filter_outliers(listings) # 4. Aggregation if listings_clean: prices_ppm2 = sorted(lot["price_per_m2"] for lot in listings_clean if lot["price_per_m2"]) median_ppm2 = _percentile(prices_ppm2, 0.5) q1_ppm2 = _percentile(prices_ppm2, 0.25) q3_ppm2 = _percentile(prices_ppm2, 0.75) median_price = int(median_ppm2 * payload.area_m2) range_low = int(q1_ppm2 * payload.area_m2) range_high = int(q3_ppm2 * payload.area_m2) n_analogs = len(listings_clean) else: median_ppm2 = 0 median_price = 0 range_low = 0 range_high = 0 n_analogs = 0 # 4b. Поправка на состояние ремонта (встреча Птицы: ремонт влияет на цену). # Аналоги — микс состояний; коэффициент сдвигает оценку под ремонт клиента. repair_coef = _repair_coefficient(payload.repair_state) repair_note = "" if listings_clean and repair_coef != 1.0: median_price = int(median_price * repair_coef) range_low = int(range_low * repair_coef) range_high = int(range_high * repair_coef) median_ppm2 = median_ppm2 * repair_coef pct = int(round((repair_coef - 1.0) * 100)) repair_note = ( f" Цена скорректирована на состояние ремонта " f"({_REPAIR_LABEL.get(payload.repair_state, '')} {pct:+d}%)." ) confidence, explanation = _compute_confidence( n_analogs, median_ppm2, q1_ppm2 if listings_clean else 0, q3_ppm2 if listings_clean else 0, fallback_used, area_widened, ) explanation = (explanation or "") + repair_note # ── Stage 3: Avito IMV evaluation as 5-th source (on-demand cached) ── imv_eval: IMVEvaluation | None = None imv_house_type = _IMV_HOUSE_TYPE_MAP.get(target_house_type) imv_renovation = _IMV_REPAIR_MAP.get(payload.repair_state) # IMV требует: address, rooms, area, floor, floor_at_home, house_type, renovation_type. # Если payload не содержит required fields — skip IMV (graceful). if ( geo is not None and geo.full_address and payload.rooms is not None and payload.area_m2 and payload.floor is not None and payload.total_floors is not None and imv_house_type is not None and imv_renovation is not None ): imv_eval = await _get_or_fetch_imv_cached( db, address=geo.full_address, rooms=payload.rooms, area_m2=payload.area_m2, floor=payload.floor, floor_at_home=payload.total_floors, house_type=imv_house_type, renovation_type=imv_renovation, has_balcony=bool(payload.has_balcony), has_loggia=False, # payload не разделяет балкон/лоджия → дефолт False ) # Include IMV в sources_used если получили sources_used_pre = sorted({lot.get("source") for lot in listings_clean if lot.get("source")}) if imv_eval is not None: sources_used_pre = sorted(set(sources_used_pre) | {"avito_imv"}) # ── Stage 8: Yandex Valuation as on-demand source (anonymous, cached 24h) ── yandex_val: YandexValuationResult | None = None if geo is not None and geo.full_address: yandex_val = await _get_or_fetch_yandex_valuation_cached( db, address=geo.full_address, ) if yandex_val is not None: sources_used_pre = sorted(set(sources_used_pre) | {"yandex_valuation"}) saved_hist = _save_yandex_history_items(db, yandex_val) logger.info( "yandex_valuation: history items processed=%d saved=%d" " (house_id=NULL — matching deferred)", len(yandex_val.history_items), saved_hist, ) # ── Stage 9: Cian Valuation as 7th source (on-demand, 24h cached, graceful if no cookies) ── cian_val: CianValuationResult | None = None if ( geo is not None and geo.full_address and payload.rooms is not None and payload.area_m2 and payload.floor is not None and payload.total_floors is not None ): try: cian_val = await estimate_via_cian_valuation( db, address=geo.full_address, total_area=payload.area_m2, rooms_count=payload.rooms, floor=payload.floor, total_floors=payload.total_floors, repair_type="cosmetic", deal_type="sale", use_cache=True, ) if cian_val is not None and cian_val.sale_price_rub: sources_used_pre = sorted(set(sources_used_pre) | {"cian_valuation"}) logger.info( "cian_valuation: price=%s accuracy=%s house_id=%s", cian_val.sale_price_rub, cian_val.sale_accuracy, cian_val.external_house_id, ) except Exception as exc: logger.warning("cian_valuation: lookup failed (graceful): %s", exc) # 5. Deals — отключены 2026-05-24 (см. Decision_TradeIn_DataQuality_8PR_Roadmap). # rosreestr_deals содержит ДДУ первичного рынка (177-200 К/м²), что искажает # оценку вторички (~114 К/м²). Возвращаемся к этому когда отделим вторичные # сделки от ДДУ в источнике. deals: list[dict[str, Any]] = [] # 6. Сохраняем в trade_in_estimates estimate_id = uuid4() now = datetime.now(tz=UTC) expires_at = now + timedelta(hours=24) analogs_lots = [_listing_to_analog(lot) for lot in listings_clean[:10]] deals_lots = [_deal_to_analog(d) for d in deals[:10]] freshness_pre = _compute_freshness_minutes(listings_clean) db.execute( text( """ INSERT INTO trade_in_estimates ( id, address, lat, lon, area_m2, rooms, floor, total_floors, year_built, house_type, repair_state, has_balcony, ownership_type, has_mortgage, client_name, client_phone, median_price, range_low, range_high, median_price_per_m2, confidence, confidence_explanation, n_analogs, analogs, actual_deals, sources_used, data_freshness_minutes, expires_at ) VALUES ( CAST(:id AS uuid), :address, :lat, :lon, :area, :rooms, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, :ownership_type, :has_mortgage, :client_name, :client_phone, :median_price, :range_low, :range_high, :median_ppm2, :confidence, :explanation, :n_analogs, CAST(:analogs_json AS jsonb), CAST(:deals_json AS jsonb), CAST(:sources_json AS jsonb), :freshness, :expires_at ) """ ), { "id": str(estimate_id), "address": geo.full_address, "lat": geo.lat, "lon": geo.lon, "area": payload.area_m2, "rooms": payload.rooms, "floor": payload.floor, "total_floors": payload.total_floors, "year_built": target_year, "house_type": target_house_type, "repair_state": payload.repair_state, "has_balcony": payload.has_balcony, "ownership_type": payload.ownership_type, "has_mortgage": payload.has_mortgage, "client_name": payload.client_name, "client_phone": payload.client_phone, "median_price": median_price, "range_low": range_low, "range_high": range_high, "median_ppm2": int(median_ppm2), "confidence": confidence, "explanation": explanation, "n_analogs": n_analogs, "analogs_json": json.dumps( [a.model_dump(mode="json") for a in analogs_lots], ensure_ascii=False ), "deals_json": json.dumps( [a.model_dump(mode="json") for a in deals_lots], ensure_ascii=False ), "sources_json": json.dumps(sources_used_pre, ensure_ascii=False), "freshness": freshness_pre, "expires_at": expires_at, }, ) # Link saved IMV evaluation к этому estimate_id атомарно с основным INSERT # (closes finding #4 from 2026-05-24 audit — prior code committed estimate first, # then UPDATEd IMV in a separate tx, racing against concurrent estimators # sharing the same cache_key). if imv_eval is not None: db.execute( text( """ UPDATE avito_imv_evaluations SET estimate_id = CAST(:estimate_id AS uuid) WHERE cache_key = :cache_key AND (estimate_id IS NULL OR estimate_id = CAST(:estimate_id AS uuid)) """ ), {"estimate_id": str(estimate_id), "cache_key": imv_eval.cache_key}, ) db.commit() logger.info( "estimate: id=%s addr=%s rooms=%d area=%.1f → median=%d (n=%d, conf=%s)%s%s", estimate_id, geo.full_address[:60], payload.rooms, payload.area_m2, median_price, n_analogs, confidence, f" imv={imv_eval.recommended_price}" if imv_eval else "", f" cian={cian_val.sale_price_rub}" if cian_val and cian_val.sale_price_rub else "", ) sources_used = sorted({lot.source for lot in analogs_lots if lot.source}) if imv_eval is not None: sources_used = sorted(set(sources_used) | {"avito_imv"}) if yandex_val is not None: sources_used = sorted(set(sources_used) | {"yandex_valuation"}) if cian_val is not None and cian_val.sale_price_rub: sources_used = sorted(set(sources_used) | {"cian_valuation"}) freshness_min = _compute_freshness_minutes(listings_clean) return AggregatedEstimate( estimate_id=estimate_id, median_price_rub=median_price, range_low_rub=range_low, range_high_rub=range_high, median_price_per_m2=int(median_ppm2), confidence=confidence, confidence_explanation=explanation, n_analogs=n_analogs, period_months=DEALS_PERIOD_MONTHS, analogs=analogs_lots, actual_deals=deals_lots, expires_at=expires_at, target_address=geo.full_address, target_lat=geo.lat, target_lon=geo.lon, sources_used=sources_used, data_freshness_minutes=freshness_min, est_days_on_market=_estimate_days_on_market(listings_clean, deals), area_m2=payload.area_m2, rooms=payload.rooms, floor=payload.floor, total_floors=payload.total_floors, year_built=target_year, house_type=target_house_type, repair_state=payload.repair_state, has_balcony=payload.has_balcony, ) def _estimate_days_on_market( listings: list[dict[str, Any]], deals: list[dict[str, Any]] ) -> int | None: """Прогноз срока продажи — медиана days_on_market по аналогам/сделкам. Возвращает None если ни у одного аналога нет данных о сроке экспозиции (наши парсеры не всегда его отдают — честно показываем «нет данных»). """ values = [ int(lot["days_on_market"]) for lot in (*listings, *deals) if lot.get("days_on_market") and int(lot["days_on_market"]) > 0 ] if len(values) < 3: return None values.sort() return values[len(values) // 2] def _compute_freshness_minutes(lots: list[dict[str, Any]]) -> int | None: """Минут с последнего парсинга — для UI «обновлено N мин назад».""" if not lots: return None from datetime import datetime as _dt now = _dt.now(tz=UTC) scraped = [lot.get("scraped_at") or lot.get("listing_date") for lot in lots] scraped_dt: list[datetime] = [] for s in scraped: if s is None: continue # listings rows из mappings — scraped_at это datetime, не date if hasattr(s, "tzinfo"): scraped_dt.append(s if s.tzinfo else s.replace(tzinfo=UTC)) if not scraped_dt: return None return int((now - max(scraped_dt)).total_seconds() / 60) # ── Internals ──────────────────────────────────────────────────────────────── def _fetch_analogs( db: Session, *, lat: float, lon: float, rooms: int, area: float, radius_m: int, area_tolerance: float = AREA_TOLERANCE, year_built: int | None = None, house_type: str | None = None, ) -> tuple[list[dict[str, Any]], bool]: """SELECT аналогов с PostGIS distance + house-match relevance. House-match (встреча Птицы — «соразмерные квартиры»): сортировка не просто по расстоянию, а по relevance-скору, где учитывается близость года постройки и совпадение типа дома. Так аналог «рядом + та же эпоха дома» побеждает аналог «чуть ближе, но дом на 30 лет старше». Стратифицированная выборка (Approach B): 1. SQL вытягивает до 300 кандидатов с per-address row_number (cap MAX_ANALOGS_PER_ADDRESS). 2. Python гарантирует MIN_ANALOGS_PER_SOURCE слотов каждому live source. 3. Оставшиеся слоты заполняются из остальных кандидатов по relevance. 4. Итоговый список отсортирован по relevance, LIMIT 50. Returns: (list_of_listings_as_dicts, fallback_radius_used_flag) """ rows = db.execute( text( """ WITH base AS ( SELECT source, source_url, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, scraped_at, ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) AS distance_m, ( ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) / 1000.0 -- CAST обязателен: target_year / target_house_type приходят NULL -- без типа → PostgreSQL "could not determine data type of parameter" -- (AmbiguousParameter). Явный тип снимает неоднозначность. + CASE WHEN :target_year IS NOT NULL AND year_built IS NOT NULL THEN abs(year_built - CAST(:target_year AS integer)) / 12.0 ELSE 0 END + CASE WHEN :target_house_type IS NOT NULL AND house_type IS NOT NULL AND house_type <> CAST(:target_house_type AS text) THEN 1.5 ELSE 0 END ) AS relevance_score, row_number() OVER ( PARTITION BY address ORDER BY ( ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) / 1000.0 + CASE WHEN :target_year IS NOT NULL AND year_built IS NOT NULL THEN abs(year_built - CAST(:target_year AS integer)) / 12.0 ELSE 0 END + CASE WHEN :target_house_type IS NOT NULL AND house_type IS NOT NULL AND house_type <> CAST(:target_house_type AS text) THEN 1.5 ELSE 0 END ) ) AS rn_addr FROM listings WHERE ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius) AND rooms = :rooms AND area_m2 BETWEEN :area_min AND :area_max AND is_active = true AND scraped_at > NOW() - (:fresh_days || ' days')::interval AND price_rub > 0 -- 2026-05-23: Avito coords теперь real (PR #487 убрал jitter после -- C-5 audit). Listings с NULL coords отфильтруются через ST_DWithin -- (geom IS NULL → не matches). geocode-missing-listings backfill -- подтягивает координаты для address-only Avito листингов. ) SELECT source, source_url, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, listing_date, days_on_market, photo_urls, scraped_at, distance_m, relevance_score FROM base WHERE rn_addr <= :max_per_addr ORDER BY relevance_score LIMIT 300 """ ), { "lat": lat, "lon": lon, "radius": radius_m, "rooms": rooms, "area_min": area * (1 - area_tolerance), "area_max": area * (1 + area_tolerance), "fresh_days": LISTINGS_FRESH_DAYS, "target_year": year_built, "target_house_type": house_type, "max_per_addr": MAX_ANALOGS_PER_ADDRESS, }, ).mappings().all() candidates: list[dict[str, Any]] = [dict(r) for r in rows] # Stratified quota: гарантируем MIN_ANALOGS_PER_SOURCE слотов каждому source. # Candidates уже отсортированы по relevance_score (лучшие первые) из SQL. guaranteed: list[dict[str, Any]] = [] guaranteed_ids: set[int] = set() # по object id, не по внешнему ключу by_source: dict[str, list[dict[str, Any]]] = {} for row in candidates: src = row.get("source") or "unknown" by_source.setdefault(src, []).append(row) for _src, src_rows in by_source.items(): quota = min(len(src_rows), MIN_ANALOGS_PER_SOURCE) for row in src_rows[:quota]: if id(row) not in guaranteed_ids: guaranteed.append(row) guaranteed_ids.add(id(row)) # Оставшиеся слоты из candidates, которые ещё не попали в guaranteed. remaining_slots = 50 - len(guaranteed) remainder: list[dict[str, Any]] = [] if remaining_slots > 0: for row in candidates: if id(row) not in guaranteed_ids: remainder.append(row) if len(remainder) >= remaining_slots: break result = guaranteed + remainder # Финальная сортировка по relevance (candidates из SQL уже отсортированы, # но guaranteed + remainder смешиваются). relevance_score присутствует в каждом dict. result.sort(key=lambda r: r.get("relevance_score") or 0.0) result = result[:50] return result, radius_m > DEFAULT_RADIUS_M def _fetch_deals( db: Session, *, lat: float, lon: float, rooms: int, area: float, radius_m: int ) -> list[dict[str, Any]]: rows = db.execute( text( """ SELECT source, address, lat, lon, rooms, area_m2, floor, total_floors, price_rub, price_per_m2, deal_date, days_on_market, ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) AS distance_m FROM deals WHERE ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, :radius) AND rooms = :rooms AND area_m2 BETWEEN :area_min AND :area_max AND deal_date > NOW() - (:months || ' months')::interval AND price_rub > 0 ORDER BY deal_date DESC LIMIT 30 """ ), { "lat": lat, "lon": lon, "radius": radius_m, "rooms": rooms, "area_min": area * (1 - AREA_TOLERANCE), "area_max": area * (1 + AREA_TOLERANCE), "months": DEALS_PERIOD_MONTHS, }, ).mappings().all() return [dict(r) for r in rows] def _filter_outliers(lots: list[dict[str, Any]]) -> list[dict[str, Any]]: """Tukey IQR rule: исключаем точки вне [Q1 - 1.5×IQR, Q3 + 1.5×IQR].""" if len(lots) < 5: return lots # на маленькой выборке нечего фильтровать prices = sorted(lot["price_per_m2"] for lot in lots if lot.get("price_per_m2")) if len(prices) < 4: return lots q1 = _percentile(prices, 0.25) q3 = _percentile(prices, 0.75) iqr = q3 - q1 low = q1 - 1.5 * iqr high = q3 + 1.5 * iqr clean = [lot for lot in lots if low <= lot.get("price_per_m2", 0) <= high] if len(clean) < len(lots): logger.info("outlier filter: %d → %d (Q1=%d Q3=%d)", len(lots), len(clean), q1, q3) return clean def _percentile(sorted_values: list[float], p: float) -> float: """Linear interpolation percentile (не округляем — оставляем float).""" if not sorted_values: return 0.0 if len(sorted_values) == 1: return float(sorted_values[0]) n = len(sorted_values) rank = p * (n - 1) lo = int(rank) hi = min(lo + 1, n - 1) frac = rank - lo return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * frac def _compute_confidence( n_analogs: int, median_ppm2: float, q1: float, q3: float, fallback_radius_used: bool, area_widened: bool = False, ) -> tuple[str, str]: """Confidence + explanation string. high — n≥10 AND IQR/median < 0.15 medium — n≥5 OR IQR/median < 0.25 low — иначе """ if median_ppm2 == 0: return "low", "Не найдено аналогов — попробуйте уточнить адрес или расширить параметры." iqr = q3 - q1 iqr_pct = iqr / median_ppm2 if median_ppm2 > 0 else 1.0 notes = [] if fallback_radius_used: notes.append("расширили радиус до 2 км") if area_widened: notes.append("расширили допуск по площади до ±25%") fallback_note = f" ({', '.join(notes)} из-за нехватки данных)" if notes else "" if n_analogs >= 10 and iqr_pct < 0.15: return ( "high", f"Найдено {n_analogs} аналогов, разброс цены ±{int(iqr_pct * 100 / 2)}% от медианы{fallback_note}.", ) # medium только если есть достаточно точек ИЛИ узкий разброс при ≥3 точках if n_analogs >= 5 or (n_analogs >= 3 and iqr_pct < 0.25): return ( "medium", f"Найдено {n_analogs} аналогов, разброс цены ±{int(iqr_pct * 100 / 2)}% от медианы{fallback_note}.", ) return ( "low", f"Только {n_analogs} аналог{'а' if 2 <= n_analogs <= 4 else 'ов' if n_analogs != 1 else ''}, " f"разброс ±{int(iqr_pct * 100 / 2)}% — рекомендуется ручная проверка{fallback_note}.", ) def _listing_to_analog(row: dict[str, Any]) -> AnalogLot: return AnalogLot( address=row.get("address") or "", area_m2=float(row.get("area_m2") or 0), rooms=int(row.get("rooms") or 0), floor=row.get("floor"), total_floors=row.get("total_floors"), price_rub=int(row["price_rub"]), price_per_m2=int(row.get("price_per_m2") or 0), listing_date=row.get("listing_date"), days_on_market=row.get("days_on_market"), photo_url=(row.get("photo_urls") or [None])[0] if isinstance(row.get("photo_urls"), list) else None, source=row.get("source"), source_url=row.get("source_url"), distance_m=int(row["distance_m"]) if row.get("distance_m") is not None else None, ) def _deal_to_analog(row: dict[str, Any]) -> AnalogLot: """deals не имеют photo_url — упрощённо.""" return AnalogLot( address=row.get("address") or "", area_m2=float(row.get("area_m2") or 0), rooms=int(row.get("rooms") or 0), floor=row.get("floor"), total_floors=row.get("total_floors"), price_rub=int(row["price_rub"]), price_per_m2=int(row.get("price_per_m2") or 0), listing_date=row.get("deal_date"), days_on_market=row.get("days_on_market"), photo_url=None, source=row.get("source"), source_url=None, # rosreestr сделки без публичной ссылки distance_m=int(row["distance_m"]) if row.get("distance_m") is not None else None, ) def _empty_estimate( payload: TradeInEstimateInput, db: Session, *, reason: str ) -> AggregatedEstimate: """Fallback когда нет данных для оценки. Сохраняет запись в БД (confidence='low', пустые analogs/deals), чтобы GET /estimate/{id} не возвращал 404. C-4 security audit. """ estimate_id = uuid4() now = datetime.now(tz=UTC) expires_at = now + timedelta(hours=24) db.execute( text( """ INSERT INTO trade_in_estimates ( id, address, area_m2, rooms, floor, total_floors, year_built, house_type, repair_state, has_balcony, ownership_type, has_mortgage, client_name, client_phone, median_price, range_low, range_high, median_price_per_m2, confidence, confidence_explanation, n_analogs, analogs, actual_deals, sources_used, expires_at ) VALUES ( CAST(:id AS uuid), :address, :area, :rooms, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, :ownership_type, :has_mortgage, :client_name, :client_phone, 0, 0, 0, 0, 'low', :explanation, 0, '[]'::jsonb, '[]'::jsonb, '[]'::jsonb, :expires_at ) """ ), { "id": str(estimate_id), "address": payload.address, "area": payload.area_m2, "rooms": payload.rooms, "floor": payload.floor, "total_floors": payload.total_floors, "year_built": payload.year_built, "house_type": payload.house_type, "repair_state": payload.repair_state, "has_balcony": payload.has_balcony, "ownership_type": payload.ownership_type, "has_mortgage": payload.has_mortgage, "client_name": payload.client_name, "client_phone": payload.client_phone, "explanation": reason, "expires_at": expires_at, }, ) db.commit() logger.info( "empty_estimate: id=%s reason=%s addr=%s", estimate_id, reason, payload.address[:60] ) return AggregatedEstimate( estimate_id=estimate_id, median_price_rub=0, range_low_rub=0, range_high_rub=0, median_price_per_m2=0, confidence="low", confidence_explanation=reason, n_analogs=0, period_months=DEALS_PERIOD_MONTHS, analogs=[], actual_deals=[], expires_at=expires_at, )