From 5051ebde2e8037cc8abcbf001c01f6f0d95b2148 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 13:08:22 +0000 Subject: [PATCH] feat(tradein): integrate Avito IMV as 5th evaluation source (on-demand cached) (#452) --- tradein-mvp/backend/app/services/estimator.py | 193 +++++++++++++++++- .../tests/test_estimator_imv_integration.py | 108 ++++++++++ 2 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_estimator_imv_integration.py diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index 7dfd2ff3..8a053c14 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -31,6 +31,13 @@ from sqlalchemy.orm import Session from app.schemas.trade_in import AggregatedEstimate, AnalogLot, TradeInEstimateInput from app.services.geocoder import GeocodeResult, geocode from app.services.house_metadata import get_house_metadata +from app.services.scrapers.avito_imv import ( + IMVAddressNotFoundError, + IMVEvaluation, + compute_imv_cache_key, + evaluate_via_imv, + save_imv_evaluation, +) logger = logging.getLogger(__name__) @@ -45,6 +52,24 @@ DEALS_PERIOD_MONTHS = 12 # сделки за последний год # Поправочные коэффициенты на состояние ремонта. Аналоги в выборке — микс # состояний (≈ "стандартный/косметический"), коэффициент сдвигает медиану под # конкретный ремонт целевой квартиры. Встреча Птицы: ремонт влияет на цену. +_IMV_HOUSE_TYPE_MAP: dict[str | None, str | None] = { + "panel": "panel", + "brick": "brick", + "monolith": "monolith", + "monolith_brick": "monolith_brick", + "monolithic": "monolith", + "block": "block", + "wood": "wood", + None: None, +} +_IMV_REPAIR_MAP: dict[str | None, str | None] = { + "needs_repair": "required", + "standard": "cosmetic", + "good": "euro", + "excellent": "designer", + None: None, +} + _REPAIR_COEF: dict[str, float] = { "needs_repair": 0.92, # требует ремонта — ниже рынка "standard": 0.98, @@ -66,6 +91,115 @@ def _repair_coefficient(repair_state: str | None) -> float: return _REPAIR_COEF.get(repair_state, 1.0) +# ── Avito IMV cache lookup (Stage 3) ──────────────────────────────────────── +IMV_CACHE_TTL_HOURS = 24 + + +async def _get_or_fetch_imv_cached( + db: Session, + *, + address: str, + rooms: int, + area_m2: float, + floor: int, + floor_at_home: int, + house_type: str, + renovation_type: str, + has_balcony: bool, + has_loggia: bool, + estimate_id_for_link: Any = None, +) -> IMVEvaluation | None: + """Cached IMV lookup. TTL 24h по cache_key (sha256 of address + params). + + 1. compute cache_key + 2. SELECT из avito_imv_evaluations WHERE cache_key = :ck AND fetched_at > NOW() - 24h + 3. Если hit → возвращаем reconstructed IMVEvaluation + 4. Cache miss → call evaluate_via_imv, save_imv_evaluation, return + + Graceful: на любой error возвращаем None (estimator продолжает без IMV). + """ + try: + cache_key = compute_imv_cache_key( + address, rooms, area_m2, floor, floor_at_home, + house_type, renovation_type, has_balcony, has_loggia, + ) + + existing = db.execute( + text( + """ + SELECT id, cache_key, address, rooms, area_m2, floor, floor_at_home, + house_type, renovation_type, has_balcony, has_loggia, + lat, lon, geo_hash, avito_address_id, avito_location_id, + avito_metro_id, avito_district_id, + recommended_price, lower_price, higher_price, market_count, + raw_response, fetched_at + FROM avito_imv_evaluations + WHERE cache_key = :ck + AND fetched_at > NOW() - (:ttl_hours || ' hours')::interval + ORDER BY fetched_at DESC + LIMIT 1 + """ + ), + {"ck": cache_key, "ttl_hours": IMV_CACHE_TTL_HOURS}, + ).mappings().first() + + if existing is not None: + logger.info( + "imv: cache HIT key=%s recommended=%d", + cache_key[:8], existing["recommended_price"], + ) + from app.services.scrapers.avito_imv import IMVGeo + return IMVEvaluation( + cache_key=existing["cache_key"], + address=existing["address"], + rooms=existing["rooms"], + area_m2=float(existing["area_m2"]), + floor=existing["floor"], + floor_at_home=existing["floor_at_home"], + house_type=existing["house_type"], + renovation_type=existing["renovation_type"], + has_balcony=existing["has_balcony"], + has_loggia=existing["has_loggia"], + geo=IMVGeo( + geo_hash=existing["geo_hash"] or "", + lat=existing["lat"], + lon=existing["lon"], + avito_address_id=existing["avito_address_id"], + avito_location_id=existing["avito_location_id"], + avito_metro_id=existing["avito_metro_id"], + avito_district_id=existing["avito_district_id"], + ), + recommended_price=existing["recommended_price"], + lower_price=existing["lower_price"], + higher_price=existing["higher_price"], + market_count=existing["market_count"], + raw_response=existing.get("raw_response"), + ) + + # Cache miss — fresh fetch + logger.info("imv: cache MISS key=%s — fetching fresh", cache_key[:8]) + result = await evaluate_via_imv( + address=address, rooms=rooms, area_m2=area_m2, + floor=floor, floor_at_home=floor_at_home, + house_type=house_type, renovation_type=renovation_type, + has_balcony=has_balcony, has_loggia=has_loggia, + ) + save_imv_evaluation(db, result, estimate_id=estimate_id_for_link) + logger.info( + "imv: fresh recommended=%d range=(%d, %d) count=%d", + result.recommended_price, result.lower_price, result.higher_price, + result.market_count or 0, + ) + return result + + except IMVAddressNotFoundError as e: + logger.warning("imv: address not found in Avito geocoder: %s", e) + return None + except Exception as e: + logger.warning("imv: fetch failed — estimator продолжает без IMV: %s", e) + return None + + # ── Public ─────────────────────────────────────────────────────────────────── async def estimate_quality( payload: TradeInEstimateInput, db: Session @@ -173,6 +307,40 @@ async def estimate_quality( ) explanation = (explanation or "") + repair_note + # ── Stage 3: Avito IMV evaluation as 5-th source (on-demand cached) ── + imv_eval: IMVEvaluation | None = None + imv_house_type = _IMV_HOUSE_TYPE_MAP.get(target_house_type) + imv_renovation = _IMV_REPAIR_MAP.get(payload.repair_state) + # IMV требует: address, rooms, area, floor, floor_at_home, house_type, renovation_type. + # Если payload не содержит required fields — skip IMV (graceful). + if ( + geo is not None + and geo.full_address + and payload.rooms is not None + and payload.area_m2 + and payload.floor is not None + and payload.total_floors is not None + and imv_house_type is not None + and imv_renovation is not None + ): + imv_eval = await _get_or_fetch_imv_cached( + db, + address=geo.full_address, + rooms=payload.rooms, + area_m2=payload.area_m2, + floor=payload.floor, + floor_at_home=payload.total_floors, + house_type=imv_house_type, + renovation_type=imv_renovation, + has_balcony=bool(payload.has_balcony), + has_loggia=False, # payload не разделяет балкон/лоджия → дефолт False + ) + + # Include IMV в sources_used если получили + sources_used_pre = sorted({lot.get("source") for lot in listings_clean if lot.get("source")}) + if imv_eval is not None: + sources_used_pre = sorted(set(sources_used_pre) | {"avito_imv"}) + # 5. Deals — фактические сделки за период deals = _fetch_deals( db, lat=geo.lat, lon=geo.lon, rooms=payload.rooms, area=payload.area_m2, @@ -186,8 +354,6 @@ async def estimate_quality( analogs_lots = [_listing_to_analog(lot) for lot in listings_clean[:10]] deals_lots = [_deal_to_analog(d) for d in deals[:10]] - - sources_used_pre = sorted({lot.get("source") for lot in listings_clean if lot.get("source")}) freshness_pre = _compute_freshness_minutes(listings_clean) db.execute( text( @@ -255,8 +421,26 @@ async def estimate_quality( ) db.commit() + # Link saved IMV evaluation к этому estimate_id (для analytics joining) + if imv_eval is not None: + try: + db.execute( + text( + """ + UPDATE avito_imv_evaluations + SET estimate_id = CAST(:estimate_id AS uuid) + WHERE cache_key = :cache_key + AND (estimate_id IS NULL OR estimate_id = CAST(:estimate_id AS uuid)) + """ + ), + {"estimate_id": str(estimate_id), "cache_key": imv_eval.cache_key}, + ) + db.commit() + except Exception as e: + logger.warning("imv: failed to link estimate_id to evaluation: %s", e) + logger.info( - "estimate: id=%s addr=%s rooms=%d area=%.1f → median=%d (n=%d, conf=%s)", + "estimate: id=%s addr=%s rooms=%d area=%.1f → median=%d (n=%d, conf=%s)%s", estimate_id, geo.full_address[:60], payload.rooms, @@ -264,9 +448,12 @@ async def estimate_quality( median_price, n_analogs, confidence, + f" imv={imv_eval.recommended_price}" if imv_eval else "", ) sources_used = sorted({lot.source for lot in analogs_lots if lot.source}) + if imv_eval is not None: + sources_used = sorted(set(sources_used) | {"avito_imv"}) freshness_min = _compute_freshness_minutes(listings_clean) return AggregatedEstimate( diff --git a/tradein-mvp/backend/tests/test_estimator_imv_integration.py b/tradein-mvp/backend/tests/test_estimator_imv_integration.py new file mode 100644 index 00000000..6d17417f --- /dev/null +++ b/tradein-mvp/backend/tests/test_estimator_imv_integration.py @@ -0,0 +1,108 @@ +"""Offline smoke for estimator IMV integration. Mocks IMV fetch + DB.""" + +import os + +# Settings требует DATABASE_URL при инициализации (fail-fast, C-3). +# Для offline unit-тестов задаём dummy DSN до любого импорта из app. +os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db") + +from unittest.mock import AsyncMock, MagicMock, patch + +import anyio + +from app.services.estimator import ( + _IMV_HOUSE_TYPE_MAP, + _IMV_REPAIR_MAP, + _get_or_fetch_imv_cached, +) + + +def test_imv_house_type_map() -> None: + assert _IMV_HOUSE_TYPE_MAP["panel"] == "panel" + assert _IMV_HOUSE_TYPE_MAP["monolithic"] == "monolith" + assert _IMV_HOUSE_TYPE_MAP[None] is None + assert _IMV_HOUSE_TYPE_MAP.get("unknown") is None + + +def test_imv_repair_map() -> None: + assert _IMV_REPAIR_MAP["needs_repair"] == "required" + assert _IMV_REPAIR_MAP["excellent"] == "designer" + assert _IMV_REPAIR_MAP[None] is None + + +def test_imv_cache_miss_calls_evaluate() -> None: + """Cache miss → calls evaluate_via_imv + save_imv_evaluation.""" + from app.services.scrapers.avito_imv import IMVEvaluation, IMVGeo + + mock_db = MagicMock() + mock_db.execute.return_value.mappings.return_value.first.return_value = None # no cache hit + + fake_result = IMVEvaluation( + cache_key="x" * 64, + address="ЕКБ test", + rooms=2, + area_m2=42.0, + floor=4, + floor_at_home=5, + house_type="panel", + renovation_type="cosmetic", + has_balcony=True, + has_loggia=False, + geo=IMVGeo(geo_hash="JWT"), + recommended_price=6_290_000, + lower_price=6_100_000, + higher_price=6_600_000, + market_count=800, + ) + + async def _run() -> None: + with ( + patch( + "app.services.estimator.evaluate_via_imv", + new=AsyncMock(return_value=fake_result), + ), + patch("app.services.estimator.save_imv_evaluation", return_value=1), + ): + result = await _get_or_fetch_imv_cached( + mock_db, + address="ЕКБ test", + rooms=2, + area_m2=42.0, + floor=4, + floor_at_home=5, + house_type="panel", + renovation_type="cosmetic", + has_balcony=True, + has_loggia=False, + ) + assert result is not None + assert result.recommended_price == 6_290_000 + + anyio.run(_run) + + +def test_imv_fetch_failure_returns_none_gracefully() -> None: + """Network failure → returns None, не raises.""" + mock_db = MagicMock() + mock_db.execute.return_value.mappings.return_value.first.return_value = None + + async def _run() -> None: + with patch( + "app.services.estimator.evaluate_via_imv", + new=AsyncMock(side_effect=RuntimeError("test net error")), + ): + result = await _get_or_fetch_imv_cached( + mock_db, + address="X", + rooms=2, + area_m2=42.0, + floor=4, + floor_at_home=5, + house_type="panel", + renovation_type="cosmetic", + has_balcony=True, + has_loggia=False, + ) + assert result is None + + anyio.run(_run)