"""avito_imv.py — Avito IMV evaluation API client (2-request flow). Stage 2d: geocode + IMV evaluation. Flow: 1) GET /web/1/coords/by_address?address= → geoHash + geo IDs 2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions Таблицы: avito_imv_evaluations (018_avito_imv_evaluations.sql) house_placement_history (017_house_placement_history.sql) """ from __future__ import annotations import hashlib import json import logging import re from dataclasses import dataclass, field from datetime import date, datetime from typing import Any from urllib.parse import urlencode from uuid import UUID from sqlalchemy import text from sqlalchemy.orm import Session logger = logging.getLogger(__name__) AVITO_BASE = "https://www.avito.ru" COORDS_ENDPOINT = "/web/1/coords/by_address" IMV_ENDPOINT = "/web/1/realty-imv/get-data" # Заголовки имитируют браузер на странице evaluation/realty _COMMON_HEADERS = { "Accept": "application/json, text/plain, */*", "Accept-Language": "ru-RU,ru;q=0.9", "Referer": "https://www.avito.ru/evaluation/realty", } # --------------------------------------------------------------------------- # Exceptions # --------------------------------------------------------------------------- class IMVAddressNotFoundError(Exception): """Raised when /coords/by_address returns no geoHash for given address.""" class IMVCityMismatchError(Exception): """Raised when locationId не входит в ожидаемый город (sanity check).""" # --------------------------------------------------------------------------- # Dataclasses # --------------------------------------------------------------------------- @dataclass class IMVGeo: geo_hash: str # JWT — не сохраняем как отдельную колонку, только в raw_response lat: float | None = None lon: float | None = None avito_address_id: int | None = None avito_location_id: int | None = None avito_metro_id: int | None = None avito_district_id: int | None = None @dataclass class IMVPlacementHistoryItem: ext_item_id: str # str(item['id']) title: str | None = None rooms: int | None = None # парсится из title (опционально) area_m2: float | None = None # парсится из title (опционально) floor: int | None = None # парсится из title (опционально) total_floors: int | None = None # парсится из title (опционально) start_price: int | None = None start_price_date: date | None = None # unix → date last_price: int | None = None last_price_date: date | None = None removed_date: date | None = None # только IMV API (в widget housePlacementHistory нет) exposure_days: int | None = None raw_payload: dict[str, Any] | None = None @dataclass class IMVSuggestion: ext_item_id: str title: str | None = None address: str | None = None price_rub: int | None = None exposure_days: int | None = None publish_date: date | None = None item_url: str | None = None metro_name: str | None = None metro_distance: str | None = None metro_color: str | None = None has_good_price_badge: bool = False raw_payload: dict[str, Any] | None = None @dataclass class IMVEvaluation: cache_key: str # sha256 address: str rooms: int area_m2: float floor: int floor_at_home: int house_type: str renovation_type: str has_balcony: bool has_loggia: bool geo: IMVGeo recommended_price: int lower_price: int higher_price: int market_count: int | None = None placement_history: list[IMVPlacementHistoryItem] = field(default_factory=list) suggestions: list[IMVSuggestion] = field(default_factory=list) search_similar_url: str | None = None raw_response: dict[str, Any] | None = None # --------------------------------------------------------------------------- # Cache key # --------------------------------------------------------------------------- def compute_imv_cache_key( address: str, rooms: int, area_m2: float, floor: int, floor_at_home: int, house_type: str, renovation_type: str, has_balcony: bool, has_loggia: bool, ) -> str: """Детерминированный sha256 ключ для 24h-кэша IMV оценки.""" key_src = ( f"{address}|{rooms}|{area_m2}|{floor}|{floor_at_home}" f"|{house_type}|{renovation_type}|{int(has_balcony)}|{int(has_loggia)}" ) return hashlib.sha256(key_src.encode("utf-8")).hexdigest() # --------------------------------------------------------------------------- # Parse helpers # --------------------------------------------------------------------------- def _unix_to_date(ts: int | float | None) -> date | None: """Конвертирует unix timestamp в date. Возвращает None для 0 и None.""" if not ts: return None try: return datetime.fromtimestamp(int(ts)).date() except (ValueError, OSError): return None # Паттерн для заголовка вида "2-к. квартира, 42 м², 4/5 эт." _TITLE_RE = re.compile( r"^(?P\d+)-к[.\s].*?(?P[\d,]+)\s*м².*?(?P\d+)/(?P\d+)\s*эт", re.IGNORECASE, ) def _parse_title(title: str | None) -> dict[str, int | float | None]: """Парсит rooms, area_m2, floor, total_floors из Avito-заголовка объявления.""" result: dict[str, int | float | None] = { "rooms": None, "area_m2": None, "floor": None, "total_floors": None, } if not title: return result m = _TITLE_RE.match(title.strip()) if not m: return result try: result["rooms"] = int(m.group("rooms")) result["area_m2"] = float(m.group("area").replace(",", ".")) result["floor"] = int(m.group("floor")) result["total_floors"] = int(m.group("total")) except (ValueError, AttributeError): pass return result def _parse_placement_item(raw: dict[str, Any]) -> IMVPlacementHistoryItem: """Парсит один элемент из placementHistory.items.""" title = raw.get("title") parsed = _parse_title(title) # Фильтруем itemImage из raw_payload (не нужны) clean_raw = {k: v for k, v in raw.items() if k not in ("itemImage",)} return IMVPlacementHistoryItem( ext_item_id=str(raw["id"]), title=title, rooms=parsed["rooms"], # type: ignore[arg-type] area_m2=parsed["area_m2"], # type: ignore[arg-type] floor=parsed["floor"], # type: ignore[arg-type] total_floors=parsed["total_floors"], # type: ignore[arg-type] start_price=raw.get("startPrice"), start_price_date=_unix_to_date(raw.get("startPriceDate")), last_price=raw.get("lastPrice"), last_price_date=_unix_to_date(raw.get("lastPriceDate")), removed_date=_unix_to_date(raw.get("removedDate")), exposure_days=raw.get("exposure"), raw_payload=clean_raw, ) def _parse_suggestion(raw: dict[str, Any]) -> IMVSuggestion: """Парсит один элемент из suggestions.items.""" metro = raw.get("metro") or {} colors: list[str] = metro.get("colors") or [] # Фильтруем imageLink из raw_payload clean_raw = {k: v for k, v in raw.items() if k not in ("imageLink",)} return IMVSuggestion( ext_item_id=str(raw["id"]), title=raw.get("title"), address=raw.get("address"), price_rub=raw.get("price"), exposure_days=raw.get("exposure"), publish_date=_unix_to_date(raw.get("publishDate")), item_url=raw.get("itemLink"), metro_name=metro.get("name"), metro_distance=metro.get("distance"), metro_color=colors[0] if colors else None, has_good_price_badge=bool(raw.get("hasGoodPriceBadge", False)), raw_payload=clean_raw, ) # --------------------------------------------------------------------------- # Main async function # --------------------------------------------------------------------------- async def evaluate_via_imv( address: str, rooms: int, area_m2: float, floor: int, floor_at_home: int, house_type: str, # panel/brick/monolith/monolith_brick/block/wood renovation_type: str, # required/cosmetic/euro/designer has_balcony: bool, has_loggia: bool, *, cffi_session: Any | None = None, ) -> IMVEvaluation: """Avito IMV — 2 HTTP requests: 1) GET /web/1/coords/by_address?address= → geoHash + geo IDs 2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions Returns IMVEvaluation с уже вычисленным cache_key. Raises: IMVAddressNotFoundError если geoHash пустой / не вернулся httpx.HTTPStatusError на 4xx/5xx """ # Импортируем здесь чтобы избежать циклических зависимостей при тестах без сети try: from curl_cffi.requests import AsyncSession as CffiAsyncSession _own_session = False if cffi_session is None: cffi_session = CffiAsyncSession(impersonate="chrome120") _own_session = True except ImportError as exc: raise RuntimeError( "curl_cffi не установлен. Добавь 'curl-cffi>=0.7.0' в pyproject.toml." ) from exc try: geo = await _geocode(cffi_session, address) evaluation = await _imv_evaluate( cffi_session, geo=geo, address=address, rooms=rooms, area_m2=area_m2, floor=floor, floor_at_home=floor_at_home, house_type=house_type, renovation_type=renovation_type, has_balcony=has_balcony, has_loggia=has_loggia, ) finally: if _own_session: await cffi_session.close() return evaluation async def _geocode(session: Any, address: str) -> IMVGeo: """Request 1: GET /web/1/coords/by_address → IMVGeo.""" params = urlencode({"address": address}) url = f"{AVITO_BASE}{COORDS_ENDPOINT}?{params}" logger.info("IMV geocode: address=%r", address) resp = await session.get(url, headers=_COMMON_HEADERS) resp.raise_for_status() data: dict[str, Any] = resp.json() geo_hash = data.get("geoHash") if not geo_hash: raise IMVAddressNotFoundError( f"Avito /coords/by_address не вернул geoHash для адреса: {address!r}" ) logger.info( "IMV geocode OK: locationId=%s lat=%s lon=%s", data.get("locationId"), data.get("latitude"), data.get("longitude"), ) return IMVGeo( geo_hash=geo_hash, lat=data.get("latitude"), lon=data.get("longitude"), avito_address_id=data.get("addressId"), avito_location_id=data.get("locationId"), avito_metro_id=data.get("metroId"), avito_district_id=data.get("districtId"), ) async def _imv_evaluate( session: Any, *, geo: IMVGeo, address: str, rooms: int, area_m2: float, floor: int, floor_at_home: int, house_type: str, renovation_type: str, has_balcony: bool, has_loggia: bool, ) -> IMVEvaluation: """Request 2: POST /web/1/realty-imv/get-data → IMVEvaluation.""" url = f"{AVITO_BASE}{IMV_ENDPOINT}" body = { "geoHash": geo.geo_hash, "floor": floor, "floorAtHome": floor_at_home, "rooms": rooms, "area": area_m2, "houseType": house_type, "renovationType": renovation_type, "hasLoggia": has_loggia, "hasBalcony": has_balcony, } headers = {**_COMMON_HEADERS, "Content-Type": "application/json"} logger.info( "IMV evaluate: address=%r rooms=%d area=%.1f floor=%d/%d", address, rooms, area_m2, floor, floor_at_home, ) resp = await session.post(url, json=body, headers=headers) resp.raise_for_status() data: dict[str, Any] = resp.json() price_block: dict[str, Any] = data.get("price") or {} recommended_price: int = price_block.get("recommendedPrice") or 0 lower_price: int = price_block.get("lowerPrice") or 0 higher_price: int = price_block.get("higherPrice") or 0 market_count: int | None = price_block.get("count") # placementHistory history_block: dict[str, Any] = data.get("placementHistory") or {} placement_history: list[IMVPlacementHistoryItem] = [] for raw_item in history_block.get("items") or []: try: placement_history.append(_parse_placement_item(raw_item)) except (KeyError, TypeError) as exc: logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc) # suggestions suggestions_block: dict[str, Any] = data.get("suggestions") or {} suggestions: list[IMVSuggestion] = [] for raw_sugg in suggestions_block.get("items") or []: try: suggestions.append(_parse_suggestion(raw_sugg)) except (KeyError, TypeError) as exc: logger.warning("IMV: не удалось распарсить suggestion: %s", exc) search_similar_url: str | None = suggestions_block.get("searchSimilarUrl") logger.info( "IMV evaluate OK: recommended=%d range=(%d, %d) count=%s history=%d suggestions=%d", recommended_price, lower_price, higher_price, market_count, len(placement_history), len(suggestions), ) cache_key = compute_imv_cache_key( address=address, rooms=rooms, area_m2=area_m2, floor=floor, floor_at_home=floor_at_home, house_type=house_type, renovation_type=renovation_type, has_balcony=has_balcony, has_loggia=has_loggia, ) return IMVEvaluation( cache_key=cache_key, address=address, rooms=rooms, area_m2=area_m2, floor=floor, floor_at_home=floor_at_home, house_type=house_type, renovation_type=renovation_type, has_balcony=has_balcony, has_loggia=has_loggia, geo=geo, recommended_price=recommended_price, lower_price=lower_price, higher_price=higher_price, market_count=market_count, placement_history=placement_history, suggestions=suggestions, search_similar_url=search_similar_url, raw_response=data, ) # --------------------------------------------------------------------------- # DB save functions # --------------------------------------------------------------------------- def save_imv_evaluation( db: Session, e: IMVEvaluation, *, estimate_id: UUID | None = None, ) -> int: """INSERT в avito_imv_evaluations ON CONFLICT(cache_key) DO UPDATE. Колонки строго по 018_avito_imv_evaluations.sql DDL. estimate_id — nullable FK на trade_in_estimates. Returns: id вставленной/обновлённой строки. """ row = db.execute( text(""" INSERT INTO avito_imv_evaluations ( cache_key, estimate_id, address, rooms, area_m2, floor, floor_at_home, house_type, renovation_type, has_balcony, has_loggia, geo_hash, lat, lon, avito_address_id, avito_location_id, avito_metro_id, avito_district_id, recommended_price, lower_price, higher_price, market_count, raw_response, fetched_at ) VALUES ( :cache_key, CAST(:estimate_id AS uuid), :address, :rooms, :area_m2, :floor, :floor_at_home, :house_type, :renovation_type, :has_balcony, :has_loggia, :geo_hash, :lat, :lon, :avito_address_id, :avito_location_id, :avito_metro_id, :avito_district_id, :recommended_price, :lower_price, :higher_price, :market_count, CAST(:raw_response AS jsonb), NOW() ) ON CONFLICT (cache_key) DO UPDATE SET estimate_id = COALESCE(EXCLUDED.estimate_id, avito_imv_evaluations.estimate_id), recommended_price = EXCLUDED.recommended_price, lower_price = EXCLUDED.lower_price, higher_price = EXCLUDED.higher_price, market_count = EXCLUDED.market_count, raw_response = EXCLUDED.raw_response, fetched_at = NOW() RETURNING id """), { "cache_key": e.cache_key, "estimate_id": str(estimate_id) if estimate_id else None, "address": e.address, "rooms": e.rooms, "area_m2": e.area_m2, "floor": e.floor, "floor_at_home": e.floor_at_home, "house_type": e.house_type, "renovation_type": e.renovation_type, "has_balcony": e.has_balcony, "has_loggia": e.has_loggia, "geo_hash": e.geo.geo_hash, "lat": e.geo.lat, "lon": e.geo.lon, "avito_address_id": e.geo.avito_address_id, "avito_location_id": e.geo.avito_location_id, "avito_metro_id": e.geo.avito_metro_id, "avito_district_id": e.geo.avito_district_id, "recommended_price": e.recommended_price, "lower_price": e.lower_price, "higher_price": e.higher_price, "market_count": e.market_count, "raw_response": ( json.dumps(e.raw_response, ensure_ascii=False) if e.raw_response else None ), }, ).fetchone() db.commit() return row.id if row else 0 def save_imv_placement_history( db: Session, eval_id: int, items: list[IMVPlacementHistoryItem], ) -> int: """INSERT в house_placement_history (source='avito_imv') с removed_date. Колонки строго по 017_house_placement_history.sql DDL. Колонка scraped_at (не fetched_at) — по DDL таблицы. house_id — NULL (backfill через отдельный процесс). eval_id принимается для логирования (не сохраняется — нет FK в 017). ON CONFLICT (source, ext_item_id) DO UPDATE — refresh dates. Returns: количество вставленных/обновлённых строк. """ count = 0 for item in items: db.execute( text(""" INSERT INTO house_placement_history ( source, ext_item_id, title, rooms, area_m2, floor, total_floors, start_price, start_price_date, last_price, last_price_date, removed_date, exposure_days, raw_payload, scraped_at ) VALUES ( 'avito_imv', :ext_item_id, :title, :rooms, :area_m2, :floor, :total_floors, :start_price, :start_price_date, :last_price, :last_price_date, :removed_date, :exposure_days, CAST(:raw_payload AS jsonb), NOW() ) ON CONFLICT (source, ext_item_id) DO UPDATE SET last_price = EXCLUDED.last_price, last_price_date = EXCLUDED.last_price_date, removed_date = EXCLUDED.removed_date, exposure_days = EXCLUDED.exposure_days, raw_payload = EXCLUDED.raw_payload, scraped_at = NOW() """), { "ext_item_id": item.ext_item_id, "title": item.title, "rooms": item.rooms, "area_m2": item.area_m2, "floor": item.floor, "total_floors": item.total_floors, "start_price": item.start_price, "start_price_date": item.start_price_date, "last_price": item.last_price, "last_price_date": item.last_price_date, "removed_date": item.removed_date, "exposure_days": item.exposure_days, "raw_payload": ( json.dumps(item.raw_payload, ensure_ascii=False) if item.raw_payload else None ), }, ) count += 1 if count: db.commit() logger.info("IMV placement history saved: eval_id=%d items=%d", eval_id, count) return count