608 lines
21 KiB
Python
608 lines
21 KiB
Python
"""avito_imv.py — Avito IMV evaluation API client (2-request flow).
|
||
|
||
Stage 2d: geocode + IMV evaluation.
|
||
|
||
Flow:
|
||
1) GET /web/1/coords/by_address?address=<urlenc> → geoHash + geo IDs
|
||
2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions
|
||
|
||
Таблицы:
|
||
avito_imv_evaluations (018_avito_imv_evaluations.sql)
|
||
house_placement_history (017_house_placement_history.sql)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import re
|
||
from dataclasses import dataclass, field
|
||
from datetime import date, datetime
|
||
from typing import Any
|
||
from urllib.parse import urlencode
|
||
from uuid import UUID
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
AVITO_BASE = "https://www.avito.ru"
|
||
COORDS_ENDPOINT = "/web/1/coords/by_address"
|
||
IMV_ENDPOINT = "/web/1/realty-imv/get-data"
|
||
|
||
# Заголовки имитируют браузер на странице evaluation/realty
|
||
_COMMON_HEADERS = {
|
||
"Accept": "application/json, text/plain, */*",
|
||
"Accept-Language": "ru-RU,ru;q=0.9",
|
||
"Referer": "https://www.avito.ru/evaluation/realty",
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Exceptions
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class IMVAddressNotFoundError(Exception):
|
||
"""Raised when /coords/by_address returns no geoHash for given address."""
|
||
|
||
|
||
class IMVCityMismatchError(Exception):
|
||
"""Raised when locationId не входит в ожидаемый город (sanity check)."""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Dataclasses
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class IMVGeo:
|
||
geo_hash: str # JWT — не сохраняем как отдельную колонку, только в raw_response
|
||
lat: float | None = None
|
||
lon: float | None = None
|
||
avito_address_id: int | None = None
|
||
avito_location_id: int | None = None
|
||
avito_metro_id: int | None = None
|
||
avito_district_id: int | None = None
|
||
|
||
|
||
@dataclass
|
||
class IMVPlacementHistoryItem:
|
||
ext_item_id: str # str(item['id'])
|
||
title: str | None = None
|
||
rooms: int | None = None # парсится из title (опционально)
|
||
area_m2: float | None = None # парсится из title (опционально)
|
||
floor: int | None = None # парсится из title (опционально)
|
||
total_floors: int | None = None # парсится из title (опционально)
|
||
start_price: int | None = None
|
||
start_price_date: date | None = None # unix → date
|
||
last_price: int | None = None
|
||
last_price_date: date | None = None
|
||
removed_date: date | None = None # только IMV API (в widget housePlacementHistory нет)
|
||
exposure_days: int | None = None
|
||
raw_payload: dict[str, Any] | None = None
|
||
|
||
|
||
@dataclass
|
||
class IMVSuggestion:
|
||
ext_item_id: str
|
||
title: str | None = None
|
||
address: str | None = None
|
||
price_rub: int | None = None
|
||
exposure_days: int | None = None
|
||
publish_date: date | None = None
|
||
item_url: str | None = None
|
||
metro_name: str | None = None
|
||
metro_distance: str | None = None
|
||
metro_color: str | None = None
|
||
has_good_price_badge: bool = False
|
||
raw_payload: dict[str, Any] | None = None
|
||
|
||
|
||
@dataclass
|
||
class IMVEvaluation:
|
||
cache_key: str # sha256
|
||
address: str
|
||
rooms: int
|
||
area_m2: float
|
||
floor: int
|
||
floor_at_home: int
|
||
house_type: str
|
||
renovation_type: str
|
||
has_balcony: bool
|
||
has_loggia: bool
|
||
|
||
geo: IMVGeo
|
||
recommended_price: int
|
||
lower_price: int
|
||
higher_price: int
|
||
market_count: int | None = None
|
||
|
||
placement_history: list[IMVPlacementHistoryItem] = field(default_factory=list)
|
||
suggestions: list[IMVSuggestion] = field(default_factory=list)
|
||
search_similar_url: str | None = None
|
||
raw_response: dict[str, Any] | None = None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Cache key
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def compute_imv_cache_key(
|
||
address: str,
|
||
rooms: int,
|
||
area_m2: float,
|
||
floor: int,
|
||
floor_at_home: int,
|
||
house_type: str,
|
||
renovation_type: str,
|
||
has_balcony: bool,
|
||
has_loggia: bool,
|
||
) -> str:
|
||
"""Детерминированный sha256 ключ для 24h-кэша IMV оценки."""
|
||
key_src = (
|
||
f"{address}|{rooms}|{area_m2}|{floor}|{floor_at_home}"
|
||
f"|{house_type}|{renovation_type}|{int(has_balcony)}|{int(has_loggia)}"
|
||
)
|
||
return hashlib.sha256(key_src.encode("utf-8")).hexdigest()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Parse helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _unix_to_date(ts: int | float | None) -> date | None:
|
||
"""Конвертирует unix timestamp в date. Возвращает None для 0 и None."""
|
||
if not ts:
|
||
return None
|
||
try:
|
||
return datetime.fromtimestamp(int(ts)).date()
|
||
except (ValueError, OSError):
|
||
return None
|
||
|
||
|
||
# Паттерн для заголовка вида "2-к. квартира, 42 м², 4/5 эт."
|
||
_TITLE_RE = re.compile(
|
||
r"^(?P<rooms>\d+)-к[.\s].*?(?P<area>[\d,]+)\s*м².*?(?P<floor>\d+)/(?P<total>\d+)\s*эт",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
def _parse_title(title: str | None) -> dict[str, int | float | None]:
|
||
"""Парсит rooms, area_m2, floor, total_floors из Avito-заголовка объявления."""
|
||
result: dict[str, int | float | None] = {
|
||
"rooms": None,
|
||
"area_m2": None,
|
||
"floor": None,
|
||
"total_floors": None,
|
||
}
|
||
if not title:
|
||
return result
|
||
m = _TITLE_RE.match(title.strip())
|
||
if not m:
|
||
return result
|
||
try:
|
||
result["rooms"] = int(m.group("rooms"))
|
||
result["area_m2"] = float(m.group("area").replace(",", "."))
|
||
result["floor"] = int(m.group("floor"))
|
||
result["total_floors"] = int(m.group("total"))
|
||
except (ValueError, AttributeError):
|
||
pass
|
||
return result
|
||
|
||
|
||
def _parse_placement_item(raw: dict[str, Any]) -> IMVPlacementHistoryItem:
|
||
"""Парсит один элемент из placementHistory.items."""
|
||
title = raw.get("title")
|
||
parsed = _parse_title(title)
|
||
|
||
# Фильтруем itemImage из raw_payload (не нужны)
|
||
clean_raw = {k: v for k, v in raw.items() if k not in ("itemImage",)}
|
||
|
||
return IMVPlacementHistoryItem(
|
||
ext_item_id=str(raw["id"]),
|
||
title=title,
|
||
rooms=parsed["rooms"], # type: ignore[arg-type]
|
||
area_m2=parsed["area_m2"], # type: ignore[arg-type]
|
||
floor=parsed["floor"], # type: ignore[arg-type]
|
||
total_floors=parsed["total_floors"], # type: ignore[arg-type]
|
||
start_price=raw.get("startPrice"),
|
||
start_price_date=_unix_to_date(raw.get("startPriceDate")),
|
||
last_price=raw.get("lastPrice"),
|
||
last_price_date=_unix_to_date(raw.get("lastPriceDate")),
|
||
removed_date=_unix_to_date(raw.get("removedDate")),
|
||
exposure_days=raw.get("exposure"),
|
||
raw_payload=clean_raw,
|
||
)
|
||
|
||
|
||
def _parse_suggestion(raw: dict[str, Any]) -> IMVSuggestion:
|
||
"""Парсит один элемент из suggestions.items."""
|
||
metro = raw.get("metro") or {}
|
||
colors: list[str] = metro.get("colors") or []
|
||
|
||
# Фильтруем imageLink из raw_payload
|
||
clean_raw = {k: v for k, v in raw.items() if k not in ("imageLink",)}
|
||
|
||
return IMVSuggestion(
|
||
ext_item_id=str(raw["id"]),
|
||
title=raw.get("title"),
|
||
address=raw.get("address"),
|
||
price_rub=raw.get("price"),
|
||
exposure_days=raw.get("exposure"),
|
||
publish_date=_unix_to_date(raw.get("publishDate")),
|
||
item_url=raw.get("itemLink"),
|
||
metro_name=metro.get("name"),
|
||
metro_distance=metro.get("distance"),
|
||
metro_color=colors[0] if colors else None,
|
||
has_good_price_badge=bool(raw.get("hasGoodPriceBadge", False)),
|
||
raw_payload=clean_raw,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main async function
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def evaluate_via_imv(
|
||
address: str,
|
||
rooms: int,
|
||
area_m2: float,
|
||
floor: int,
|
||
floor_at_home: int,
|
||
house_type: str, # panel/brick/monolith/monolith_brick/block/wood
|
||
renovation_type: str, # required/cosmetic/euro/designer
|
||
has_balcony: bool,
|
||
has_loggia: bool,
|
||
*,
|
||
cffi_session: Any | None = None,
|
||
) -> IMVEvaluation:
|
||
"""Avito IMV — 2 HTTP requests:
|
||
|
||
1) GET /web/1/coords/by_address?address=<urlenc> → geoHash + geo IDs
|
||
2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions
|
||
|
||
Returns IMVEvaluation с уже вычисленным cache_key.
|
||
|
||
Raises:
|
||
IMVAddressNotFoundError если geoHash пустой / не вернулся
|
||
httpx.HTTPStatusError на 4xx/5xx
|
||
"""
|
||
# Импортируем здесь чтобы избежать циклических зависимостей при тестах без сети
|
||
try:
|
||
from curl_cffi.requests import AsyncSession as CffiAsyncSession
|
||
|
||
_own_session = False
|
||
if cffi_session is None:
|
||
cffi_session = CffiAsyncSession(impersonate="chrome120")
|
||
_own_session = True
|
||
except ImportError as exc:
|
||
raise RuntimeError(
|
||
"curl_cffi не установлен. Добавь 'curl-cffi>=0.7.0' в pyproject.toml."
|
||
) from exc
|
||
|
||
try:
|
||
geo = await _geocode(cffi_session, address)
|
||
evaluation = await _imv_evaluate(
|
||
cffi_session,
|
||
geo=geo,
|
||
address=address,
|
||
rooms=rooms,
|
||
area_m2=area_m2,
|
||
floor=floor,
|
||
floor_at_home=floor_at_home,
|
||
house_type=house_type,
|
||
renovation_type=renovation_type,
|
||
has_balcony=has_balcony,
|
||
has_loggia=has_loggia,
|
||
)
|
||
finally:
|
||
if _own_session:
|
||
await cffi_session.close()
|
||
|
||
return evaluation
|
||
|
||
|
||
async def _geocode(session: Any, address: str) -> IMVGeo:
|
||
"""Request 1: GET /web/1/coords/by_address → IMVGeo."""
|
||
params = urlencode({"address": address})
|
||
url = f"{AVITO_BASE}{COORDS_ENDPOINT}?{params}"
|
||
logger.info("IMV geocode: address=%r", address)
|
||
|
||
resp = await session.get(url, headers=_COMMON_HEADERS)
|
||
resp.raise_for_status()
|
||
data: dict[str, Any] = resp.json()
|
||
|
||
geo_hash = data.get("geoHash")
|
||
if not geo_hash:
|
||
raise IMVAddressNotFoundError(
|
||
f"Avito /coords/by_address не вернул geoHash для адреса: {address!r}"
|
||
)
|
||
|
||
logger.info(
|
||
"IMV geocode OK: locationId=%s lat=%s lon=%s",
|
||
data.get("locationId"),
|
||
data.get("latitude"),
|
||
data.get("longitude"),
|
||
)
|
||
|
||
return IMVGeo(
|
||
geo_hash=geo_hash,
|
||
lat=data.get("latitude"),
|
||
lon=data.get("longitude"),
|
||
avito_address_id=data.get("addressId"),
|
||
avito_location_id=data.get("locationId"),
|
||
avito_metro_id=data.get("metroId"),
|
||
avito_district_id=data.get("districtId"),
|
||
)
|
||
|
||
|
||
async def _imv_evaluate(
|
||
session: Any,
|
||
*,
|
||
geo: IMVGeo,
|
||
address: str,
|
||
rooms: int,
|
||
area_m2: float,
|
||
floor: int,
|
||
floor_at_home: int,
|
||
house_type: str,
|
||
renovation_type: str,
|
||
has_balcony: bool,
|
||
has_loggia: bool,
|
||
) -> IMVEvaluation:
|
||
"""Request 2: POST /web/1/realty-imv/get-data → IMVEvaluation."""
|
||
url = f"{AVITO_BASE}{IMV_ENDPOINT}"
|
||
body = {
|
||
"geoHash": geo.geo_hash,
|
||
"floor": floor,
|
||
"floorAtHome": floor_at_home,
|
||
"rooms": rooms,
|
||
"area": area_m2,
|
||
"houseType": house_type,
|
||
"renovationType": renovation_type,
|
||
"hasLoggia": has_loggia,
|
||
"hasBalcony": has_balcony,
|
||
}
|
||
headers = {**_COMMON_HEADERS, "Content-Type": "application/json"}
|
||
|
||
logger.info(
|
||
"IMV evaluate: address=%r rooms=%d area=%.1f floor=%d/%d",
|
||
address,
|
||
rooms,
|
||
area_m2,
|
||
floor,
|
||
floor_at_home,
|
||
)
|
||
|
||
resp = await session.post(url, json=body, headers=headers)
|
||
resp.raise_for_status()
|
||
data: dict[str, Any] = resp.json()
|
||
|
||
price_block: dict[str, Any] = data.get("price") or {}
|
||
recommended_price: int = price_block.get("recommendedPrice") or 0
|
||
lower_price: int = price_block.get("lowerPrice") or 0
|
||
higher_price: int = price_block.get("higherPrice") or 0
|
||
market_count: int | None = price_block.get("count")
|
||
|
||
# placementHistory
|
||
history_block: dict[str, Any] = data.get("placementHistory") or {}
|
||
placement_history: list[IMVPlacementHistoryItem] = []
|
||
for raw_item in history_block.get("items") or []:
|
||
try:
|
||
placement_history.append(_parse_placement_item(raw_item))
|
||
except (KeyError, TypeError) as exc:
|
||
logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc)
|
||
|
||
# suggestions
|
||
suggestions_block: dict[str, Any] = data.get("suggestions") or {}
|
||
suggestions: list[IMVSuggestion] = []
|
||
for raw_sugg in suggestions_block.get("items") or []:
|
||
try:
|
||
suggestions.append(_parse_suggestion(raw_sugg))
|
||
except (KeyError, TypeError) as exc:
|
||
logger.warning("IMV: не удалось распарсить suggestion: %s", exc)
|
||
|
||
search_similar_url: str | None = suggestions_block.get("searchSimilarUrl")
|
||
|
||
logger.info(
|
||
"IMV evaluate OK: recommended=%d range=(%d, %d) count=%s history=%d suggestions=%d",
|
||
recommended_price,
|
||
lower_price,
|
||
higher_price,
|
||
market_count,
|
||
len(placement_history),
|
||
len(suggestions),
|
||
)
|
||
|
||
cache_key = compute_imv_cache_key(
|
||
address=address,
|
||
rooms=rooms,
|
||
area_m2=area_m2,
|
||
floor=floor,
|
||
floor_at_home=floor_at_home,
|
||
house_type=house_type,
|
||
renovation_type=renovation_type,
|
||
has_balcony=has_balcony,
|
||
has_loggia=has_loggia,
|
||
)
|
||
|
||
return IMVEvaluation(
|
||
cache_key=cache_key,
|
||
address=address,
|
||
rooms=rooms,
|
||
area_m2=area_m2,
|
||
floor=floor,
|
||
floor_at_home=floor_at_home,
|
||
house_type=house_type,
|
||
renovation_type=renovation_type,
|
||
has_balcony=has_balcony,
|
||
has_loggia=has_loggia,
|
||
geo=geo,
|
||
recommended_price=recommended_price,
|
||
lower_price=lower_price,
|
||
higher_price=higher_price,
|
||
market_count=market_count,
|
||
placement_history=placement_history,
|
||
suggestions=suggestions,
|
||
search_similar_url=search_similar_url,
|
||
raw_response=data,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# DB save functions
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def save_imv_evaluation(
|
||
db: Session,
|
||
e: IMVEvaluation,
|
||
*,
|
||
estimate_id: UUID | None = None,
|
||
) -> int:
|
||
"""INSERT в avito_imv_evaluations ON CONFLICT(cache_key) DO UPDATE.
|
||
|
||
Колонки строго по 018_avito_imv_evaluations.sql DDL.
|
||
estimate_id — nullable FK на trade_in_estimates.
|
||
|
||
Returns: id вставленной/обновлённой строки.
|
||
"""
|
||
row = db.execute(
|
||
text("""
|
||
INSERT INTO avito_imv_evaluations (
|
||
cache_key, estimate_id,
|
||
address, rooms, area_m2, floor, floor_at_home,
|
||
house_type, renovation_type, has_balcony, has_loggia,
|
||
geo_hash,
|
||
lat, lon, avito_address_id, avito_location_id,
|
||
avito_metro_id, avito_district_id,
|
||
recommended_price, lower_price, higher_price, market_count,
|
||
raw_response, fetched_at
|
||
) VALUES (
|
||
:cache_key, CAST(:estimate_id AS uuid),
|
||
:address, :rooms, :area_m2, :floor, :floor_at_home,
|
||
:house_type, :renovation_type, :has_balcony, :has_loggia,
|
||
:geo_hash,
|
||
:lat, :lon, :avito_address_id, :avito_location_id,
|
||
:avito_metro_id, :avito_district_id,
|
||
:recommended_price, :lower_price, :higher_price, :market_count,
|
||
CAST(:raw_response AS jsonb), NOW()
|
||
)
|
||
ON CONFLICT (cache_key) DO UPDATE SET
|
||
estimate_id = COALESCE(EXCLUDED.estimate_id, avito_imv_evaluations.estimate_id),
|
||
recommended_price = EXCLUDED.recommended_price,
|
||
lower_price = EXCLUDED.lower_price,
|
||
higher_price = EXCLUDED.higher_price,
|
||
market_count = EXCLUDED.market_count,
|
||
raw_response = EXCLUDED.raw_response,
|
||
fetched_at = NOW()
|
||
RETURNING id
|
||
"""),
|
||
{
|
||
"cache_key": e.cache_key,
|
||
"estimate_id": str(estimate_id) if estimate_id else None,
|
||
"address": e.address,
|
||
"rooms": e.rooms,
|
||
"area_m2": e.area_m2,
|
||
"floor": e.floor,
|
||
"floor_at_home": e.floor_at_home,
|
||
"house_type": e.house_type,
|
||
"renovation_type": e.renovation_type,
|
||
"has_balcony": e.has_balcony,
|
||
"has_loggia": e.has_loggia,
|
||
"geo_hash": e.geo.geo_hash,
|
||
"lat": e.geo.lat,
|
||
"lon": e.geo.lon,
|
||
"avito_address_id": e.geo.avito_address_id,
|
||
"avito_location_id": e.geo.avito_location_id,
|
||
"avito_metro_id": e.geo.avito_metro_id,
|
||
"avito_district_id": e.geo.avito_district_id,
|
||
"recommended_price": e.recommended_price,
|
||
"lower_price": e.lower_price,
|
||
"higher_price": e.higher_price,
|
||
"market_count": e.market_count,
|
||
"raw_response": (
|
||
json.dumps(e.raw_response, ensure_ascii=False) if e.raw_response else None
|
||
),
|
||
},
|
||
).fetchone()
|
||
db.commit()
|
||
return row.id if row else 0
|
||
|
||
|
||
def save_imv_placement_history(
|
||
db: Session,
|
||
eval_id: int,
|
||
items: list[IMVPlacementHistoryItem],
|
||
) -> int:
|
||
"""INSERT в house_placement_history (source='avito_imv') с removed_date.
|
||
|
||
Колонки строго по 017_house_placement_history.sql DDL.
|
||
Колонка scraped_at (не fetched_at) — по DDL таблицы.
|
||
house_id — NULL (backfill через отдельный процесс).
|
||
eval_id принимается для логирования (не сохраняется — нет FK в 017).
|
||
|
||
ON CONFLICT (source, ext_item_id) DO UPDATE — refresh dates.
|
||
Returns: количество вставленных/обновлённых строк.
|
||
"""
|
||
count = 0
|
||
for item in items:
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO house_placement_history (
|
||
source, ext_item_id,
|
||
title, rooms, area_m2, floor, total_floors,
|
||
start_price, start_price_date,
|
||
last_price, last_price_date,
|
||
removed_date, exposure_days,
|
||
raw_payload, scraped_at
|
||
) VALUES (
|
||
'avito_imv', :ext_item_id,
|
||
:title, :rooms, :area_m2, :floor, :total_floors,
|
||
:start_price, :start_price_date,
|
||
:last_price, :last_price_date,
|
||
:removed_date, :exposure_days,
|
||
CAST(:raw_payload AS jsonb), NOW()
|
||
)
|
||
ON CONFLICT (source, ext_item_id) DO UPDATE SET
|
||
last_price = EXCLUDED.last_price,
|
||
last_price_date = EXCLUDED.last_price_date,
|
||
removed_date = EXCLUDED.removed_date,
|
||
exposure_days = EXCLUDED.exposure_days,
|
||
raw_payload = EXCLUDED.raw_payload,
|
||
scraped_at = NOW()
|
||
"""),
|
||
{
|
||
"ext_item_id": item.ext_item_id,
|
||
"title": item.title,
|
||
"rooms": item.rooms,
|
||
"area_m2": item.area_m2,
|
||
"floor": item.floor,
|
||
"total_floors": item.total_floors,
|
||
"start_price": item.start_price,
|
||
"start_price_date": item.start_price_date,
|
||
"last_price": item.last_price,
|
||
"last_price_date": item.last_price_date,
|
||
"removed_date": item.removed_date,
|
||
"exposure_days": item.exposure_days,
|
||
"raw_payload": (
|
||
json.dumps(item.raw_payload, ensure_ascii=False)
|
||
if item.raw_payload
|
||
else None
|
||
),
|
||
},
|
||
)
|
||
count += 1
|
||
|
||
if count:
|
||
db.commit()
|
||
logger.info("IMV placement history saved: eval_id=%d items=%d", eval_id, count)
|
||
|
||
return count
|