feat(tradein): avito_imv.py — Avito IMV evaluation API client (2 requests)
Stage 2d of AvitoScraper_v2.
- evaluate_via_imv(address, rooms, area, floor, ...) — async 2-request flow:
1) GET /web/1/coords/by_address → geoHash + geo IDs
2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions
- IMVEvaluation dataclass: cache_key (sha256), geo, price (recommended/lower/higher/market_count),
placement_history with removed_date, suggestions
- compute_imv_cache_key(...) — deterministic hash for 24h cache lookup
- save_imv_evaluation(db, e) — UPSERT в avito_imv_evaluations ON CONFLICT(cache_key)
- save_imv_placement_history(db, items) — INSERT с source='avito_imv' и removed_date NOT NULL
(отличие от Stage 2c widget save flow)
- IMVAddressNotFoundError / IMVCityMismatchError exceptions
- pytest offline smoke (cache_key determinism + unix→date + dataclass construct)
Note: suggestions saving в listings skipped — Stage 3 решит when integrating.
Note: house_placement_history использует scraped_at (не fetched_at) — по DDL 017.
avito_imv_evaluations сохраняет geo_hash в отдельную колонку — по DDL 018.
This commit is contained in:
lekss361
parent
a2953e97c4
commit
375f785100
3 changed files with 895 additions and 0 deletions
608
tradein-mvp/backend/app/services/scrapers/avito_imv.py
Normal file
608
tradein-mvp/backend/app/services/scrapers/avito_imv.py
Normal file
|
|
@ -0,0 +1,608 @@
|
|||
"""avito_imv.py — Avito IMV evaluation API client (2-request flow).
|
||||
|
||||
Stage 2d: geocode + IMV evaluation.
|
||||
|
||||
Flow:
|
||||
1) GET /web/1/coords/by_address?address=<urlenc> → geoHash + geo IDs
|
||||
2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions
|
||||
|
||||
Таблицы:
|
||||
avito_imv_evaluations (018_avito_imv_evaluations.sql)
|
||||
house_placement_history (017_house_placement_history.sql)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
AVITO_BASE = "https://www.avito.ru"
|
||||
COORDS_ENDPOINT = "/web/1/coords/by_address"
|
||||
IMV_ENDPOINT = "/web/1/realty-imv/get-data"
|
||||
|
||||
# Заголовки имитируют браузер на странице evaluation/realty
|
||||
_COMMON_HEADERS = {
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9",
|
||||
"Referer": "https://www.avito.ru/evaluation/realty",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class IMVAddressNotFoundError(Exception):
|
||||
"""Raised when /coords/by_address returns no geoHash for given address."""
|
||||
|
||||
|
||||
class IMVCityMismatchError(Exception):
|
||||
"""Raised when locationId не входит в ожидаемый город (sanity check)."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class IMVGeo:
|
||||
geo_hash: str # JWT — не сохраняем как отдельную колонку, только в raw_response
|
||||
lat: float | None = None
|
||||
lon: float | None = None
|
||||
avito_address_id: int | None = None
|
||||
avito_location_id: int | None = None
|
||||
avito_metro_id: int | None = None
|
||||
avito_district_id: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IMVPlacementHistoryItem:
|
||||
ext_item_id: str # str(item['id'])
|
||||
title: str | None = None
|
||||
rooms: int | None = None # парсится из title (опционально)
|
||||
area_m2: float | None = None # парсится из title (опционально)
|
||||
floor: int | None = None # парсится из title (опционально)
|
||||
total_floors: int | None = None # парсится из title (опционально)
|
||||
start_price: int | None = None
|
||||
start_price_date: date | None = None # unix → date
|
||||
last_price: int | None = None
|
||||
last_price_date: date | None = None
|
||||
removed_date: date | None = None # только IMV API (в widget housePlacementHistory нет)
|
||||
exposure_days: int | None = None
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IMVSuggestion:
|
||||
ext_item_id: str
|
||||
title: str | None = None
|
||||
address: str | None = None
|
||||
price_rub: int | None = None
|
||||
exposure_days: int | None = None
|
||||
publish_date: date | None = None
|
||||
item_url: str | None = None
|
||||
metro_name: str | None = None
|
||||
metro_distance: str | None = None
|
||||
metro_color: str | None = None
|
||||
has_good_price_badge: bool = False
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IMVEvaluation:
|
||||
cache_key: str # sha256
|
||||
address: str
|
||||
rooms: int
|
||||
area_m2: float
|
||||
floor: int
|
||||
floor_at_home: int
|
||||
house_type: str
|
||||
renovation_type: str
|
||||
has_balcony: bool
|
||||
has_loggia: bool
|
||||
|
||||
geo: IMVGeo
|
||||
recommended_price: int
|
||||
lower_price: int
|
||||
higher_price: int
|
||||
market_count: int | None = None
|
||||
|
||||
placement_history: list[IMVPlacementHistoryItem] = field(default_factory=list)
|
||||
suggestions: list[IMVSuggestion] = field(default_factory=list)
|
||||
search_similar_url: str | None = None
|
||||
raw_response: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cache key
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_imv_cache_key(
|
||||
address: str,
|
||||
rooms: int,
|
||||
area_m2: float,
|
||||
floor: int,
|
||||
floor_at_home: int,
|
||||
house_type: str,
|
||||
renovation_type: str,
|
||||
has_balcony: bool,
|
||||
has_loggia: bool,
|
||||
) -> str:
|
||||
"""Детерминированный sha256 ключ для 24h-кэша IMV оценки."""
|
||||
key_src = (
|
||||
f"{address}|{rooms}|{area_m2}|{floor}|{floor_at_home}"
|
||||
f"|{house_type}|{renovation_type}|{int(has_balcony)}|{int(has_loggia)}"
|
||||
)
|
||||
return hashlib.sha256(key_src.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _unix_to_date(ts: int | float | None) -> date | None:
|
||||
"""Конвертирует unix timestamp в date. Возвращает None для 0 и None."""
|
||||
if not ts:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromtimestamp(int(ts)).date()
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
# Паттерн для заголовка вида "2-к. квартира, 42 м², 4/5 эт."
|
||||
_TITLE_RE = re.compile(
|
||||
r"^(?P<rooms>\d+)-к[.\s].*?(?P<area>[\d,]+)\s*м².*?(?P<floor>\d+)/(?P<total>\d+)\s*эт",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_title(title: str | None) -> dict[str, int | float | None]:
|
||||
"""Парсит rooms, area_m2, floor, total_floors из Avito-заголовка объявления."""
|
||||
result: dict[str, int | float | None] = {
|
||||
"rooms": None,
|
||||
"area_m2": None,
|
||||
"floor": None,
|
||||
"total_floors": None,
|
||||
}
|
||||
if not title:
|
||||
return result
|
||||
m = _TITLE_RE.match(title.strip())
|
||||
if not m:
|
||||
return result
|
||||
try:
|
||||
result["rooms"] = int(m.group("rooms"))
|
||||
result["area_m2"] = float(m.group("area").replace(",", "."))
|
||||
result["floor"] = int(m.group("floor"))
|
||||
result["total_floors"] = int(m.group("total"))
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def _parse_placement_item(raw: dict[str, Any]) -> IMVPlacementHistoryItem:
|
||||
"""Парсит один элемент из placementHistory.items."""
|
||||
title = raw.get("title")
|
||||
parsed = _parse_title(title)
|
||||
|
||||
# Фильтруем itemImage из raw_payload (не нужны)
|
||||
clean_raw = {k: v for k, v in raw.items() if k not in ("itemImage",)}
|
||||
|
||||
return IMVPlacementHistoryItem(
|
||||
ext_item_id=str(raw["id"]),
|
||||
title=title,
|
||||
rooms=parsed["rooms"], # type: ignore[arg-type]
|
||||
area_m2=parsed["area_m2"], # type: ignore[arg-type]
|
||||
floor=parsed["floor"], # type: ignore[arg-type]
|
||||
total_floors=parsed["total_floors"], # type: ignore[arg-type]
|
||||
start_price=raw.get("startPrice"),
|
||||
start_price_date=_unix_to_date(raw.get("startPriceDate")),
|
||||
last_price=raw.get("lastPrice"),
|
||||
last_price_date=_unix_to_date(raw.get("lastPriceDate")),
|
||||
removed_date=_unix_to_date(raw.get("removedDate")),
|
||||
exposure_days=raw.get("exposure"),
|
||||
raw_payload=clean_raw,
|
||||
)
|
||||
|
||||
|
||||
def _parse_suggestion(raw: dict[str, Any]) -> IMVSuggestion:
|
||||
"""Парсит один элемент из suggestions.items."""
|
||||
metro = raw.get("metro") or {}
|
||||
colors: list[str] = metro.get("colors") or []
|
||||
|
||||
# Фильтруем imageLink из raw_payload
|
||||
clean_raw = {k: v for k, v in raw.items() if k not in ("imageLink",)}
|
||||
|
||||
return IMVSuggestion(
|
||||
ext_item_id=str(raw["id"]),
|
||||
title=raw.get("title"),
|
||||
address=raw.get("address"),
|
||||
price_rub=raw.get("price"),
|
||||
exposure_days=raw.get("exposure"),
|
||||
publish_date=_unix_to_date(raw.get("publishDate")),
|
||||
item_url=raw.get("itemLink"),
|
||||
metro_name=metro.get("name"),
|
||||
metro_distance=metro.get("distance"),
|
||||
metro_color=colors[0] if colors else None,
|
||||
has_good_price_badge=bool(raw.get("hasGoodPriceBadge", False)),
|
||||
raw_payload=clean_raw,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main async function
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def evaluate_via_imv(
|
||||
address: str,
|
||||
rooms: int,
|
||||
area_m2: float,
|
||||
floor: int,
|
||||
floor_at_home: int,
|
||||
house_type: str, # panel/brick/monolith/monolith_brick/block/wood
|
||||
renovation_type: str, # required/cosmetic/euro/designer
|
||||
has_balcony: bool,
|
||||
has_loggia: bool,
|
||||
*,
|
||||
cffi_session: Any | None = None,
|
||||
) -> IMVEvaluation:
|
||||
"""Avito IMV — 2 HTTP requests:
|
||||
|
||||
1) GET /web/1/coords/by_address?address=<urlenc> → geoHash + geo IDs
|
||||
2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions
|
||||
|
||||
Returns IMVEvaluation с уже вычисленным cache_key.
|
||||
|
||||
Raises:
|
||||
IMVAddressNotFoundError если geoHash пустой / не вернулся
|
||||
httpx.HTTPStatusError на 4xx/5xx
|
||||
"""
|
||||
# Импортируем здесь чтобы избежать циклических зависимостей при тестах без сети
|
||||
try:
|
||||
from curl_cffi.requests import AsyncSession as CffiAsyncSession
|
||||
|
||||
_own_session = False
|
||||
if cffi_session is None:
|
||||
cffi_session = CffiAsyncSession(impersonate="chrome120")
|
||||
_own_session = True
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"curl_cffi не установлен. Добавь 'curl-cffi>=0.7.0' в pyproject.toml."
|
||||
) from exc
|
||||
|
||||
try:
|
||||
geo = await _geocode(cffi_session, address)
|
||||
evaluation = await _imv_evaluate(
|
||||
cffi_session,
|
||||
geo=geo,
|
||||
address=address,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
floor_at_home=floor_at_home,
|
||||
house_type=house_type,
|
||||
renovation_type=renovation_type,
|
||||
has_balcony=has_balcony,
|
||||
has_loggia=has_loggia,
|
||||
)
|
||||
finally:
|
||||
if _own_session:
|
||||
await cffi_session.close()
|
||||
|
||||
return evaluation
|
||||
|
||||
|
||||
async def _geocode(session: Any, address: str) -> IMVGeo:
|
||||
"""Request 1: GET /web/1/coords/by_address → IMVGeo."""
|
||||
params = urlencode({"address": address})
|
||||
url = f"{AVITO_BASE}{COORDS_ENDPOINT}?{params}"
|
||||
logger.info("IMV geocode: address=%r", address)
|
||||
|
||||
resp = await session.get(url, headers=_COMMON_HEADERS)
|
||||
resp.raise_for_status()
|
||||
data: dict[str, Any] = resp.json()
|
||||
|
||||
geo_hash = data.get("geoHash")
|
||||
if not geo_hash:
|
||||
raise IMVAddressNotFoundError(
|
||||
f"Avito /coords/by_address не вернул geoHash для адреса: {address!r}"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"IMV geocode OK: locationId=%s lat=%s lon=%s",
|
||||
data.get("locationId"),
|
||||
data.get("latitude"),
|
||||
data.get("longitude"),
|
||||
)
|
||||
|
||||
return IMVGeo(
|
||||
geo_hash=geo_hash,
|
||||
lat=data.get("latitude"),
|
||||
lon=data.get("longitude"),
|
||||
avito_address_id=data.get("addressId"),
|
||||
avito_location_id=data.get("locationId"),
|
||||
avito_metro_id=data.get("metroId"),
|
||||
avito_district_id=data.get("districtId"),
|
||||
)
|
||||
|
||||
|
||||
async def _imv_evaluate(
|
||||
session: Any,
|
||||
*,
|
||||
geo: IMVGeo,
|
||||
address: str,
|
||||
rooms: int,
|
||||
area_m2: float,
|
||||
floor: int,
|
||||
floor_at_home: int,
|
||||
house_type: str,
|
||||
renovation_type: str,
|
||||
has_balcony: bool,
|
||||
has_loggia: bool,
|
||||
) -> IMVEvaluation:
|
||||
"""Request 2: POST /web/1/realty-imv/get-data → IMVEvaluation."""
|
||||
url = f"{AVITO_BASE}{IMV_ENDPOINT}"
|
||||
body = {
|
||||
"geoHash": geo.geo_hash,
|
||||
"floor": floor,
|
||||
"floorAtHome": floor_at_home,
|
||||
"rooms": rooms,
|
||||
"area": area_m2,
|
||||
"houseType": house_type,
|
||||
"renovationType": renovation_type,
|
||||
"hasLoggia": has_loggia,
|
||||
"hasBalcony": has_balcony,
|
||||
}
|
||||
headers = {**_COMMON_HEADERS, "Content-Type": "application/json"}
|
||||
|
||||
logger.info(
|
||||
"IMV evaluate: address=%r rooms=%d area=%.1f floor=%d/%d",
|
||||
address,
|
||||
rooms,
|
||||
area_m2,
|
||||
floor,
|
||||
floor_at_home,
|
||||
)
|
||||
|
||||
resp = await session.post(url, json=body, headers=headers)
|
||||
resp.raise_for_status()
|
||||
data: dict[str, Any] = resp.json()
|
||||
|
||||
price_block: dict[str, Any] = data.get("price") or {}
|
||||
recommended_price: int = price_block.get("recommendedPrice") or 0
|
||||
lower_price: int = price_block.get("lowerPrice") or 0
|
||||
higher_price: int = price_block.get("higherPrice") or 0
|
||||
market_count: int | None = price_block.get("count")
|
||||
|
||||
# placementHistory
|
||||
history_block: dict[str, Any] = data.get("placementHistory") or {}
|
||||
placement_history: list[IMVPlacementHistoryItem] = []
|
||||
for raw_item in history_block.get("items") or []:
|
||||
try:
|
||||
placement_history.append(_parse_placement_item(raw_item))
|
||||
except (KeyError, TypeError) as exc:
|
||||
logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc)
|
||||
|
||||
# suggestions
|
||||
suggestions_block: dict[str, Any] = data.get("suggestions") or {}
|
||||
suggestions: list[IMVSuggestion] = []
|
||||
for raw_sugg in suggestions_block.get("items") or []:
|
||||
try:
|
||||
suggestions.append(_parse_suggestion(raw_sugg))
|
||||
except (KeyError, TypeError) as exc:
|
||||
logger.warning("IMV: не удалось распарсить suggestion: %s", exc)
|
||||
|
||||
search_similar_url: str | None = suggestions_block.get("searchSimilarUrl")
|
||||
|
||||
logger.info(
|
||||
"IMV evaluate OK: recommended=%d range=(%d, %d) count=%s history=%d suggestions=%d",
|
||||
recommended_price,
|
||||
lower_price,
|
||||
higher_price,
|
||||
market_count,
|
||||
len(placement_history),
|
||||
len(suggestions),
|
||||
)
|
||||
|
||||
cache_key = compute_imv_cache_key(
|
||||
address=address,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
floor_at_home=floor_at_home,
|
||||
house_type=house_type,
|
||||
renovation_type=renovation_type,
|
||||
has_balcony=has_balcony,
|
||||
has_loggia=has_loggia,
|
||||
)
|
||||
|
||||
return IMVEvaluation(
|
||||
cache_key=cache_key,
|
||||
address=address,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
floor_at_home=floor_at_home,
|
||||
house_type=house_type,
|
||||
renovation_type=renovation_type,
|
||||
has_balcony=has_balcony,
|
||||
has_loggia=has_loggia,
|
||||
geo=geo,
|
||||
recommended_price=recommended_price,
|
||||
lower_price=lower_price,
|
||||
higher_price=higher_price,
|
||||
market_count=market_count,
|
||||
placement_history=placement_history,
|
||||
suggestions=suggestions,
|
||||
search_similar_url=search_similar_url,
|
||||
raw_response=data,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DB save functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def save_imv_evaluation(
|
||||
db: Session,
|
||||
e: IMVEvaluation,
|
||||
*,
|
||||
estimate_id: UUID | None = None,
|
||||
) -> int:
|
||||
"""INSERT в avito_imv_evaluations ON CONFLICT(cache_key) DO UPDATE.
|
||||
|
||||
Колонки строго по 018_avito_imv_evaluations.sql DDL.
|
||||
estimate_id — nullable FK на trade_in_estimates.
|
||||
|
||||
Returns: id вставленной/обновлённой строки.
|
||||
"""
|
||||
row = db.execute(
|
||||
text("""
|
||||
INSERT INTO avito_imv_evaluations (
|
||||
cache_key, estimate_id,
|
||||
address, rooms, area_m2, floor, floor_at_home,
|
||||
house_type, renovation_type, has_balcony, has_loggia,
|
||||
geo_hash,
|
||||
lat, lon, avito_address_id, avito_location_id,
|
||||
avito_metro_id, avito_district_id,
|
||||
recommended_price, lower_price, higher_price, market_count,
|
||||
raw_response, fetched_at
|
||||
) VALUES (
|
||||
:cache_key, CAST(:estimate_id AS uuid),
|
||||
:address, :rooms, :area_m2, :floor, :floor_at_home,
|
||||
:house_type, :renovation_type, :has_balcony, :has_loggia,
|
||||
:geo_hash,
|
||||
:lat, :lon, :avito_address_id, :avito_location_id,
|
||||
:avito_metro_id, :avito_district_id,
|
||||
:recommended_price, :lower_price, :higher_price, :market_count,
|
||||
CAST(:raw_response AS jsonb), NOW()
|
||||
)
|
||||
ON CONFLICT (cache_key) DO UPDATE SET
|
||||
estimate_id = COALESCE(EXCLUDED.estimate_id, avito_imv_evaluations.estimate_id),
|
||||
recommended_price = EXCLUDED.recommended_price,
|
||||
lower_price = EXCLUDED.lower_price,
|
||||
higher_price = EXCLUDED.higher_price,
|
||||
market_count = EXCLUDED.market_count,
|
||||
raw_response = EXCLUDED.raw_response,
|
||||
fetched_at = NOW()
|
||||
RETURNING id
|
||||
"""),
|
||||
{
|
||||
"cache_key": e.cache_key,
|
||||
"estimate_id": str(estimate_id) if estimate_id else None,
|
||||
"address": e.address,
|
||||
"rooms": e.rooms,
|
||||
"area_m2": e.area_m2,
|
||||
"floor": e.floor,
|
||||
"floor_at_home": e.floor_at_home,
|
||||
"house_type": e.house_type,
|
||||
"renovation_type": e.renovation_type,
|
||||
"has_balcony": e.has_balcony,
|
||||
"has_loggia": e.has_loggia,
|
||||
"geo_hash": e.geo.geo_hash,
|
||||
"lat": e.geo.lat,
|
||||
"lon": e.geo.lon,
|
||||
"avito_address_id": e.geo.avito_address_id,
|
||||
"avito_location_id": e.geo.avito_location_id,
|
||||
"avito_metro_id": e.geo.avito_metro_id,
|
||||
"avito_district_id": e.geo.avito_district_id,
|
||||
"recommended_price": e.recommended_price,
|
||||
"lower_price": e.lower_price,
|
||||
"higher_price": e.higher_price,
|
||||
"market_count": e.market_count,
|
||||
"raw_response": (
|
||||
json.dumps(e.raw_response, ensure_ascii=False) if e.raw_response else None
|
||||
),
|
||||
},
|
||||
).fetchone()
|
||||
db.commit()
|
||||
return row.id if row else 0
|
||||
|
||||
|
||||
def save_imv_placement_history(
|
||||
db: Session,
|
||||
eval_id: int,
|
||||
items: list[IMVPlacementHistoryItem],
|
||||
) -> int:
|
||||
"""INSERT в house_placement_history (source='avito_imv') с removed_date.
|
||||
|
||||
Колонки строго по 017_house_placement_history.sql DDL.
|
||||
Колонка scraped_at (не fetched_at) — по DDL таблицы.
|
||||
house_id — NULL (backfill через отдельный процесс).
|
||||
eval_id принимается для логирования (не сохраняется — нет FK в 017).
|
||||
|
||||
ON CONFLICT (source, ext_item_id) DO UPDATE — refresh dates.
|
||||
Returns: количество вставленных/обновлённых строк.
|
||||
"""
|
||||
count = 0
|
||||
for item in items:
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO house_placement_history (
|
||||
source, ext_item_id,
|
||||
title, rooms, area_m2, floor, total_floors,
|
||||
start_price, start_price_date,
|
||||
last_price, last_price_date,
|
||||
removed_date, exposure_days,
|
||||
raw_payload, scraped_at
|
||||
) VALUES (
|
||||
'avito_imv', :ext_item_id,
|
||||
:title, :rooms, :area_m2, :floor, :total_floors,
|
||||
:start_price, :start_price_date,
|
||||
:last_price, :last_price_date,
|
||||
:removed_date, :exposure_days,
|
||||
CAST(:raw_payload AS jsonb), NOW()
|
||||
)
|
||||
ON CONFLICT (source, ext_item_id) DO UPDATE SET
|
||||
last_price = EXCLUDED.last_price,
|
||||
last_price_date = EXCLUDED.last_price_date,
|
||||
removed_date = EXCLUDED.removed_date,
|
||||
exposure_days = EXCLUDED.exposure_days,
|
||||
raw_payload = EXCLUDED.raw_payload,
|
||||
scraped_at = NOW()
|
||||
"""),
|
||||
{
|
||||
"ext_item_id": item.ext_item_id,
|
||||
"title": item.title,
|
||||
"rooms": item.rooms,
|
||||
"area_m2": item.area_m2,
|
||||
"floor": item.floor,
|
||||
"total_floors": item.total_floors,
|
||||
"start_price": item.start_price,
|
||||
"start_price_date": item.start_price_date,
|
||||
"last_price": item.last_price,
|
||||
"last_price_date": item.last_price_date,
|
||||
"removed_date": item.removed_date,
|
||||
"exposure_days": item.exposure_days,
|
||||
"raw_payload": (
|
||||
json.dumps(item.raw_payload, ensure_ascii=False)
|
||||
if item.raw_payload
|
||||
else None
|
||||
),
|
||||
},
|
||||
)
|
||||
count += 1
|
||||
|
||||
if count:
|
||||
db.commit()
|
||||
logger.info("IMV placement history saved: eval_id=%d items=%d", eval_id, count)
|
||||
|
||||
return count
|
||||
0
tradein-mvp/backend/tests/__init__.py
Normal file
0
tradein-mvp/backend/tests/__init__.py
Normal file
287
tradein-mvp/backend/tests/test_avito_imv_parse.py
Normal file
287
tradein-mvp/backend/tests/test_avito_imv_parse.py
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
"""Offline smoke tests для avito_imv модуля.
|
||||
|
||||
Без сети — только cache_key вычисление, парсинг заголовков, unix→date, dataclass конструирование.
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.avito_imv import (
|
||||
IMVEvaluation,
|
||||
IMVGeo,
|
||||
IMVPlacementHistoryItem,
|
||||
IMVSuggestion,
|
||||
_parse_placement_item,
|
||||
_parse_suggestion,
|
||||
_parse_title,
|
||||
_unix_to_date,
|
||||
compute_imv_cache_key,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cache_key
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cache_key_deterministic():
|
||||
k1 = compute_imv_cache_key(
|
||||
"ЕКБ ул. Чайковского, 66А", 2, 42.0, 4, 5, "panel", "cosmetic", True, False
|
||||
)
|
||||
k2 = compute_imv_cache_key(
|
||||
"ЕКБ ул. Чайковского, 66А", 2, 42.0, 4, 5, "panel", "cosmetic", True, False
|
||||
)
|
||||
assert k1 == k2
|
||||
assert len(k1) == 64 # sha256 hex
|
||||
|
||||
|
||||
def test_cache_key_differs_by_rooms():
|
||||
k1 = compute_imv_cache_key(
|
||||
"ЕКБ ул. Чайковского, 66А", 2, 42.0, 4, 5, "panel", "cosmetic", True, False
|
||||
)
|
||||
k3 = compute_imv_cache_key(
|
||||
"ЕКБ ул. Чайковского, 66А", 3, 42.0, 4, 5, "panel", "cosmetic", True, False
|
||||
)
|
||||
assert k1 != k3
|
||||
|
||||
|
||||
def test_cache_key_differs_by_balcony():
|
||||
k_with = compute_imv_cache_key(
|
||||
"ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", True, False
|
||||
)
|
||||
k_without = compute_imv_cache_key(
|
||||
"ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", False, False
|
||||
)
|
||||
assert k_with != k_without
|
||||
|
||||
|
||||
def test_cache_key_differs_by_address():
|
||||
k1 = compute_imv_cache_key("ул. А, 1", 2, 42.0, 4, 5, "panel", "cosmetic", True, False)
|
||||
k2 = compute_imv_cache_key("ул. Б, 2", 2, 42.0, 4, 5, "panel", "cosmetic", True, False)
|
||||
assert k1 != k2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _unix_to_date
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unix_date_none():
|
||||
assert _unix_to_date(None) is None
|
||||
|
||||
|
||||
def test_unix_date_zero():
|
||||
assert _unix_to_date(0) is None
|
||||
|
||||
|
||||
def test_unix_date_positive():
|
||||
result = _unix_to_date(1715000000)
|
||||
assert isinstance(result, date)
|
||||
# 1715000000 примерно 2024-05-06 UTC+5 (Екатеринбург)
|
||||
assert result.year == 2024
|
||||
assert result.month in (5, 6) # зависит от локали
|
||||
|
||||
|
||||
def test_unix_date_float():
|
||||
result = _unix_to_date(1697000000.0)
|
||||
assert isinstance(result, date)
|
||||
assert result.year == 2023
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_title
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_title_full():
|
||||
parsed = _parse_title("2-к. квартира, 42 м², 4/5 эт.")
|
||||
assert parsed["rooms"] == 2
|
||||
assert parsed["area_m2"] == 42.0
|
||||
assert parsed["floor"] == 4
|
||||
assert parsed["total_floors"] == 5
|
||||
|
||||
|
||||
def test_parse_title_comma_decimal():
|
||||
parsed = _parse_title("1-к. квартира, 27,1 м², 2/5 эт.")
|
||||
assert parsed["rooms"] == 1
|
||||
assert parsed["area_m2"] == pytest.approx(27.1)
|
||||
assert parsed["floor"] == 2
|
||||
assert parsed["total_floors"] == 5
|
||||
|
||||
|
||||
def test_parse_title_none():
|
||||
parsed = _parse_title(None)
|
||||
assert parsed["rooms"] is None
|
||||
assert parsed["area_m2"] is None
|
||||
|
||||
|
||||
def test_parse_title_no_match():
|
||||
parsed = _parse_title("студия, 20 м²")
|
||||
assert parsed["rooms"] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_placement_item
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_placement_item_full():
|
||||
raw = {
|
||||
"id": 9876543210,
|
||||
"title": "2-к. квартира, 42 м², 4/5 эт.",
|
||||
"startPrice": 6500000,
|
||||
"startPriceDate": 1697000000,
|
||||
"lastPrice": 6290000,
|
||||
"lastPriceDate": 1715000000,
|
||||
"removedDate": 1715500000,
|
||||
"exposure": 210,
|
||||
"itemImage": {"url": "https://avito.ru/img.jpg"}, # должен быть отфильтрован
|
||||
}
|
||||
item = _parse_placement_item(raw)
|
||||
|
||||
assert item.ext_item_id == "9876543210"
|
||||
assert item.title == "2-к. квартира, 42 м², 4/5 эт."
|
||||
assert item.rooms == 2
|
||||
assert item.area_m2 == 42.0
|
||||
assert item.floor == 4
|
||||
assert item.total_floors == 5
|
||||
assert item.start_price == 6500000
|
||||
assert isinstance(item.start_price_date, date)
|
||||
assert item.last_price == 6290000
|
||||
assert isinstance(item.last_price_date, date)
|
||||
assert isinstance(item.removed_date, date)
|
||||
assert item.exposure_days == 210
|
||||
# itemImage должен быть отфильтрован из raw_payload
|
||||
assert item.raw_payload is not None
|
||||
assert "itemImage" not in item.raw_payload
|
||||
|
||||
|
||||
def test_parse_placement_item_no_removed_date():
|
||||
raw = {
|
||||
"id": 111,
|
||||
"title": "1-к. квартира, 30 м², 2/9 эт.",
|
||||
"startPrice": 3000000,
|
||||
"startPriceDate": 1697000000,
|
||||
"lastPrice": 2900000,
|
||||
"lastPriceDate": 1715000000,
|
||||
"exposure": 50,
|
||||
}
|
||||
item = _parse_placement_item(raw)
|
||||
assert item.removed_date is None
|
||||
assert item.ext_item_id == "111"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_suggestion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_suggestion_full():
|
||||
raw = {
|
||||
"id": 7986882804,
|
||||
"title": "2-к. квартира, 42 м², 4/5 эт.",
|
||||
"address": "Авиационная улица, 63к2, Екатеринбург",
|
||||
"price": 6300000,
|
||||
"exposure": 14,
|
||||
"publishDate": 1716000000,
|
||||
"itemLink": "/ekaterinburg/kvartiry/some-path",
|
||||
"metro": {"name": "Чкаловская", "distance": "2,7 км", "colors": ["#cf2734"]},
|
||||
"hasGoodPriceBadge": False,
|
||||
"imageLink": "https://avito.ru/img.jpg", # должен быть отфильтрован
|
||||
}
|
||||
sugg = _parse_suggestion(raw)
|
||||
|
||||
assert sugg.ext_item_id == "7986882804"
|
||||
assert sugg.price_rub == 6300000
|
||||
assert sugg.metro_name == "Чкаловская"
|
||||
assert sugg.metro_distance == "2,7 км"
|
||||
assert sugg.metro_color == "#cf2734"
|
||||
assert sugg.item_url == "/ekaterinburg/kvartiry/some-path"
|
||||
assert sugg.has_good_price_badge is False
|
||||
assert isinstance(sugg.publish_date, date)
|
||||
# imageLink должен быть отфильтрован
|
||||
assert sugg.raw_payload is not None
|
||||
assert "imageLink" not in sugg.raw_payload
|
||||
|
||||
|
||||
def test_parse_suggestion_no_metro():
|
||||
raw = {
|
||||
"id": 999,
|
||||
"title": "1-к. квартира, 25 м², 3/5 эт.",
|
||||
"price": 4000000,
|
||||
"exposure": 5,
|
||||
"publishDate": 1716000000,
|
||||
"itemLink": "/path",
|
||||
}
|
||||
sugg = _parse_suggestion(raw)
|
||||
assert sugg.metro_name is None
|
||||
assert sugg.metro_color is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IMVEvaluation dataclass конструирование
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_dataclass_minimal_construct():
|
||||
e = IMVEvaluation(
|
||||
cache_key="a" * 64,
|
||||
address="X",
|
||||
rooms=2,
|
||||
area_m2=42.0,
|
||||
floor=4,
|
||||
floor_at_home=5,
|
||||
house_type="panel",
|
||||
renovation_type="cosmetic",
|
||||
has_balcony=True,
|
||||
has_loggia=False,
|
||||
geo=IMVGeo(geo_hash="JWT.TOKEN.HERE"),
|
||||
recommended_price=6_290_000,
|
||||
lower_price=6_100_000,
|
||||
higher_price=6_600_000,
|
||||
)
|
||||
assert e.recommended_price == 6_290_000
|
||||
assert e.lower_price == 6_100_000
|
||||
assert e.higher_price == 6_600_000
|
||||
assert e.geo.geo_hash == "JWT.TOKEN.HERE"
|
||||
assert e.placement_history == []
|
||||
assert e.suggestions == []
|
||||
assert e.market_count is None
|
||||
assert e.raw_response is None
|
||||
|
||||
|
||||
def test_dataclass_with_history():
|
||||
item = IMVPlacementHistoryItem(
|
||||
ext_item_id="123",
|
||||
title="2-к. квартира, 42 м², 4/5 эт.",
|
||||
rooms=2,
|
||||
area_m2=42.0,
|
||||
floor=4,
|
||||
total_floors=5,
|
||||
start_price=6500000,
|
||||
last_price=6290000,
|
||||
removed_date=date(2024, 5, 12),
|
||||
exposure_days=210,
|
||||
)
|
||||
e = IMVEvaluation(
|
||||
cache_key="b" * 64,
|
||||
address="Y",
|
||||
rooms=2,
|
||||
area_m2=42.0,
|
||||
floor=4,
|
||||
floor_at_home=5,
|
||||
house_type="panel",
|
||||
renovation_type="euro",
|
||||
has_balcony=False,
|
||||
has_loggia=True,
|
||||
geo=IMVGeo(geo_hash="JWT2"),
|
||||
recommended_price=6_000_000,
|
||||
lower_price=5_800_000,
|
||||
higher_price=6_300_000,
|
||||
market_count=800,
|
||||
placement_history=[item],
|
||||
)
|
||||
assert len(e.placement_history) == 1
|
||||
assert e.placement_history[0].removed_date == date(2024, 5, 12)
|
||||
assert e.market_count == 800
|
||||
Loading…
Add table
Reference in a new issue