diff --git a/tradein-mvp/backend/app/services/scrapers/avito_imv.py b/tradein-mvp/backend/app/services/scrapers/avito_imv.py index 815ce4c0..bd9b8aa6 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_imv.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_imv.py @@ -1,10 +1,11 @@ -"""avito_imv.py — Avito IMV evaluation API client (2-request flow). +"""avito_imv.py — Avito IMV evaluation API client (3-request flow, 2026-05+). Stage 2d: geocode + IMV evaluation. -Flow: - 1) GET /web/1/coords/by_address?address= → geoHash + geo IDs - 2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions +Flow (current contract): + 1) GET /web/1/coords/by_address?address= → normalize + lat/lon + 2) POST /js/v2/geo/position → rich JWT (geoFieldsHash) + 3) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions Таблицы: avito_imv_evaluations (018_avito_imv_evaluations.sql) @@ -30,6 +31,7 @@ logger = logging.getLogger(__name__) AVITO_BASE = "https://www.avito.ru" COORDS_ENDPOINT = "/web/1/coords/by_address" +POSITION_ENDPOINT = "/js/v2/geo/position" IMV_ENDPOINT = "/web/1/realty-imv/get-data" # Заголовки имитируют браузер на странице evaluation/realty @@ -271,16 +273,19 @@ async def evaluate_via_imv( *, cffi_session: Any | None = None, ) -> IMVEvaluation: - """Avito IMV — 2 HTTP requests: + """Avito IMV — 3 HTTP requests (current contract 2026-05+): - 1) GET /web/1/coords/by_address?address= → geoHash + geo IDs - 2) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions + 1) GET /web/1/coords/by_address?address= → normalize + lat/lon + 2) POST /js/v2/geo/position → rich JWT (geoFieldsHash) + 3) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions Returns IMVEvaluation с уже вычисленным cache_key. Raises: - IMVAddressNotFoundError если geoHash пустой / не вернулся - httpx.HTTPStatusError на 4xx/5xx + IMVAddressNotFoundError если point/normalizedAddress пустые (step 1) + или geoFieldsHash не вернулся (step 2) + IMVAuthError на 401/403 + IMVTransientError на 5xx / network errors """ # Импортируем здесь чтобы избежать циклических зависимостей при тестах без сети try: @@ -333,40 +338,83 @@ def _raise_for_status_categorized(resp: Any, context: str) -> None: async def _geocode(session: Any, address: str) -> IMVGeo: - """Request 1: GET /web/1/coords/by_address → IMVGeo.""" + """Two-step Avito IMV geocoder (current contract 2026-05+). + + Step A: GET /web/1/coords/by_address → normalize + lat/lon + Step B: POST /js/v2/geo/position → rich JWT (geoFieldsHash) with + addressId/locationId/metroId/districtId + + JWT from step B is what /realty-imv/get-data expects as `geoHash`. + """ + # Step A — normalize + coords params = urlencode({"address": address}) - url = f"{AVITO_BASE}{COORDS_ENDPOINT}?{params}" - logger.info("IMV geocode: address=%r", address) + url_a = f"{AVITO_BASE}{COORDS_ENDPOINT}?{params}" + logger.info("IMV geocode A: address=%r", address) try: - resp = await session.get(url, headers=_COMMON_HEADERS) + resp_a = await session.get(url_a, headers=_COMMON_HEADERS) except Exception as exc: - # Timeout / connection error — retryable - raise IMVTransientError(f"Avito geocode network error: {exc}") from exc - _raise_for_status_categorized(resp, "geocode") - data: dict[str, Any] = resp.json() + raise IMVTransientError(f"Avito geocode A network error: {exc}") from exc + _raise_for_status_categorized(resp_a, "geocode-A") + data_a: dict[str, Any] = resp_a.json() - geo_hash = data.get("geoHash") - if not geo_hash: + point = data_a.get("point") or {} + lat = point.get("latitude") + lon = point.get("longitude") + normalized = data_a.get("normalizedAddress") + + if not lat or not lon or not normalized: raise IMVAddressNotFoundError( - f"Avito /coords/by_address не вернул geoHash для адреса: {address!r}" + f"Avito /coords/by_address не вернул point для {address!r}" + ) + + # Step B — rich JWT + url_b = f"{AVITO_BASE}{POSITION_ENDPOINT}" + payload_b = { + "address": normalized, + "addressId": "", + "categoryId": 24, # 24 = недвижимость / квартиры + "isRadius": False, + "latitude": lat, + "longitude": lon, + } + logger.info("IMV geocode B: position for %r", normalized[:60]) + + try: + resp_b = await session.post( + url_b, + headers={**_COMMON_HEADERS, "Content-Type": "application/json"}, + json=payload_b, + ) + except Exception as exc: + raise IMVTransientError(f"Avito geocode B network error: {exc}") from exc + _raise_for_status_categorized(resp_b, "geocode-B") + data_b: dict[str, Any] = resp_b.json() + + jwt = data_b.get("geoFieldsHash") + if not jwt: + raise IMVAddressNotFoundError( + f"Avito /js/v2/geo/position не вернул geoFieldsHash для {normalized!r}: " + f"errorText={data_b.get('errorText')!r}" ) logger.info( - "IMV geocode OK: locationId=%s lat=%s lon=%s", - data.get("locationId"), - data.get("latitude"), - data.get("longitude"), + "IMV geocode OK: addressId=%s locationId=%s metroId=%s lat=%s lon=%s", + data_b.get("addressId"), + data_b.get("locationId"), + data_b.get("metroId"), + lat, + lon, ) return IMVGeo( - geo_hash=geo_hash, - lat=data.get("latitude"), - lon=data.get("longitude"), - avito_address_id=data.get("addressId"), - avito_location_id=data.get("locationId"), - avito_metro_id=data.get("metroId"), - avito_district_id=data.get("districtId"), + geo_hash=jwt, + lat=lat, + lon=lon, + avito_address_id=data_b.get("addressId"), + avito_location_id=data_b.get("locationId"), + avito_metro_id=data_b.get("metroId"), + avito_district_id=data_b.get("districtId"), )