fix(tradein-imv): harden Avito IMV anti-bot path + offline contract tests (#565) #620
4 changed files with 469 additions and 50 deletions
|
|
@ -2,10 +2,15 @@
|
|||
|
||||
Stage 2d: geocode + IMV evaluation.
|
||||
|
||||
Flow (current contract):
|
||||
Flow (current contract, подтверждён живым capture 2026-05 — fixtures
|
||||
tests/fixtures/avito_imv_geo_position.json + avito_imv_getdata.json):
|
||||
0) GET /evaluation/realty → warm-up (seed anti-bot cookies)
|
||||
1) GET /web/1/coords/by_address?address=<urlenc> → normalize + lat/lon
|
||||
2) POST /js/v2/geo/position → rich JWT (geoFieldsHash)
|
||||
3) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions
|
||||
3) POST /web/1/realty-imv/get-data → price + placementHistory? + suggestions
|
||||
|
||||
Anti-bot: zerkalim AvitoScraper (avito.py) — chrome120 TLS + document-заголовки +
|
||||
warm-up GET + XHR Sec-Fetch заголовки. Bare-session XHR ловят 403 с server-IP.
|
||||
|
||||
Таблицы:
|
||||
avito_imv_evaluations (018_avito_imv_evaluations.sql)
|
||||
|
|
@ -33,14 +38,37 @@ AVITO_BASE = "https://www.avito.ru"
|
|||
COORDS_ENDPOINT = "/web/1/coords/by_address"
|
||||
POSITION_ENDPOINT = "/js/v2/geo/position"
|
||||
IMV_ENDPOINT = "/web/1/realty-imv/get-data"
|
||||
# Тёплая страница для seed anti-bot cookies (srv_id/_avisc/u/...) перед XHR.
|
||||
WARMUP_URL = f"{AVITO_BASE}/evaluation/realty"
|
||||
|
||||
# Заголовки имитируют браузер на странице evaluation/realty
|
||||
# Заголовки document-навигации (warm-up GET) — зеркалят production-набор
|
||||
# из avito.py (AvitoScraper.__aenter__). Нужны чтобы пройти TLS+header anti-bot.
|
||||
_DOC_HEADERS = {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
"Cache-Control": "max-age=0",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
# Заголовки XHR/fetch для API-шагов (coords/position/get-data). Имитируют
|
||||
# fetch() со страницы /evaluation/realty: JSON Accept + same-origin Sec-Fetch.
|
||||
_COMMON_HEADERS = {
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
"Referer": "https://www.avito.ru/evaluation/realty",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-origin",
|
||||
}
|
||||
|
||||
# Timeout для всех Avito-вызовов (без него curl_cffi висит на anti-bot stall).
|
||||
_HTTP_TIMEOUT_SEC = 25
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
|
|
@ -255,6 +283,83 @@ def _parse_suggestion(raw: dict[str, Any]) -> IMVSuggestion:
|
|||
)
|
||||
|
||||
|
||||
def _parse_geo_position(data_b: dict[str, Any], *, lat: float, lon: float) -> IMVGeo:
|
||||
"""Парсит ответ POST /js/v2/geo/position (step B) → IMVGeo.
|
||||
|
||||
Контракт (подтверждён живым capture 2026-05, fixture avito_imv_geo_position.json):
|
||||
{address, addressId, districtId, geoFieldsHash, latitude, longitude,
|
||||
locationId, parentLocationId, metroId?, errorText, ...}
|
||||
|
||||
`geoFieldsHash` — JWT, payload которого кодирует {latitude, longitude,
|
||||
address, addressId, locationId, districtId}; это то, что step C ожидает как
|
||||
`geoHash`. metroId присутствует только для метро-городов (None для ЕКБ-окраин).
|
||||
|
||||
Raises:
|
||||
IMVAddressNotFoundError если geoFieldsHash пустой/отсутствует.
|
||||
"""
|
||||
jwt = data_b.get("geoFieldsHash")
|
||||
if not jwt:
|
||||
raise IMVAddressNotFoundError(
|
||||
f"Avito /js/v2/geo/position не вернул geoFieldsHash для "
|
||||
f"{str(data_b.get('address'))[:60]!r}: errorText={data_b.get('errorText')!r}"
|
||||
)
|
||||
return IMVGeo(
|
||||
geo_hash=jwt,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
avito_address_id=data_b.get("addressId"),
|
||||
avito_location_id=data_b.get("locationId"),
|
||||
avito_metro_id=data_b.get("metroId"),
|
||||
avito_district_id=data_b.get("districtId"),
|
||||
)
|
||||
|
||||
|
||||
def _parse_price(data: dict[str, Any]) -> tuple[int, int, int, int | None]:
|
||||
"""Парсит top-level `price` блок из realty-imv/get-data (step C).
|
||||
|
||||
Контракт (fixture avito_imv_getdata.json):
|
||||
price = {recommendedPrice, lowerPrice, higherPrice, count, addItemUrl}
|
||||
|
||||
Returns: (recommended_price, lower_price, higher_price, market_count).
|
||||
Отсутствующие цены → 0 (caller трактует как "нет оценки"); count → None.
|
||||
"""
|
||||
price_block: dict[str, Any] = data.get("price") or {}
|
||||
recommended_price: int = price_block.get("recommendedPrice") or 0
|
||||
lower_price: int = price_block.get("lowerPrice") or 0
|
||||
higher_price: int = price_block.get("higherPrice") or 0
|
||||
market_count: int | None = price_block.get("count")
|
||||
return recommended_price, lower_price, higher_price, market_count
|
||||
|
||||
|
||||
def _parse_placement_history(data: dict[str, Any]) -> list[IMVPlacementHistoryItem]:
|
||||
"""Парсит опциональный `placementHistory.items` блок (step C).
|
||||
|
||||
placementHistory отсутствует для части адресов (живой capture: проспект
|
||||
Ленина вернул только itemParams/price/suggestions) — тогда возвращаем [].
|
||||
Битые элементы пропускаем с warning, не роняя весь ответ.
|
||||
"""
|
||||
history_block: dict[str, Any] = data.get("placementHistory") or {}
|
||||
items: list[IMVPlacementHistoryItem] = []
|
||||
for raw_item in history_block.get("items") or []:
|
||||
try:
|
||||
items.append(_parse_placement_item(raw_item))
|
||||
except (KeyError, TypeError) as exc:
|
||||
logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc)
|
||||
return items
|
||||
|
||||
|
||||
def _parse_suggestions(data: dict[str, Any]) -> tuple[list[IMVSuggestion], str | None]:
|
||||
"""Парсит `suggestions` блок (step C) → (items, searchSimilarUrl)."""
|
||||
suggestions_block: dict[str, Any] = data.get("suggestions") or {}
|
||||
suggestions: list[IMVSuggestion] = []
|
||||
for raw_sugg in suggestions_block.get("items") or []:
|
||||
try:
|
||||
suggestions.append(_parse_suggestion(raw_sugg))
|
||||
except (KeyError, TypeError) as exc:
|
||||
logger.warning("IMV: не удалось распарсить suggestion: %s", exc)
|
||||
return suggestions, suggestions_block.get("searchSimilarUrl")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main async function
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -293,7 +398,14 @@ async def evaluate_via_imv(
|
|||
|
||||
_own_session = False
|
||||
if cffi_session is None:
|
||||
cffi_session = CffiAsyncSession(impersonate="chrome120")
|
||||
# Зеркалим production-набор из AvitoScraper.__aenter__ (avito.py:150-162):
|
||||
# chrome120 TLS + document-заголовки + timeout. Затем warm-up GET для
|
||||
# seed anti-bot cookies — bare-session XHR Avito банит на server-IP.
|
||||
cffi_session = CffiAsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=_HTTP_TIMEOUT_SEC,
|
||||
headers=_DOC_HEADERS,
|
||||
)
|
||||
_own_session = True
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
|
|
@ -301,6 +413,8 @@ async def evaluate_via_imv(
|
|||
) from exc
|
||||
|
||||
try:
|
||||
if _own_session:
|
||||
await _warmup(cffi_session)
|
||||
geo = await _geocode(cffi_session, address)
|
||||
evaluation = await _imv_evaluate(
|
||||
cffi_session,
|
||||
|
|
@ -322,6 +436,22 @@ async def evaluate_via_imv(
|
|||
return evaluation
|
||||
|
||||
|
||||
async def _warmup(session: Any) -> None:
|
||||
"""GET /evaluation/realty чтобы seed anti-bot cookies (srv_id/_avisc/u/...).
|
||||
|
||||
Без warm-up XHR-шаги (coords/position/get-data) c server-IP ловят 403 anti-bot.
|
||||
Best-effort: не роняем оценку если страница недоступна — последующие шаги
|
||||
всё равно попробуют и дадут типизированную ошибку.
|
||||
"""
|
||||
try:
|
||||
resp = await session.get(WARMUP_URL, headers=_DOC_HEADERS)
|
||||
logger.info("IMV warm-up GET /evaluation/realty → HTTP %d", resp.status_code)
|
||||
except Exception as exc:
|
||||
# warm-up опционален: логируем и продолжаем — шаги ниже дадут типизированную
|
||||
# ошибку, если cookies реально нужны. Не re-raise намеренно.
|
||||
logger.warning("IMV warm-up failed (продолжаем без cookies): %s", exc)
|
||||
|
||||
|
||||
def _raise_for_status_categorized(resp: Any, context: str) -> None:
|
||||
"""Raise typed IMV exception based on HTTP status code.
|
||||
|
||||
|
|
@ -393,31 +523,17 @@ async def _geocode(session: Any, address: str) -> IMVGeo:
|
|||
_raise_for_status_categorized(resp_b, "geocode-B")
|
||||
data_b: dict[str, Any] = resp_b.json()
|
||||
|
||||
jwt = data_b.get("geoFieldsHash")
|
||||
if not jwt:
|
||||
raise IMVAddressNotFoundError(
|
||||
f"Avito /js/v2/geo/position не вернул geoFieldsHash для {normalized!r}: "
|
||||
f"errorText={data_b.get('errorText')!r}"
|
||||
)
|
||||
geo = _parse_geo_position(data_b, lat=lat, lon=lon) # raises IMVAddressNotFoundError
|
||||
|
||||
logger.info(
|
||||
"IMV geocode OK: addressId=%s locationId=%s metroId=%s lat=%s lon=%s",
|
||||
data_b.get("addressId"),
|
||||
data_b.get("locationId"),
|
||||
data_b.get("metroId"),
|
||||
geo.avito_address_id,
|
||||
geo.avito_location_id,
|
||||
geo.avito_metro_id,
|
||||
lat,
|
||||
lon,
|
||||
)
|
||||
|
||||
return IMVGeo(
|
||||
geo_hash=jwt,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
avito_address_id=data_b.get("addressId"),
|
||||
avito_location_id=data_b.get("locationId"),
|
||||
avito_metro_id=data_b.get("metroId"),
|
||||
avito_district_id=data_b.get("districtId"),
|
||||
)
|
||||
return geo
|
||||
|
||||
|
||||
async def _imv_evaluate(
|
||||
|
|
@ -465,31 +581,9 @@ async def _imv_evaluate(
|
|||
_raise_for_status_categorized(resp, "imv-evaluate")
|
||||
data: dict[str, Any] = resp.json()
|
||||
|
||||
price_block: dict[str, Any] = data.get("price") or {}
|
||||
recommended_price: int = price_block.get("recommendedPrice") or 0
|
||||
lower_price: int = price_block.get("lowerPrice") or 0
|
||||
higher_price: int = price_block.get("higherPrice") or 0
|
||||
market_count: int | None = price_block.get("count")
|
||||
|
||||
# placementHistory
|
||||
history_block: dict[str, Any] = data.get("placementHistory") or {}
|
||||
placement_history: list[IMVPlacementHistoryItem] = []
|
||||
for raw_item in history_block.get("items") or []:
|
||||
try:
|
||||
placement_history.append(_parse_placement_item(raw_item))
|
||||
except (KeyError, TypeError) as exc:
|
||||
logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc)
|
||||
|
||||
# suggestions
|
||||
suggestions_block: dict[str, Any] = data.get("suggestions") or {}
|
||||
suggestions: list[IMVSuggestion] = []
|
||||
for raw_sugg in suggestions_block.get("items") or []:
|
||||
try:
|
||||
suggestions.append(_parse_suggestion(raw_sugg))
|
||||
except (KeyError, TypeError) as exc:
|
||||
logger.warning("IMV: не удалось распарсить suggestion: %s", exc)
|
||||
|
||||
search_similar_url: str | None = suggestions_block.get("searchSimilarUrl")
|
||||
recommended_price, lower_price, higher_price, market_count = _parse_price(data)
|
||||
placement_history = _parse_placement_history(data)
|
||||
suggestions, search_similar_url = _parse_suggestions(data)
|
||||
|
||||
logger.info(
|
||||
"IMV evaluate OK: recommended=%d range=(%d, %d) count=%s history=%d suggestions=%d",
|
||||
|
|
|
|||
13
tradein-mvp/backend/tests/fixtures/avito_imv_geo_position.json
vendored
Normal file
13
tradein-mvp/backend/tests/fixtures/avito_imv_geo_position.json
vendored
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"address": "Свердловская область, Екатеринбург, улица Малышева, 125",
|
||||
"addressId": 22319477,
|
||||
"closeSuggest": true,
|
||||
"districtId": 789,
|
||||
"errorText": "",
|
||||
"geoFieldsHash": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJsYXRpdHVkZSI6NTYuODQwODcyLCJsb25naXR1ZGUiOjYwLjY1NDA3NywiYWRkcmVzcyI6ItCh0LLQtdGA0LTQu9C-0LLRgdC60LDRjyDQvtCx0LvQsNGB0YLRjCwg0JXQutCw0YLQtdGA0LjQvdCx0YPRgNCzLCDRg9C70LjRhtCwINCc0LDQu9GL0YjQtdCy0LAsIDEyNSIsImFkZHJlc3NJZCI6MjIzMTk0NzcsImxvY2F0aW9uSWQiOjY1NDA3MCwiZGlzdHJpY3RJZCI6Nzg5fQ.wkkTyzUmHikPyD_hMJkZIIoXmyIyGhvqpZQyKSXJ6V4",
|
||||
"isFullAddress": true,
|
||||
"latitude": 56.840872,
|
||||
"locationId": 654070,
|
||||
"longitude": 60.654077,
|
||||
"parentLocationId": 653700
|
||||
}
|
||||
157
tradein-mvp/backend/tests/fixtures/avito_imv_getdata.json
vendored
Normal file
157
tradein-mvp/backend/tests/fixtures/avito_imv_getdata.json
vendored
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
{
|
||||
"itemParams": {
|
||||
"categoryID": 24,
|
||||
"rawParams": "{\"110688\":458589,\"110710\":472001,\"118593\":[1714472],\"496\":5123,\"497\":5190,\"498\":5245,\"499\":5254,\"549\":5697,\"578\":54}"
|
||||
},
|
||||
"placementHistory": {
|
||||
"items": [
|
||||
{
|
||||
"exposure": 115,
|
||||
"id": 7334693344,
|
||||
"itemImage": "https://60.img.avito.st/image/1/1.pLl9lba3CFBbN2agdtqcl8k2CFrDpgs-yzY.knU6Ae7h8FdvPtmtqDIsoCetYbBMeBNHHd7Z3Mu4ct0",
|
||||
"lastPrice": 6800000,
|
||||
"lastPriceDate": 1755648000,
|
||||
"removedDate": 1756077885,
|
||||
"startPrice": 7000000,
|
||||
"startPriceDate": 1745712000,
|
||||
"title": "2-к. квартира, 43 м², 5/5 эт."
|
||||
}
|
||||
]
|
||||
},
|
||||
"price": {
|
||||
"addItemUrl": "/additem?category_id=24¶ms%5B110688%5D=458589¶ms%5B110710%5D=472001¶ms%5B118593%5D=1714472¶ms%5B201%5D=1059¶ms%5B496%5D=5123¶ms%5B497%5D=5190¶ms%5B498%5D=5245¶ms%5B499%5D=5254¶ms%5B549%5D=5697¶ms%5B578%5D=54.000000",
|
||||
"count": 800,
|
||||
"higherPrice": 7738500,
|
||||
"lowerPrice": 7148900,
|
||||
"recommendedPrice": 7370000
|
||||
},
|
||||
"suggestions": {
|
||||
"items": [
|
||||
{
|
||||
"address": "Свердловская область, Екатеринбург, Студенческая улица, 70",
|
||||
"exposure": 63,
|
||||
"hasGoodPriceBadge": false,
|
||||
"id": 8000753763,
|
||||
"imageLink": "https://80.img.avito.st/image/1/1._ByNPLa1UPX7nIL1-VP1R7ucUvUzl5L0651S9w.8SVb7wAlUUR_4CrkmaCoNyDgKKpkduXIvG2ystoh9EU",
|
||||
"isFavorite": false,
|
||||
"itemLink": "/ekaterinburg/kvartiry/3-k._kvartira_616_m_15_et._8000753763",
|
||||
"price": 7400000,
|
||||
"publishDate": 1772614160,
|
||||
"title": "3-к. квартира, 61,6 м², 1/5 эт."
|
||||
},
|
||||
{
|
||||
"address": "Свердловская обл., муниципальное образование Екатеринбург, Екатеринбург, ул. Софьи Ковалевской, 12",
|
||||
"exposure": 13,
|
||||
"hasGoodPriceBadge": true,
|
||||
"id": 8060842441,
|
||||
"imageLink": "https://90.img.avito.st/image/1/1.kg1foLa1PuQpAOzkd86GKmcAPOThC_zlOQE85g.k3bmz3PqoQnYHj81udy1rgF9iy6rmv4pkCx5024vLeA",
|
||||
"isFavorite": false,
|
||||
"itemLink": "/ekaterinburg/kvartiry/1-k._kvartira_545_m_24_et._8060842441",
|
||||
"price": 7490000,
|
||||
"publishDate": 1778847849,
|
||||
"title": "1-к. квартира, 54,5 м², 2/4 эт."
|
||||
},
|
||||
{
|
||||
"address": "Свердловская область, Екатеринбург, улица Малышева, 109",
|
||||
"exposure": 28,
|
||||
"hasGoodPriceBadge": false,
|
||||
"id": 8066712904,
|
||||
"imageLink": "https://20.img.avito.st/image/1/1.kYR7Rba1PW0N5e9tVW-QnELlP23F7v9sHeQ_bw.8emEbUAnDfDmjFC4t4lyFpKJc1ZPOte3I3elhIkqseg?cqp=2.nqHHyj8nxExgujaMxUsx0_L17NSXZ1ZQKCIQgy1FRSfAZeHHzJ12cn_5SQ7YtQ4=",
|
||||
"isFavorite": false,
|
||||
"itemLink": "/ekaterinburg/kvartiry/3-k._kvartira_52_m_35_et._8066712904",
|
||||
"metro": {
|
||||
"colors": [
|
||||
"#0A6F20"
|
||||
],
|
||||
"distance": "2,6 км",
|
||||
"name": "Площадь 1905 года"
|
||||
},
|
||||
"price": 7520000,
|
||||
"publishDate": 1777491918,
|
||||
"title": "3-к. квартира, 52 м², 3/5 эт."
|
||||
},
|
||||
{
|
||||
"address": "Свердловская обл., муниципальное образование Екатеринбург, Екатеринбург, пр-т Ленина, 103",
|
||||
"exposure": 9,
|
||||
"hasGoodPriceBadge": false,
|
||||
"id": 8120591680,
|
||||
"imageLink": "https://20.img.avito.st/image/1/1.lR-6X7a1OfbM_-v2nA2GQoP_O_YE9Pv33P479A.cG8CmMm3YH5JmMgDlCIIrqA-jWy9cDH46kaAMAnU7Nk?cqp=2.nqHHyj8nxExgujaMxUsx0_L17NSXZ1ZQKCIQgy1FRSfAZeHHzJ12cn_5SQ7YtQ4=",
|
||||
"isFavorite": false,
|
||||
"itemLink": "/ekaterinburg/kvartiry/2-k._kvartira_551_m_15_et._8120591680",
|
||||
"metro": {
|
||||
"colors": [
|
||||
"#0A6F20"
|
||||
],
|
||||
"distance": "2,9 км",
|
||||
"name": "Динамо"
|
||||
},
|
||||
"price": 7530000,
|
||||
"publishDate": 1779216343,
|
||||
"title": "2-к. квартира, 55,1 м², 1/5 эт."
|
||||
},
|
||||
{
|
||||
"address": "Свердловская область, Екатеринбург, улица Коминтерна, 20",
|
||||
"exposure": 101,
|
||||
"hasGoodPriceBadge": false,
|
||||
"id": 7938143586,
|
||||
"imageLink": "https://60.img.avito.st/image/1/1.BL5067a1qFcCS3pXcPdnzTxKqlfKQGpWEkqqVQ.NN0956Nx_wZJueNDEnfdVWPeyve6hi_bxAUHVVmd65k",
|
||||
"isFavorite": false,
|
||||
"itemLink": "/ekaterinburg/kvartiry/3-k._kvartira_549_m_25_et._7938143586",
|
||||
"metro": {
|
||||
"colors": [
|
||||
"#0A6F20"
|
||||
],
|
||||
"distance": "3,4 км",
|
||||
"name": "Геологическая"
|
||||
},
|
||||
"price": 7190000,
|
||||
"publishDate": 1771241225,
|
||||
"title": "3-к. квартира, 54,9 м², 2/5 эт."
|
||||
},
|
||||
{
|
||||
"address": "Свердловская область, Екатеринбург, улица Малышева, 130Б",
|
||||
"exposure": 244,
|
||||
"hasGoodPriceBadge": true,
|
||||
"id": 4638238123,
|
||||
"imageLink": "https://30.img.avito.st/image/1/1.PpTjQ7a1kn2V40B9kxd2xebikH1d6FB8heKQfw.fCUNFxjvSx8cYylDSxHgGqQRGf7eYAxmvFs28MjGkxc?cqp=2.nqHHyj8nxExgujaMxUsx0_L17NSXZ1ZQKCIQgy1FRSfAZeHHzJ12cn_5SQ7YtQ4=",
|
||||
"isFavorite": false,
|
||||
"itemLink": "/ekaterinburg/kvartiry/4-k._kvartira_562_m_15_et._4638238123",
|
||||
"metro": {
|
||||
"colors": [
|
||||
"#0A6F20"
|
||||
],
|
||||
"distance": "3,1 км",
|
||||
"name": "Площадь 1905 года"
|
||||
},
|
||||
"price": 7000000,
|
||||
"publishDate": 1738063015,
|
||||
"title": "4-к. квартира, 56,2 м², 1/5 эт."
|
||||
},
|
||||
{
|
||||
"address": "Свердловская обл., муниципальное образование Екатеринбург, Екатеринбург, Студенческая ул., 80",
|
||||
"exposure": 8,
|
||||
"hasGoodPriceBadge": false,
|
||||
"id": 8131017429,
|
||||
"imageLink": "https://40.img.avito.st/image/1/1.4fku77a1TRBYT58QYPiv3xdPTxCQRI8RSE5PEg.pEjigFdQ2vb_8pkuReSJ1w9xXHSegiJjXXG8-7uJzYQ",
|
||||
"isFavorite": false,
|
||||
"itemLink": "/ekaterinburg/kvartiry/1-k._kvartira_41_m_1125_et._8131017429",
|
||||
"price": 7000000,
|
||||
"publishDate": 1779276588,
|
||||
"title": "1-к. квартира, 41 м², 11/25 эт."
|
||||
},
|
||||
{
|
||||
"address": "Свердловская область, Екатеринбург, Комсомольская улица, 50",
|
||||
"exposure": 102,
|
||||
"hasGoodPriceBadge": false,
|
||||
"id": 7901596052,
|
||||
"imageLink": "https://30.img.avito.st/image/1/1.plJjuba1CrsVGdi7I7uPXSoYCLvdEsi6BRgIuQ.gy3OfVsh4ZQEyPnlX9Kse1VQDKqYHgUR6H00mt6oBvM?cqp=2.nqHHyj8nxExgujaMxUsx0_L17NSXZ1ZQKCIQgy1FRSfAZeHHzJ12cn_5SQ7YtQ4=",
|
||||
"isFavorite": false,
|
||||
"itemLink": "/ekaterinburg/kvartiry/4-k._kvartira_729_m_15_et._7901596052",
|
||||
"price": 6990000,
|
||||
"publishDate": 1770308156,
|
||||
"title": "4-к. квартира, 72,9 м², 1/5 эт."
|
||||
}
|
||||
],
|
||||
"searchSimilarUrl": "/ekaterinburg/kvartiry/prodam/panelnyy_dom-ASgBAgICAkSSA8YQ5Af6UQ?f=ASgBAQECBESSA8YQ5Af6Uay~DaTHNYK9DtCk0QEBQMoINIJZgFmEWQJFhAkXeyJmcm9tIjo0My4yLCJ0byI6NjQuOH3GmgwdeyJmcm9tIjo1ODk2MDAwLCJ0byI6ODg0NDAwMH0"
|
||||
}
|
||||
}
|
||||
|
|
@ -3,22 +3,35 @@
|
|||
Без сети — только cache_key вычисление, парсинг заголовков, unix→date, dataclass конструирование.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.avito_imv import (
|
||||
IMVAddressNotFoundError,
|
||||
IMVEvaluation,
|
||||
IMVGeo,
|
||||
IMVPlacementHistoryItem,
|
||||
IMVSuggestion,
|
||||
_parse_geo_position,
|
||||
_parse_placement_history,
|
||||
_parse_placement_item,
|
||||
_parse_price,
|
||||
_parse_suggestion,
|
||||
_parse_suggestions,
|
||||
_parse_title,
|
||||
_unix_to_date,
|
||||
compute_imv_cache_key,
|
||||
)
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def _load_fixture(name: str) -> dict:
|
||||
with (_FIXTURES / name).open(encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cache_key
|
||||
|
|
@ -285,3 +298,145 @@ def test_dataclass_with_history():
|
|||
assert len(e.placement_history) == 1
|
||||
assert e.placement_history[0].removed_date == date(2024, 5, 12)
|
||||
assert e.market_count == 800
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_geo_position (step B) — driven by live-captured fixture (issue #565)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_geo_position_from_fixture():
|
||||
"""Real /js/v2/geo/position response (ЕКБ, ул. Малышева 125) → IMVGeo."""
|
||||
data_b = _load_fixture("avito_imv_geo_position.json")
|
||||
geo = _parse_geo_position(data_b, lat=56.840872, lon=60.654077)
|
||||
|
||||
assert geo.geo_hash # non-empty JWT
|
||||
assert geo.geo_hash.startswith("eyJ") # base64url JWT header
|
||||
assert geo.geo_hash.count(".") == 2 # header.payload.signature
|
||||
assert geo.avito_address_id == 22319477
|
||||
assert geo.avito_location_id == 654070
|
||||
assert geo.avito_district_id == 789
|
||||
assert geo.lat == pytest.approx(56.840872)
|
||||
assert geo.lon == pytest.approx(60.654077)
|
||||
|
||||
|
||||
def test_parse_geo_position_ekb_has_no_metro():
|
||||
"""ЕКБ-окраина не отдаёт metroId — должен быть None, не падать."""
|
||||
data_b = _load_fixture("avito_imv_geo_position.json")
|
||||
geo = _parse_geo_position(data_b, lat=56.84, lon=60.65)
|
||||
assert geo.avito_metro_id is None
|
||||
|
||||
|
||||
def test_parse_geo_position_missing_hash_raises():
|
||||
"""Пустой geoFieldsHash → IMVAddressNotFoundError (graceful skip в estimator)."""
|
||||
with pytest.raises(IMVAddressNotFoundError):
|
||||
_parse_geo_position(
|
||||
{"address": "X", "errorText": "not found", "geoFieldsHash": ""},
|
||||
lat=1.0,
|
||||
lon=2.0,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_price / placement / suggestions (step C) — live-captured fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_price_from_fixture():
|
||||
"""Real realty-imv/get-data price block populates all three prices + count."""
|
||||
data = _load_fixture("avito_imv_getdata.json")
|
||||
recommended, lower, higher, count = _parse_price(data)
|
||||
|
||||
assert recommended == 7370000
|
||||
assert lower == 7148900
|
||||
assert higher == 7738500
|
||||
assert count == 800
|
||||
assert lower < recommended < higher
|
||||
|
||||
|
||||
def test_parse_price_missing_block_defaults_zero():
|
||||
"""Нет price блока (degenerate ответ) → нули, не KeyError."""
|
||||
recommended, lower, higher, count = _parse_price({})
|
||||
assert (recommended, lower, higher) == (0, 0, 0)
|
||||
assert count is None
|
||||
|
||||
|
||||
def test_parse_placement_history_from_fixture():
|
||||
"""placementHistory.items парсятся в IMVPlacementHistoryItem с removed_date."""
|
||||
data = _load_fixture("avito_imv_getdata.json")
|
||||
items = _parse_placement_history(data)
|
||||
|
||||
assert len(items) >= 1
|
||||
first = items[0]
|
||||
assert first.ext_item_id # str(id)
|
||||
assert first.start_price and first.last_price
|
||||
assert isinstance(first.start_price_date, date)
|
||||
assert isinstance(first.removed_date, date)
|
||||
# raw_payload очищен от тяжёлого itemImage
|
||||
assert first.raw_payload is not None
|
||||
assert "itemImage" not in first.raw_payload
|
||||
|
||||
|
||||
def test_parse_placement_history_absent_block_returns_empty():
|
||||
"""placementHistory отсутствует (как у проспект Ленина) → [] без падения."""
|
||||
data = _load_fixture("avito_imv_getdata.json")
|
||||
data.pop("placementHistory", None)
|
||||
assert _parse_placement_history(data) == []
|
||||
|
||||
|
||||
def test_parse_suggestions_from_fixture():
|
||||
"""suggestions.items парсятся + searchSimilarUrl извлекается."""
|
||||
data = _load_fixture("avito_imv_getdata.json")
|
||||
suggestions, search_url = _parse_suggestions(data)
|
||||
|
||||
assert len(suggestions) >= 1
|
||||
assert all(s.ext_item_id for s in suggestions)
|
||||
assert all(s.price_rub for s in suggestions)
|
||||
# raw_payload очищен от imageLink
|
||||
assert "imageLink" not in (suggestions[0].raw_payload or {})
|
||||
assert search_url is not None
|
||||
assert search_url.startswith("/")
|
||||
|
||||
|
||||
def test_fixtures_feed_full_evaluation_shape():
|
||||
"""Сквозной оффлайн-тест: fixtures B+C → заполненный IMVEvaluation.
|
||||
|
||||
Зеркалит то, что _geocode + _imv_evaluate собирают из живого ответа,
|
||||
без сети. Регрессионный guard для контракта step B/C (issue #565).
|
||||
"""
|
||||
data_b = _load_fixture("avito_imv_geo_position.json")
|
||||
data_c = _load_fixture("avito_imv_getdata.json")
|
||||
|
||||
geo = _parse_geo_position(data_b, lat=56.840872, lon=60.654077)
|
||||
recommended, lower, higher, count = _parse_price(data_c)
|
||||
history = _parse_placement_history(data_c)
|
||||
suggestions, search_url = _parse_suggestions(data_c)
|
||||
|
||||
e = IMVEvaluation(
|
||||
cache_key="c" * 64,
|
||||
address="Екатеринбург, улица Малышева, 125",
|
||||
rooms=2,
|
||||
area_m2=54.0,
|
||||
floor=4,
|
||||
floor_at_home=9,
|
||||
house_type="panel",
|
||||
renovation_type="cosmetic",
|
||||
has_balcony=True,
|
||||
has_loggia=False,
|
||||
geo=geo,
|
||||
recommended_price=recommended,
|
||||
lower_price=lower,
|
||||
higher_price=higher,
|
||||
market_count=count,
|
||||
placement_history=history,
|
||||
suggestions=suggestions,
|
||||
search_similar_url=search_url,
|
||||
raw_response=data_c,
|
||||
)
|
||||
|
||||
assert e.geo.geo_hash # non-empty
|
||||
assert e.recommended_price == 7370000
|
||||
assert e.lower_price == 7148900
|
||||
assert e.higher_price == 7738500
|
||||
assert e.market_count == 800
|
||||
assert len(e.suggestions) >= 1
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue