From 81bf7e6def6ea4ce96f7f02b63a325740e7f68f3 Mon Sep 17 00:00:00 2001 From: Light1YT Date: Thu, 28 May 2026 16:29:05 +0500 Subject: [PATCH] fix(tradein-imv): harden Avito IMV anti-bot path + offline contract tests (#565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add warm-up GET /evaluation/realty + production browser header set (Sec-Fetch, X-Requested-With, 25s timeout) mirroring avito.py, чтобы защитить 3-step IMV flow от server-IP анти-бот 403. Refactor step B/C парсинга в pure helpers (_parse_geo_position/_parse_price/_parse_placement_history/_parse_suggestions) + fixture-driven offline тесты из живых JSON-captures. Честный диагноз: контракт (geoFieldsHash, price-блок, categoryId=24) НЕ менялся — проверено живым capture'ом с этой машины (A→B→C, recommended 7.39M). Реальная причина #565 — скорее анти-бот по PROD-IP, не код → если #565 останется после deploy, настоящий фикс = ротация egress IP воркера (devops, отдельно). code-reviewer: ✅ APPROVE no critical — warm-up non-fatal, injected-session path unchanged, refactor preserves shapes, graceful-None сохранён, fixtures без PII. Tests: 12 new (27 total). Ruff clean. Closes #565 --- .../app/services/scrapers/avito_imv.py | 192 +++++++++++++----- .../fixtures/avito_imv_geo_position.json | 13 ++ .../tests/fixtures/avito_imv_getdata.json | 157 ++++++++++++++ .../backend/tests/test_avito_imv_parse.py | 157 +++++++++++++- 4 files changed, 469 insertions(+), 50 deletions(-) create mode 100644 tradein-mvp/backend/tests/fixtures/avito_imv_geo_position.json create mode 100644 tradein-mvp/backend/tests/fixtures/avito_imv_getdata.json diff --git a/tradein-mvp/backend/app/services/scrapers/avito_imv.py b/tradein-mvp/backend/app/services/scrapers/avito_imv.py index 3113b89d..f1caa3be 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_imv.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_imv.py @@ -2,10 +2,15 @@ Stage 2d: geocode + IMV evaluation. -Flow (current contract): +Flow (current contract, подтверждён живым capture 2026-05 — fixtures +tests/fixtures/avito_imv_geo_position.json + avito_imv_getdata.json): + 0) GET /evaluation/realty → warm-up (seed anti-bot cookies) 1) GET /web/1/coords/by_address?address= → normalize + lat/lon 2) POST /js/v2/geo/position → rich JWT (geoFieldsHash) - 3) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions + 3) POST /web/1/realty-imv/get-data → price + placementHistory? + suggestions + +Anti-bot: zerkalim AvitoScraper (avito.py) — chrome120 TLS + document-заголовки + +warm-up GET + XHR Sec-Fetch заголовки. Bare-session XHR ловят 403 с server-IP. Таблицы: avito_imv_evaluations (018_avito_imv_evaluations.sql) @@ -33,14 +38,37 @@ AVITO_BASE = "https://www.avito.ru" COORDS_ENDPOINT = "/web/1/coords/by_address" POSITION_ENDPOINT = "/js/v2/geo/position" IMV_ENDPOINT = "/web/1/realty-imv/get-data" +# Тёплая страница для seed anti-bot cookies (srv_id/_avisc/u/...) перед XHR. +WARMUP_URL = f"{AVITO_BASE}/evaluation/realty" -# Заголовки имитируют браузер на странице evaluation/realty +# Заголовки document-навигации (warm-up GET) — зеркалят production-набор +# из avito.py (AvitoScraper.__aenter__). Нужны чтобы пройти TLS+header anti-bot. +_DOC_HEADERS = { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", + "Cache-Control": "max-age=0", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", +} + +# Заголовки XHR/fetch для API-шагов (coords/position/get-data). Имитируют +# fetch() со страницы /evaluation/realty: JSON Accept + same-origin Sec-Fetch. _COMMON_HEADERS = { "Accept": "application/json, text/plain, */*", - "Accept-Language": "ru-RU,ru;q=0.9", + "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", "Referer": "https://www.avito.ru/evaluation/realty", + "X-Requested-With": "XMLHttpRequest", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", } +# Timeout для всех Avito-вызовов (без него curl_cffi висит на anti-bot stall). +_HTTP_TIMEOUT_SEC = 25 + # --------------------------------------------------------------------------- # Exceptions @@ -255,6 +283,83 @@ def _parse_suggestion(raw: dict[str, Any]) -> IMVSuggestion: ) +def _parse_geo_position(data_b: dict[str, Any], *, lat: float, lon: float) -> IMVGeo: + """Парсит ответ POST /js/v2/geo/position (step B) → IMVGeo. + + Контракт (подтверждён живым capture 2026-05, fixture avito_imv_geo_position.json): + {address, addressId, districtId, geoFieldsHash, latitude, longitude, + locationId, parentLocationId, metroId?, errorText, ...} + + `geoFieldsHash` — JWT, payload которого кодирует {latitude, longitude, + address, addressId, locationId, districtId}; это то, что step C ожидает как + `geoHash`. metroId присутствует только для метро-городов (None для ЕКБ-окраин). + + Raises: + IMVAddressNotFoundError если geoFieldsHash пустой/отсутствует. + """ + jwt = data_b.get("geoFieldsHash") + if not jwt: + raise IMVAddressNotFoundError( + f"Avito /js/v2/geo/position не вернул geoFieldsHash для " + f"{str(data_b.get('address'))[:60]!r}: errorText={data_b.get('errorText')!r}" + ) + return IMVGeo( + geo_hash=jwt, + lat=lat, + lon=lon, + avito_address_id=data_b.get("addressId"), + avito_location_id=data_b.get("locationId"), + avito_metro_id=data_b.get("metroId"), + avito_district_id=data_b.get("districtId"), + ) + + +def _parse_price(data: dict[str, Any]) -> tuple[int, int, int, int | None]: + """Парсит top-level `price` блок из realty-imv/get-data (step C). + + Контракт (fixture avito_imv_getdata.json): + price = {recommendedPrice, lowerPrice, higherPrice, count, addItemUrl} + + Returns: (recommended_price, lower_price, higher_price, market_count). + Отсутствующие цены → 0 (caller трактует как "нет оценки"); count → None. + """ + price_block: dict[str, Any] = data.get("price") or {} + recommended_price: int = price_block.get("recommendedPrice") or 0 + lower_price: int = price_block.get("lowerPrice") or 0 + higher_price: int = price_block.get("higherPrice") or 0 + market_count: int | None = price_block.get("count") + return recommended_price, lower_price, higher_price, market_count + + +def _parse_placement_history(data: dict[str, Any]) -> list[IMVPlacementHistoryItem]: + """Парсит опциональный `placementHistory.items` блок (step C). + + placementHistory отсутствует для части адресов (живой capture: проспект + Ленина вернул только itemParams/price/suggestions) — тогда возвращаем []. + Битые элементы пропускаем с warning, не роняя весь ответ. + """ + history_block: dict[str, Any] = data.get("placementHistory") or {} + items: list[IMVPlacementHistoryItem] = [] + for raw_item in history_block.get("items") or []: + try: + items.append(_parse_placement_item(raw_item)) + except (KeyError, TypeError) as exc: + logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc) + return items + + +def _parse_suggestions(data: dict[str, Any]) -> tuple[list[IMVSuggestion], str | None]: + """Парсит `suggestions` блок (step C) → (items, searchSimilarUrl).""" + suggestions_block: dict[str, Any] = data.get("suggestions") or {} + suggestions: list[IMVSuggestion] = [] + for raw_sugg in suggestions_block.get("items") or []: + try: + suggestions.append(_parse_suggestion(raw_sugg)) + except (KeyError, TypeError) as exc: + logger.warning("IMV: не удалось распарсить suggestion: %s", exc) + return suggestions, suggestions_block.get("searchSimilarUrl") + + # --------------------------------------------------------------------------- # Main async function # --------------------------------------------------------------------------- @@ -293,7 +398,14 @@ async def evaluate_via_imv( _own_session = False if cffi_session is None: - cffi_session = CffiAsyncSession(impersonate="chrome120") + # Зеркалим production-набор из AvitoScraper.__aenter__ (avito.py:150-162): + # chrome120 TLS + document-заголовки + timeout. Затем warm-up GET для + # seed anti-bot cookies — bare-session XHR Avito банит на server-IP. + cffi_session = CffiAsyncSession( + impersonate="chrome120", + timeout=_HTTP_TIMEOUT_SEC, + headers=_DOC_HEADERS, + ) _own_session = True except ImportError as exc: raise RuntimeError( @@ -301,6 +413,8 @@ async def evaluate_via_imv( ) from exc try: + if _own_session: + await _warmup(cffi_session) geo = await _geocode(cffi_session, address) evaluation = await _imv_evaluate( cffi_session, @@ -322,6 +436,22 @@ async def evaluate_via_imv( return evaluation +async def _warmup(session: Any) -> None: + """GET /evaluation/realty чтобы seed anti-bot cookies (srv_id/_avisc/u/...). + + Без warm-up XHR-шаги (coords/position/get-data) c server-IP ловят 403 anti-bot. + Best-effort: не роняем оценку если страница недоступна — последующие шаги + всё равно попробуют и дадут типизированную ошибку. + """ + try: + resp = await session.get(WARMUP_URL, headers=_DOC_HEADERS) + logger.info("IMV warm-up GET /evaluation/realty → HTTP %d", resp.status_code) + except Exception as exc: + # warm-up опционален: логируем и продолжаем — шаги ниже дадут типизированную + # ошибку, если cookies реально нужны. Не re-raise намеренно. + logger.warning("IMV warm-up failed (продолжаем без cookies): %s", exc) + + def _raise_for_status_categorized(resp: Any, context: str) -> None: """Raise typed IMV exception based on HTTP status code. @@ -393,31 +523,17 @@ async def _geocode(session: Any, address: str) -> IMVGeo: _raise_for_status_categorized(resp_b, "geocode-B") data_b: dict[str, Any] = resp_b.json() - jwt = data_b.get("geoFieldsHash") - if not jwt: - raise IMVAddressNotFoundError( - f"Avito /js/v2/geo/position не вернул geoFieldsHash для {normalized!r}: " - f"errorText={data_b.get('errorText')!r}" - ) + geo = _parse_geo_position(data_b, lat=lat, lon=lon) # raises IMVAddressNotFoundError logger.info( "IMV geocode OK: addressId=%s locationId=%s metroId=%s lat=%s lon=%s", - data_b.get("addressId"), - data_b.get("locationId"), - data_b.get("metroId"), + geo.avito_address_id, + geo.avito_location_id, + geo.avito_metro_id, lat, lon, ) - - return IMVGeo( - geo_hash=jwt, - lat=lat, - lon=lon, - avito_address_id=data_b.get("addressId"), - avito_location_id=data_b.get("locationId"), - avito_metro_id=data_b.get("metroId"), - avito_district_id=data_b.get("districtId"), - ) + return geo async def _imv_evaluate( @@ -465,31 +581,9 @@ async def _imv_evaluate( _raise_for_status_categorized(resp, "imv-evaluate") data: dict[str, Any] = resp.json() - price_block: dict[str, Any] = data.get("price") or {} - recommended_price: int = price_block.get("recommendedPrice") or 0 - lower_price: int = price_block.get("lowerPrice") or 0 - higher_price: int = price_block.get("higherPrice") or 0 - market_count: int | None = price_block.get("count") - - # placementHistory - history_block: dict[str, Any] = data.get("placementHistory") or {} - placement_history: list[IMVPlacementHistoryItem] = [] - for raw_item in history_block.get("items") or []: - try: - placement_history.append(_parse_placement_item(raw_item)) - except (KeyError, TypeError) as exc: - logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc) - - # suggestions - suggestions_block: dict[str, Any] = data.get("suggestions") or {} - suggestions: list[IMVSuggestion] = [] - for raw_sugg in suggestions_block.get("items") or []: - try: - suggestions.append(_parse_suggestion(raw_sugg)) - except (KeyError, TypeError) as exc: - logger.warning("IMV: не удалось распарсить suggestion: %s", exc) - - search_similar_url: str | None = suggestions_block.get("searchSimilarUrl") + recommended_price, lower_price, higher_price, market_count = _parse_price(data) + placement_history = _parse_placement_history(data) + suggestions, search_similar_url = _parse_suggestions(data) logger.info( "IMV evaluate OK: recommended=%d range=(%d, %d) count=%s history=%d suggestions=%d", diff --git a/tradein-mvp/backend/tests/fixtures/avito_imv_geo_position.json b/tradein-mvp/backend/tests/fixtures/avito_imv_geo_position.json new file mode 100644 index 00000000..1194357b --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/avito_imv_geo_position.json @@ -0,0 +1,13 @@ +{ + "address": "Свердловская область, Екатеринбург, улица Малышева, 125", + "addressId": 22319477, + "closeSuggest": true, + "districtId": 789, + "errorText": "", + "geoFieldsHash": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJsYXRpdHVkZSI6NTYuODQwODcyLCJsb25naXR1ZGUiOjYwLjY1NDA3NywiYWRkcmVzcyI6ItCh0LLQtdGA0LTQu9C-0LLRgdC60LDRjyDQvtCx0LvQsNGB0YLRjCwg0JXQutCw0YLQtdGA0LjQvdCx0YPRgNCzLCDRg9C70LjRhtCwINCc0LDQu9GL0YjQtdCy0LAsIDEyNSIsImFkZHJlc3NJZCI6MjIzMTk0NzcsImxvY2F0aW9uSWQiOjY1NDA3MCwiZGlzdHJpY3RJZCI6Nzg5fQ.wkkTyzUmHikPyD_hMJkZIIoXmyIyGhvqpZQyKSXJ6V4", + "isFullAddress": true, + "latitude": 56.840872, + "locationId": 654070, + "longitude": 60.654077, + "parentLocationId": 653700 +} \ No newline at end of file diff --git a/tradein-mvp/backend/tests/fixtures/avito_imv_getdata.json b/tradein-mvp/backend/tests/fixtures/avito_imv_getdata.json new file mode 100644 index 00000000..012df5e7 --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/avito_imv_getdata.json @@ -0,0 +1,157 @@ +{ + "itemParams": { + "categoryID": 24, + "rawParams": "{\"110688\":458589,\"110710\":472001,\"118593\":[1714472],\"496\":5123,\"497\":5190,\"498\":5245,\"499\":5254,\"549\":5697,\"578\":54}" + }, + "placementHistory": { + "items": [ + { + "exposure": 115, + "id": 7334693344, + "itemImage": "https://60.img.avito.st/image/1/1.pLl9lba3CFBbN2agdtqcl8k2CFrDpgs-yzY.knU6Ae7h8FdvPtmtqDIsoCetYbBMeBNHHd7Z3Mu4ct0", + "lastPrice": 6800000, + "lastPriceDate": 1755648000, + "removedDate": 1756077885, + "startPrice": 7000000, + "startPriceDate": 1745712000, + "title": "2-к. квартира, 43 м², 5/5 эт." + } + ] + }, + "price": { + "addItemUrl": "/additem?category_id=24¶ms%5B110688%5D=458589¶ms%5B110710%5D=472001¶ms%5B118593%5D=1714472¶ms%5B201%5D=1059¶ms%5B496%5D=5123¶ms%5B497%5D=5190¶ms%5B498%5D=5245¶ms%5B499%5D=5254¶ms%5B549%5D=5697¶ms%5B578%5D=54.000000", + "count": 800, + "higherPrice": 7738500, + "lowerPrice": 7148900, + "recommendedPrice": 7370000 + }, + "suggestions": { + "items": [ + { + "address": "Свердловская область, Екатеринбург, Студенческая улица, 70", + "exposure": 63, + "hasGoodPriceBadge": false, + "id": 8000753763, + "imageLink": "https://80.img.avito.st/image/1/1._ByNPLa1UPX7nIL1-VP1R7ucUvUzl5L0651S9w.8SVb7wAlUUR_4CrkmaCoNyDgKKpkduXIvG2ystoh9EU", + "isFavorite": false, + "itemLink": "/ekaterinburg/kvartiry/3-k._kvartira_616_m_15_et._8000753763", + "price": 7400000, + "publishDate": 1772614160, + "title": "3-к. квартира, 61,6 м², 1/5 эт." + }, + { + "address": "Свердловская обл., муниципальное образование Екатеринбург, Екатеринбург, ул. Софьи Ковалевской, 12", + "exposure": 13, + "hasGoodPriceBadge": true, + "id": 8060842441, + "imageLink": "https://90.img.avito.st/image/1/1.kg1foLa1PuQpAOzkd86GKmcAPOThC_zlOQE85g.k3bmz3PqoQnYHj81udy1rgF9iy6rmv4pkCx5024vLeA", + "isFavorite": false, + "itemLink": "/ekaterinburg/kvartiry/1-k._kvartira_545_m_24_et._8060842441", + "price": 7490000, + "publishDate": 1778847849, + "title": "1-к. квартира, 54,5 м², 2/4 эт." + }, + { + "address": "Свердловская область, Екатеринбург, улица Малышева, 109", + "exposure": 28, + "hasGoodPriceBadge": false, + "id": 8066712904, + "imageLink": "https://20.img.avito.st/image/1/1.kYR7Rba1PW0N5e9tVW-QnELlP23F7v9sHeQ_bw.8emEbUAnDfDmjFC4t4lyFpKJc1ZPOte3I3elhIkqseg?cqp=2.nqHHyj8nxExgujaMxUsx0_L17NSXZ1ZQKCIQgy1FRSfAZeHHzJ12cn_5SQ7YtQ4=", + "isFavorite": false, + "itemLink": "/ekaterinburg/kvartiry/3-k._kvartira_52_m_35_et._8066712904", + "metro": { + "colors": [ + "#0A6F20" + ], + "distance": "2,6 км", + "name": "Площадь 1905 года" + }, + "price": 7520000, + "publishDate": 1777491918, + "title": "3-к. квартира, 52 м², 3/5 эт." + }, + { + "address": "Свердловская обл., муниципальное образование Екатеринбург, Екатеринбург, пр-т Ленина, 103", + "exposure": 9, + "hasGoodPriceBadge": false, + "id": 8120591680, + "imageLink": "https://20.img.avito.st/image/1/1.lR-6X7a1OfbM_-v2nA2GQoP_O_YE9Pv33P479A.cG8CmMm3YH5JmMgDlCIIrqA-jWy9cDH46kaAMAnU7Nk?cqp=2.nqHHyj8nxExgujaMxUsx0_L17NSXZ1ZQKCIQgy1FRSfAZeHHzJ12cn_5SQ7YtQ4=", + "isFavorite": false, + "itemLink": "/ekaterinburg/kvartiry/2-k._kvartira_551_m_15_et._8120591680", + "metro": { + "colors": [ + "#0A6F20" + ], + "distance": "2,9 км", + "name": "Динамо" + }, + "price": 7530000, + "publishDate": 1779216343, + "title": "2-к. квартира, 55,1 м², 1/5 эт." + }, + { + "address": "Свердловская область, Екатеринбург, улица Коминтерна, 20", + "exposure": 101, + "hasGoodPriceBadge": false, + "id": 7938143586, + "imageLink": "https://60.img.avito.st/image/1/1.BL5067a1qFcCS3pXcPdnzTxKqlfKQGpWEkqqVQ.NN0956Nx_wZJueNDEnfdVWPeyve6hi_bxAUHVVmd65k", + "isFavorite": false, + "itemLink": "/ekaterinburg/kvartiry/3-k._kvartira_549_m_25_et._7938143586", + "metro": { + "colors": [ + "#0A6F20" + ], + "distance": "3,4 км", + "name": "Геологическая" + }, + "price": 7190000, + "publishDate": 1771241225, + "title": "3-к. квартира, 54,9 м², 2/5 эт." + }, + { + "address": "Свердловская область, Екатеринбург, улица Малышева, 130Б", + "exposure": 244, + "hasGoodPriceBadge": true, + "id": 4638238123, + "imageLink": "https://30.img.avito.st/image/1/1.PpTjQ7a1kn2V40B9kxd2xebikH1d6FB8heKQfw.fCUNFxjvSx8cYylDSxHgGqQRGf7eYAxmvFs28MjGkxc?cqp=2.nqHHyj8nxExgujaMxUsx0_L17NSXZ1ZQKCIQgy1FRSfAZeHHzJ12cn_5SQ7YtQ4=", + "isFavorite": false, + "itemLink": "/ekaterinburg/kvartiry/4-k._kvartira_562_m_15_et._4638238123", + "metro": { + "colors": [ + "#0A6F20" + ], + "distance": "3,1 км", + "name": "Площадь 1905 года" + }, + "price": 7000000, + "publishDate": 1738063015, + "title": "4-к. квартира, 56,2 м², 1/5 эт." + }, + { + "address": "Свердловская обл., муниципальное образование Екатеринбург, Екатеринбург, Студенческая ул., 80", + "exposure": 8, + "hasGoodPriceBadge": false, + "id": 8131017429, + "imageLink": "https://40.img.avito.st/image/1/1.4fku77a1TRBYT58QYPiv3xdPTxCQRI8RSE5PEg.pEjigFdQ2vb_8pkuReSJ1w9xXHSegiJjXXG8-7uJzYQ", + "isFavorite": false, + "itemLink": "/ekaterinburg/kvartiry/1-k._kvartira_41_m_1125_et._8131017429", + "price": 7000000, + "publishDate": 1779276588, + "title": "1-к. квартира, 41 м², 11/25 эт." + }, + { + "address": "Свердловская область, Екатеринбург, Комсомольская улица, 50", + "exposure": 102, + "hasGoodPriceBadge": false, + "id": 7901596052, + "imageLink": "https://30.img.avito.st/image/1/1.plJjuba1CrsVGdi7I7uPXSoYCLvdEsi6BRgIuQ.gy3OfVsh4ZQEyPnlX9Kse1VQDKqYHgUR6H00mt6oBvM?cqp=2.nqHHyj8nxExgujaMxUsx0_L17NSXZ1ZQKCIQgy1FRSfAZeHHzJ12cn_5SQ7YtQ4=", + "isFavorite": false, + "itemLink": "/ekaterinburg/kvartiry/4-k._kvartira_729_m_15_et._7901596052", + "price": 6990000, + "publishDate": 1770308156, + "title": "4-к. квартира, 72,9 м², 1/5 эт." + } + ], + "searchSimilarUrl": "/ekaterinburg/kvartiry/prodam/panelnyy_dom-ASgBAgICAkSSA8YQ5Af6UQ?f=ASgBAQECBESSA8YQ5Af6Uay~DaTHNYK9DtCk0QEBQMoINIJZgFmEWQJFhAkXeyJmcm9tIjo0My4yLCJ0byI6NjQuOH3GmgwdeyJmcm9tIjo1ODk2MDAwLCJ0byI6ODg0NDAwMH0" + } +} \ No newline at end of file diff --git a/tradein-mvp/backend/tests/test_avito_imv_parse.py b/tradein-mvp/backend/tests/test_avito_imv_parse.py index ea132f51..e129bdcc 100644 --- a/tradein-mvp/backend/tests/test_avito_imv_parse.py +++ b/tradein-mvp/backend/tests/test_avito_imv_parse.py @@ -3,22 +3,35 @@ Без сети — только cache_key вычисление, парсинг заголовков, unix→date, dataclass конструирование. """ +import json from datetime import date +from pathlib import Path import pytest from app.services.scrapers.avito_imv import ( + IMVAddressNotFoundError, IMVEvaluation, IMVGeo, IMVPlacementHistoryItem, - IMVSuggestion, + _parse_geo_position, + _parse_placement_history, _parse_placement_item, + _parse_price, _parse_suggestion, + _parse_suggestions, _parse_title, _unix_to_date, compute_imv_cache_key, ) +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _load_fixture(name: str) -> dict: + with (_FIXTURES / name).open(encoding="utf-8") as f: + return json.load(f) + # --------------------------------------------------------------------------- # cache_key @@ -285,3 +298,145 @@ def test_dataclass_with_history(): assert len(e.placement_history) == 1 assert e.placement_history[0].removed_date == date(2024, 5, 12) assert e.market_count == 800 + + +# --------------------------------------------------------------------------- +# _parse_geo_position (step B) — driven by live-captured fixture (issue #565) +# --------------------------------------------------------------------------- + + +def test_parse_geo_position_from_fixture(): + """Real /js/v2/geo/position response (ЕКБ, ул. Малышева 125) → IMVGeo.""" + data_b = _load_fixture("avito_imv_geo_position.json") + geo = _parse_geo_position(data_b, lat=56.840872, lon=60.654077) + + assert geo.geo_hash # non-empty JWT + assert geo.geo_hash.startswith("eyJ") # base64url JWT header + assert geo.geo_hash.count(".") == 2 # header.payload.signature + assert geo.avito_address_id == 22319477 + assert geo.avito_location_id == 654070 + assert geo.avito_district_id == 789 + assert geo.lat == pytest.approx(56.840872) + assert geo.lon == pytest.approx(60.654077) + + +def test_parse_geo_position_ekb_has_no_metro(): + """ЕКБ-окраина не отдаёт metroId — должен быть None, не падать.""" + data_b = _load_fixture("avito_imv_geo_position.json") + geo = _parse_geo_position(data_b, lat=56.84, lon=60.65) + assert geo.avito_metro_id is None + + +def test_parse_geo_position_missing_hash_raises(): + """Пустой geoFieldsHash → IMVAddressNotFoundError (graceful skip в estimator).""" + with pytest.raises(IMVAddressNotFoundError): + _parse_geo_position( + {"address": "X", "errorText": "not found", "geoFieldsHash": ""}, + lat=1.0, + lon=2.0, + ) + + +# --------------------------------------------------------------------------- +# _parse_price / placement / suggestions (step C) — live-captured fixture +# --------------------------------------------------------------------------- + + +def test_parse_price_from_fixture(): + """Real realty-imv/get-data price block populates all three prices + count.""" + data = _load_fixture("avito_imv_getdata.json") + recommended, lower, higher, count = _parse_price(data) + + assert recommended == 7370000 + assert lower == 7148900 + assert higher == 7738500 + assert count == 800 + assert lower < recommended < higher + + +def test_parse_price_missing_block_defaults_zero(): + """Нет price блока (degenerate ответ) → нули, не KeyError.""" + recommended, lower, higher, count = _parse_price({}) + assert (recommended, lower, higher) == (0, 0, 0) + assert count is None + + +def test_parse_placement_history_from_fixture(): + """placementHistory.items парсятся в IMVPlacementHistoryItem с removed_date.""" + data = _load_fixture("avito_imv_getdata.json") + items = _parse_placement_history(data) + + assert len(items) >= 1 + first = items[0] + assert first.ext_item_id # str(id) + assert first.start_price and first.last_price + assert isinstance(first.start_price_date, date) + assert isinstance(first.removed_date, date) + # raw_payload очищен от тяжёлого itemImage + assert first.raw_payload is not None + assert "itemImage" not in first.raw_payload + + +def test_parse_placement_history_absent_block_returns_empty(): + """placementHistory отсутствует (как у проспект Ленина) → [] без падения.""" + data = _load_fixture("avito_imv_getdata.json") + data.pop("placementHistory", None) + assert _parse_placement_history(data) == [] + + +def test_parse_suggestions_from_fixture(): + """suggestions.items парсятся + searchSimilarUrl извлекается.""" + data = _load_fixture("avito_imv_getdata.json") + suggestions, search_url = _parse_suggestions(data) + + assert len(suggestions) >= 1 + assert all(s.ext_item_id for s in suggestions) + assert all(s.price_rub for s in suggestions) + # raw_payload очищен от imageLink + assert "imageLink" not in (suggestions[0].raw_payload or {}) + assert search_url is not None + assert search_url.startswith("/") + + +def test_fixtures_feed_full_evaluation_shape(): + """Сквозной оффлайн-тест: fixtures B+C → заполненный IMVEvaluation. + + Зеркалит то, что _geocode + _imv_evaluate собирают из живого ответа, + без сети. Регрессионный guard для контракта step B/C (issue #565). + """ + data_b = _load_fixture("avito_imv_geo_position.json") + data_c = _load_fixture("avito_imv_getdata.json") + + geo = _parse_geo_position(data_b, lat=56.840872, lon=60.654077) + recommended, lower, higher, count = _parse_price(data_c) + history = _parse_placement_history(data_c) + suggestions, search_url = _parse_suggestions(data_c) + + e = IMVEvaluation( + cache_key="c" * 64, + address="Екатеринбург, улица Малышева, 125", + rooms=2, + area_m2=54.0, + floor=4, + floor_at_home=9, + house_type="panel", + renovation_type="cosmetic", + has_balcony=True, + has_loggia=False, + geo=geo, + recommended_price=recommended, + lower_price=lower, + higher_price=higher, + market_count=count, + placement_history=history, + suggestions=suggestions, + search_similar_url=search_url, + raw_response=data_c, + ) + + assert e.geo.geo_hash # non-empty + assert e.recommended_price == 7370000 + assert e.lower_price == 7148900 + assert e.higher_price == 7738500 + assert e.market_count == 800 + assert len(e.suggestions) >= 1