From 7f23a9b3f1f362e74e322311622232e78602572c Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 11:04:24 +0000 Subject: [PATCH] feat(tradein-imv): retry with cleaned address + categorize errors (#505) --- tradein-mvp/backend/app/services/estimator.py | 51 ++++++++++++ .../app/services/scrapers/avito_imv.py | 38 ++++++++- tradein-mvp/scripts/probe-imv-by-address.py | 83 +++++++++++++++++++ 3 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 tradein-mvp/scripts/probe-imv-by-address.py diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index 0e6aad9a..52d85280 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -22,6 +22,7 @@ from __future__ import annotations import hashlib import json import logging +import re from datetime import UTC, datetime, timedelta from typing import Any from uuid import uuid4 @@ -34,7 +35,9 @@ from app.services.geocoder import GeocodeResult, geocode from app.services.house_metadata import get_house_metadata from app.services.scrapers.avito_imv import ( IMVAddressNotFoundError, + IMVAuthError, IMVEvaluation, + IMVTransientError, compute_imv_cache_key, evaluate_via_imv, save_imv_evaluation, @@ -105,6 +108,13 @@ def _repair_coefficient(repair_state: str | None) -> float: # ── Avito IMV cache lookup (Stage 3) ──────────────────────────────────────── IMV_CACHE_TTL_HOURS = 24 +# Префиксы в адресе, которые Avito-геокодер не распознаёт (не жилые назначения). +# Пример: "Склад, ул. Заводская, д. 44-а" → "ул. Заводская, д. 44-а" +_NOISE_PREFIX_RE = re.compile( + r"(Склад|Гараж|Подсобка|Нежилое|Помещение|Цех),\s*", + flags=re.IGNORECASE, +) + YANDEX_VALUATION_CACHE_TTL_HOURS = 24 YANDEX_VALUATION_DEFAULT_CATEGORY = "APARTMENT" YANDEX_VALUATION_DEFAULT_TYPE = "SELL" @@ -209,6 +219,47 @@ async def _get_or_fetch_imv_cached( except IMVAddressNotFoundError as e: logger.warning("imv: address not found in Avito geocoder: %s", e) + # Retry once with noise prefixes stripped (e.g. "Склад, ул. X" → "ул. X") + cleaned = _NOISE_PREFIX_RE.sub("", address) + if cleaned != address: + logger.info( + "imv: retry with cleaned address %r → %r", + address[:60], + cleaned[:60], + ) + try: + result = await evaluate_via_imv( + address=cleaned, + rooms=rooms, + area_m2=area_m2, + floor=floor, + floor_at_home=floor_at_home, + house_type=house_type, + renovation_type=renovation_type, + has_balcony=has_balcony, + has_loggia=has_loggia, + ) + save_imv_evaluation(db, result, estimate_id=estimate_id_for_link) + logger.info( + "imv: retry OK recommended=%d range=(%d, %d) count=%d", + result.recommended_price, + result.lower_price, + result.higher_price, + result.market_count or 0, + ) + return result + except IMVAddressNotFoundError: + logger.warning("imv: cleaned address also not found — giving up") + except Exception as retry_exc: + logger.warning("imv: retry failed: %s", retry_exc) + return None + except IMVAuthError as e: + logger.error( + "imv: auth/quota error — manual action required: %s", e, + ) + return None + except IMVTransientError as e: + logger.warning("imv: transient error, skipping retry in estimator context: %s", e) return None except Exception as e: logger.warning("imv: fetch failed — estimator продолжает без IMV: %s", e) diff --git a/tradein-mvp/backend/app/services/scrapers/avito_imv.py b/tradein-mvp/backend/app/services/scrapers/avito_imv.py index c56c3562..815ce4c0 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_imv.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_imv.py @@ -53,6 +53,14 @@ class IMVCityMismatchError(Exception): """Raised when locationId не входит в ожидаемый город (sanity check).""" +class IMVAuthError(Exception): + """Avito API rejected auth / quota exceeded (HTTP 401 / 403).""" + + +class IMVTransientError(Exception): + """Network / 5xx / timeout — retryable.""" + + # --------------------------------------------------------------------------- # Dataclasses # --------------------------------------------------------------------------- @@ -309,14 +317,33 @@ async def evaluate_via_imv( return evaluation +def _raise_for_status_categorized(resp: Any, context: str) -> None: + """Raise typed IMV exception based on HTTP status code. + + Используется вместо bare raise_for_status() чтобы caller мог различать + auth-failure (требует human attention) от transient (retryable). + """ + status = resp.status_code + if status in (401, 403): + raise IMVAuthError(f"Avito {context}: HTTP {status} — auth rejected / quota exceeded") + if status >= 500: + raise IMVTransientError(f"Avito {context}: HTTP {status} — transient server error") + if status >= 400: + resp.raise_for_status() # other 4xx → standard HTTPStatusError + + async def _geocode(session: Any, address: str) -> IMVGeo: """Request 1: GET /web/1/coords/by_address → IMVGeo.""" params = urlencode({"address": address}) url = f"{AVITO_BASE}{COORDS_ENDPOINT}?{params}" logger.info("IMV geocode: address=%r", address) - resp = await session.get(url, headers=_COMMON_HEADERS) - resp.raise_for_status() + try: + resp = await session.get(url, headers=_COMMON_HEADERS) + except Exception as exc: + # Timeout / connection error — retryable + raise IMVTransientError(f"Avito geocode network error: {exc}") from exc + _raise_for_status_categorized(resp, "geocode") data: dict[str, Any] = resp.json() geo_hash = data.get("geoHash") @@ -381,8 +408,11 @@ async def _imv_evaluate( floor_at_home, ) - resp = await session.post(url, json=body, headers=headers) - resp.raise_for_status() + try: + resp = await session.post(url, json=body, headers=headers) + except Exception as exc: + raise IMVTransientError(f"Avito IMV network error: {exc}") from exc + _raise_for_status_categorized(resp, "imv-evaluate") data: dict[str, Any] = resp.json() price_block: dict[str, Any] = data.get("price") or {} diff --git a/tradein-mvp/scripts/probe-imv-by-address.py b/tradein-mvp/scripts/probe-imv-by-address.py new file mode 100644 index 00000000..2cc7f272 --- /dev/null +++ b/tradein-mvp/scripts/probe-imv-by-address.py @@ -0,0 +1,83 @@ +"""Diagnose IMV API behavior for a specific address. + +Usage: + python scripts/probe-imv-by-address.py "Свердловская область, г. Екатеринбург, Склад, ул. Заводская, д. 44-а" + +Prints: raw API response, error category, cleaned-address retry result. +""" +import asyncio +import logging +import sys + +logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") + + +async def main(address: str) -> None: + from app.services.scrapers.avito_imv import ( + IMVAddressNotFoundError, + IMVAuthError, + IMVTransientError, + evaluate_via_imv, + ) + + # Default params for ЕКБ 3-room 60м² + params = dict( + rooms=3, + area_m2=60.0, + floor=5, + floor_at_home=17, + house_type="panel", + renovation_type="cosmetic", + has_balcony=False, + has_loggia=False, + ) + + print(f"\n=== Attempt 1: original address ===\n{address}\n") + try: + result = await evaluate_via_imv(address=address, **params) # type: ignore[arg-type] + print(f"OK: recommended={result.recommended_price}") + return + except IMVAddressNotFoundError as e: + print(f"FAIL: address not found (IMVAddressNotFoundError): {e}") + except IMVAuthError as e: + print(f"FAIL: auth/quota error (IMVAuthError): {e}") + return + except IMVTransientError as e: + print(f"FAIL: transient/network error (IMVTransientError): {e}") + return + except Exception as e: + print(f"FAIL: {type(e).__name__}: {e}") + return + + import re + + cleaned = re.sub( + r"(Склад|Гараж|Подсобка|Нежилое|Помещение|Цех),\s*", + "", + address, + flags=re.IGNORECASE, + ) + if cleaned != address: + print(f"\n=== Attempt 2: cleaned address ===\n{cleaned}\n") + try: + result = await evaluate_via_imv(address=cleaned, **params) # type: ignore[arg-type] + print(f"OK on retry: recommended={result.recommended_price}") + except IMVAddressNotFoundError as e: + print(f"FAIL on retry (IMVAddressNotFoundError): {e}") + except IMVAuthError as e: + print(f"FAIL on retry (IMVAuthError — quota/auth): {e}") + except IMVTransientError as e: + print(f"FAIL on retry (IMVTransientError — network/5xx): {e}") + except Exception as e: + print(f"FAIL on retry: {type(e).__name__}: {e}") + else: + print("\nNo noise prefix found — cleaned address identical, skipping retry.") + + +if __name__ == "__main__": + _address = ( + sys.argv[1] + if len(sys.argv) > 1 + else "Свердловская область, г. Екатеринбург, Склад, ул. Заводская, д. 44-а" + ) + asyncio.run(main(_address))