diff --git a/tradein-mvp/backend/app/services/scrapers/avito_imv.py b/tradein-mvp/backend/app/services/scrapers/avito_imv.py deleted file mode 100644 index bb8bb481..00000000 --- a/tradein-mvp/backend/app/services/scrapers/avito_imv.py +++ /dev/null @@ -1,853 +0,0 @@ -"""avito_imv.py — Avito IMV evaluation API client (3-request flow, 2026-05+). - -Stage 2d: geocode + IMV evaluation. - -Flow (current contract, подтверждён живым capture 2026-05 — fixtures -tests/fixtures/avito_imv_geo_position.json + avito_imv_getdata.json): - 0) GET /evaluation/realty → warm-up (seed anti-bot cookies) - 1) GET /web/1/coords/by_address?address= → normalize + lat/lon - 2) POST /js/v2/geo/position → rich JWT (geoFieldsHash) - 3) POST /web/1/realty-imv/get-data → price + placementHistory? + suggestions - -Anti-bot: zerkalim AvitoScraper (avito.py) — chrome120 TLS + document-заголовки + -warm-up GET + XHR Sec-Fetch заголовки. Bare-session XHR ловят 403 с server-IP. - -Таблицы: - avito_imv_evaluations (018_avito_imv_evaluations.sql) - house_placement_history (017_house_placement_history.sql) -""" - -from __future__ import annotations - -import hashlib -import json -import logging -import re -from dataclasses import dataclass, field -from datetime import date -from typing import TYPE_CHECKING, Any -from urllib.parse import urlencode -from uuid import UUID - -from sqlalchemy import text -from sqlalchemy.orm import Session - -from app.services.scrapers.avito_shared import _unix_to_date - -if TYPE_CHECKING: - # Импорт под TYPE_CHECKING (как в avito_detail.py/avito_houses.py) — избегаем - # hard import cycle browser_fetcher → config → ... и лишней зависимости в - # offline-тестах парсинга, которым browser не нужен. - from app.services.scrapers.browser_fetcher import BrowserFetcher - -logger = logging.getLogger(__name__) - -AVITO_BASE = "https://www.avito.ru" -COORDS_ENDPOINT = "/web/1/coords/by_address" -POSITION_ENDPOINT = "/js/v2/geo/position" -IMV_ENDPOINT = "/web/1/realty-imv/get-data" -# Тёплая страница для seed anti-bot cookies (srv_id/_avisc/u/...) перед XHR. -WARMUP_URL = f"{AVITO_BASE}/evaluation/realty" - -# Заголовки document-навигации (warm-up GET) — зеркалят production-набор -# из avito.py (AvitoScraper.__aenter__). Нужны чтобы пройти TLS+header anti-bot. -_DOC_HEADERS = { - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - "Cache-Control": "max-age=0", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", -} - -# Заголовки XHR/fetch для API-шагов (coords/position/get-data). Имитируют -# fetch() со страницы /evaluation/realty: JSON Accept + same-origin Sec-Fetch. -_COMMON_HEADERS = { - "Accept": "application/json, text/plain, */*", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - "Referer": "https://www.avito.ru/evaluation/realty", - "X-Requested-With": "XMLHttpRequest", - "Sec-Fetch-Dest": "empty", - "Sec-Fetch-Mode": "cors", - "Sec-Fetch-Site": "same-origin", -} - -# Timeout для всех Avito-вызовов (без него curl_cffi висит на anti-bot stall). -_HTTP_TIMEOUT_SEC = 25 - - -# --------------------------------------------------------------------------- -# Exceptions -# --------------------------------------------------------------------------- - - -class IMVAddressNotFoundError(Exception): - """Raised when /coords/by_address returns no geoHash for given address.""" - - -class IMVCityMismatchError(Exception): - """Raised when locationId не входит в ожидаемый город (sanity check).""" - - -class IMVAuthError(Exception): - """Avito API rejected auth / quota exceeded (HTTP 401 / 403).""" - - -class IMVTransientError(Exception): - """Network / 5xx / timeout — retryable.""" - - -# --------------------------------------------------------------------------- -# Dataclasses -# --------------------------------------------------------------------------- - - -@dataclass -class IMVGeo: - geo_hash: str # JWT — не сохраняем как отдельную колонку, только в raw_response - lat: float | None = None - lon: float | None = None - avito_address_id: int | None = None - avito_location_id: int | None = None - avito_metro_id: int | None = None - avito_district_id: int | None = None - - -@dataclass -class IMVPlacementHistoryItem: - ext_item_id: str # str(item['id']) - title: str | None = None - rooms: int | None = None # парсится из title (опционально) - area_m2: float | None = None # парсится из title (опционально) - floor: int | None = None # парсится из title (опционально) - total_floors: int | None = None # парсится из title (опционально) - start_price: int | None = None - start_price_date: date | None = None # unix → date - last_price: int | None = None - last_price_date: date | None = None - removed_date: date | None = None # только IMV API (в widget housePlacementHistory нет) - exposure_days: int | None = None - raw_payload: dict[str, Any] | None = None - - -@dataclass -class IMVSuggestion: - ext_item_id: str - title: str | None = None - address: str | None = None - price_rub: int | None = None - exposure_days: int | None = None - publish_date: date | None = None - item_url: str | None = None - metro_name: str | None = None - metro_distance: str | None = None - metro_color: str | None = None - has_good_price_badge: bool = False - raw_payload: dict[str, Any] | None = None - - -@dataclass -class IMVEvaluation: - cache_key: str # sha256 - address: str - rooms: int - area_m2: float - floor: int - floor_at_home: int - house_type: str - renovation_type: str - has_balcony: bool - has_loggia: bool - - geo: IMVGeo - recommended_price: int - lower_price: int - higher_price: int - market_count: int | None = None - - placement_history: list[IMVPlacementHistoryItem] = field(default_factory=list) - suggestions: list[IMVSuggestion] = field(default_factory=list) - search_similar_url: str | None = None - raw_response: dict[str, Any] | None = None - - -# --------------------------------------------------------------------------- -# Cache key -# --------------------------------------------------------------------------- - - -def compute_imv_cache_key( - address: str, - rooms: int, - area_m2: float, - floor: int, - floor_at_home: int, - house_type: str, - renovation_type: str, - has_balcony: bool, - has_loggia: bool, -) -> str: - """Детерминированный sha256 ключ для 24h-кэша IMV оценки.""" - key_src = ( - f"{address}|{rooms}|{area_m2}|{floor}|{floor_at_home}" - f"|{house_type}|{renovation_type}|{int(has_balcony)}|{int(has_loggia)}" - ) - return hashlib.sha256(key_src.encode("utf-8")).hexdigest() - - -# --------------------------------------------------------------------------- -# Parse helpers -# --------------------------------------------------------------------------- - - -# Паттерн для заголовка вида "2-к. квартира, 42 м², 4/5 эт." -_TITLE_RE = re.compile( - r"^(?P\d+)-к[.\s].*?(?P[\d,]+)\s*м².*?(?P\d+)/(?P\d+)\s*эт", - re.IGNORECASE, -) - - -def _parse_title(title: str | None) -> dict[str, int | float | None]: - """Парсит rooms, area_m2, floor, total_floors из Avito-заголовка объявления.""" - result: dict[str, int | float | None] = { - "rooms": None, - "area_m2": None, - "floor": None, - "total_floors": None, - } - if not title: - return result - m = _TITLE_RE.match(title.strip()) - if not m: - return result - try: - result["rooms"] = int(m.group("rooms")) - result["area_m2"] = float(m.group("area").replace(",", ".")) - result["floor"] = int(m.group("floor")) - result["total_floors"] = int(m.group("total")) - except (ValueError, AttributeError): - pass - return result - - -def _parse_placement_item(raw: dict[str, Any]) -> IMVPlacementHistoryItem: - """Парсит один элемент из placementHistory.items.""" - title = raw.get("title") - parsed = _parse_title(title) - - # Фильтруем itemImage из raw_payload (не нужны) - clean_raw = {k: v for k, v in raw.items() if k not in ("itemImage",)} - - return IMVPlacementHistoryItem( - ext_item_id=str(raw["id"]), - title=title, - rooms=parsed["rooms"], # type: ignore[arg-type] - area_m2=parsed["area_m2"], # type: ignore[arg-type] - floor=parsed["floor"], # type: ignore[arg-type] - total_floors=parsed["total_floors"], # type: ignore[arg-type] - start_price=raw.get("startPrice"), - start_price_date=_unix_to_date(raw.get("startPriceDate")), - last_price=raw.get("lastPrice"), - last_price_date=_unix_to_date(raw.get("lastPriceDate")), - removed_date=_unix_to_date(raw.get("removedDate")), - exposure_days=raw.get("exposure"), - raw_payload=clean_raw, - ) - - -def _parse_suggestion(raw: dict[str, Any]) -> IMVSuggestion: - """Парсит один элемент из suggestions.items.""" - metro = raw.get("metro") or {} - colors: list[str] = metro.get("colors") or [] - - # Фильтруем imageLink из raw_payload - clean_raw = {k: v for k, v in raw.items() if k not in ("imageLink",)} - - return IMVSuggestion( - ext_item_id=str(raw["id"]), - title=raw.get("title"), - address=raw.get("address"), - price_rub=raw.get("price"), - exposure_days=raw.get("exposure"), - publish_date=_unix_to_date(raw.get("publishDate")), - item_url=raw.get("itemLink"), - metro_name=metro.get("name"), - metro_distance=metro.get("distance"), - metro_color=colors[0] if colors else None, - has_good_price_badge=bool(raw.get("hasGoodPriceBadge", False)), - raw_payload=clean_raw, - ) - - -def _parse_geo_position(data_b: dict[str, Any], *, lat: float, lon: float) -> IMVGeo: - """Парсит ответ POST /js/v2/geo/position (step B) → IMVGeo. - - Контракт (подтверждён живым capture 2026-05, fixture avito_imv_geo_position.json): - {address, addressId, districtId, geoFieldsHash, latitude, longitude, - locationId, parentLocationId, metroId?, errorText, ...} - - `geoFieldsHash` — JWT, payload которого кодирует {latitude, longitude, - address, addressId, locationId, districtId}; это то, что step C ожидает как - `geoHash`. metroId присутствует только для метро-городов (None для ЕКБ-окраин). - - Raises: - IMVAddressNotFoundError если geoFieldsHash пустой/отсутствует. - """ - jwt = data_b.get("geoFieldsHash") - if not jwt: - raise IMVAddressNotFoundError( - f"Avito /js/v2/geo/position не вернул geoFieldsHash для " - f"{str(data_b.get('address'))[:60]!r}: errorText={data_b.get('errorText')!r}" - ) - return IMVGeo( - geo_hash=jwt, - lat=lat, - lon=lon, - avito_address_id=data_b.get("addressId"), - avito_location_id=data_b.get("locationId"), - avito_metro_id=data_b.get("metroId"), - avito_district_id=data_b.get("districtId"), - ) - - -def _parse_price(data: dict[str, Any]) -> tuple[int, int, int, int | None]: - """Парсит top-level `price` блок из realty-imv/get-data (step C). - - Контракт (fixture avito_imv_getdata.json): - price = {recommendedPrice, lowerPrice, higherPrice, count, addItemUrl} - - Returns: (recommended_price, lower_price, higher_price, market_count). - Отсутствующие цены → 0 (caller трактует как "нет оценки"); count → None. - """ - price_block: dict[str, Any] = data.get("price") or {} - recommended_price: int = price_block.get("recommendedPrice") or 0 - lower_price: int = price_block.get("lowerPrice") or 0 - higher_price: int = price_block.get("higherPrice") or 0 - market_count: int | None = price_block.get("count") - return recommended_price, lower_price, higher_price, market_count - - -def _parse_placement_history(data: dict[str, Any]) -> list[IMVPlacementHistoryItem]: - """Парсит опциональный `placementHistory.items` блок (step C). - - placementHistory отсутствует для части адресов (живой capture: проспект - Ленина вернул только itemParams/price/suggestions) — тогда возвращаем []. - Битые элементы пропускаем с warning, не роняя весь ответ. - """ - history_block: dict[str, Any] = data.get("placementHistory") or {} - items: list[IMVPlacementHistoryItem] = [] - for raw_item in history_block.get("items") or []: - try: - items.append(_parse_placement_item(raw_item)) - except (KeyError, TypeError) as exc: - logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc) - return items - - -def _parse_suggestions(data: dict[str, Any]) -> tuple[list[IMVSuggestion], str | None]: - """Парсит `suggestions` блок (step C) → (items, searchSimilarUrl).""" - suggestions_block: dict[str, Any] = data.get("suggestions") or {} - suggestions: list[IMVSuggestion] = [] - for raw_sugg in suggestions_block.get("items") or []: - try: - suggestions.append(_parse_suggestion(raw_sugg)) - except (KeyError, TypeError) as exc: - logger.warning("IMV: не удалось распарсить suggestion: %s", exc) - return suggestions, suggestions_block.get("searchSimilarUrl") - - -# --------------------------------------------------------------------------- -# Browser-transport adapter (#915 Stage 3) -# --------------------------------------------------------------------------- - - -class _BrowserResponse: - """Мимикрия под curl_cffi Response поверх ответа BrowserFetcher.fetch_json. - - IMV-флоу читает только .status_code / .text / .json() / .raise_for_status() — - адаптируем ровно эти четыре (подтверждено по avito_imv.py: warm-up, _geocode, - _imv_evaluate, _raise_for_status_categorized). - """ - - def __init__(self, status: int, body: str) -> None: - self.status_code = status - self.text = body - - def json(self) -> Any: - return json.loads(self.text) - - def raise_for_status(self) -> None: - if self.status_code >= 400: - # _raise_for_status_categorized уже обрабатывает 401/403/5xx ДО вызова - # raise_for_status(); сюда попадают только "прочие 4xx" (fall-through). - # Зеркалим curl_cffi/httpx-семантику ровно настолько, чтобы caller - # получил исключение. - raise RuntimeError(f"HTTP {self.status_code}") - - -class _BrowserSessionAdapter: - """Адаптер: IMV-флоу думает что это curl_cffi-сессия, а ходит через камуфокс - in-page fetch (same-origin avito.ru). Куки/JWT живут в контексте браузера - провайдера, поэтому 4 запроса IMV переиспользуют warmed-сессию. Прокси и - fingerprint — из sidecar (обходит datacenter-403, #562/#853).""" - - def __init__(self, browser_fetcher: BrowserFetcher, *, origin: str) -> None: - self._bf = browser_fetcher - self._origin = origin - - async def get(self, url: str, *, headers: dict | None = None) -> _BrowserResponse: - r = await self._bf.fetch_json(url, method="GET", headers=headers, origin=self._origin) - return _BrowserResponse(r["status"], r["body"]) - - async def post( - self, url: str, *, headers: dict | None = None, json: Any = None - ) -> _BrowserResponse: - import json as _json - - body = _json.dumps(json) if json is not None else None - r = await self._bf.fetch_json( - url, method="POST", headers=headers, body=body, origin=self._origin - ) - return _BrowserResponse(r["status"], r["body"]) - - async def close(self) -> None: - # browser принадлежит вызывающему (backfill открывает один на батч) — no-op. - return None - - -# --------------------------------------------------------------------------- -# Main async function -# --------------------------------------------------------------------------- - - -async def evaluate_via_imv( - address: str, - rooms: int, - area_m2: float, - floor: int, - floor_at_home: int, - house_type: str, # panel/brick/monolith/monolith_brick/block/wood - renovation_type: str, # required/cosmetic/euro/designer - has_balcony: bool, - has_loggia: bool, - *, - cffi_session: Any | None = None, - browser_fetcher: BrowserFetcher | None = None, -) -> IMVEvaluation: - """Avito IMV — 3 HTTP requests (current contract 2026-05+): - - 1) GET /web/1/coords/by_address?address= → normalize + lat/lon - 2) POST /js/v2/geo/position → rich JWT (geoFieldsHash) - 3) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions - - Returns IMVEvaluation с уже вычисленным cache_key. - - Raises: - IMVAddressNotFoundError если point/normalizedAddress пустые (step 1) - или geoFieldsHash не вернулся (step 2) - IMVAuthError на 401/403 - IMVTransientError на 5xx / network errors - """ - # #915 Stage 3: browser-транспорт. Если передан browser_fetcher — ходим через - # камуфокс in-page fetch (same-origin avito.ru), а не curl_cffi. Прокси + - # warmed-cookies + реальный fingerprint из sidecar обходят datacenter-403 - # (#562/#853). _own_session=True → finally дёрнет adapter.close() (no-op, - # браузер принадлежит caller'у — backfill открывает один на батч). Этот путь - # НЕ требует curl_cffi, поэтому импорт пакета остаётся только в else-ветке. - if browser_fetcher is not None: - cffi_session = _BrowserSessionAdapter(browser_fetcher, origin=WARMUP_URL) - _own_session = True - else: - # Импортируем здесь чтобы избежать циклических зависимостей при тестах без сети - try: - from curl_cffi.requests import AsyncSession as CffiAsyncSession - - from app.core.config import settings - - _own_session = False - if cffi_session is None: - # Зеркалим production-набор из AvitoScraper.__aenter__ (avito.py:150-162): - # chrome120 TLS + document-заголовки + timeout. Затем warm-up GET для - # seed anti-bot cookies — bare-session XHR Avito банит на server-IP. - # Mobile proxy wiring (#806 follow-up): только для own-session (переданная - # сессия уже с прокси от AvitoScraper). proxy=None → прямое подключение. - _proxy_url = settings.scraper_proxy_url - _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None - cffi_session = CffiAsyncSession( - impersonate="chrome120", - timeout=_HTTP_TIMEOUT_SEC, - proxies=_proxies, - headers=_DOC_HEADERS, - ) - _own_session = True - except ImportError as exc: - raise RuntimeError( - "curl_cffi не установлен. Добавь 'curl-cffi>=0.7.0' в pyproject.toml." - ) from exc - - try: - if _own_session: - await _warmup(cffi_session) - geo = await _geocode(cffi_session, address) - evaluation = await _imv_evaluate( - cffi_session, - geo=geo, - address=address, - rooms=rooms, - area_m2=area_m2, - floor=floor, - floor_at_home=floor_at_home, - house_type=house_type, - renovation_type=renovation_type, - has_balcony=has_balcony, - has_loggia=has_loggia, - ) - finally: - if _own_session: - await cffi_session.close() - - return evaluation - - -async def _warmup(session: Any) -> None: - """GET /evaluation/realty чтобы seed anti-bot cookies (srv_id/_avisc/u/...). - - Без warm-up XHR-шаги (coords/position/get-data) c server-IP ловят 403 anti-bot. - Best-effort: не роняем оценку если страница недоступна — последующие шаги - всё равно попробуют и дадут типизированную ошибку. - """ - try: - resp = await session.get(WARMUP_URL, headers=_DOC_HEADERS) - logger.info("IMV warm-up GET /evaluation/realty → HTTP %d", resp.status_code) - except Exception as exc: - # warm-up опционален: логируем и продолжаем — шаги ниже дадут типизированную - # ошибку, если cookies реально нужны. Не re-raise намеренно. - logger.warning("IMV warm-up failed (продолжаем без cookies): %s", exc) - - -def _raise_for_status_categorized(resp: Any, context: str) -> None: - """Raise typed IMV exception based on HTTP status code. - - Используется вместо bare raise_for_status() чтобы caller мог различать - auth-failure (требует human attention) от transient (retryable). - """ - status = resp.status_code - if status in (401, 403): - raise IMVAuthError(f"Avito {context}: HTTP {status} — auth rejected / quota exceeded") - if status >= 500: - raise IMVTransientError(f"Avito {context}: HTTP {status} — transient server error") - if status >= 400: - body_preview = (resp.text or "")[:200] - logger.warning("Avito %s HTTP %d: body=%r", context, status, body_preview) - resp.raise_for_status() # other 4xx → standard HTTPStatusError - - -async def _geocode(session: Any, address: str) -> IMVGeo: - """Two-step Avito IMV geocoder (current contract 2026-05+). - - Step A: GET /web/1/coords/by_address → normalize + lat/lon - Step B: POST /js/v2/geo/position → rich JWT (geoFieldsHash) with - addressId/locationId/metroId/districtId - - JWT from step B is what /realty-imv/get-data expects as `geoHash`. - """ - # Step A — normalize + coords - params = urlencode({"address": address}) - url_a = f"{AVITO_BASE}{COORDS_ENDPOINT}?{params}" - logger.info("IMV geocode A: address=%r", address) - - try: - resp_a = await session.get(url_a, headers=_COMMON_HEADERS) - except Exception as exc: - raise IMVTransientError(f"Avito geocode A network error: {exc}") from exc - _raise_for_status_categorized(resp_a, "geocode-A") - data_a: dict[str, Any] = resp_a.json() - - point = data_a.get("point") or {} - lat = point.get("latitude") - lon = point.get("longitude") - normalized = data_a.get("normalizedAddress") - - if not lat or not lon or not normalized: - raise IMVAddressNotFoundError(f"Avito /coords/by_address не вернул point для {address!r}") - - # Step B — rich JWT - url_b = f"{AVITO_BASE}{POSITION_ENDPOINT}" - payload_b = { - "address": normalized, - "addressId": "", - "categoryId": 24, # 24 = недвижимость / квартиры - "isRadius": False, - "latitude": lat, - "longitude": lon, - } - logger.info("IMV geocode B: position for %r", normalized[:60]) - - try: - resp_b = await session.post( - url_b, - headers={**_COMMON_HEADERS, "Content-Type": "application/json"}, - json=payload_b, - ) - except Exception as exc: - raise IMVTransientError(f"Avito geocode B network error: {exc}") from exc - _raise_for_status_categorized(resp_b, "geocode-B") - data_b: dict[str, Any] = resp_b.json() - - geo = _parse_geo_position(data_b, lat=lat, lon=lon) # raises IMVAddressNotFoundError - - logger.info( - "IMV geocode OK: addressId=%s locationId=%s metroId=%s lat=%s lon=%s", - geo.avito_address_id, - geo.avito_location_id, - geo.avito_metro_id, - lat, - lon, - ) - return geo - - -async def _imv_evaluate( - session: Any, - *, - geo: IMVGeo, - address: str, - rooms: int, - area_m2: float, - floor: int, - floor_at_home: int, - house_type: str, - renovation_type: str, - has_balcony: bool, - has_loggia: bool, -) -> IMVEvaluation: - """Request 2: POST /web/1/realty-imv/get-data → IMVEvaluation.""" - url = f"{AVITO_BASE}{IMV_ENDPOINT}" - body = { - "geoHash": geo.geo_hash, - "floor": floor, - "floorAtHome": floor_at_home, - "rooms": rooms, - "area": area_m2, - "houseType": house_type, - "renovationType": renovation_type, - "hasLoggia": has_loggia, - "hasBalcony": has_balcony, - } - headers = {**_COMMON_HEADERS, "Content-Type": "application/json"} - - logger.info( - "IMV evaluate: address=%r rooms=%d area=%.1f floor=%d/%d", - address, - rooms, - area_m2, - floor, - floor_at_home, - ) - - try: - resp = await session.post(url, json=body, headers=headers) - except Exception as exc: - raise IMVTransientError(f"Avito IMV network error: {exc}") from exc - _raise_for_status_categorized(resp, "imv-evaluate") - data: dict[str, Any] = resp.json() - - recommended_price, lower_price, higher_price, market_count = _parse_price(data) - placement_history = _parse_placement_history(data) - suggestions, search_similar_url = _parse_suggestions(data) - - logger.info( - "IMV evaluate OK: recommended=%d range=(%d, %d) count=%s history=%d suggestions=%d", - recommended_price, - lower_price, - higher_price, - market_count, - len(placement_history), - len(suggestions), - ) - - cache_key = compute_imv_cache_key( - address=address, - rooms=rooms, - area_m2=area_m2, - floor=floor, - floor_at_home=floor_at_home, - house_type=house_type, - renovation_type=renovation_type, - has_balcony=has_balcony, - has_loggia=has_loggia, - ) - - return IMVEvaluation( - cache_key=cache_key, - address=address, - rooms=rooms, - area_m2=area_m2, - floor=floor, - floor_at_home=floor_at_home, - house_type=house_type, - renovation_type=renovation_type, - has_balcony=has_balcony, - has_loggia=has_loggia, - geo=geo, - recommended_price=recommended_price, - lower_price=lower_price, - higher_price=higher_price, - market_count=market_count, - placement_history=placement_history, - suggestions=suggestions, - search_similar_url=search_similar_url, - raw_response=data, - ) - - -# --------------------------------------------------------------------------- -# DB save functions -# --------------------------------------------------------------------------- - - -def save_imv_evaluation( - db: Session, - e: IMVEvaluation, - *, - estimate_id: UUID | None = None, -) -> int: - """INSERT в avito_imv_evaluations ON CONFLICT(cache_key) DO UPDATE. - - Колонки строго по 018_avito_imv_evaluations.sql DDL. - estimate_id — nullable FK на trade_in_estimates. - - Returns: id вставленной/обновлённой строки. - """ - row = db.execute( - text(""" - INSERT INTO avito_imv_evaluations ( - cache_key, estimate_id, - address, rooms, area_m2, floor, floor_at_home, - house_type, renovation_type, has_balcony, has_loggia, - geo_hash, - lat, lon, avito_address_id, avito_location_id, - avito_metro_id, avito_district_id, - recommended_price, lower_price, higher_price, market_count, - raw_response, fetched_at - ) VALUES ( - :cache_key, CAST(:estimate_id AS uuid), - :address, :rooms, :area_m2, :floor, :floor_at_home, - :house_type, :renovation_type, :has_balcony, :has_loggia, - :geo_hash, - :lat, :lon, :avito_address_id, :avito_location_id, - :avito_metro_id, :avito_district_id, - :recommended_price, :lower_price, :higher_price, :market_count, - CAST(:raw_response AS jsonb), NOW() - ) - ON CONFLICT (cache_key) DO UPDATE SET - estimate_id = COALESCE(EXCLUDED.estimate_id, avito_imv_evaluations.estimate_id), - recommended_price = EXCLUDED.recommended_price, - lower_price = EXCLUDED.lower_price, - higher_price = EXCLUDED.higher_price, - market_count = EXCLUDED.market_count, - raw_response = EXCLUDED.raw_response, - fetched_at = NOW() - RETURNING id - """), - { - "cache_key": e.cache_key, - "estimate_id": str(estimate_id) if estimate_id else None, - "address": e.address, - "rooms": e.rooms, - "area_m2": e.area_m2, - "floor": e.floor, - "floor_at_home": e.floor_at_home, - "house_type": e.house_type, - "renovation_type": e.renovation_type, - "has_balcony": e.has_balcony, - "has_loggia": e.has_loggia, - "geo_hash": e.geo.geo_hash, - "lat": e.geo.lat, - "lon": e.geo.lon, - "avito_address_id": e.geo.avito_address_id, - "avito_location_id": e.geo.avito_location_id, - "avito_metro_id": e.geo.avito_metro_id, - "avito_district_id": e.geo.avito_district_id, - "recommended_price": e.recommended_price, - "lower_price": e.lower_price, - "higher_price": e.higher_price, - "market_count": e.market_count, - "raw_response": ( - json.dumps(e.raw_response, ensure_ascii=False) if e.raw_response else None - ), - }, - ).fetchone() - db.commit() - return row.id if row else 0 - - -def save_imv_placement_history( - db: Session, - eval_id: int, - items: list[IMVPlacementHistoryItem], -) -> int: - """INSERT в house_placement_history (source='avito_imv') с removed_date. - - Колонки строго по 017_house_placement_history.sql DDL. - Колонка scraped_at (не fetched_at) — по DDL таблицы. - house_id — NULL (backfill через отдельный процесс). - eval_id принимается для логирования (не сохраняется — нет FK в 017). - - ON CONFLICT (source, ext_item_id) DO UPDATE — refresh dates. - Returns: количество вставленных/обновлённых строк. - """ - count = 0 - for item in items: - db.execute( - text(""" - INSERT INTO house_placement_history ( - source, ext_item_id, - title, rooms, area_m2, floor, total_floors, - start_price, start_price_date, - last_price, last_price_date, - removed_date, exposure_days, - raw_payload, scraped_at - ) VALUES ( - 'avito_imv', :ext_item_id, - :title, :rooms, :area_m2, :floor, :total_floors, - :start_price, :start_price_date, - :last_price, :last_price_date, - :removed_date, :exposure_days, - CAST(:raw_payload AS jsonb), NOW() - ) - ON CONFLICT (source, ext_item_id) DO UPDATE SET - last_price = EXCLUDED.last_price, - last_price_date = EXCLUDED.last_price_date, - removed_date = EXCLUDED.removed_date, - exposure_days = EXCLUDED.exposure_days, - raw_payload = EXCLUDED.raw_payload, - scraped_at = NOW() - """), - { - "ext_item_id": item.ext_item_id, - "title": item.title, - "rooms": item.rooms, - "area_m2": item.area_m2, - "floor": item.floor, - "total_floors": item.total_floors, - "start_price": item.start_price, - "start_price_date": item.start_price_date, - "last_price": item.last_price, - "last_price_date": item.last_price_date, - "removed_date": item.removed_date, - "exposure_days": item.exposure_days, - "raw_payload": ( - json.dumps(item.raw_payload, ensure_ascii=False) if item.raw_payload else None - ), - }, - ) - count += 1 - - if count: - db.commit() - logger.info("IMV placement history saved: eval_id=%d items=%d", eval_id, count) - - return count diff --git a/tradein-mvp/backend/app/services/scrapers/domclick_detail.py b/tradein-mvp/backend/app/services/scrapers/domclick_detail.py deleted file mode 100644 index 041df810..00000000 --- a/tradein-mvp/backend/app/services/scrapers/domclick_detail.py +++ /dev/null @@ -1,584 +0,0 @@ -"""DomClick.ru Layer B — detail-card enrichment (#1846 follow-up). - -Layer A (domclick.py) собирает базовые поля через BFF JSON API. Layer B дочитывает -страницу карточки (``https://ekaterinburg.domclick.ru/card/sale__flat__``) и -обогащает listings: renovation/repair_state, living/kitchen area, balconies, -sale_type, owners_count, encumbrances, year_built, views и priceHistory. - -Транспорт — BrowserFetcher через QRATOR-clean прокси (source="cian"): card-хосты -DomClick заблокированы QRATOR через domclick/avito-прокси, но чисты через cian -(verified live 2026-06-27). curl_cffi на card-хосте отдаёт 401/стаб. fetch_detail -принимает уже открытый browser_fetcher; жизненным циклом сессии и выбором source -владеет оркестратор. - -SSR-quirk ---------- -HTML карточки встраивает ``window.__SSR_STATE__ = {...}`` — это JS object literal, -а НЕ чистый JSON: в value-позиции встречается bare ``undefined`` (json.loads падает). -_extract_ssr_state делает balanced-brace scan (с пропуском скобок внутри строк) и -заменяет bare ``undefined`` → ``null`` строго в value-позиции, не трогая строки. - -Форма SSR-стейта подтверждена на живой карточке 2075729321 (2026-06-27): -productCard.objectInfo.{renovation,livingArea,kitchenArea}, productCard.priceInfo. -priceHistory ({date ISO8601+tz, price, diff, state}), productCard.egrnData -(snake_case owners_count/collateral/collateral_sber), productCard.legalOptions. -saleType, productCard.viewsCount/callsCount, houseInfo.info.{wallType,floorType}, -и ТОП-УРОВНЕМ pricePrediction (DomClick AVM → raw_extra.avm). -Schedule для Layer B ships DORMANT — включается вручную после prod-smoke. -""" - -from __future__ import annotations - -import json -import logging -import re -from dataclasses import dataclass, field -from datetime import UTC, datetime -from typing import TYPE_CHECKING, Any - -from sqlalchemy import text -from sqlalchemy.orm import Session - -from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError -from app.services.scrapers.repair_state_normalizer import ( - infer_repair_state_from_text, - normalize_repair_state, -) - -if TYPE_CHECKING: - from app.services.scrapers.browser_fetcher import BrowserFetcher - -logger = logging.getLogger(__name__) - -# ── Маркеры anti-bot challenge (DataDome / QRATOR) ─────────────────────────── -# Зеркало идеи Layer A: если __SSR_STATE__ отсутствует И страница похожа на -# challenge — это блок, а не parse-failure. -_BLOCK_MARKERS: tuple[str, ...] = ("qrator", "captcha", "datadome", "access denied") - -# ── item_id из URL карточки: .../card/sale__flat__2075729321 ───────────────── -_ITEM_ID_RE = re.compile(r"sale__flat__(\d+)") - -# ── SSR assignment marker ──────────────────────────────────────────────────── -_SSR_ASSIGN_RE = re.compile(r"window\.__SSR_STATE__\s*=\s*") - -# ── bare `undefined` (value-позиция) → null, не трогая строки ───────────────── -_UNDEFINED_RE = re.compile(r"(?<=[:\[,\s])undefined(?=[\s,\]\}])") - -# ── renovation (RU) → каноничный repair_state enum ─────────────────────────── -# ЛОКАЛЬНАЯ карта (не трогаем shared _RAW_TO_ENUM в repair_state_normalizer). -# Значения уже каноничны; прогоняются через normalize_repair_state для валидации. -_RENOVATION_MAP: dict[str, str] = { - "евроремонт": "good", - "косметический": "standard", - "дизайнерский": "excellent", - "черновая": "needs_repair", - "без отделки": "needs_repair", - "требует ремонта": "needs_repair", - "предчистовая": "needs_repair", -} - - -# ── DomClickDetailEnrichment ────────────────────────────────────────────────── -@dataclass -class DomClickDetailEnrichment: - """Результат парсинга detail-карточки DomClick (Layer B). - - save_detail_enrichment() пишет подмножество в listings (+ offer_price_history). - Все поля None-safe: отсутствующее → None (не крэш). - """ - - item_id: str | None = None - source_url: str | None = None - - repair_state: str | None = None # enum: needs_repair/standard/good/excellent - repair_type: str | None = None # сырое RU-значение renovation - living_area_m2: float | None = None - kitchen_area_m2: float | None = None - balconies_count: int | None = None - has_balcony: bool | None = None - sale_type: str | None = None - owners_count: int | None = None - encumbrances_clean: bool | None = None # True=чисто, False=есть обременение - year_built: int | None = None - views_total: int | None = None - - price_changes: list[dict[str, Any]] = field(default_factory=list) - raw_extra: dict[str, Any] = field(default_factory=dict) - - -# ── SSR state extraction ────────────────────────────────────────────────────── - - -def _scan_balanced_object(s: str, start: int) -> str: - """Сканирует от '{' по индексу ``start`` до парной '}'. - - Отслеживает глубину по {/}, ИГНОРИРУЯ скобки внутри JS-строк (одинарные и - двойные кавычки с ``\\``-эскейпами). Возвращает литерал ``{...}`` включительно. - - Raises: - DomClickParseError: если парная скобка не найдена (truncated HTML). - """ - depth = 0 - in_string = False - quote_char = "" - escaped = False - for i in range(start, len(s)): - ch = s[i] - if in_string: - if escaped: - escaped = False - elif ch == "\\": - escaped = True - elif ch == quote_char: - in_string = False - continue - if ch in ('"', "'"): - in_string = True - quote_char = ch - elif ch == "{": - depth += 1 - elif ch == "}": - depth -= 1 - if depth == 0: - return s[start : i + 1] - raise DomClickParseError("__SSR_STATE__ object not balanced (no matching brace)") - - -def _extract_ssr_state(html: str) -> dict[str, Any]: - """Извлекает ``window.__SSR_STATE__`` object-литерал из HTML карточки. - - Шаги: - 1. Находим присваивание ``window.__SSR_STATE__ =``. Если нет — различаем - challenge-страницу (→ DomClickBlockedError) и просто отсутствие - стейта (→ DomClickParseError). - 2. Balanced-brace scan от первого '{' → литерал. - 3. Санитайз bare ``undefined`` → ``null`` (только value-позиция). - 4. json.loads. - - Raises: - DomClickBlockedError: страница без стейта + маркеры anti-bot. - DomClickParseError: стейт не найден / не сбалансирован / json.loads упал. - """ - match = _SSR_ASSIGN_RE.search(html) - if match is None: - html_lower = html.lower() - if any(marker in html_lower for marker in _BLOCK_MARKERS): - raise DomClickBlockedError( - "DomClick detail: challenge page detected (no __SSR_STATE__, " - f"markers checked: {_BLOCK_MARKERS})" - ) - raise DomClickParseError("__SSR_STATE__ not found") - - brace_start = html.find("{", match.end()) - if brace_start == -1: - raise DomClickParseError("__SSR_STATE__ assignment has no opening '{'") - - literal = _scan_balanced_object(html, brace_start) - sanitized = _UNDEFINED_RE.sub("null", literal) - try: - data = json.loads(sanitized) - except json.JSONDecodeError as exc: - raise DomClickParseError( - f"__SSR_STATE__ json.loads failed: {exc} (head={sanitized[:200]!r})" - ) from exc - if not isinstance(data, dict): - raise DomClickParseError(f"__SSR_STATE__ is not an object: {type(data)}") - return data - - -# ── Value coercion helpers ──────────────────────────────────────────────────── - - -def _to_float(value: Any) -> float | None: - """None-safe float. bool/неконвертируемое → None.""" - if value is None or isinstance(value, bool): - return None - try: - return float(value) - except (ValueError, TypeError): - return None - - -def _to_int(value: Any) -> int | None: - """None-safe int. bool/неконвертируемое → None.""" - if value is None or isinstance(value, bool): - return None - try: - return int(value) - except (ValueError, TypeError): - return None - - -def _parse_change_time(value: Any) -> datetime | None: - """Нормализует дату изменения цены → timezone-aware datetime. - - Адаптация Layer A ``_parse_publish_date`` (которая возвращала date): здесь - поддерживаем epoch (сек/мс), числовую строку и ISO8601. Real priceHistory.date - приходит как ``2026-04-15T09:02:26.548728+03:00`` — fromisoformat СОХРАНЯЕТ - исходный offset (+03:00), НЕ конвертирует в UTC; naive-значения получают UTC. - epoch трактуем как UTC. Невалидное → None (caller пропускает запись). - """ - if value is None or isinstance(value, bool): - return None - # epoch (число): >1e12 трактуем как миллисекунды - if isinstance(value, int | float): - ts = float(value) - if ts > 1e12: - ts /= 1000.0 - try: - return datetime.fromtimestamp(ts, tz=UTC) - except (ValueError, OSError, OverflowError): - return None - s = str(value).strip() - if not s: - return None - if s.isdigit(): - ts = float(s) - if ts > 1e12: - ts /= 1000.0 - try: - return datetime.fromtimestamp(ts, tz=UTC) - except (ValueError, OSError, OverflowError): - return None - # ISO8601 (поддержим trailing 'Z') - try: - dt = datetime.fromisoformat(s.replace("Z", "+00:00")) - except (ValueError, TypeError): - return None - if dt.tzinfo is None: - dt = dt.replace(tzinfo=UTC) - return dt - - -def _map_repair_state(renovation: str | None) -> str | None: - """RU renovation → каноничный enum. Нет совпадения в карте → инференс из текста.""" - if not renovation: - return None - mapped = _RENOVATION_MAP.get(renovation.strip().lower()) - if mapped is not None: - return normalize_repair_state(mapped) - return infer_repair_state_from_text(renovation) - - -def _compact(d: dict[str, Any]) -> dict[str, Any]: - """Выкидывает None-значения, чтобы raw_extra оставался компактным.""" - return {k: v for k, v in d.items() if v is not None} - - -def _pick(d: dict[str, Any], *keys: str) -> Any: - """Первое не-None значение среди ключей (defensive snake/camel spellings).""" - for k in keys: - v = d.get(k) - if v is not None: - return v - return None - - -# ── item_id ─────────────────────────────────────────────────────────────────── - - -def _extract_item_id(url: str) -> str | None: - """sale__flat__ из URL карточки.""" - m = _ITEM_ID_RE.search(url or "") - return m.group(1) if m else None - - -# ── parse_detail_html ────────────────────────────────────────────────────────── - - -def parse_detail_html(html: str, source_url: str) -> DomClickDetailEnrichment: - """Pure parser detail-карточки DomClick (без сети/БД). - - _extract_ssr_state может бросить DomClickBlockedError/DomClickParseError — - они propagate (оркестратор различает блок и parse-failure). Дальнейшая - навигация по полям обёрнута: per-field сюрприз формы → минимальный - enrichment (item_id+url), а не крэш. - """ - state = _extract_ssr_state(html) - item_id = _extract_item_id(source_url) - try: - pc = state.get("productCard") or {} - oi = pc.get("objectInfo") or {} - hi = (state.get("houseInfo") or {}).get("info") or {} - egrn = pc.get("egrnData") or {} - - # repair: enum + сырое значение - renovation = oi.get("renovation") - repair_type = renovation if isinstance(renovation, str) else None - repair_state = _map_repair_state(repair_type) - - # площади - living_area_m2 = _to_float(oi.get("livingArea")) - kitchen_area_m2 = _to_float(oi.get("kitchenArea")) - - # балконы: int → count+флаг; иное (строка/др.) → raw_extra, counts None - balconies = oi.get("balconies") - balconies_count: int | None = None - has_balcony: bool | None = None - balconies_raw: Any = None - if isinstance(balconies, bool): - balconies_raw = balconies - elif isinstance(balconies, int): - balconies_count = balconies - has_balcony = balconies > 0 - elif balconies is not None: - balconies_raw = balconies - - sale_type = (pc.get("legalOptions") or {}).get("saleType") - - # owners_count — защищаемся от двух написаний ключа - owners_count = _to_int(egrn.get("owners_count")) - if owners_count is None: - owners_count = _to_int(egrn.get("ownersCount")) - - # encumbrances_clean (ИНВЕРСИЯ): collateral/collateral_sber обозначают - # НАЛИЧИЕ обременения. Любой truthy → clean=False; оба отсутствуют/false - # при наличии egrnData → True; egrnData нет → None. - collateral = egrn.get("collateral") - collateral_sber = egrn.get("collateral_sber") - if egrn: - encumbrances_clean: bool | None = not bool(collateral or collateral_sber) - else: - encumbrances_clean = None - - year_built = _to_int(hi.get("buildYear")) - views_total = _to_int(pc.get("viewsCount")) - - # priceHistory → нормализованные записи (skip при битой дате / без цены) - raw_history = (pc.get("priceInfo") or {}).get("priceHistory") or [] - price_changes: list[dict[str, Any]] = [] - if isinstance(raw_history, list): - for entry in raw_history: - if not isinstance(entry, dict): - continue - change_time = _parse_change_time(entry.get("date")) - price_rub = _to_int(entry.get("price")) - # Skip при нераспарсенной дате ИЛИ отсутствующей/невалидной цене. - if change_time is None or price_rub is None: - continue - diff = entry.get("diff") - diff_percent = ( - diff if isinstance(diff, int | float) and not isinstance(diff, bool) else None - ) - price_changes.append( - { - "change_time": change_time, - "price_rub": price_rub, - "diff_percent": diff_percent, - } - ) - - # AVM (Layer C, тот же fetch) — DomClick price prediction живёт ТОП-УРОВНЕМ - # в SSR-стейте. Может быть плоским или обёрнут в .result/.data — defensively - # разворачиваем. В raw_extra пишем только присутствующие поля. - avm_root = state.get("pricePrediction") or {} - avm_src = avm_root - if isinstance(avm_root, dict): - avm_src = avm_root.get("result") or avm_root.get("data") or avm_root - if not isinstance(avm_src, dict): - avm_src = {} - avm = _compact( - { - "market_price": _pick(avm_src, "market_price", "marketPrice"), - "min": _pick(avm_src, "min_market_price", "minMarketPrice"), - "max": _pick(avm_src, "max_market_price", "maxMarketPrice"), - "repair_quality": _pick(avm_src, "repair_quality", "repairQuality"), - } - ) - - # raw_extra → merge в listings.raw_payload. wall/floor type ОСТАЮТСЯ тут, - # НЕ пишем в listings.house_type (кросс-источниковый словарь грязный). - # egrn area key — точное написание не подтверждено; пробуем несколько. - egrn_area = egrn.get("area") or egrn.get("rosreestrArea") or egrn.get("object_area") - demand = _compact( - { - "calls": pc.get("callsCount"), - "favorites": pc.get("favoriteOfferUsersCount"), - "duplicates": pc.get("duplicatesOfferCount"), - } - ) - raw_extra = _compact( - { - "wall_type": hi.get("wallType") or (pc.get("house") or {}).get("wallType"), - "floor_type": hi.get("floorType"), - "entrance_count": hi.get("entranceCount"), - "quarters_count": hi.get("quartersCount"), - "energy_efficiency": hi.get("energyEfficiency"), - "building_series": hi.get("buildingSeries"), - "domclick_building_guid": (pc.get("address") or {}).get("guid"), - "egrn_area": egrn_area, - "demand": demand or None, - "collateral": collateral, - "collateral_sber": collateral_sber, - "balconies_raw": balconies_raw, - "avm": avm or None, - } - ) - - return DomClickDetailEnrichment( - item_id=item_id, - source_url=source_url, - repair_state=repair_state, - repair_type=repair_type, - living_area_m2=living_area_m2, - kitchen_area_m2=kitchen_area_m2, - balconies_count=balconies_count, - has_balcony=has_balcony, - sale_type=sale_type if isinstance(sale_type, str) else None, - owners_count=owners_count, - encumbrances_clean=encumbrances_clean, - year_built=year_built, - views_total=views_total, - price_changes=price_changes, - raw_extra=raw_extra, - ) - except Exception: - logger.warning( - "domclick_detail: field navigation failed for %s — minimal enrichment", - source_url, - exc_info=True, - ) - return DomClickDetailEnrichment(item_id=item_id, source_url=source_url) - - -# ── fetch_detail ──────────────────────────────────────────────────────────────── - - -async def fetch_detail( - card_url: str, - *, - browser_fetcher: BrowserFetcher, -) -> DomClickDetailEnrichment: - """Fetch абсолютного URL карточки через BrowserFetcher → parse_detail_html. - - card_url уже абсолютный (listings.source_url из Layer A), URL не строим. - Сессией владеет оркестратор (передаёт уже открытый browser_fetcher). - - Raises: - DomClickBlockedError: anti-bot challenge ИЛИ ошибка браузерного fetch - (нет curl-fallback — DataDome даёт 401 на curl). Оркестратор считает - это блоком и помечает listing failed. - DomClickParseError: HTTP 200, но SSR-стейт не парсится (дрейф схемы). - """ - try: - html = await browser_fetcher.fetch(card_url) - except DomClickBlockedError: - raise - except Exception as exc: - raise DomClickBlockedError( - f"DomClick detail browser fetch failed for {card_url}: {exc}" - ) from exc - return parse_detail_html(html, card_url) - - -# ── save_detail_enrichment ────────────────────────────────────────────────────── - - -def save_detail_enrichment( - db: Session, - listing_id: int, - e: DomClickDetailEnrichment, -) -> bool: - """Persist enrichment в одной транзакции (commit в конце; rollback — за caller). - - 1. UPDATE listings SET (COALESCE — None не затирает существующее), - raw_payload || raw_extra, detail_enriched_at=NOW() WHERE id=listing_id. - 2. Для каждой записи priceHistory — INSERT INTO offer_price_history - (ON CONFLICT DO NOTHING → идемпотентно). source='domklik'. - - Snapshot (upsert_listing_snapshot) НЕ пишем: enrichment не несёт текущую цену, - а detail-backfill цену не меняет — snapshot остаётся за SERP-путём (note). - - Returns: - True если UPDATE задел строку (listing найден), иначе False. - """ - raw_extra_json = json.dumps(e.raw_extra or {}, ensure_ascii=False, default=str) - - result = db.execute( - text(""" - UPDATE listings SET - repair_state = COALESCE(:repair_state, repair_state), - repair_type = COALESCE(:repair_type, repair_type), - living_area_m2 = COALESCE(:living_area_m2, living_area_m2), - kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2), - balconies_count = COALESCE(:balconies_count, balconies_count), - has_balcony = COALESCE(:has_balcony, has_balcony), - sale_type = COALESCE(:sale_type, sale_type), - owners_count = COALESCE(:owners_count, owners_count), - encumbrances_clean = COALESCE(:encumbrances_clean, encumbrances_clean), - year_built = COALESCE(:year_built, year_built), - views_total = COALESCE(:views_total, views_total), - raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_extra AS jsonb), - detail_enriched_at = NOW() - WHERE id = CAST(:lid AS bigint) - """), - { - "lid": listing_id, - "repair_state": e.repair_state, - "repair_type": e.repair_type, - "living_area_m2": e.living_area_m2, - "kitchen_area_m2": e.kitchen_area_m2, - "balconies_count": e.balconies_count, - "has_balcony": e.has_balcony, - "sale_type": e.sale_type, - "owners_count": e.owners_count, - "encumbrances_clean": e.encumbrances_clean, - "year_built": e.year_built, - "views_total": e.views_total, - "raw_extra": raw_extra_json, - }, - ) - - # priceHistory — INSERT, де-дуп по (listing_id, change_time). SAVEPOINT per-row - # (backend.md): битый cast одной записи не валит весь UPDATE+commit. - for change in e.price_changes: - ct = change.get("change_time") - price = change.get("price_rub") - if not ct or not price: - continue - try: - with db.begin_nested(): - db.execute( - text(""" - INSERT INTO offer_price_history - (listing_id, change_time, price_rub, diff_percent, source) - VALUES ( - CAST(:lid AS bigint), - CAST(:ct AS timestamptz), - CAST(:price AS numeric), - CAST(:diff AS numeric), - 'domklik' - ) - ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq - DO NOTHING - """), - { - "lid": listing_id, - "ct": ct, - "price": price, - "diff": change.get("diff_percent"), - }, - ) - except Exception as exc: - logger.warning( - "domclick_detail: price-history insert failed listing_id=%s ct=%s: %s", - listing_id, - ct, - exc, - ) - - db.commit() - found = result.rowcount > 0 - if not found: - logger.warning( - "domclick_detail save: listing not found id=%s (item_id=%s)", - listing_id, - e.item_id, - ) - else: - logger.info( - "domclick_detail save: listing id=%s enriched (price_changes=%d)", - listing_id, - len(e.price_changes), - ) - return found diff --git a/tradein-mvp/backend/app/services/scrapers/ekb_geoportal_client.py b/tradein-mvp/backend/app/services/scrapers/ekb_geoportal_client.py deleted file mode 100644 index 5557da3b..00000000 --- a/tradein-mvp/backend/app/services/scrapers/ekb_geoportal_client.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Клиент городского геопортала Екатеринбурга (геопортал.екатеринбург.рф). - -Открытый GeoServer WFS, без авторизации. Слой `portal:portal_geo_topo_build` — -футпринты зданий ЕКБ (намного полнее чем NSPD cad_buildings, который пропускает -~70% зданий города). - -ВАЖНО: сервер отдаёт 403 на дефолтный User-Agent curl/httpx. Нужны браузерные -заголовки (UA + Referer + Accept) — тогда 200. - -Геометрия `geoloc` в EPSG:4326 (MultiPolygon footprint). BBOX axis order = -lon_min,lat_min,lon_max,lat_max. Сервер режет каждый ответ на 20000 features — -лоадер тайлит мелко, чтобы не упереться. - -Центроид считается чистым Python'ом (area-weighted, без shapely — shapely не входит -в зависимости tradein-mvp backend). - -Используется лоадером app.tasks.ekb_geoportal_ingest. -""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass - -import httpx - -logger = logging.getLogger(__name__) - -# Punycode-хост геопортал.екатеринбург.рф -GEOPORTAL_HOST = "https://xn--80afgznagjs.xn--80acgfbsl1azdqr.xn--p1ai" -WFS_PATH = "/api/gis/ows/3/portal/wfs" -BUILD_LAYER = "portal:portal_geo_topo_build" - -# Браузерные заголовки — без них хост отдаёт 403. -_BROWSER_HEADERS = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/148.0.0.0 Safari/537.36" - ), - "Referer": f"{GEOPORTAL_HOST}/", - "Accept": "application/json", -} - - -@dataclass(frozen=True, slots=True) -class TopoBuilding: - """Одна часть-фича здания из слоя portal_geo_topo_build. - - Здание может быть разбито на несколько part-features (одинаковые street/house, - разные key) — лоадер группирует их по (street_norm, house_norm). - """ - - street: str - house: str - floors: int | None - purpose: str | None - material: str | None - key: int | None - lat: float - lon: float - - -def _parse_int(value: object) -> int | None: - if value is None: - return None - try: - return int(value) - except (ValueError, TypeError): - return None - - -def _parse_str(value: object) -> str | None: - if value is None: - return None - s = str(value).strip() - return s or None - - -def _ring_centroid(ring: list) -> tuple[float, float, float] | None: - """Area-weighted centroid of one linear ring → (cx, cy, signed_area2). - - signed_area2 = 2× signed area (используется как вес при усреднении колец/полигонов). - Возвращает None если кольцо вырождено (нулевая площадь). - """ - n = len(ring) - if n < 3: - return None - a2 = 0.0 # 2× signed area - cx = 0.0 - cy = 0.0 - for i in range(n): - x0, y0 = ring[i][0], ring[i][1] - x1, y1 = ring[(i + 1) % n][0], ring[(i + 1) % n][1] - cross = x0 * y1 - x1 * y0 - a2 += cross - cx += (x0 + x1) * cross - cy += (y0 + y1) * cross - if a2 == 0.0: - return None - cx /= 3.0 * a2 - cy /= 3.0 * a2 - return (cx, cy, a2) - - -def _multipolygon_centroid(coords: list) -> tuple[float, float] | None: - """Центроид GeoJSON MultiPolygon/Polygon-частей: усреднение по |площади| внешних колец. - - coords — список полигонов, каждый = [outer_ring, hole1, ...]. Берём только - внешнее кольцо каждого полигона (дыр у footprint'ов зданий практически нет, - их вклад в положение центра пренебрежим). Возвращает (lon, lat) или None. - """ - sum_w = 0.0 - sum_x = 0.0 - sum_y = 0.0 - for polygon in coords: - if not polygon: - continue - outer = polygon[0] - c = _ring_centroid(outer) - if c is None: - continue - cx, cy, a2 = c - w = abs(a2) - sum_w += w - sum_x += cx * w - sum_y += cy * w - if sum_w == 0.0: - return None - return (sum_x / sum_w, sum_y / sum_w) - - -def _geometry_centroid(geom: dict) -> tuple[float, float] | None: - """Центроид GeoJSON geometry (MultiPolygon или Polygon) → (lon, lat) или None.""" - gtype = geom.get("type") - coords = geom.get("coordinates") - if not coords: - return None - if gtype == "MultiPolygon": - return _multipolygon_centroid(coords) - if gtype == "Polygon": - # Polygon coordinates = [outer_ring, hole, ...] → обернём в один полигон. - return _multipolygon_centroid([coords]) - return None - - -def _feature_to_building(feature: dict) -> TopoBuilding | None: - """GeoJSON Feature → TopoBuilding. None если нет улицы/дома или геометрии.""" - props = feature.get("properties") or {} - street = _parse_str(props.get("topo_street")) - house = _parse_str(props.get("topo_num_house")) - if not street or not house: - return None - - geom = feature.get("geometry") - if not isinstance(geom, dict): - return None - try: - centroid = _geometry_centroid(geom) - except Exception: - logger.warning("geoportal: bad geometry for street=%r house=%r", street, house) - return None - if centroid is None: - return None - lon, lat = centroid - - return TopoBuilding( - street=street, - house=house, - floors=_parse_int(props.get("topo_floor")), - purpose=_parse_str(props.get("topo_purpose")), - material=_parse_str(props.get("topo_material")), - key=_parse_int(props.get("key")), - lat=lat, - lon=lon, - ) - - -def parse_feature_collection(payload: dict) -> list[TopoBuilding]: - """GeoJSON FeatureCollection → list[TopoBuilding]. Пропускает невалидные.""" - features = payload.get("features") - if not isinstance(features, list): - return [] - out: list[TopoBuilding] = [] - for feature in features: - if not isinstance(feature, dict): - continue - building = _feature_to_building(feature) - if building is not None: - out.append(building) - return out - - -class EkbGeoportalClient: - """Async WFS-клиент геопортала ЕКБ. Один клиент переиспользуется лоадером.""" - - def __init__(self, *, timeout: float = 30.0) -> None: - self._timeout = timeout - - async def fetch_topo_buildings( - self, - bbox: tuple[float, float, float, float], - *, - count: int = 20000, - ) -> list[TopoBuilding]: - """Запрашивает здания в bbox (lon_min, lat_min, lon_max, lat_max). - - Graceful: на non-200/parse-error логирует warning и возвращает []. - """ - lon_min, lat_min, lon_max, lat_max = bbox - cql = f"BBOX(geoloc,{lon_min},{lat_min},{lon_max},{lat_max})" - params = { - "service": "WFS", - "version": "2.0.0", - "request": "GetFeature", - "typeNames": BUILD_LAYER, - "outputFormat": "application/json", - "CQL_FILTER": cql, - "count": str(count), - } - url = f"{GEOPORTAL_HOST}{WFS_PATH}" - try: - # Геопортал ЕКБ отдаёт сертификат российского CA (Минцифры), которого нет в - # стандартном trust store httpx → TLS verify падает. Данные публичные, - # read-only → verify=False (тот же подход, что curl -k при разведке). - async with httpx.AsyncClient( - timeout=self._timeout, headers=_BROWSER_HEADERS, verify=False - ) as client: - resp = await client.get(url, params=params) - if resp.status_code != 200: - logger.warning("geoportal WFS → HTTP %d for bbox=%s", resp.status_code, bbox) - return [] - payload = resp.json() - except Exception as exc: - logger.warning("geoportal WFS fetch/parse failed for bbox=%s: %s", bbox, exc) - return [] - - return parse_feature_collection(payload) diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py deleted file mode 100644 index 08b88a45..00000000 --- a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py +++ /dev/null @@ -1,662 +0,0 @@ -"""Yandex Realty detail page scraper. - -Fetches /offer// and extracts Product JSON-LD + DOM sections into -a DetailEnrichment Pydantic model. Used by enrichment pipeline -(Wave 5+ matching / Wave 6 estimator). -""" - -from __future__ import annotations - -import json -import logging -import re -from datetime import date -from typing import Any - -from pydantic import BaseModel, Field -from selectolax.parser import HTMLParser, Node -from sqlalchemy import text -from sqlalchemy.orm import Session - -from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.base import BaseScraper -from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text -from app.services.scrapers.yandex_helpers import ( - RE_AGENCY_FOUNDED, - RE_AGENCY_OBJECTS, - RE_METRO_WALK, - RE_VIEWS, - RE_YEAR, - find_ld_by_type, - parse_house_type, - parse_ru_date, - parse_rub, -) - -logger = logging.getLogger(__name__) - - -# ── Pydantic models ─────────────────────────────────────────────────────────── - - -class MetroStation(BaseModel): - name: str - walk_min: int | None = None - - -class DetailEnrichment(BaseModel): - """Enrichment payload from a Yandex detail page.""" - - offer_id: str - source_url: str - - # Pricing — Product JSON-LD `offers.price` is the exact int - price_rub: int | None = None - price_per_m2: int | None = None - - # Title + basic params - title: str | None = None - rooms: int | None = None - area_m2: float | None = None - living_area_m2: float | None = None - kitchen_area_m2: float | None = None - ceiling_height: float | None = None # meters, e.g. 2.55 - floor: int | None = None - total_floors: int | None = None - - # Address (full) - address: str | None = None - - # Description (full text) - description: str | None = None - - # Repair state — enum, inferred from description text (#622). - # Yandex не отдаёт структурного поля ремонта, поэтому только инференс. - repair_state: str | None = None - - # Stats - views_total: int | None = None - publish_date: date | None = None - publish_date_relative: str | None = None - - # Agency block (OfferCardAuthorInfo) - agency_name: str | None = None - agency_founded_year: int | None = None - agency_objects_count: int | None = None - seller_name: str | None = None # last text line before "Агентство «...»" - - # Metro stations from "Расположение" section - metro_stations: list[MetroStation] = Field(default_factory=list) - - # Photos — 8 sizes from Product.image[] - photo_urls: list[str] = Field(default_factory=list) - - # Newbuilding linkage - newbuilding_url: str | None = None - newbuilding_id: str | None = None - - # NLP from description (best-effort) - house_type_nlp: str | None = None - year_built_hint: int | None = None - metro_walk_min: int | None = None - - # Raw payload (trimmed) - raw_payload: dict[str, Any] | None = None - - -# ── Scraper ─────────────────────────────────────────────────────────────────── - - -class YandexDetailScraper(BaseScraper): - """Detail page scraper for realty.yandex.ru.""" - - name = "yandex_detail" - base_url = "https://realty.yandex.ru" - request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings - - def __init__(self) -> None: - super().__init__() - self.request_delay_sec = get_scraper_delay(self.name) - - # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError - async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override] - raise NotImplementedError( - "YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead." - ) - - async def fetch_detail(self, offer_url: str) -> DetailEnrichment | None: - try: - response = await self._http_get(offer_url) - except Exception: - logger.exception("yandex detail fetch failed: %s", offer_url) - return None - if response.status_code != 200: - logger.warning("yandex detail returned %d for %s", response.status_code, offer_url) - return None - result = self.parse(response.text, offer_url=offer_url) - await self.sleep_between_requests() - return result - - def parse(self, html: str, offer_url: str) -> DetailEnrichment | None: - offer_id_match = re.search(r"/offer/(\d+)/?", offer_url) - if not offer_id_match: - logger.warning("offer_url has no /offer//: %s", offer_url) - return None - offer_id = offer_id_match.group(1) - - tree = HTMLParser(html) - - # --- Product JSON-LD (authoritative price) --- - product = find_ld_by_type(html, "Product") or {} - offers_ld = product.get("offers") or {} - if isinstance(offers_ld, list) and offers_ld: - offers_ld = offers_ld[0] - price_ld = offers_ld.get("price") if isinstance(offers_ld, dict) else None - try: - price_rub = int(price_ld) if price_ld else None - except (TypeError, ValueError): - price_rub = None - - # Photos from JSON-LD image[] (typically 8 size variants) - images = product.get("image") or [] - if isinstance(images, str): - images = [images] - photo_urls = [u for u in images if isinstance(u, str)] - - # --- Title + summary --- - title_node = tree.css_first("h1") - title = title_node.text(strip=True) if title_node else None - - rooms, area_m2, floor, total_floors = _parse_title(title or "") - - # --- Structural offer card (window.INITIAL_STATE → offerCard.card) --- - # Authoritative source for area/floor/ceiling/kitchen — the h1 title - # misses non-standard layouts (студия / свободная планировка) and never - # carries ceiling/kitchen/living. Title stays as fallback below. - living_area_m2: float | None = None - kitchen_area_m2: float | None = None - ceiling_height: float | None = None - card = _extract_offer_card(html, offer_id) - if card is not None: - ( - c_rooms, - c_area, - c_living, - c_kitchen, - c_ceiling, - c_floor, - c_total_floors, - ) = _parse_card_fields(card) - # Structural source wins; title only fills the gaps it left. - rooms = c_rooms if c_rooms is not None else rooms - area_m2 = c_area if c_area is not None else area_m2 - living_area_m2 = c_living - kitchen_area_m2 = c_kitchen - ceiling_height = c_ceiling - floor = c_floor if c_floor is not None else floor - total_floors = c_total_floors if c_total_floors is not None else total_floors - - # --- OfferCardSummary text block --- - summary_node = tree.css_first('[data-test="OfferCardSummary"]') - summary_text = summary_node.text(strip=True) if summary_node else "" - - # Views + relative publish date from summary text - views_match = RE_VIEWS.search(summary_text) - views_total = int(views_match.group(1)) if views_match else None - publish_date = parse_ru_date(summary_text) - publish_date_relative = _extract_relative_date(summary_text) - - # price_per_m2 — from summary text if absent in LD - price_per_m2: int | None = None - ppm2_match = re.search(r"(\d[\d\s]+)\s*₽\s*за\s*м²", summary_text) - if ppm2_match: - price_per_m2 = parse_rub(ppm2_match.group(1)) - - # --- OfferCardAuthorInfo (agency block) --- - author_node = tree.css_first('[data-test="OfferCardAuthorInfo"]') - agency_name: str | None = None - agency_founded_year: int | None = None - agency_objects_count: int | None = None - seller_name: str | None = None - if author_node is not None: - author_text = author_node.text(strip=True) - agency_h2 = author_node.css_first("h2") - agency_name = agency_h2.text(strip=True) if agency_h2 else None - founded_m = RE_AGENCY_FOUNDED.search(author_text) - if founded_m: - agency_founded_year = int(founded_m.group(1)) - objects_m = RE_AGENCY_OBJECTS.search(author_text) - if objects_m: - agency_objects_count = int(objects_m.group(1)) - # seller_name — last text line before "Агентство" - seller_name = _extract_seller_name(summary_text, agency_name) - - # --- Description section (after H2 "Описание") --- - description = _find_section_text(tree, "Описание") - - # --- Repair state: инференс из описания (#622), Yandex без структурного поля --- - repair_state = infer_repair_state_from_text(description or summary_text) - - # --- Address --- - address = _extract_address(summary_text) - - # --- Metro stations from "Расположение" section --- - location_text = _find_section_text(tree, "Расположение") or "" - metro_stations = _parse_metro_stations(location_text) - - # --- Newbuilding link --- - nb_url: str | None = None - nb_id: str | None = None - nb_link = tree.css_first('a[href*="/kupit/novostrojka/"]') - if nb_link is not None: - nb_href = nb_link.attributes.get("href", "") - nb_match = re.search(r"/novostrojka/[\w-]+?-(\d+)/?", nb_href) - if nb_match: - nb_id = nb_match.group(1) - nb_url = ( - nb_href if nb_href.startswith("http") else f"https://realty.yandex.ru{nb_href}" - ) - - # --- NLP best-effort from description --- - nlp_text = description or summary_text - house_type_nlp = parse_house_type(nlp_text) - year_hint_m = RE_YEAR.search(nlp_text or "") - year_built_hint = int(year_hint_m.group(1)) if year_hint_m else None - walk_m = RE_METRO_WALK.search(nlp_text or "") - metro_walk_min = int(walk_m.group(1)) if walk_m else None - - return DetailEnrichment( - offer_id=offer_id, - source_url=offer_url, - price_rub=price_rub, - price_per_m2=price_per_m2, - title=title, - rooms=rooms, - area_m2=area_m2, - living_area_m2=living_area_m2, - kitchen_area_m2=kitchen_area_m2, - ceiling_height=ceiling_height, - floor=floor, - total_floors=total_floors, - address=address, - description=description, - repair_state=repair_state, - views_total=views_total, - publish_date=publish_date, - publish_date_relative=publish_date_relative, - agency_name=agency_name, - agency_founded_year=agency_founded_year, - agency_objects_count=agency_objects_count, - seller_name=seller_name, - metro_stations=metro_stations, - photo_urls=photo_urls, - newbuilding_url=nb_url, - newbuilding_id=nb_id, - house_type_nlp=house_type_nlp, - year_built_hint=year_built_hint, - metro_walk_min=metro_walk_min, - raw_payload={ - "summary_text": summary_text[:1000], - "description_len": len(description) if description else 0, - "photo_count": len(photo_urls), - }, - ) - - -# ── Helpers ─────────────────────────────────────────────────────────────────── - - -def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int | None]: - """Extract (rooms, area_m2, floor, total_floors) from h1 text.""" - rooms: int | None = None - area_m2: float | None = None - floor: int | None = None - total_floors: int | None = None - - area_m = re.search(r"(\d+[.,]?\d*)\s*м²", title) - if area_m: - area_m2 = float(area_m.group(1).replace(",", ".")) - - if re.search(r"студи[яюй]", title, re.IGNORECASE): - rooms = 0 - else: - rooms_m = re.search(r"(\d+)\s*-?\s*комнатн", title, re.IGNORECASE) - if rooms_m: - try: - rooms = int(rooms_m.group(1)) - except ValueError: - pass - - floor_m = re.search(r"(\d+)\s+этаж\s+из\s+(\d+)", title, re.IGNORECASE) - if floor_m: - floor = int(floor_m.group(1)) - total_floors = int(floor_m.group(2)) - - return rooms, area_m2, floor, total_floors - - -def _extract_js_object(html: str, marker: str) -> str | None: - """Return the JSON object literal assigned to `marker` (e.g. window.INITIAL_STATE). - - Brace-matches from the first ``{`` after ``marker = ...`` to its balanced - close, respecting string literals/escapes. Returns the raw JSON text or None. - """ - i = html.find(marker) - if i < 0: - return None - eq = html.find("=", i) - if eq < 0: - return None - start = html.find("{", eq) - if start < 0: - return None - bal = 0 - in_str = False - esc = False - k = start - n = len(html) - while k < n: - c = html[k] - if in_str: - if esc: - esc = False - elif c == "\\": - esc = True - elif c == '"': - in_str = False - else: - if c == '"': - in_str = True - elif c == "{": - bal += 1 - elif c == "}": - bal -= 1 - if bal == 0: - return html[start : k + 1] - k += 1 - return None - - -def _find_card_by_offer_id(obj: Any, offer_id: str) -> dict[str, Any] | None: - """Recursively locate the offer dict whose offerId matches and carries `area`.""" - if isinstance(obj, dict): - if obj.get("offerId") == offer_id and "area" in obj: - return obj - for v in obj.values(): - found = _find_card_by_offer_id(v, offer_id) - if found is not None: - return found - elif isinstance(obj, list): - for v in obj: - found = _find_card_by_offer_id(v, offer_id) - if found is not None: - return found - return None - - -def _extract_offer_card(html: str, offer_id: str) -> dict[str, Any] | None: - """Extract the offer card object from window.INITIAL_STATE. - - Yandex embeds full structured offer data in ``window.INITIAL_STATE`` under - ``offerCard.card``. This is the authoritative source for area / floor / - ceiling / kitchen — unlike the h1 title which misses non-standard layouts - and never carries ceiling/kitchen. Returns the card dict or None. - """ - blob = _extract_js_object(html, "window.INITIAL_STATE") - if not blob: - return None - try: - state = json.loads(blob) - except (json.JSONDecodeError, ValueError): - logger.warning("yandex detail: INITIAL_STATE failed to parse for offer %s", offer_id) - return None - # Fast path: canonical location. - offer_card = state.get("offerCard") if isinstance(state, dict) else None - if isinstance(offer_card, dict): - card = offer_card.get("card") - if isinstance(card, dict) and card.get("offerId") == offer_id: - return card - # Fallback: deep search (page structure may differ). - return _find_card_by_offer_id(state, offer_id) - - -def _space_value(node: Any) -> float | None: - """Yandex area fields are ``{"value": N, "unit": "SQUARE_METER"}`` → float.""" - if isinstance(node, dict): - val = node.get("value") - if isinstance(val, int | float): - return float(val) - elif isinstance(node, int | float): - return float(node) - return None - - -def _parse_card_fields( - card: dict[str, Any], -) -> tuple[ - int | None, - float | None, - float | None, - float | None, - float | None, - int | None, - int | None, -]: - """Extract (rooms, area, living, kitchen, ceiling, floor, total_floors) from card. - - rooms: ``roomsTotal``, or 0 when ``house.studio`` is truthy (студии не - несут roomsTotal). floor: first element of ``floorsOffered``. - """ - area = _space_value(card.get("area")) - living = _space_value(card.get("livingSpace")) - kitchen = _space_value(card.get("kitchenSpace")) - - ceiling_raw = card.get("ceilingHeight") - ceiling: float | None = None - if isinstance(ceiling_raw, int | float): - ceiling = float(ceiling_raw) - - rooms_raw = card.get("roomsTotal") - rooms: int | None = None - if isinstance(rooms_raw, int): - rooms = rooms_raw - house = card.get("house") - if isinstance(house, dict) and house.get("studio"): - rooms = 0 - - total_floors_raw = card.get("floorsTotal") - total_floors = total_floors_raw if isinstance(total_floors_raw, int) else None - - floor: int | None = None - floors_offered = card.get("floorsOffered") - if isinstance(floors_offered, list) and floors_offered: - first = floors_offered[0] - if isinstance(first, int): - floor = first - - return rooms, area, living, kitchen, ceiling, floor, total_floors - - -def _find_section_text(tree: HTMLParser, heading: str) -> str | None: - """Find the text content of a
/
whose preceding h2/h3 matches heading. - - Yandex page structure varies; this scans h2/h3 nodes, then returns the - concatenated text of subsequent sibling blocks until the next heading. - """ - for h in tree.css("h2, h3"): - if heading.lower() in (h.text(strip=True) or "").lower(): - # collect subsequent siblings until the next h2/h3 - parts: list[str] = [] - node: Node | None = h.next - while node is not None: - tag = (node.tag or "").lower() - if tag in {"h2", "h3"}: - break - txt = node.text(strip=True) if hasattr(node, "text") else "" - if txt: - parts.append(txt) - node = node.next - return " ".join(parts).strip() or None - return None - - -def _extract_address(summary_text: str) -> str | None: - """Best-effort address extraction from summary block.""" - # Pattern: "Россия, Свердловская область, Екатеринбург, улица Х, д. N" - m = re.search(r"(Россия[^•]+?)(?:•|\d+\s+просмотр|$)", summary_text) - if m: - addr = m.group(1).strip().rstrip(",").strip() - return addr if len(addr) > 10 else None - return None - - -def _parse_metro_stations(location_text: str) -> list[MetroStation]: - """Parse 'Уральская 11 мин. Динамо 16 мин.' → list of MetroStation.""" - stations: list[MetroStation] = [] - # name (1+ Cyrillic words) + space + N + space + мин(.|у|ут) - for m in re.finditer(r"([А-ЯЁ][А-Яа-яё\s-]+?)\s+(\d+)\s*мин", location_text): - name = m.group(1).strip() - if 2 <= len(name) <= 40: - stations.append(MetroStation(name=name, walk_min=int(m.group(2)))) - if len(stations) >= 5: - break - return stations - - -def _extract_relative_date(summary_text: str) -> str | None: - """Capture phrases like '6 часов назад' / 'вчера' / '3 дня назад'.""" - m = re.search( - r"(\d+\s+(?:минут|час|часов|часа|день|дня|дней|недел[ьюи])\s+назад" - r"|вчера|сегодня|позавчера)", - summary_text, - re.IGNORECASE, - ) - return m.group(1).strip() if m else None - - -def _extract_seller_name(summary_text: str, agency_name: str | None) -> str | None: - """Heuristic: line right before 'Агентство ...' in summary text.""" - if not agency_name or agency_name not in summary_text: - return None - head = summary_text.split(agency_name, 1)[0] - # last short token sequence (likely "Имя Фамилия") - m = re.findall(r"([А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+){1,2})", head) - return m[-1] if m else None - - -# ── Save helper ─────────────────────────────────────────────────────────────── - - -def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) -> bool: - """Persist Yandex DetailEnrichment to listings row. - - UPDATE listings SET = COALESCE(:val, ), ..., detail_enriched_at = NOW() - WHERE id = listing_id. - - COALESCE semantics: keeps existing non-NULL value if new value is NULL (never - overwrites a populated column with NULL). area_m2 from detail is more accurate - than SERP, but COALESCE preserves SERP value if detail returns NULL — acceptable. - - Returns True if the UPDATE touched at least one row (listing_id found in DB). - """ - metro_json: str | None = None - if e.metro_stations: - metro_json = json.dumps( - [s.model_dump(exclude_none=True) for s in e.metro_stations], - ensure_ascii=False, - ) - - photo_json: str | None = None - if e.photo_urls: - photo_json = json.dumps(e.photo_urls, ensure_ascii=False) - - result = db.execute( - text(""" - UPDATE listings SET - rooms = COALESCE(CAST(:rooms AS int), rooms), - area_m2 = COALESCE(CAST(:area_m2 AS numeric), area_m2), - living_area_m2 = COALESCE( - CAST(:living_area_m2 AS numeric), - living_area_m2 - ), - kitchen_area_m2 = COALESCE( - CAST(:kitchen_area_m2 AS numeric), - kitchen_area_m2 - ), - ceiling_height = COALESCE( - CAST(:ceiling_height AS numeric), - ceiling_height - ), - floor = COALESCE(CAST(:floor AS int), floor), - total_floors = COALESCE(CAST(:total_floors AS int), total_floors), - address = COALESCE(CAST(:address AS text), address), - description = COALESCE(CAST(:description AS text), description), - repair_state = COALESCE(CAST(:repair_state AS text),repair_state), - publish_date = COALESCE(CAST(:publish_date AS date),publish_date), - views_total_yandex = COALESCE(CAST(:views_total AS int), views_total_yandex), - publish_date_relative= COALESCE( - CAST(:pub_date_rel AS text), - publish_date_relative - ), - agency_name = COALESCE(CAST(:agency_name AS text), agency_name), - agency_founded_year = COALESCE( - CAST(:agency_founded_year AS int), - agency_founded_year - ), - agency_objects_count = COALESCE( - CAST(:agency_objects_count AS int), - agency_objects_count - ), - metro_stations = COALESCE( - CAST(:metro_stations AS jsonb), - metro_stations - ), - photo_urls = COALESCE( - CAST(:photo_urls AS jsonb), - photo_urls - ), - newbuilding_url = COALESCE( - CAST(:newbuilding_url AS text), - newbuilding_url - ), - newbuilding_id = COALESCE( - CAST(:newbuilding_id AS text), - newbuilding_id - ), - detail_enriched_at = NOW() - WHERE id = CAST(:listing_id AS bigint) - """), - { - "listing_id": listing_id, - "rooms": e.rooms, - "area_m2": e.area_m2, - "living_area_m2": e.living_area_m2, - "kitchen_area_m2": e.kitchen_area_m2, - "ceiling_height": e.ceiling_height, - "floor": e.floor, - "total_floors": e.total_floors, - "address": e.address, - "description": e.description, - "repair_state": e.repair_state, - "publish_date": e.publish_date, - "views_total": e.views_total, - "pub_date_rel": e.publish_date_relative, - "agency_name": e.agency_name, - "agency_founded_year": e.agency_founded_year, - "agency_objects_count": e.agency_objects_count, - "metro_stations": metro_json, - "photo_urls": photo_json, - "newbuilding_url": e.newbuilding_url, - "newbuilding_id": e.newbuilding_id, - }, - ) - db.commit() - saved = (result.rowcount or 0) > 0 - logger.info( - "yandex detail enrichment saved listing_id=%s (metro=%d photos=%d saved=%s)", - listing_id, - len(e.metro_stations), - len(e.photo_urls), - saved, - ) - return saved diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py deleted file mode 100644 index 91033dd4..00000000 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ /dev/null @@ -1,526 +0,0 @@ -"""Yandex Realty anonymous valuation/history scraper. - -URL pattern: - GET /otsenka-kvartiry-po-adresu-onlayn/?address=...&offerCategory=...&offerType=...&page=N - -No cookies, no auth required (unlike Cian Calculator or Avito IMV). - -Extracts: -- House metadata: year_built, total_floors, house_type, ceiling_height, has_lift, total_objects -- Historical offers: area, rooms, floor, start/last price + per-m2, publish_date DMY, exposure - -Two-strategy history extraction: -1. data-test container selectors (CSS, if Yandex adds them) -2. Fallback: text chunks around DD.MM.YYYY date matches with dedup by (date, area, floor) -""" - -from __future__ import annotations - -import logging -import re -from datetime import date -from typing import Any -from urllib.parse import urlencode - -from curl_cffi.requests import AsyncSession as _CurlCffiSession -from pydantic import BaseModel, Field -from selectolax.parser import HTMLParser - -from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.base import BaseScraper -from app.services.scrapers.yandex_helpers import ( - parse_dmy, - parse_house_type, - parse_rub, -) - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Models -# --------------------------------------------------------------------------- - - -class ValuationHouseMeta(BaseModel): - """House metadata extracted from Yandex valuation page.""" - - year_built: int | None = None - total_floors: int | None = None - house_type: str | None = None # panel/brick/monolith/... - ceiling_height: float | None = None # in meters, e.g. 2.5 - has_lift: bool | None = None - total_objects: int | None = None # 'N объектов' (full archive count) - has_panorama: bool = False # 'Панорама' label present - - def validate_match( - self, - expected_year_built: int | None = None, - expected_total_floors: int | None = None, - ) -> float: - """Return confidence 0..1 that this house meta matches expected values. - - Used after fetching valuation by address to detect when Yandex returned a - different house (ambiguous address geocoding). Tolerance ±1 year, ±1 floor. - - Both expected=None → 1.0 (no check). Mismatch on any dimension → 0.0 for it. - """ - score = 0.0 - checks = 0 - if expected_year_built is not None: - checks += 1 - if self.year_built is not None and abs(self.year_built - expected_year_built) <= 1: - score += 1.0 - if expected_total_floors is not None: - checks += 1 - if ( - self.total_floors is not None - and abs(self.total_floors - expected_total_floors) <= 1 - ): - score += 1.0 - if checks == 0: - return 1.0 - return score / checks - - -class ValuationHistoryItem(BaseModel): - """One historical offer entry from the valuation page.""" - - area_m2: float | None = None - rooms: int | None = None - floor: int | None = None - start_price: int | None = None - start_price_per_m2: int | None = None - last_price: int | None = None - last_price_per_m2: int | None = None - publish_date: date | None = None - removed_date: date | None = None # ← NEW - exposure_days: int | None = None - status: str | None = None # 'В продаже' / 'Снято' - - -class YandexValuationResult(BaseModel): - """Full result of one /otsenka-... GET.""" - - address: str - offer_category: str - offer_type: str - page: int - source_url: str - house: ValuationHouseMeta - history_items: list[ValuationHistoryItem] = Field(default_factory=list) - raw_payload: dict[str, Any] | None = None - - -# --------------------------------------------------------------------------- -# Regex constants -# --------------------------------------------------------------------------- - -RE_YEAR_BUILT = re.compile(r"Дом\s+(\d{4})\s+года", re.IGNORECASE) -RE_FLOORS = re.compile(r"(\d+)\s+этажей", re.IGNORECASE) -RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE) -RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE) - -# Lookbehind blocks digit-only adjacency (rejects year-concat like '20244,6') -# while still allowing tokens preceded by punctuation/letter. Min 2 digits -# rejects sub-fragments like '2,2' that come from inside '52,2'. Max 4 digits -# whole part catches obvious junk (penthouse extremes are <500 m² in practice; -# sanity cap from PR #538 backs this up). -RE_ITEM_AREA = re.compile(r"(? None: - super().__init__() - self.request_delay_sec = get_scraper_delay(self.name) - self._cffi_session: _CurlCffiSession | None = None - - async def __aenter__(self) -> YandexValuationScraper: # type: ignore[override] - # Override: Yandex valuation endpoint gates SSR data on Chrome TLS - # fingerprint. Plain httpx returns shell HTML (CSR-only). - # Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py - # Mobile proxy wiring (#806 follow-up): route via mobile proxy to avoid - # datacenter-IP blocks. proxy=None → прямое подключение (dev). - from app.core.config import settings - - _proxy_url = settings.scraper_proxy_url - _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None - self._cffi_session = _CurlCffiSession(impersonate="chrome120", proxies=_proxies) - return self - - async def __aexit__(self, *args: object) -> None: # type: ignore[override] - if self._cffi_session is not None: - await self._cffi_session.close() - self._cffi_session = None - - async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override] - """curl_cffi-based GET with Chrome120 impersonation. - - Returns curl_cffi Response (compatible API: .status_code, .text). - Caller must check status_code; no automatic retry (BaseScraper.retry - decorator is per-method and not inherited cleanly when overridden). - """ - if self._cffi_session is None: - raise RuntimeError("YandexValuationScraper must be used as async context manager") - kwargs.setdefault("timeout", 30) - return await self._cffi_session.get(url, **kwargs) - - async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override] - raise NotImplementedError( - "YandexValuationScraper is address-based; use fetch_house_history() instead." - ) - - async def fetch_house_history( - self, - address: str, - offer_category: str = "APARTMENT", - offer_type: str = "SELL", - page: int = 1, - ) -> YandexValuationResult | None: - """Fetch and parse one page of house history for the given address. - - Args: - address: Postal address string (will be URL-encoded). - offer_category: 'APARTMENT' / 'ROOMS' / etc. - offer_type: 'SELL' / 'RENT'. - page: 1-based pagination index. - - Returns: - YandexValuationResult or None on HTTP error / network failure. - """ - params = { - "address": address, - "offerCategory": offer_category, - "offerType": offer_type, - "page": page, - } - url = f"{self.base_url}{self.valuation_path}?{urlencode(params)}" - try: - response = await self._http_get(url) - except Exception: - logger.exception("yandex valuation fetch failed: %s", url) - return None - if response.status_code != 200: - logger.warning("yandex valuation returned %d for %s", response.status_code, url) - return None - result = self.parse( - response.text, - address=address, - offer_category=offer_category, - offer_type=offer_type, - page=page, - source_url=url, - ) - await self.sleep_between_requests() - return result - - def parse( - self, - html: str, - address: str, - offer_category: str, - offer_type: str, - page: int, - source_url: str, - ) -> YandexValuationResult: - """Parse raw HTML into YandexValuationResult. Pure function — usable in unit tests.""" - html_normalized = html.replace("\xa0", " ") - tree = HTMLParser(html_normalized) - body = tree.body - body_text = (body.text(strip=True) if body else "").replace("\xa0", " ") - - house = self._parse_house_meta(body_text) - history_items = self._parse_history_items(tree, body_text) - - return YandexValuationResult( - address=address, - offer_category=offer_category, - offer_type=offer_type, - page=page, - source_url=source_url, - house=house, - history_items=history_items, - raw_payload={ - "body_len": len(body_text), - "items_count": len(history_items), - }, - ) - - @staticmethod - def _parse_house_meta(body_text: str) -> ValuationHouseMeta: - """Extract house-level metadata from page body text.""" - year_m = RE_YEAR_BUILT.search(body_text) - floors_m = RE_FLOORS.search(body_text) - ceiling_m = RE_CEILING.search(body_text) - objects_m = RE_TOTAL_OBJECTS.search(body_text) - return ValuationHouseMeta( - year_built=int(year_m.group(1)) if year_m else None, - total_floors=int(floors_m.group(1)) if floors_m else None, - house_type=parse_house_type(body_text), - ceiling_height=(float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None), - has_lift="Лифт" in body_text, - total_objects=int(objects_m.group(1)) if objects_m else None, - has_panorama="Панорама" in body_text, - ) - - def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]: - """Extract historical offer items. - - Strategy 1 (preferred, structured): each row at .OffersArchiveSearchOffers__row, - with 6 cells (.OffersArchiveSearchOffers__cell) in fixed order: - [0] area+rooms, [1] floor, [2] start_price+ppm2, [3] last_price+ppm2, - [4] publish_date+exposure, [5] status/removed_date. - Yields per-cell parsing — no chunked-text ambiguity. - - Strategy 2 (last-resort fallback): legacy chunked-text scan around - DD.MM.YYYY anchors. Kept for the rare case Yandex changes class names; - triggers a logger.warning so we notice. - """ - items: list[ValuationHistoryItem] = [] - for row in tree.css(".OffersArchiveSearchOffers__row"): - item = self._parse_row_cells(row) - if item: - items.append(item) - if items: - return items - # Fallback (only if structured selector failed) - logger.warning( - "yandex_valuation: .OffersArchiveSearchOffers__row matched 0 items — " - "falling back to chunked-text scan" - ) - return self._parse_items_from_chunked_text(body_text) - - @classmethod - def _parse_row_cells(cls, row) -> ValuationHistoryItem | None: - """Parse one .OffersArchiveSearchOffers__row into a ValuationHistoryItem. - - Cell layout is positional (no labels). Returns None if the row doesn't - match the expected 6-cell structure (defensive — Yandex may A/B-test). - """ - cells = row.css(".OffersArchiveSearchOffers__cell") - if len(cells) < 5: - return None - cell_texts = [(c.text(strip=True) or "").replace("\xa0", " ") for c in cells] - - # [0] photo (skipped — empty cell) - # [1] title — single cell contains area + rooms + floor jammed together, - # e.g. "48,5 м², 1-комнатная19 этаж" (real Yandex layout, 2026-05-24) - title_text = cell_texts[1] if len(cell_texts) > 1 else "" - - area_m = RE_ITEM_AREA.search(title_text) - area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None - # Sanity drop (belt-and-suspenders from PR #538) - if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0): - area_m2 = None - - rooms: int | None = None - if RE_ITEM_STUDIO.search(title_text): - rooms = 0 - else: - rooms_m = RE_ITEM_ROOMS.search(title_text) - if rooms_m: - rooms = int(rooms_m.group(1)) - - floor_m = RE_ITEM_FLOOR.search(title_text) - floor: int | None = int(floor_m.group(1)) if floor_m else None - - # [2] start price + ppm2 — "5,1 млн ₽105 155 ₽ за м²" (no separator) - start_price: int | None = None - start_ppm2: int | None = None - if len(cell_texts) > 2: - tokens = _RE_PRICE_TOKEN.findall(cell_texts[2]) - if tokens: - start_price = parse_rub(tokens[0]) - ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[2]) - if ppm2_tokens: - start_ppm2 = parse_rub(ppm2_tokens[0]) - - # [3] last price + ppm2 - last_price: int | None = None - last_ppm2: int | None = None - if len(cell_texts) > 3: - tokens = _RE_PRICE_TOKEN.findall(cell_texts[3]) - if tokens: - last_price = parse_rub(tokens[0]) - ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[3]) - if ppm2_tokens: - last_ppm2 = parse_rub(ppm2_tokens[0]) - - # [4] publish_date + exposure ("23.10.2023В экспозиции 945 дней") - publish_date = None - exposure_days: int | None = None - if len(cell_texts) > 4: - publish_date = parse_dmy(cell_texts[4]) - expo_m = RE_ITEM_EXPOSURE.search(cell_texts[4]) - if expo_m: - exposure_days = int(expo_m.group(1)) - - # [5] status or removed_date - removed_date = None - status: str | None = None - if len(cell_texts) > 5: - last_cell = cell_texts[5] - status_m = RE_ITEM_STATUS.search(last_cell) - if status_m: - status = status_m.group(1) - removed_date = parse_dmy(last_cell) - - # Sort dates chronologically (in case Yandex flips them — same fix as PR #541) - if publish_date and removed_date and removed_date < publish_date: - publish_date, removed_date = removed_date, publish_date - - # Row is valid only if we got area OR start_price - if area_m2 is None and start_price is None: - return None - - return ValuationHistoryItem( - area_m2=area_m2, - rooms=rooms, - floor=floor, - start_price=start_price, - start_price_per_m2=start_ppm2, - last_price=last_price, - last_price_per_m2=last_ppm2, - publish_date=publish_date, - removed_date=removed_date, - exposure_days=exposure_days, - status=status, - ) - - @staticmethod - def _parse_item_text(text: str) -> ValuationHistoryItem | None: - """Parse a single text block into a ValuationHistoryItem. - - Returns None if neither area nor start_price can be extracted - (item is considered invalid/noise). - """ - if not text or len(text) < 20: - return None - - # area + rooms - area_m = RE_ITEM_AREA.search(text) - area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None - # Sanity cap — Yandex page text sometimes concatenates digits across DOM - # boundaries (e.g. "2025\xa0106,7 м²" → 2_025_106.7). Flats over 10_000 m² - # are impossible; DB column is NUMERIC(8,2) which overflows at 10^6. - # Drop the value rather than block the whole save batch. - if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0): - logger.warning( - "yandex_valuation: dropping nonsensical area_m2=%s from item chunk", - area_m2, - ) - area_m2 = None - - rooms: int | None = None - if RE_ITEM_STUDIO.search(text): - rooms = 0 - else: - rooms_m = RE_ITEM_ROOMS.search(text) - if rooms_m: - rooms = int(rooms_m.group(1)) - - floor_m = RE_ITEM_FLOOR.search(text) - floor = int(floor_m.group(1)) if floor_m else None - - # Prices — find first 2 price tokens (start, last) - price_tokens = _RE_PRICE_TOKEN.findall(text) - start_price = parse_rub(price_tokens[0]) if len(price_tokens) >= 1 else None - last_price = parse_rub(price_tokens[1]) if len(price_tokens) >= 2 else None - - ppm2_tokens = _RE_PPM2_TOKEN.findall(text) - start_ppm2 = parse_rub(ppm2_tokens[0]) if len(ppm2_tokens) >= 1 else None - last_ppm2 = parse_rub(ppm2_tokens[1]) if len(ppm2_tokens) >= 2 else None - - # Extract ALL DD.MM.YYYY dates: first → publish_date, second → removed_date - date_matches = list(re.finditer(r"\d{2}\.\d{2}\.\d{4}", text)) - dates_parsed: list[date] = [] - for m in date_matches: - d = parse_dmy(m.group(0)) - if d is not None: - dates_parsed.append(d) - # Sort dates chronologically — chunked-text scan returns them in page-text - # order, but semantically publish_date is earliest and removed_date is - # latest. Without this, ~1% of rows on prod show removed < publish. - if dates_parsed: - dates_sorted = sorted(d for d in dates_parsed if d is not None) - publish_date = dates_sorted[0] if dates_sorted else None - removed_date = dates_sorted[1] if len(dates_sorted) >= 2 else None - else: - publish_date = None - removed_date = None - expo_m = RE_ITEM_EXPOSURE.search(text) - exposure_days = int(expo_m.group(1)) if expo_m else None - - status_m = RE_ITEM_STATUS.search(text) - status = status_m.group(1) if status_m else None - - # Item is valid only if we got at least area or price - if area_m2 is None and start_price is None: - return None - - return ValuationHistoryItem( - area_m2=area_m2, - rooms=rooms, - floor=floor, - start_price=start_price, - start_price_per_m2=start_ppm2, - last_price=last_price, - last_price_per_m2=last_ppm2, - publish_date=publish_date, - removed_date=removed_date, - exposure_days=exposure_days, - status=status, - ) - - @classmethod - def _parse_items_from_chunked_text(cls, body_text: str) -> list[ValuationHistoryItem]: - """Fallback: split body text into chunks around DD.MM.YYYY dates and parse each chunk. - - Window: 200 chars before + 100 chars after each date match. - Deduplicates by (publish_date, area_m2, floor). - Caps output at 30 items per page to avoid runaway extraction. - """ - items: list[ValuationHistoryItem] = [] - for m in re.finditer(r"\d{2}\.\d{2}\.\d{4}", body_text): - start = max(0, m.start() - 200) - end = min(len(body_text), m.end() + 100) - chunk = body_text[start:end] - item = cls._parse_item_text(chunk) - if item: - items.append(item) - - # Dedup by (publish_date, area_m2, floor) - seen: set[tuple[Any, Any, Any]] = set() - deduped: list[ValuationHistoryItem] = [] - for item in items: - key = (item.publish_date, item.area_m2, item.floor) - if key not in seen: - seen.add(key) - deduped.append(item) - return deduped[:30] diff --git a/tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py index 8540169f..540d4766 100644 --- a/tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py +++ b/tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py @@ -1,18 +1,22 @@ """Parity-proof: admin.py Avito debug-роуты (#2305, Group A) — legacy vs scraper_kit. -`app/api/v1/admin.py::scrape_avito_house` / `scrape_avito_imv` теперь зовут -`scraper_kit.providers.avito.{houses,imv}` вместо `app.services.scrapers.avito_*` -(тот же `/scrape` root-роут — `scraper_kit.providers.avito.serp.AvitoScraper`). -Каждая пара функций ниже — чистое, offline-тестируемое parsing-ядро, которое -реально используется соответствующим debug-эндпоинтом (не сам HTTP-fetch — -см. `tests/support/README.md`: "Live network/DB в parity-тестах избегайте"). +`app/api/v1/admin.py::scrape_avito_house` теперь зовёт `scraper_kit.providers.avito.houses` +вместо `app.services.scrapers.avito_houses` (тот же `/scrape` root-роут — +`scraper_kit.providers.avito.serp.AvitoScraper`). Каждая пара функций ниже — чистое, +offline-тестируемое parsing-ядро, которое реально используется соответствующим +debug-эндпоинтом (не сам HTTP-fetch — см. `tests/support/README.md`: "Live network/DB +в parity-тестах избегайте"). - `_parse_house_page` — `scrape_avito_house` → `fetch_house_catalog` парсит виджет `housePage` этой функцией. -- `_parse_price` — `scrape_avito_imv` → `evaluate_via_imv` парсит top-level `price` - блок ответа `realty-imv/get-data` этой функцией. - `_extract_rooms_from_title` / `_extract_area_from_title` — использует `AvitoScraper` (SERP-парсинг `/scrape` root-роута) для title-fallback полей. + +Легаси `app.services.scrapers.avito_imv` (`scrape_avito_imv` → `evaluate_via_imv` → +`_parse_price`) удалён (#2277 финальный шаг scraper_kit-миграции, 0 runtime +importers) — parity-проверка `_parse_price` убрана вместе с ним; kit-сторона +`_parse_price` покрыта отдельно (`tests/scrapers/test_avito_imv_kit_parity.py` / +`tests/test_avito_imv_parse.py`). """ from __future__ import annotations @@ -25,7 +29,6 @@ import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from scraper_kit.providers.avito.houses import _parse_house_page as kit_parse_house_page -from scraper_kit.providers.avito.imv import _parse_price as kit_parse_price from scraper_kit.providers.avito.serp import ( _extract_area_from_title as kit_extract_area_from_title, ) @@ -38,7 +41,6 @@ from app.services.scrapers.avito import ( _extract_rooms_from_title as legacy_extract_rooms_from_title, ) from app.services.scrapers.avito_houses import _parse_house_page as legacy_parse_house_page -from app.services.scrapers.avito_imv import _parse_price as legacy_parse_price from tests.support.parity import assert_parity _HOUSE_WIDGET = { @@ -71,16 +73,6 @@ _HOUSE_WIDGET = { } } -_IMV_GETDATA = { - "price": { - "recommendedPrice": 8500000, - "lowerPrice": 7800000, - "higherPrice": 9200000, - "count": 42, - "addItemUrl": "/add", - } -} - def test_avito_house_page_parse_are_class_distinct() -> None: """Sanity: legacy/kit HouseInfo — разные классы, обычный `==` даёт False.""" @@ -101,15 +93,6 @@ def test_parity_avito_house_page() -> None: ) -def test_parity_avito_imv_price() -> None: - """`scrape_avito_imv` (admin.py) → evaluate_via_imv → _parse_price.""" - assert_parity( - legacy_fn=legacy_parse_price, - kit_fn=kit_parse_price, - fixtures=[(_IMV_GETDATA,)], - ) - - def test_parity_avito_serp_title_extract() -> None: """`/scrape` root-роут (admin.py) → AvitoScraper SERP title-fallback fields.""" titles = [ diff --git a/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py index 5da9e12e..b37e5d57 100644 --- a/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py +++ b/tradein-mvp/backend/tests/scrapers/test_admin_domclick_ingest_kit_parity.py @@ -12,12 +12,13 @@ Issue #2307 (Group D, PR #2318) портировал `domclick_detail.py` в kit с этой задачей и явно оставил `scripts/ingest_domclick_jsonl.py` untouched ("the only other domclick_detail caller is scripts/ingest_domclick_jsonl.py, also untouched") — поэтому переключение этого caller'а сделано ЗДЕСЬ, в рамках -#2305, теперь когда kit-эквивалент существует. `parse_detail_html`/exception- -parity для domclick_detail уже исчерпывающе доказаны в -`tests/scrapers/test_domclick_detail_kit_parity.py` (issue #2307) — этот файл -НЕ дублирует то доказательство, а фокусируется на конкретной точке потребления: -`ScrapedLot`/`DomClickDetailEnrichment`, сконструированные ИЗ ТЕХ ЖЕ kwargs, что -и `_build_lot()`/`_build_enrichment()` в самом скрипте. +#2305, теперь когда kit-эквивалент существует. + +Легаси `app.services.scrapers.domclick_detail` удалён (#2277 финальный шаг +scraper_kit-миграции, 0 runtime importers, `scripts/ingest_domclick_jsonl.py` уже +100% на kit-стороне для `DomClickDetailEnrichment`) — legacy-vs-kit parity для +`DomClickDetailEnrichment`-конструирования убрана вместе с ним; `ScrapedLot` +parity (`app.services.scrapers.base`, всё ещё легаси/живой модуль) остаётся. """ from __future__ import annotations @@ -27,14 +28,8 @@ import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from scraper_kit.base import ScrapedLot as KitScrapedLot -from scraper_kit.providers.domclick.detail import ( - DomClickDetailEnrichment as KitDomClickDetailEnrichment, -) from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot -from app.services.scrapers.domclick_detail import ( - DomClickDetailEnrichment as LegacyDomClickDetailEnrichment, -) from tests.support.parity import assert_parity # Аналог одной ok-записи JSONL-раннера (см. ingest_domclick_jsonl._build_lot). @@ -56,23 +51,6 @@ _LOT_KWARGS = { "raw_payload": {"is_sber_collateral": False, "square_price": 156538, "avm": {}}, } -# Аналог одной ok-записи JSONL-раннера (см. ingest_domclick_jsonl._build_enrichment). -_ENRICHMENT_KWARGS = { - "item_id": "2075729321", - "source_url": "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321", - "repair_state": "euro", - "repair_type": "евроремонт", - "living_area_m2": 32.1, - "kitchen_area_m2": 9.5, - "sale_type": "free", - "owners_count": 1, - "encumbrances_clean": True, - "year_built": 2015, - "views_total": 338, - "price_changes": [{"change_time": "2026-01-15T00:00:00", "price_rub": 8500000}], - "raw_extra": {"wall_type": "кирпично-монолитный", "calls": 5}, -} - def _build_legacy_lot(rec: dict[str, object]) -> LegacyScrapedLot: return LegacyScrapedLot(**rec) # type: ignore[arg-type] @@ -82,14 +60,6 @@ def _build_kit_lot(rec: dict[str, object]) -> KitScrapedLot: return KitScrapedLot(**rec) # type: ignore[arg-type] -def _build_legacy_enrichment(rec: dict[str, object]) -> LegacyDomClickDetailEnrichment: - return LegacyDomClickDetailEnrichment(**rec) # type: ignore[arg-type] - - -def _build_kit_enrichment(rec: dict[str, object]) -> KitDomClickDetailEnrichment: - return KitDomClickDetailEnrichment(**rec) # type: ignore[arg-type] - - def test_scraped_lot_are_class_distinct() -> None: """Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False.""" legacy_result = _build_legacy_lot(_LOT_KWARGS) @@ -106,14 +76,3 @@ def test_parity_ingest_domclick_scraped_lot() -> None: kit_fn=_build_kit_lot, fixtures=[_LOT_KWARGS], ) - - -def test_parity_ingest_domclick_enrichment() -> None: - """`ingest_domclick_jsonl._build_enrichment()` kwargs → - DomClickDetailEnrichment(legacy) vs DomClickDetailEnrichment(kit) — - см. module docstring про переключение этого caller'а в рамках #2305.""" - assert_parity( - legacy_fn=_build_legacy_enrichment, - kit_fn=_build_kit_enrichment, - fixtures=[_ENRICHMENT_KWARGS], - ) diff --git a/tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py index 92470a0e..67cd2bce 100644 --- a/tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py +++ b/tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py @@ -7,13 +7,16 @@ чистое, offline-тестируемое parsing-ядро (см. `tests/support/README.md`: "Live network/DB в parity-тестах избегайте"). -- `_parse_card_fields` — `scrape_yandex_detail` → `YandexDetailScraper.fetch_detail` - извлекает (rooms, area, living, kitchen, ceiling, floor, total_floors) этой функцией. -- `YandexValuationScraper._parse_house_meta` (@staticmethod) — `scrape_yandex_valuation` - → `fetch_house_history` извлекает house-level метаданные этой функцией. - `_entity_to_lot` — `/scrape` root-роут → `YandexRealtyScraper.fetch_around` конвертирует gate-API entity → ScrapedLot этой функцией (также доказывает ScrapedLot-контракт, общий с `save_listings`/`scripts/ingest_domclick_jsonl.py`). + +Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277 +финальный шаг scraper_kit-миграции, 0 runtime importers) — `_parse_card_fields` / +`_parse_house_meta` parity-проверки убраны отсюда вместе с ними; kit-side coverage +для обеих функций сохранена standalone в `tests/test_yandex_detail_structural.py` / +`tests/test_yandex_valuation.py`. `yandex_realty` остаётся легаси (не мигрирован) — +его `_entity_to_lot` parity-проверка не тронута. """ from __future__ import annotations @@ -22,39 +25,11 @@ import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") -from scraper_kit.providers.yandex.detail import _parse_card_fields as kit_parse_card_fields from scraper_kit.providers.yandex.serp import _entity_to_lot as kit_entity_to_lot -from scraper_kit.providers.yandex.valuation import ( - YandexValuationScraper as KitYandexValuationScraper, -) -from app.services.scrapers.yandex_detail import _parse_card_fields as legacy_parse_card_fields from app.services.scrapers.yandex_realty import _entity_to_lot as legacy_entity_to_lot -from app.services.scrapers.yandex_valuation import ( - YandexValuationScraper as LegacyYandexValuationScraper, -) from tests.support.parity import assert_parity -_CARD = { - "roomsTotal": 2, - "area": {"value": 54.3}, - "livingSpace": {"value": 32.1}, - "kitchenSpace": {"value": 9.5}, - "ceilingHeight": 2.7, - "floorsOffered": [5], - "floorsTotal": 9, -} -_CARD_STUDIO = { - "house": {"studio": True}, - "area": {"value": 25.0}, - "floorsOffered": [1], - "floorsTotal": 5, -} - -_VALUATION_BODY_TEXT = ( - "Дом 2015 года. 9 этажей. 2,7 м потолки. Лифт есть. 120 объектов продажи. Панорама двора." -) - # gate-API entity (realty.yandex.ru), без creationDate/author — исключает # time-dependent поля (days_on_market) из фикстуры (deterministic offline). _YANDEX_ENTITY = { @@ -77,33 +52,6 @@ _YANDEX_ENTITY = { } -def test_parity_yandex_detail_card_fields() -> None: - """`scrape_yandex_detail` (admin.py) → fetch_detail → _parse_card_fields.""" - assert_parity( - legacy_fn=legacy_parse_card_fields, - kit_fn=kit_parse_card_fields, - fixtures=[(_CARD,), (_CARD_STUDIO,)], - ) - - -def test_yandex_valuation_house_meta_are_class_distinct() -> None: - """Sanity: legacy/kit ValuationHouseMeta — разные классы, обычный `==` даёт False.""" - legacy_result = LegacyYandexValuationScraper._parse_house_meta(_VALUATION_BODY_TEXT) - kit_result = KitYandexValuationScraper._parse_house_meta(_VALUATION_BODY_TEXT) - - assert type(legacy_result) is not type(kit_result) - assert legacy_result.model_dump() == kit_result.model_dump() - - -def test_parity_yandex_valuation_house_meta() -> None: - """`scrape_yandex_valuation` (admin.py) → fetch_house_history → _parse_house_meta.""" - assert_parity( - legacy_fn=LegacyYandexValuationScraper._parse_house_meta, - kit_fn=KitYandexValuationScraper._parse_house_meta, - fixtures=[_VALUATION_BODY_TEXT], - ) - - def test_yandex_entity_to_lot_are_class_distinct() -> None: """Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False.""" legacy_result = legacy_entity_to_lot(_YANDEX_ENTITY) diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_imv_browser_transport.py b/tradein-mvp/backend/tests/scrapers/test_avito_imv_browser_transport.py index ca3d61be..48a98a51 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_imv_browser_transport.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_imv_browser_transport.py @@ -10,6 +10,10 @@ 403 на get-data → IMVAuthError. browser_fetcher мокается AsyncMock'ом — реальный HTTP/камуфокс не поднимается. + +Легаси `app.services.scrapers.avito_imv` удалён (#2277 финальный шаг scraper_kit- +миграции, 0 runtime importers) — тесты переведены на kit-эквивалент +`scraper_kit.providers.avito.imv` без изменения покрытия. """ from __future__ import annotations @@ -20,8 +24,7 @@ from typing import Any from unittest.mock import AsyncMock import pytest - -from app.services.scrapers.avito_imv import ( +from scraper_kit.providers.avito.imv import ( AVITO_BASE, WARMUP_URL, IMVAuthError, diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_imv_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_avito_imv_kit_parity.py index b0fbdabe..3cc4e08e 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_imv_kit_parity.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_imv_kit_parity.py @@ -1,46 +1,31 @@ -"""Golden-parity + config-footgun proof for `evaluate_via_imv` — issue #2334. +"""Config-footgun regression for kit `evaluate_via_imv` — issue #2334. -Group E1 of the scraper_kit migration epic (#2277 → #2308 → #2334): migrates -`app/services/scrapers/avito_imv.py` readiness onto its -`scraper_kit.providers.avito.imv` equivalent. Unlike prior groups (#2305/#2306), -`evaluate_via_imv` had ZERO parity coverage before this file — `_parse_price` alone -is covered by `tests/scrapers/test_admin_avito_kit_parity.py` (Group A, admin debug -route), but the full 3-request async flow (geocode A/B + evaluate C) that -`app/services/estimator.py` and `app/services/house_imv_backfill.py` depend on was -never proven byte-for-byte identical between legacy and kit. +Group E1 of the scraper_kit migration epic (#2277 → #2308 → #2334 → final deletion +step). Legacy `app/services/scrapers/avito_imv.py` was deleted once the kit provider +(`scraper_kit.providers.avito.imv`) reached golden parity and no runtime caller +referenced the legacy module anymore — the legacy-vs-kit comparison tests that used +to live here were removed together with the legacy import. -Two independent things are proven here: +What remains is a standalone kit-only regression, still worth keeping on its own: -1. **Golden parity** — legacy `evaluate_via_imv` and kit `evaluate_via_imv` give the - IDENTICAL `IMVEvaluation` on the SAME input, using REAL Avito API responses - recorded live (`tests/fixtures/avito_imv_geo_position.json` + - `avito_imv_getdata.json`, see `tests/test_avito_imv_parse.py` for provenance — - issue #565, 2026-05 capture, ЕКБ ул. Малышева 125). Both sides run through the - `browser_fetcher=` transport (see `tests/scrapers/test_avito_imv_browser_transport.py`) - so the test is offline/deterministic and never touches `curl_cffi`/proxy config — - that's a SEPARATE axis, covered below. - -2. **Config-footgun regression** — CONFIRMED by recon before this issue started: - kit `evaluate_via_imv(config: ScraperConfig | None = None, ...)` only reads - `config.scraper_proxy_url` when `config is not None`; legacy unconditionally reads - `settings.scraper_proxy_url` itself. Call the kit fn WITHOUT `config=` → own - curl_cffi session gets built with NO proxy (`proxies=None`) even though a real - proxy is configured — a silent behaviour change vs legacy. WITH - `config=RealScraperConfig()` (the established DI pattern, see - `app/services/cian_price_history.py` / `tests/test_scraper_kit_pricehistory_session_parity.py:: - test_fetch_detail_own_session_proxy_wiring_parity` for precedent) the proxy is - used identically to legacy. NOTE: neither `estimator.py` nor - `house_imv_backfill.py` is switched to the kit import in this issue (see report in - PR/issue #2334) — these tests exist to prove the KIT SIDE is safe to wire up - whenever that switch happens (#2337 for estimator.py). +**Config-footgun regression** — CONFIRMED by recon before issue #2334 started: +kit `evaluate_via_imv(config: ScraperConfig | None = None, ...)` only reads +`config.scraper_proxy_url` when `config is not None`; the deleted legacy module +unconditionally read `settings.scraper_proxy_url` itself. Call the kit fn WITHOUT +`config=` → own curl_cffi session gets built with NO proxy (`proxies=None`) even +though a real proxy is configured — a silent behaviour change vs the old code. WITH +`config=RealScraperConfig()` (the established DI pattern, see +`app/services/cian_price_history.py` / `tests/test_scraper_kit_pricehistory_session_parity.py:: +test_fetch_detail_own_session_proxy_wiring_parity` for precedent) the proxy is wired +correctly. NOTE: neither `estimator.py` nor `house_imv_backfill.py` needs this +config= kwarg wired for `evaluate_via_imv` specifically — these tests exist to prove +the KIT SIDE stays safe against a "simplify away the config kwarg" regression. """ from __future__ import annotations import asyncio -import json import os -from pathlib import Path from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -54,173 +39,10 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/ from scraper_kit.providers.avito.imv import ( IMVAddressNotFoundError as KitIMVAddressNotFoundError, ) -from scraper_kit.providers.avito.imv import _parse_geo_position as kit_parse_geo_position -from scraper_kit.providers.avito.imv import _parse_placement_history as kit_parse_history -from scraper_kit.providers.avito.imv import _parse_suggestions as kit_parse_suggestions -from scraper_kit.providers.avito.imv import compute_imv_cache_key as kit_cache_key from scraper_kit.providers.avito.imv import evaluate_via_imv as kit_evaluate_via_imv from app.core.config import settings from app.services.scraper_adapters import RealScraperConfig -from app.services.scrapers.avito_imv import _parse_geo_position as legacy_parse_geo_position -from app.services.scrapers.avito_imv import _parse_placement_history as legacy_parse_history -from app.services.scrapers.avito_imv import _parse_suggestions as legacy_parse_suggestions -from app.services.scrapers.avito_imv import compute_imv_cache_key as legacy_cache_key -from app.services.scrapers.avito_imv import evaluate_via_imv as legacy_evaluate_via_imv -from tests.support.parity import ParityMismatchError, assert_parity - -_FIXTURES = Path(__file__).parent.parent / "fixtures" - - -def _load_fixture(name: str) -> dict[str, Any]: - with (_FIXTURES / name).open(encoding="utf-8") as f: - return json.load(f) - - -# --------------------------------------------------------------------------- -# Cheap pure-function parity (closes the gap beyond `_parse_price`, already -# covered by tests/scrapers/test_admin_avito_kit_parity.py Group A). -# --------------------------------------------------------------------------- - - -def test_compute_imv_cache_key_parity() -> None: - assert_parity( - legacy_fn=legacy_cache_key, - kit_fn=kit_cache_key, - fixtures=[ - ("ЕКБ ул. Малышева, 125", 2, 54.0, 4, 9, "panel", "cosmetic", True, False), - ], - ) - - -def test_parse_geo_position_parity_real_fixture() -> None: - data_b = _load_fixture("avito_imv_geo_position.json") - assert_parity( - legacy_fn=lambda: legacy_parse_geo_position(data_b, lat=56.840872, lon=60.654077), - kit_fn=lambda: kit_parse_geo_position(data_b, lat=56.840872, lon=60.654077), - fixtures=[()], - ) - - -def test_parse_placement_history_parity_real_fixture() -> None: - data_c = _load_fixture("avito_imv_getdata.json") - assert_parity( - legacy_fn=legacy_parse_history, - kit_fn=kit_parse_history, - fixtures=[(data_c,)], - ) - - -def test_parse_suggestions_parity_real_fixture() -> None: - data_c = _load_fixture("avito_imv_getdata.json") - assert_parity( - legacy_fn=legacy_parse_suggestions, - kit_fn=kit_parse_suggestions, - fixtures=[(data_c,)], - ) - - -# --------------------------------------------------------------------------- -# Golden parity — full evaluate_via_imv() flow on REAL captured API fixtures -# --------------------------------------------------------------------------- - -# Same address/lat/lon/lot-params as tests/test_avito_imv_parse.py:: -# test_fixtures_feed_full_evaluation_shape — keeps this test consistent with the -# established "reference realistic case" derived from the live 2026-05 capture -# (issue #565), rather than inventing a new ad-hoc fixture. -_ADDRESS = "Свердловская область, Екатеринбург, улица Малышева, 125" -_LAT = 56.840872 -_LON = 60.654077 -_LOT_PARAMS: dict[str, Any] = { - "rooms": 2, - "area_m2": 54.0, - "floor": 4, - "floor_at_home": 9, - "house_type": "panel", - "renovation_type": "cosmetic", - "has_balcony": True, - "has_loggia": False, -} - - -def _canned_browser_responses() -> list[dict[str, Any]]: - """4 responses in call order: warm-up GET, coords A, position B, get-data C. - - B and C bodies are the REAL live-captured Avito responses (fixtures/*.json); - A is synthesized with the SAME lat/lon as the fixture B (both refer to the same - ул. Малышева, 125 lookup) so the flow is internally consistent end-to-end. - """ - warmup = {"status": 200, "body": "warm"} - coords_a_body = { - "normalizedAddress": _ADDRESS, - "point": {"latitude": _LAT, "longitude": _LON}, - } - coords_a = {"status": 200, "body": json.dumps(coords_a_body)} - position_b = {"status": 200, "body": json.dumps(_load_fixture("avito_imv_geo_position.json"))} - getdata_c = {"status": 200, "body": json.dumps(_load_fixture("avito_imv_getdata.json"))} - return [warmup, coords_a, position_b, getdata_c] - - -def _run_legacy_evaluation() -> Any: - bf = AsyncMock() - bf.fetch_json = AsyncMock(side_effect=_canned_browser_responses()) - return asyncio.run(legacy_evaluate_via_imv(address=_ADDRESS, browser_fetcher=bf, **_LOT_PARAMS)) - - -def _run_kit_evaluation() -> Any: - bf = AsyncMock() - bf.fetch_json = AsyncMock(side_effect=_canned_browser_responses()) - return asyncio.run(kit_evaluate_via_imv(address=_ADDRESS, browser_fetcher=bf, **_LOT_PARAMS)) - - -def test_legacy_and_kit_imv_evaluation_are_class_distinct() -> None: - """Sanity-check ДО harness'а — same reason as test_avito_detail_kit_parity.py: - legacy/kit IMVEvaluation are different classes (different modules), so plain - `==` is always False even on identical field values. - """ - legacy_result = _run_legacy_evaluation() - kit_result = _run_kit_evaluation() - - assert type(legacy_result) is not type(kit_result) - assert legacy_result != kit_result - assert legacy_result.recommended_price == kit_result.recommended_price - assert legacy_result.geo.geo_hash == kit_result.geo.geo_hash - - -def test_evaluate_via_imv_golden_parity_real_fixtures() -> None: - """Proves kit evaluate_via_imv gives BYTE-IDENTICAL IMVEvaluation to legacy on a - real recorded Avito API response (no synthetic мокы для price/history/suggestions — - только step A синтетика для координат, зеркалящих ту же самую фикстуру B). - """ - assert_parity( - legacy_fn=_run_legacy_evaluation, - kit_fn=_run_kit_evaluation, - fixtures=[()], - ) - - -def test_parity_harness_catches_imv_field_divergence() -> None: - """Доказывает, что harness реально ЛОВИТ расхождение на этой dataclass-форме — - мутируем kit-результат (recommended_price) и убеждаемся в ParityMismatchError, - называющем именно это поле (mirrors test_avito_detail_kit_parity.py pattern). - """ - import dataclasses - - legacy_result = _run_legacy_evaluation() - kit_result = _run_kit_evaluation() - mutated_kit_result = dataclasses.replace( - kit_result, recommended_price=kit_result.recommended_price + 1 - ) - - with pytest.raises(ParityMismatchError) as exc_info: - assert_parity( - legacy_fn=lambda: legacy_result, - kit_fn=lambda: mutated_kit_result, - fixtures=[()], - ) - - assert "recommended_price" in str(exc_info.value) - # --------------------------------------------------------------------------- # Config-footgun regression (#2334 CONFIRMED recon finding) @@ -229,7 +51,8 @@ def test_parity_harness_catches_imv_field_divergence() -> None: # evaluate_via_imv's own-curl_cffi-session path (browser_fetcher=None, the default) # is the one that reads proxy config. Mirrors # tests/test_scraper_proxy.py::test_avito_imv_own_session_receives_proxies (legacy -# reference) and tests/test_scraper_kit_pricehistory_session_parity.py:: +# reference, now gone with the legacy module) and +# tests/test_scraper_kit_pricehistory_session_parity.py:: # test_fetch_detail_own_session_proxy_wiring_parity (kit config= precedent for a # sibling provider, #2306). @@ -287,10 +110,10 @@ def test_kit_evaluate_via_imv_without_config_drops_proxy(monkeypatch: pytest.Mon def test_kit_evaluate_via_imv_with_config_uses_proxy(monkeypatch: pytest.MonkeyPatch) -> None: """With config=RealScraperConfig() (the established DI pattern for this epic) the - kit path reads the SAME settings.scraper_proxy_url as legacy and wires it into the - curl_cffi session identically — this is the regression guard: if someone later - "simplifies away" the config kwarg threading in evaluate_via_imv, this test fails - on `captured.get("proxies") is None` instead of the expected proxy dict. + kit path reads the SAME settings.scraper_proxy_url as legacy used to and wires it + into the curl_cffi session identically — this is the regression guard: if someone + later "simplifies away" the config kwarg threading in evaluate_via_imv, this test + fails on `captured.get("proxies") is None` instead of the expected proxy dict. """ monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://test-proxy.local:8080") mock_session = _mock_own_session_flow() diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py b/tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py index e5225014..683339be 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py @@ -4,12 +4,19 @@ До фикса было 3 независимые копии с разными timezone: - avito.py (inline в `_build_sort_timestamp_map`) — MSK (документировано #726) - avito_houses.py — UTC - - avito_imv.py — naive/system-local (`datetime.fromtimestamp(ts)` без tz) + - avito_imv.py (легаси, удалён #2277) — naive/system-local + (`datetime.fromtimestamp(ts)` без tz) Один и тот же epoch мог давать РАЗНЫЕ `date` в разных путях (пример из живой fixture avito_imv_getdata.json: removedDate=1756077885 → UTC 2025-08-24, MSK 2025-08-25). Офлайн: без сети/curl_cffi/БД, только импорт + чистые функции. + +Легаси `app.services.scrapers.avito_imv` удалён (#2277 финальный шаг scraper_kit- +миграции, 0 runtime importers) — IMV-ветка consolidation-проверки переведена на +kit-эквивалент `scraper_kit.providers.avito.imv`, который использует ТОТ ЖЕ паттерн +(`from scraper_kit.providers.avito.shared import _unix_to_date`) — регрессия +"3 независимые копии с разными TZ" остаётся под наблюдением и на kit-стороне. """ from __future__ import annotations @@ -19,7 +26,10 @@ from datetime import date os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") -from app.services.scrapers import avito, avito_houses, avito_imv, avito_shared +from scraper_kit.providers.avito import imv as kit_avito_imv +from scraper_kit.providers.avito import shared as kit_avito_shared + +from app.services.scrapers import avito, avito_houses, avito_shared # Живой пример из fixtures/avito_imv_getdata.json (placementHistory.items[0].removedDate) — # именно тот timestamp, на котором UTC и MSK расходились в date() ДО фикса @@ -39,27 +49,29 @@ def test_avito_houses_reexports_shared_unix_to_date() -> None: assert avito_houses._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE -def test_avito_imv_reexports_shared_unix_to_date() -> None: - """avito_imv.py больше не держит свою копию (была naive/system-local) — общая функция.""" - assert avito_imv._unix_to_date is avito_shared._unix_to_date - assert avito_imv._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE +def test_kit_avito_imv_reexports_shared_unix_to_date() -> None: + """kit imv.py не держит свою копию — импортирует общую функцию (тот же паттерн, + что и легаси avito_houses.py/avito_shared.py после фикса #2129).""" + assert kit_avito_imv._unix_to_date is kit_avito_shared._unix_to_date + assert kit_avito_imv._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE def test_avito_serp_sort_timestamp_matches_shared_date() -> None: """avito.py путь (SERP `sortTimeStamp`, epoch-МС) для того же момента времени даёт ТУ ЖЕ дату, что и houses/IMV путь (epoch-секунды) — единственная разница - в единицах измерения epoch, TZ теперь общая (MSK) во всех трёх путях.""" + в единицах измерения epoch, TZ теперь общая (MSK) во всех путях (легаси и kit).""" html = ( f'
{{"id":8043936560,"sortTimeStamp":{_EPOCH_SECONDS * 1000},' f'"addressDetailed":{{"foo":"bar"}}}}
' ) ts_map = avito._build_sort_timestamp_map(html) assert ts_map["8043936560"] == _EXPECTED_MSK_DATE - # Все три пути согласованы на одном моменте времени: + # Легаси avito/avito_houses/avito_shared + kit avito_imv все согласованы на одном + # моменте времени: assert ( ts_map["8043936560"] == avito_houses._unix_to_date(_EPOCH_SECONDS) - == avito_imv._unix_to_date(_EPOCH_SECONDS) + == kit_avito_imv._unix_to_date(_EPOCH_SECONDS) == avito_shared._unix_to_date(_EPOCH_SECONDS) ) diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py b/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py index 3216ec29..12e05ec5 100644 --- a/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py +++ b/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py @@ -1,4 +1,4 @@ -"""Unit-тесты для DomClick Layer B detail-scraper (domclick_detail.py). +"""Unit-тесты для DomClick Layer B detail-scraper (scraper_kit.providers.domclick.detail). Offline: parse/SSR-тесты без сети/БД; save-тест через MagicMock (зеркало test_cian_detail.py — в проекте нет live-DB фикстуры, идемпотентность @@ -19,6 +19,14 @@ test_cian_detail.py — в проекте нет live-DB фикстуры, ид item_id из URL. - fetch_detail: browser-branch + проброс ошибок. - save_detail_enrichment: UPDATE-параметры, detail_enriched_at, price-history INSERT. + +Легаси `app.services.scrapers.domclick_detail` удалён (#2277 финальный шаг +scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент +`scraper_kit.providers.domclick.detail` без изменения покрытия. Exceptions тоже +переведены на `scraper_kit.domclick_exceptions` (kit держит свою независимую копию, +`parse_detail_html`(kit) поднимает именно её — legacy `app.services.scrapers. +domclick_exceptions` остаётся живым модулем для `domclick.py`, но здесь больше не +подходящий класс для `pytest.raises`). """ from __future__ import annotations @@ -28,15 +36,14 @@ from datetime import UTC, datetime, timedelta from unittest.mock import AsyncMock, MagicMock import pytest - -from app.services.scrapers.domclick_detail import ( +from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError +from scraper_kit.providers.domclick.detail import ( DomClickDetailEnrichment, _extract_ssr_state, fetch_detail, parse_detail_html, save_detail_enrichment, ) -from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError _CARD_URL = "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321" diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick_detail_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_domclick_detail_kit_parity.py deleted file mode 100644 index fae5c7aa..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_domclick_detail_kit_parity.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Parity-proof: `app.services.scrapers.domclick_detail` vs -`scraper_kit.providers.domclick.detail` (issue #2307, Group D). - -`parse_detail_html` — parsing-ядро без сети/БД (SSR-стейт → dataclass), поэтому -сравнивается offline на фиксированной HTML-фикстуре через -`tests.support.parity.assert_parity` (harness #2304). Legacy и kit тела функции -на момент написания теста ИДЕНТИЧНЫ (см. audit -`Scraper_Kit_Legacy_Dependency_Audit_0703` в vault) — различаются только -import-пути зависимостей (`app.services.scrapers.*` vs `scraper_kit.*`). - -HTML-фикстура — независимая копия литерала из -`tests/scrapers/test_domclick_detail.py::_HTML` (тот же live-card ground truth -2075729321, 2026-06-27), чтобы этот тест не зависел от другого test-модуля. -""" - -# SSR-JSON фикстура содержит длинные строки - -from __future__ import annotations - -import dataclasses -import os - -import pytest - -# app.services.scrapers.domclick_detail не читает settings напрямую, но соседние -# модули пакета app.services.scrapers делают это при импорте app.core.config — -# фиктивный DSN мостит офлайн-запуск без CI-уровневого DATABASE_URL (mirror -# tests/scrapers/test_avito_detail_kit_parity.py). -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") - -from scraper_kit.domclick_exceptions import ( - DomClickBlockedError as KitDomClickBlockedError, -) -from scraper_kit.domclick_exceptions import ( - DomClickParseError as KitDomClickParseError, -) -from scraper_kit.providers.domclick.detail import ( - parse_detail_html as kit_parse_detail_html, -) - -from app.services.scrapers.domclick_detail import ( - parse_detail_html as legacy_parse_detail_html, -) -from app.services.scrapers.domclick_exceptions import ( - DomClickBlockedError as LegacyDomClickBlockedError, -) -from app.services.scrapers.domclick_exceptions import ( - DomClickParseError as LegacyDomClickParseError, -) -from tests.support.parity import ParityMismatchError, assert_parity - -_CARD_URL = "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321" - -# Real __SSR_STATE__ shape (ground truth, live card 2075729321, 2026-06-27). -# bare `undefined` (value-position), in-string "undefined", '{' внутри строки и -# эскейп-кавычки — написан вручную (json.dumps использовать нельзя, нужен -# именно НЕвалидный-для-json undefined). -_SSR_LITERAL = """{ - "productCard": { - "objectInfo": { - "area": 38.0, - "rooms": 1, - "floor": 5, - "isApartment": false, - "renovation": "евроремонт", - "livingArea": 22.37, - "kitchenArea": 10.13, - "description": "Состояние пока undefined.", - "note": "угол {da}", - "quote": "сказал \\"привет\\" сосед" - }, - "legalOptions": {"saleType": "Свободная продажа"}, - "egrnData": { - "area": 38.2, - "floor": 5, - "owners_count": 1, - "collateral": true, - "collateral_sber": false - }, - "priceInfo": { - "priceHistory": [ -{"date": "2026-04-15T09:02:26.548728+03:00", "price": 13400000, "diff": -400000, "state": "less"}, - {"date": "not-a-date", "price": 4800000, "diff": -1.0, "state": "less"}, - {"date": "2026-05-01T10:00:00Z", "price": null, "diff": 0, "state": "more"} - ] - }, - "viewsCount": 338, - "callsCount": 5, - "favoriteOfferUsersCount": 12, - "duplicatesOfferCount": 0, - "address": {"guid": "abc-guid-123"}, - "unknownField": undefined - }, - "houseInfo": { - "info": { - "buildYear": 2015, - "wallType": "Кирпично-монолитный", - "floorType": "Железобетонный", - "entranceCount": 3, - "quartersCount": 120, - "energyEfficiency": "B", - "buildingSeries": "индивидуальный" - } - }, - "pricePrediction": { - "market_price": 13000000, - "min_market_price": 12000000, - "max_market_price": 14000000, - "rent_short": 50000, - "rent_long": 35000, - "repair_quality": "good" - } -}""" - -_HTML = ( - "" -) - - -def test_legacy_and_kit_parse_detail_html_are_class_distinct() -> None: - """Sanity-check ДО harness'а: legacy/kit DomClickDetailEnrichment — РАЗНЫЕ - классы (разные модули) → обычный `==` даёт False, хотя данные идентичны. - Именно это harness обходит через структурное сравнение (dataclasses.fields). - """ - legacy_result = legacy_parse_detail_html(_HTML, _CARD_URL) - kit_result = kit_parse_detail_html(_HTML, _CARD_URL) - - assert type(legacy_result) is not type(kit_result) - assert legacy_result != kit_result # dataclass __eq__ проверяет class identity первым - assert dataclasses.asdict(legacy_result) == dataclasses.asdict(kit_result) - - -def test_parity_harness_confirms_domclick_detail_parse_equivalence() -> None: - """Реальное использование harness'а для DomClick Layer B (issue #2307).""" - assert_parity( - legacy_fn=legacy_parse_detail_html, - kit_fn=kit_parse_detail_html, - fixtures=[(_HTML, _CARD_URL)], - ) - - -def test_parity_harness_catches_real_field_divergence() -> None: - """Harness реально ЛОВИТ расхождение на реальной форме — мутируем один - field (``living_area_m2``) в kit-результате и убеждаемся, что harness - падает с ``ParityMismatchError``, называющим именно этот field. - """ - legacy_result = legacy_parse_detail_html(_HTML, _CARD_URL) - kit_result = kit_parse_detail_html(_HTML, _CARD_URL) - assert kit_result.living_area_m2 is not None # sanity: поле реально распарсилось - - mutated_kit_result = dataclasses.replace( - kit_result, living_area_m2=kit_result.living_area_m2 + 1 - ) - - with pytest.raises(ParityMismatchError) as exc_info: - assert_parity( - legacy_fn=lambda: legacy_result, - kit_fn=lambda: mutated_kit_result, - fixtures=[()], - ) - - assert "living_area_m2" in str(exc_info.value) - - -def test_legacy_and_kit_exceptions_behave_identically_on_challenge_page() -> None: - """Anti-bot challenge page (no __SSR_STATE__, block-маркеры) → оба пути - поднимают семантически эквивалентный DomClickBlockedError (разные классы, - те же условия детекции — зеркало `test_scraper_kit_domclick_golden_parity.py` - для Layer A exceptions). - """ - challenge_html = "Access denied by DataDome captcha" - - with pytest.raises(LegacyDomClickBlockedError): - legacy_parse_detail_html(challenge_html, _CARD_URL) - with pytest.raises(KitDomClickBlockedError): - kit_parse_detail_html(challenge_html, _CARD_URL) - - -def test_legacy_and_kit_exceptions_behave_identically_on_missing_state() -> None: - """HTML без __SSR_STATE__ и без block-маркеров → оба пути поднимают - DomClickParseError (дрейф схемы / повреждённый ответ, не блок).""" - no_state_html = "no ssr state here" - - with pytest.raises(LegacyDomClickParseError): - legacy_parse_detail_html(no_state_html, _CARD_URL) - with pytest.raises(KitDomClickParseError): - kit_parse_detail_html(no_state_html, _CARD_URL) diff --git a/tradein-mvp/backend/tests/scrapers/test_ekb_geoportal_client_kit_parity.py b/tradein-mvp/backend/tests/scrapers/test_ekb_geoportal_client_kit_parity.py deleted file mode 100644 index 02d0a5c6..00000000 --- a/tradein-mvp/backend/tests/scrapers/test_ekb_geoportal_client_kit_parity.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Parity-proof: `app.services.scrapers.ekb_geoportal_client` vs -`scraper_kit.providers.ekb_geoportal.client` (issue #2307, Group D). - -`parse_feature_collection` (GeoJSON FeatureCollection → list[TopoBuilding], -area-weighted centroid) — чистая функция без сети/БД, поэтому сравнивается -offline через `tests.support.parity.assert_parity` (harness #2304). Legacy и -kit тела функции на момент написания теста ИДЕНТИЧНЫ (см. audit -`Scraper_Kit_Legacy_Dependency_Audit_0703` в vault) — единственное отличие — -import-путь модуля. - -GeoJSON-фикстуры — независимая копия из -`tests/test_ekb_geoportal_ingest.py::test_parse_feature_collection_basic`, -чтобы этот тест не зависел от другого test-модуля. -""" - -from __future__ import annotations - -import dataclasses - -import pytest -from scraper_kit.providers.ekb_geoportal.client import ( - TopoBuilding as KitTopoBuilding, -) -from scraper_kit.providers.ekb_geoportal.client import ( - parse_feature_collection as kit_parse_feature_collection, -) - -from app.services.scrapers.ekb_geoportal_client import ( - TopoBuilding as LegacyTopoBuilding, -) -from app.services.scrapers.ekb_geoportal_client import ( - parse_feature_collection as legacy_parse_feature_collection, -) -from tests.support.parity import ParityMismatchError, assert_parity - -# A unit square [0,0]-[2,2] → centroid (1, 1) in (lon, lat). -_SQUARE = [[[[0.0, 0.0], [2.0, 0.0], [2.0, 2.0], [0.0, 2.0], [0.0, 0.0]]]] - - -def _multipolygon(coords: list) -> dict: - return {"type": "MultiPolygon", "coordinates": coords} - - -_FEATURE_COLLECTION = { - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "geometry": _multipolygon(_SQUARE), - "properties": { - "topo_street": "Космонавтов", - "topo_num_house": "7б", - "topo_floor": 9, - "topo_purpose": "жилое", - "topo_material": "3", - "key": 12345, - }, - }, - { - # без street/house — должен быть пропущен обоими путями - "type": "Feature", - "geometry": _multipolygon(_SQUARE), - "properties": {"topo_street": "", "topo_num_house": "5"}, - }, - ], -} - - -def test_legacy_and_kit_topo_building_are_class_distinct() -> None: - """Sanity-check ДО harness'а: legacy/kit TopoBuilding — РАЗНЫЕ классы - (разные модули) → обычный `==` даёт False, хотя данные идентичны.""" - legacy_result = legacy_parse_feature_collection(_FEATURE_COLLECTION) - kit_result = kit_parse_feature_collection(_FEATURE_COLLECTION) - - assert len(legacy_result) == len(kit_result) == 1 - assert type(legacy_result[0]) is not type(kit_result[0]) - assert legacy_result[0] != kit_result[0] # dataclass __eq__ проверяет class identity первым - assert dataclasses.asdict(legacy_result[0]) == dataclasses.asdict(kit_result[0]) - - -def test_parity_harness_confirms_ekb_geoportal_parse_equivalence() -> None: - """Реальное использование harness'а для EKB Geoportal client (issue #2307).""" - assert_parity( - legacy_fn=legacy_parse_feature_collection, - kit_fn=kit_parse_feature_collection, - fixtures=[(_FEATURE_COLLECTION,)], - ) - - -def test_parity_harness_catches_real_field_divergence() -> None: - """Harness реально ЛОВИТ расхождение — мутируем ``floors`` в kit-результате - и убеждаемся, что harness падает с ``ParityMismatchError``, называющим - именно этот field.""" - legacy_result = legacy_parse_feature_collection(_FEATURE_COLLECTION) - kit_result = kit_parse_feature_collection(_FEATURE_COLLECTION) - assert kit_result[0].floors == 9 # sanity: поле реально распарсилось - - mutated_kit_building = dataclasses.replace(kit_result[0], floors=1) - assert isinstance(mutated_kit_building, KitTopoBuilding) - assert isinstance(legacy_result[0], LegacyTopoBuilding) - - with pytest.raises(ParityMismatchError) as exc_info: - assert_parity( - legacy_fn=lambda: legacy_result, - kit_fn=lambda: [mutated_kit_building], - fixtures=[()], - ) - - assert "floors" in str(exc_info.value) - - -def test_legacy_and_kit_empty_and_malformed_inputs_match() -> None: - """Graceful-degradation ветки (пустой/битый payload) дают одинаковый [] - на обоих путях.""" - for payload in ({}, {"features": "nope"}, {"features": [None, 42]}): - assert legacy_parse_feature_collection(payload) == [] - assert kit_parse_feature_collection(payload) == [] diff --git a/tradein-mvp/backend/tests/scrapers/test_yandex_valuation_kit_migration.py b/tradein-mvp/backend/tests/scrapers/test_yandex_valuation_kit_migration.py index 59c0d1f5..a758b1df 100644 --- a/tradein-mvp/backend/tests/scrapers/test_yandex_valuation_kit_migration.py +++ b/tradein-mvp/backend/tests/scrapers/test_yandex_valuation_kit_migration.py @@ -1,24 +1,18 @@ """Issue #2336 (Group E3 of scraper_kit migration epic #2277/#2308). `scraper_kit.providers.yandex.valuation.YandexValuationScraper` is already a -strangler-copy (#2133) of `app.services.scrapers.yandex_valuation`, and already used -via the kit path in `admin.py::scrape_yandex_valuation` (#2305, Group A) with the -established DI convention: `config=RealScraperConfig()`, -`delay_provider=get_scraper_delay`, `proxy_provider=_kit_proxy_provider()`. +strangler-copy (#2133) of the (now deleted, #2277 final step) +`app.services.scrapers.yandex_valuation`, and already used via the kit path in +`admin.py::scrape_yandex_valuation` (#2305, Group A) with the established DI +convention: `config=RealScraperConfig()`, `delay_provider=get_scraper_delay`, +`proxy_provider=_kit_proxy_provider()`. -`app/services/estimator.py` (the real, paying-user-facing caller) still imports the -LEGACY module directly and instantiates `YandexValuationScraper()` with NO arguments -(`estimator.py:598`). Switching that call site to the kit class is #2337's job — out -of scope here. This file exists to prove #2336's two confirmed risk patterns for -whoever does that switch, so a regression is caught in CI rather than silently in prod: +This file proves #2336's two confirmed risk patterns for any future call site that +constructs the kit scraper directly, so a regression is caught in CI rather than +silently in prod: 1. **Mandatory `config: ScraperConfig`** (same shape as the cian_valuation footgun, - #2335) — omitting it raises `TypeError`. `estimator.py`'s call site sits inside a - blanket `except Exception` (see `_get_or_fetch_yandex_valuation_cached`, lines - ~596-606) that logs via `logger.warning(..., e)` — no `exc_info`/traceback, and at - the SAME log severity as ordinary transient fetch failures. A `TypeError` from a - forgotten `config=` would read like "Yandex source unavailable" in prod logs, not - "wiring bug", even though the message text itself does end up in the log line. + #2335) — omitting it raises `TypeError`. 2. **Optional `delay_provider`** — if omitted, silently falls back to the hardcoded class-default `request_delay_sec = 5.0`, discarding whatever an admin configured via @@ -26,11 +20,10 @@ whoever does that switch, so a regression is caught in CI rather than silently i `max(per_source, global)`). No exception, no log — just quietly weaker anti-ban throttling if an admin had raised it after ban trouble. -Also extends `tests/test_scraper_kit_yandex_golden_parity.py::test_valuation_parse_parity` -(which already proves `.parse()` output identity using a duck-typed `SimpleNamespace` -config) to the REAL production construction path — `RealScraperConfig()` + -`get_scraper_delay` — via `tests/support/parity.assert_parity` (issue #2304 harness), -per the epic's parity-gate convention. +Legacy-vs-kit parity coverage (previously here, using +`app.services.scrapers.yandex_valuation`) was removed together with the legacy module +deletion — `.parse()` correctness itself is still covered kit-only by +`tests/test_yandex_valuation.py` / `tests/test_yandex_valuation_save.py`. """ from __future__ import annotations @@ -40,10 +33,9 @@ from unittest.mock import patch import pytest -# Old app.services.scrapers.yandex_valuation and app.services.scraper_adapters both -# import app.core.config.settings=Settings(), which requires DATABASE_URL. Offline -# parsing/construction never touches a real DB — a fake DSN is enough (same trick as -# tests/test_scraper_kit_yandex_golden_parity.py). +# app.services.scraper_adapters imports app.core.config.settings=Settings(), which +# requires DATABASE_URL. Offline construction never touches a real DB — a fake DSN +# is enough. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from scraper_kit.providers.yandex.valuation import ( @@ -52,29 +44,6 @@ from scraper_kit.providers.yandex.valuation import ( from app.services import scraper_settings from app.services.scraper_adapters import RealScraperConfig -from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.yandex_valuation import ( - YandexValuationScraper as LegacyYandexValuationScraper, -) -from tests.support.parity import assert_parity - -_VALUATION_HTML = """ - -Дом 2016 года 25 этажей 2,7 м потолки Лифт 42 объекта Панорама -48,5 м², 1-комнатная, 5 этаж 5,1 млн ₽ 105 155 ₽ за м² 23.10.2023 -В экспозиции 945 дней Снято 24.05.2024 -32 м², студия, 2 этаж 3,2 млн ₽ 100 000 ₽ за м² 01.02.2024 В продаже - -""" - -_PARSE_KWARGS = { - "address": "Екатеринбург, ул. Ленина, 5", - "offer_category": "APARTMENT", - "offer_type": "SELL", - "page": 1, - "source_url": "https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address=test", -} - # --------------------------------------------------------------------------- # Footgun #1 — mandatory `config` @@ -149,62 +118,3 @@ def test_kit_valuation_delay_provider_receives_scraper_name() -> None: ) assert calls == ["yandex_valuation"] assert scraper.request_delay_sec == 9.0 - - -# --------------------------------------------------------------------------- -# Extended parity: full instantiation/config-wiring path, not just parsing -# --------------------------------------------------------------------------- - - -def test_full_construction_and_parse_parity_with_real_config_wiring() -> None: - """Legacy no-arg ctor vs kit ctor with the REAL production DI convention. - - `tests/test_scraper_kit_yandex_golden_parity.py::test_valuation_parse_parity` - already proves `.parse()` output identity using a duck-typed `SimpleNamespace` - config. This test additionally proves the REAL wiring convention used in - production (`config=RealScraperConfig()`, `delay_provider=get_scraper_delay` -- - same as `admin.py::scrape_yandex_valuation`, #2305 Group A) constructs without - error and does not change parsing output vs. the legacy no-arg constructor. - """ - - def _legacy_parse(html: str) -> object: - return LegacyYandexValuationScraper().parse(html, **_PARSE_KWARGS) - - def _kit_parse(html: str) -> object: - scraper = KitYandexValuationScraper( - config=RealScraperConfig(), delay_provider=get_scraper_delay - ) - return scraper.parse(html, **_PARSE_KWARGS) - - assert_parity( - legacy_fn=_legacy_parse, - kit_fn=_kit_parse, - fixtures=[_VALUATION_HTML], - ) - - -def test_parity_harness_catches_valuation_construction_divergence() -> None: - """Sanity: assert_parity actually fails if kit construction changes parsed output. - - Proves the extended-coverage test above is not vacuously green -- mutates the - kit-side house metadata and confirms `assert_parity` reports the exact field. - """ - from tests.support.parity import ParityMismatchError - - scraper = KitYandexValuationScraper( - config=RealScraperConfig(), delay_provider=get_scraper_delay - ) - kit_result = scraper.parse(_VALUATION_HTML, **_PARSE_KWARGS) - mutated = kit_result.model_copy( - update={"house": kit_result.house.model_copy(update={"year_built": 1900})} - ) - - with pytest.raises(ParityMismatchError) as exc_info: - assert_parity( - legacy_fn=lambda: LegacyYandexValuationScraper().parse( - _VALUATION_HTML, **_PARSE_KWARGS - ), - kit_fn=lambda: mutated, - fixtures=[()], - ) - assert "year_built" in str(exc_info.value) diff --git a/tradein-mvp/backend/tests/tasks/test_yandex_detail_backfill.py b/tradein-mvp/backend/tests/tasks/test_yandex_detail_backfill.py index c801ea2e..7714ac7f 100644 --- a/tradein-mvp/backend/tests/tasks/test_yandex_detail_backfill.py +++ b/tradein-mvp/backend/tests/tasks/test_yandex_detail_backfill.py @@ -1,7 +1,12 @@ -"""Tests for app.tasks.yandex_detail_backfill. +"""Tests for app.tasks.yandex_detail_backfill (the KIT task — already imports +`scraper_kit.providers.yandex.detail.{YandexDetailScraper,save_detail_enrichment}`). Fetch mechanism changed: curl_cffi AsyncSession(chrome120+proxy) + YandexDetailScraper.parse (instead of old fetch_detail path). Mocks target session.get and scraper.parse. + +Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг +scraper_kit-миграции) — `save_detail_enrichment` coverage-тесты внизу файла +переведены на kit-эквивалент (тот же, что реально вызывает эта задача). """ from __future__ import annotations @@ -337,13 +342,14 @@ async def test_backfill_no_proxy_when_settings_none() -> None: # --------------------------------------------------------------------------- -# Tests: save_detail_enrichment (unchanged — kept for coverage) +# Tests: save_detail_enrichment (kit — same function the task above actually calls; +# легаси `app.services.scrapers.yandex_detail` удалён #2277 финальный шаг миграции) # --------------------------------------------------------------------------- def test_save_detail_enrichment_maps_fields_to_update() -> None: """save_detail_enrichment calls db.execute with UPDATE and commits.""" - from app.services.scrapers.yandex_detail import ( + from scraper_kit.providers.yandex.detail import ( DetailEnrichment, MetroStation, save_detail_enrichment, @@ -403,7 +409,7 @@ def test_save_detail_enrichment_maps_fields_to_update() -> None: def test_save_detail_enrichment_empty_metro_and_photos() -> None: """Empty metro_stations and photo_urls -> NULL passed for both jsonb columns.""" - from app.services.scrapers.yandex_detail import DetailEnrichment, save_detail_enrichment + from scraper_kit.providers.yandex.detail import DetailEnrichment, save_detail_enrichment enrichment = DetailEnrichment( offer_id="99", @@ -424,7 +430,7 @@ def test_save_detail_enrichment_empty_metro_and_photos() -> None: def test_save_detail_enrichment_rowcount_zero_returns_false() -> None: """rowcount=0 (listing_id not found) -> returns False.""" - from app.services.scrapers.yandex_detail import DetailEnrichment, save_detail_enrichment + from scraper_kit.providers.yandex.detail import DetailEnrichment, save_detail_enrichment enrichment = DetailEnrichment( offer_id="404", diff --git a/tradein-mvp/backend/tests/test_avito_imv_parse.py b/tradein-mvp/backend/tests/test_avito_imv_parse.py index e129bdcc..217d8c44 100644 --- a/tradein-mvp/backend/tests/test_avito_imv_parse.py +++ b/tradein-mvp/backend/tests/test_avito_imv_parse.py @@ -1,6 +1,10 @@ -"""Offline smoke tests для avito_imv модуля. +"""Offline smoke tests для scraper_kit avito IMV провайдера. -Без сети — только cache_key вычисление, парсинг заголовков, unix→date, dataclass конструирование. +Без сети — только cache_key вычисление, парсинг заголовков, unix→date, dataclass +конструирование. Легаси `app.services.scrapers.avito_imv` удалён (#2277 финальный шаг +scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент +`scraper_kit.providers.avito.imv`, регрессия по контракту step B/C (issue #565) +сохранена без изменений. """ import json @@ -8,8 +12,7 @@ from datetime import date from pathlib import Path import pytest - -from app.services.scrapers.avito_imv import ( +from scraper_kit.providers.avito.imv import ( IMVAddressNotFoundError, IMVEvaluation, IMVGeo, @@ -60,12 +63,8 @@ def test_cache_key_differs_by_rooms(): def test_cache_key_differs_by_balcony(): - k_with = compute_imv_cache_key( - "ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", True, False - ) - k_without = compute_imv_cache_key( - "ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", False, False - ) + k_with = compute_imv_cache_key("ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", True, False) + k_without = compute_imv_cache_key("ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", False, False) assert k_with != k_without diff --git a/tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py b/tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py index 6496c2c2..638e3bfa 100644 --- a/tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py +++ b/tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py @@ -9,6 +9,11 @@ Covers: (provider="cache", confidence="exact"); parse-fail / no-row → falls through. All network + DB mocked. + +Легаси `app.services.scrapers.ekb_geoportal_client` удалён (#2277 финальный шаг +scraper_kit-миграции, 0 runtime importers) — client-тесты переведены на kit- +эквивалент `scraper_kit.providers.ekb_geoportal.client` (уже используемый живым +`app.tasks.ekb_geoportal_ingest`), покрытие не изменилось. """ from __future__ import annotations @@ -25,10 +30,11 @@ _wp_mock = MagicMock() sys.modules.setdefault("weasyprint", _wp_mock) -from app.services.scrapers.ekb_geoportal_client import ( # noqa: E402 +from scraper_kit.providers.ekb_geoportal.client import ( # noqa: E402 TopoBuilding, parse_feature_collection, ) + from app.tasks.ekb_geoportal_ingest import ( # noqa: E402 _group_buildings, _tile_grid, diff --git a/tradein-mvp/backend/tests/test_extval_house_id_write_path.py b/tradein-mvp/backend/tests/test_extval_house_id_write_path.py index 13acf2f2..2038ae30 100644 --- a/tradein-mvp/backend/tests/test_extval_house_id_write_path.py +++ b/tradein-mvp/backend/tests/test_extval_house_id_write_path.py @@ -6,6 +6,12 @@ Best-effort: house_id=None → колонка NULL, сохранение НЕ п DB — MagicMock, записываем параметры db.execute (никакого реального Postgres, как в test_yandex_valuation_save / test_backfill_house_coords). + +Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг +scraper_kit-миграции) — `_yandex_result()` переведён на kit-эквивалент +`scraper_kit.providers.yandex.valuation` (тот же тип, что `estimator.py` реально +использует через `YandexValuationScraper`, который здесь пропатчен). `cian_valuation` +остаётся легаси (не мигрирован) — его импорт не тронут. """ from __future__ import annotations @@ -88,7 +94,7 @@ def _make_db_cache_miss() -> MagicMock: def _yandex_result(): - from app.services.scrapers.yandex_valuation import ( + from scraper_kit.providers.yandex.valuation import ( ValuationHouseMeta, YandexValuationResult, ) diff --git a/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py index 4d7d37a3..26edff4b 100644 --- a/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py +++ b/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py @@ -6,7 +6,9 @@ Migrates legacy scraper imports of these files to their `scraper_kit` equivalent - app/services/house_imv_backfill.py → BrowserFetcher now from `scraper_kit.browser_fetcher` (was `app.services.scrapers.browser_fetcher`). - avito_imv itself stays legacy (valuation dependency, issue #2308 — separate). + avito_imv itself was still legacy at the time (valuation dependency, issue + #2308 — separate); migrated to `scraper_kit.providers.avito.imv` in #2334, + then the legacy module was deleted entirely in #2277's final step. - app/tasks/avito_detail_backfill.py → AvitoScraper/BrowserFetcher/avito exceptions/fetch_detail/research_in_session/save_detail_enrichment now from `scraper_kit.*`. `build_warmed_session`/`_AVITO_WARM_SEARCH_URL` stay legacy — diff --git a/tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py index 244878d8..85bfb7b2 100644 --- a/tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py +++ b/tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py @@ -10,20 +10,25 @@ house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→ - SERP `_entity_to_lot` (gate-API entity → ScrapedLot) на репрезентативных entity; - SERP `_parse_gate_json` (payload → list[ScrapedLot]) с новостройкой/вторичкой; - `normalize_house_type` (SCREAMING/camelCase → канон); - - detail `parse` (INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment) на - реальных offer-HTML фикстурах; - - valuation `parse` (house-meta + history → YandexValuationResult) на синтетике; - newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) на синтетике; - `_build_url` (gate-API URL builder) — детерминированность. Идентичность результата = kit-парсер верно скопирован (развязка не задела логику). + +Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277 +финальный шаг scraper_kit-миграции, 0 runtime importers) — detail/valuation +parity-проверки убраны отсюда вместе с ними. Kit-side coverage для detail/valuation +`.parse()` сохранена без изменений в `tests/test_yandex_detail.py` / +`tests/test_yandex_detail_structural.py` / `tests/test_yandex_valuation.py` / +`tests/test_yandex_valuation_save.py` (переведены на kit-эквиваленты тем же PR). +`yandex_realty` / `yandex_newbuilding` / `house_type_normalizer` остаются легаси +(не мигрированы) — их parity-проверки не тронуты. """ from __future__ import annotations import inspect import os -from pathlib import Path from types import SimpleNamespace from typing import Any @@ -33,7 +38,6 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/ # НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном # тесте; ruff сортирует scraper_kit (first-party) выше app-импортов. -from scraper_kit.providers.yandex.detail import YandexDetailScraper as NewDetailScraper from scraper_kit.providers.yandex.newbuilding import ( YandexNewbuildingScraper as NewNewbuildingScraper, ) @@ -41,25 +45,16 @@ from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewRealtySc from scraper_kit.providers.yandex.serp import _entity_to_lot as new_entity_to_lot from scraper_kit.providers.yandex.serp import _parse_gate_json as new_parse_gate_json from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type -from scraper_kit.providers.yandex.valuation import ( - YandexValuationScraper as NewValuationScraper, -) from app.services.scrapers.house_type_normalizer import ( normalize_house_type as old_normalize_house_type, ) -from app.services.scrapers.yandex_detail import YandexDetailScraper as OldDetailScraper from app.services.scrapers.yandex_newbuilding import ( YandexNewbuildingScraper as OldNewbuildingScraper, ) from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldRealtyScraper from app.services.scrapers.yandex_realty import _entity_to_lot as old_entity_to_lot from app.services.scrapers.yandex_realty import _parse_gate_json as old_parse_gate_json -from app.services.scrapers.yandex_valuation import ( - YandexValuationScraper as OldValuationScraper, -) - -_FIXTURES = Path(__file__).parent / "fixtures" # Инжектируемый конфиг для kit-скрапперов (duck-typed namespace, structurally # удовлетворяет scraper_kit.contracts.ScraperConfig в объёме читаемых полей). @@ -268,61 +263,6 @@ def test_normalize_house_type_parity() -> None: assert old_normalize_house_type(raw) == new_normalize_house_type(raw) -# ── detail parse parity (реальные offer-HTML фикстуры) ─────────────────────── - -_SECONDARY_ID = "3402418396407468801" -_STUDIO_ID = "7811691822126445498" - - -def _load_offer(offer_id: str) -> str: - return (_FIXTURES / f"yandex_offer_{offer_id}.html").read_text(encoding="utf-8") - - -def _offer_url(offer_id: str) -> str: - return f"https://realty.yandex.ru/offer/{offer_id}/" - - -def test_detail_parse_parity() -> None: - """INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment идентичен.""" - old_scraper = OldDetailScraper() - new_scraper = NewDetailScraper() - for offer_id in (_SECONDARY_ID, _STUDIO_ID): - html = _load_offer(offer_id) - url = _offer_url(offer_id) - old_res = old_scraper.parse(html, offer_url=url) - new_res = new_scraper.parse(html, offer_url=url) - assert old_res is not None and new_res is not None - assert old_res.model_dump() == new_res.model_dump() - - -# ── valuation parse parity (синтетика) ─────────────────────────────────────── - -_VALUATION_HTML = """ - -Дом 2016 года 25 этажей 2,7 м потолки Лифт 42 объекта Панорама -48,5 м², 1-комнатная, 5 этаж 5,1 млн ₽ 105 155 ₽ за м² 23.10.2023 -В экспозиции 945 дней Снято 24.05.2024 -32 м², студия, 2 этаж 3,2 млн ₽ 100 000 ₽ за м² 01.02.2024 В продаже - -""" - - -def test_valuation_parse_parity() -> None: - """house-meta + history (chunked-text fallback) → идентичный результат.""" - old_scraper = OldValuationScraper() - new_scraper = NewValuationScraper(config=_KIT_CONFIG) - kwargs = dict( - address="Екатеринбург, ул. Ленина, 5", - offer_category="APARTMENT", - offer_type="SELL", - page=1, - source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address=test", - ) - old_res = old_scraper.parse(_VALUATION_HTML, **kwargs) - new_res = new_scraper.parse(_VALUATION_HTML, **kwargs) - assert old_res.model_dump() == new_res.model_dump() - - # ── newbuilding parse parity (синтетика) ───────────────────────────────────── _NEWBUILDING_HTML = """ diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py index 025e4b39..f6eb5cf3 100644 --- a/tradein-mvp/backend/tests/test_scraper_proxy.py +++ b/tradein-mvp/backend/tests/test_scraper_proxy.py @@ -5,9 +5,17 @@ Covers: - AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent - None/empty → direct connection (no proxies dict) - _avito_proxies() helper returns correct dict shape - - per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation, + - per-scraper proxy wiring: YandexRealty, YandexValuation (kit), CianValuation, CianNewbuilding (fetch_newbuilding — now BrowserFetcher, proxy is browser-level), - AvitoIMV, AvitoDetail + AvitoDetail + +Легаси `app.services.scrapers.yandex_valuation` / `avito_imv` удалены (#2277 +финальный шаг scraper_kit-миграции). YandexValuation proxy-wiring тесты переведены +на kit-эквивалент (`config=` DI вместо патча `app.core.config.settings`). +`test_avito_imv_own_session_receives_proxies` (легаси, patch-based) удалён — +эквивалентная и более точная kit-регрессия (config footgun: с/без `config=`) уже +покрыта в `tests/scrapers/test_avito_imv_kit_parity.py:: +test_kit_evaluate_via_imv_{with,without}_config_{uses,drops}_proxy`. """ from __future__ import annotations @@ -202,8 +210,8 @@ async def test_yandex_realty_opens_browser_fetcher(monkeypatch): @pytest.mark.asyncio async def test_yandex_valuation_session_receives_proxies(): - """YandexValuationScraper.__aenter__ forwards proxies= kwarg.""" - from app.services.scrapers.yandex_valuation import YandexValuationScraper + """Kit YandexValuationScraper.__aenter__ forwards proxies= kwarg from config.""" + from scraper_kit.providers.yandex.valuation import YandexValuationScraper mock_session = MagicMock() mock_session.close = AsyncMock() @@ -213,18 +221,18 @@ async def test_yandex_valuation_session_receives_proxies(): captured_kwargs.update(kwargs) return mock_session - with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session): - with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): - scraper = YandexValuationScraper() - await scraper.__aenter__() + cfg = SimpleNamespace(use_proxy_pool_curl=False, scraper_proxy_url=_PROXY_URL) + with patch("scraper_kit.providers.yandex.valuation._CurlCffiSession", _fake_session): + async with YandexValuationScraper(cfg): + pass assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES @pytest.mark.asyncio async def test_yandex_valuation_session_no_proxies_when_none(): - """YandexValuationScraper.__aenter__: proxies=None when no proxy configured.""" - from app.services.scrapers.yandex_valuation import YandexValuationScraper + """Kit YandexValuationScraper.__aenter__: proxies=None when no proxy configured.""" + from scraper_kit.providers.yandex.valuation import YandexValuationScraper mock_session = MagicMock() mock_session.close = AsyncMock() @@ -234,10 +242,10 @@ async def test_yandex_valuation_session_no_proxies_when_none(): captured_kwargs.update(kwargs) return mock_session - with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session): - with patch("app.core.config.settings", _mock_settings(None)): - scraper = YandexValuationScraper() - await scraper.__aenter__() + cfg = SimpleNamespace(use_proxy_pool_curl=False, scraper_proxy_url=None) + with patch("scraper_kit.providers.yandex.valuation._CurlCffiSession", _fake_session): + async with YandexValuationScraper(cfg): + pass assert captured_kwargs.get("proxies") is None @@ -384,56 +392,6 @@ async def test_avito_detail_shared_session_not_patched(): assert ctor_calls == [] -# ── AvitoIMV — evaluate_via_imv own-session ────────────────────────────────── - - -@pytest.mark.asyncio -async def test_avito_imv_own_session_receives_proxies(): - """evaluate_via_imv creates own session with proxies= when none passed. - - `CffiAsyncSession` is imported locally inside evaluate_via_imv as - `from curl_cffi.requests import AsyncSession as CffiAsyncSession`, so we - patch `curl_cffi.requests.AsyncSession` (the source) to intercept it. - """ - from app.services.scrapers import avito_imv - - mock_session = MagicMock() - mock_warmup_resp = MagicMock() - mock_warmup_resp.status_code = 200 - mock_geo_resp = MagicMock() - mock_geo_resp.status_code = 200 - mock_geo_resp.json = MagicMock(return_value={"result": {"point": None}}) - mock_session.get = AsyncMock(return_value=mock_warmup_resp) - mock_session.post = AsyncMock(return_value=mock_geo_resp) - mock_session.close = AsyncMock() - - captured_kwargs: dict = {} - - def _fake_session(*args, **kwargs): - captured_kwargs.update(kwargs) - return mock_session - - # Patch the source so the local `from curl_cffi.requests import AsyncSession` picks it up - with patch("curl_cffi.requests.AsyncSession", _fake_session): - with patch("app.core.config.settings", _mock_settings(_PROXY_URL)): - try: - await avito_imv.evaluate_via_imv( - address="ул. Тургенева, 4", - rooms=2, - area_m2=50.0, - floor=3, - floor_at_home=9, - house_type="panel", - renovation_type="cosmetic", - has_balcony=False, - has_loggia=False, - ) - except Exception: - pass # network/parse failures expected in unit test - - assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES - - # ── Settings.cian_proxy_url property ───────────────────────────────────────── # cian_proxy_url: CIAN_PROXY_URL > scraper_proxy_url (fallback) > None. diff --git a/tradein-mvp/backend/tests/test_yandex_detail.py b/tradein-mvp/backend/tests/test_yandex_detail.py index be458c72..c3431aa5 100644 --- a/tradein-mvp/backend/tests/test_yandex_detail.py +++ b/tradein-mvp/backend/tests/test_yandex_detail.py @@ -1,6 +1,10 @@ """Unit tests for YandexDetailScraper — Product JSON-LD + DOM detail parser. All tests run against hand-crafted HTML fixtures. No live network access. + +Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг +scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент +`scraper_kit.providers.yandex.detail` без изменения покрытия. """ from __future__ import annotations @@ -8,8 +12,7 @@ from __future__ import annotations import json import pytest - -from app.services.scrapers.yandex_detail import ( +from scraper_kit.providers.yandex.detail import ( DetailEnrichment, MetroStation, YandexDetailScraper, @@ -213,7 +216,7 @@ class TestAgencyBlock:

АН Городской риелтор

Год основания 1998. Продали 150 объектов.

""" - summary = "Иван Петров АН Городской риелтор • 50 просмотров" " • опубликовано 3 марта 2025" + summary = "Иван Петров АН Городской риелтор • 50 просмотров • опубликовано 3 марта 2025" html = _make_html(author_block=author_html, summary_text=summary) result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) assert result is not None diff --git a/tradein-mvp/backend/tests/test_yandex_detail_structural.py b/tradein-mvp/backend/tests/test_yandex_detail_structural.py index 4547f519..2af5be65 100644 --- a/tradein-mvp/backend/tests/test_yandex_detail_structural.py +++ b/tradein-mvp/backend/tests/test_yandex_detail_structural.py @@ -8,6 +8,10 @@ never carries ceiling/kitchen — which is why DB area coverage sat at ~63%. Fixtures are trimmed real offer pages fetched via the headless browser: - yandex_offer_3402418396407468801.html — 1-комн. вторичка (full chars) - yandex_offer_7811691822126445498.html — студия / новостройка + +Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг +scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент +`scraper_kit.providers.yandex.detail` без изменения покрытия. """ from __future__ import annotations @@ -15,8 +19,7 @@ from __future__ import annotations from pathlib import Path import pytest - -from app.services.scrapers.yandex_detail import ( +from scraper_kit.providers.yandex.detail import ( YandexDetailScraper, _extract_offer_card, _parse_card_fields, diff --git a/tradein-mvp/backend/tests/test_yandex_history_area_filter.py b/tradein-mvp/backend/tests/test_yandex_history_area_filter.py index ce31b8f3..3bc90192 100644 --- a/tradein-mvp/backend/tests/test_yandex_history_area_filter.py +++ b/tradein-mvp/backend/tests/test_yandex_history_area_filter.py @@ -5,6 +5,10 @@ _save_yandex_history_items должен: - сохранить валидные items (area_m2 > 0) без потерь - не ронять батч из-за одного битого item - лого кол-во отброшенных items + +Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг +scraper_kit-миграции) — `result` типы переведены на kit-эквивалент +`scraper_kit.providers.yandex.valuation`. """ from __future__ import annotations @@ -15,13 +19,14 @@ from unittest.mock import MagicMock, patch os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -from app.services.estimator import _save_yandex_history_items -from app.services.scrapers.yandex_valuation import ( +from scraper_kit.providers.yandex.valuation import ( ValuationHistoryItem, ValuationHouseMeta, YandexValuationResult, ) +from app.services.estimator import _save_yandex_history_items + def _make_result(items: list[ValuationHistoryItem]) -> YandexValuationResult: return YandexValuationResult( diff --git a/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py b/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py index 3b262ebc..ea6b61ef 100644 --- a/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py +++ b/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py @@ -1,10 +1,27 @@ -"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings.""" +"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings. + +Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277 +финальный шаг scraper_kit-миграции). `yandex_realty` / `yandex_newbuilding` пока +остаются легаси (не мигрированы) — их delay-wiring тесты не тронуты. +`yandex_detail`/`yandex_valuation` переведены на kit-эквиваленты, которые используют +инжектируемый `delay_provider=` (strangler DI, #2133) вместо прямого импорта +`get_scraper_delay` — тест адаптирован под реальный контракт kit-класса. +""" + +from types import SimpleNamespace from unittest.mock import patch -from app.services.scrapers.yandex_detail import YandexDetailScraper +from scraper_kit.providers.yandex.detail import YandexDetailScraper +from scraper_kit.providers.yandex.valuation import YandexValuationScraper + from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper from app.services.scrapers.yandex_realty import YandexRealtyScraper -from app.services.scrapers.yandex_valuation import YandexValuationScraper + +_KIT_CONFIG = SimpleNamespace( + scraper_proxy_url=None, + yandex_proxy_rotate_url=None, + avito_proxy_rotate_url=None, +) def test_yandex_realty_picks_up_delay(): @@ -14,22 +31,23 @@ def test_yandex_realty_picks_up_delay(): def test_yandex_detail_picks_up_delay(): - with patch("app.services.scrapers.yandex_detail.get_scraper_delay", return_value=9.5): - s = YandexDetailScraper() - assert s.request_delay_sec == 9.5 + """Kit YandexDetailScraper: delay_provider callable вызывается с self.name.""" + s = YandexDetailScraper(delay_provider=lambda _name: 9.5) + assert s.request_delay_sec == 9.5 def test_yandex_newbuilding_picks_up_delay(): - with patch( - "app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5 - ): + with patch("app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5): s = YandexNewbuildingScraper() assert s.request_delay_sec == 9.5 def test_yandex_valuation_picks_up_delay(): - with patch( - "app.services.scrapers.yandex_valuation.get_scraper_delay", return_value=9.5 - ): - s = YandexValuationScraper() - assert s.request_delay_sec == 9.5 + """Kit YandexValuationScraper: delay_provider callable вызывается с self.name. + + Mandatory-config footgun и полное покрытие delay_provider wiring уже проверены + в tests/scrapers/test_yandex_valuation_kit_migration.py — здесь просто smoke, + зеркалящий остальные Yandex-скрапперы в этом файле. + """ + s = YandexValuationScraper(config=_KIT_CONFIG, delay_provider=lambda _name: 9.5) + assert s.request_delay_sec == 9.5 diff --git a/tradein-mvp/backend/tests/test_yandex_valuation.py b/tradein-mvp/backend/tests/test_yandex_valuation.py index c32e4aec..1e34b15d 100644 --- a/tradein-mvp/backend/tests/test_yandex_valuation.py +++ b/tradein-mvp/backend/tests/test_yandex_valuation.py @@ -3,19 +3,32 @@ Fixture HTML simulates the Yandex valuation page body text containing: - House meta block (year, floors, type, ceiling, lift, total objects, panorama) - 2-3 historical offer entries with full structure + +Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг +scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент +`scraper_kit.providers.yandex.valuation` без изменения покрытия. Kit-конструктор +требует обязательный `config:` (см. `tests/scrapers/test_yandex_valuation_kit_migration.py` +для footgun-регрессии) — используем тот же duck-typed `SimpleNamespace`, что и +`tests/test_scraper_kit_yandex_golden_parity.py`. """ from __future__ import annotations from datetime import date +from types import SimpleNamespace import pytest - -from app.services.scrapers.yandex_valuation import ( +from scraper_kit.providers.yandex.valuation import ( YandexValuationResult, YandexValuationScraper, ) +_KIT_CONFIG = SimpleNamespace( + scraper_proxy_url=None, + yandex_proxy_rotate_url=None, + avito_proxy_rotate_url=None, +) + # --------------------------------------------------------------------------- # Fixture helpers # --------------------------------------------------------------------------- @@ -59,7 +72,7 @@ FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\ def test_parse_house_meta_full(): - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) meta = scraper._parse_house_meta(_HOUSE_META_BLOCK) assert meta.year_built == 1981 assert meta.total_floors == 9 @@ -176,7 +189,7 @@ def test_parse_items_from_chunked_text_multiple_items(): def test_parse_full_result_address_propagated(): - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) result = scraper.parse( FULL_FIXTURE_HTML, address="Екатеринбург, ул. Ленина, 1", @@ -197,7 +210,7 @@ def test_parse_full_result_address_propagated(): def test_parse_full_result_house_meta_populated(): - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) result = scraper.parse( FULL_FIXTURE_HTML, address="test", @@ -219,7 +232,7 @@ def test_parse_full_result_house_meta_populated(): @pytest.mark.asyncio async def test_fetch_around_raises_not_implemented(): - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) with pytest.raises(NotImplementedError): await scraper.fetch_around(lat=56.8, lon=60.6) @@ -287,7 +300,7 @@ def test_parse_normalizes_nbsp_for_price_per_m2(): "4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² " "" ) - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) result = scraper.parse( html, address="test", @@ -303,21 +316,21 @@ def test_parse_normalizes_nbsp_for_price_per_m2(): def test_validate_match_full_match(): - from app.services.scrapers.yandex_valuation import ValuationHouseMeta + from scraper_kit.providers.yandex.valuation import ValuationHouseMeta meta = ValuationHouseMeta(year_built=2025, total_floors=25) assert meta.validate_match(2025, 25) == 1.0 def test_validate_match_year_mismatch(): - from app.services.scrapers.yandex_valuation import ValuationHouseMeta + from scraper_kit.providers.yandex.valuation import ValuationHouseMeta meta = ValuationHouseMeta(year_built=1984, total_floors=9) assert meta.validate_match(2025, 25) == 0.0 def test_validate_match_partial(): - from app.services.scrapers.yandex_valuation import ValuationHouseMeta + from scraper_kit.providers.yandex.valuation import ValuationHouseMeta meta = ValuationHouseMeta(year_built=2025, total_floors=10) # year matches, floors mismatch (>1 tolerance) → 0.5 @@ -325,7 +338,7 @@ def test_validate_match_partial(): def test_validate_match_tolerance_one_year(): - from app.services.scrapers.yandex_valuation import ValuationHouseMeta + from scraper_kit.providers.yandex.valuation import ValuationHouseMeta meta = ValuationHouseMeta(year_built=2024, total_floors=25) # ±1 year tolerance @@ -333,7 +346,7 @@ def test_validate_match_tolerance_one_year(): def test_validate_match_no_expected_returns_one(): - from app.services.scrapers.yandex_valuation import ValuationHouseMeta + from scraper_kit.providers.yandex.valuation import ValuationHouseMeta meta = ValuationHouseMeta(year_built=None, total_floors=None) assert meta.validate_match(None, None) == 1.0 @@ -400,9 +413,9 @@ def test_area_regex_rejects_year_concat(): item = YandexValuationScraper._parse_item_text(text) # The "202452,2" token has digits jammed before it (no separator), so the # tightened regex must NOT match it. - assert item is None or item.area_m2 is None, ( - f"expected no area match for concat token, got {item.area_m2 if item else 'None item'}" - ) + assert ( + item is None or item.area_m2 is None + ), f"expected no area match for concat token, got {item.area_m2 if item else 'None item'}" def test_area_regex_accepts_isolated_token(): @@ -458,9 +471,9 @@ def test_area_regex_two_komn_chunk(): text = "8 000 000 ₽ за м²2-комнатная 52,2 м² 5 этаж 10.05.2024 8 000 000 ₽ В продаже" item = YandexValuationScraper._parse_item_text(text) assert item is not None - assert item.area_m2 == 52.2, ( - f"2-комн area must parse despite preceding tokens, got {item.area_m2}" - ) + assert ( + item.area_m2 == 52.2 + ), f"2-комн area must parse despite preceding tokens, got {item.area_m2}" def test_area_regex_still_blocks_year_concat(): @@ -511,7 +524,7 @@ _DOM_FIXTURE_ROW_HTML = """ def test_dom_parser_extracts_two_rows(): - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) house_meta = "Дом 2020 года 25 этажей Монолитное здание " result = scraper.parse( f"{house_meta}{_DOM_FIXTURE_ROW_HTML}", @@ -561,7 +574,7 @@ def test_dom_parser_studio_row(): "" "" ) - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) result = scraper.parse( html, address="t", @@ -586,7 +599,7 @@ def test_dom_parser_fallback_when_no_rows(): "10.05.2024 9 000 000 ₽ В продаже" "" ) - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) result = scraper.parse( html, address="t", @@ -610,7 +623,7 @@ def test_dom_parser_skips_malformed_row(): "" "" ) - scraper = YandexValuationScraper() + scraper = YandexValuationScraper(config=_KIT_CONFIG) result = scraper.parse( html, address="t", diff --git a/tradein-mvp/backend/tests/test_yandex_valuation_save.py b/tradein-mvp/backend/tests/test_yandex_valuation_save.py index ff138d91..9079f0df 100644 --- a/tradein-mvp/backend/tests/test_yandex_valuation_save.py +++ b/tradein-mvp/backend/tests/test_yandex_valuation_save.py @@ -1,4 +1,10 @@ -"""Tests for _save_yandex_history_items — house_id linking + new columns.""" +"""Tests for _save_yandex_history_items — house_id linking + new columns. + +Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг +scraper_kit-миграции) — `result` типы переведены на kit-эквивалент +`scraper_kit.providers.yandex.valuation`, тот же тип, что `estimator.py` реально +принимает в `_save_yandex_history_items`. +""" from __future__ import annotations @@ -11,14 +17,14 @@ from datetime import date from unittest.mock import MagicMock, patch import pytest - -from app.services.estimator import _save_yandex_history_items -from app.services.scrapers.yandex_valuation import ( +from scraper_kit.providers.yandex.valuation import ( ValuationHistoryItem, ValuationHouseMeta, YandexValuationResult, ) +from app.services.estimator import _save_yandex_history_items + def _make_result(items: list[ValuationHistoryItem]) -> YandexValuationResult: return YandexValuationResult(