chore(tradein/scraper-kit): удалить 5 orphaned legacy-модулей (#2277 финальный шаг, частичный) #2394
28 changed files with 267 additions and 3826 deletions
|
|
@ -1,853 +0,0 @@
|
|||
"""avito_imv.py — Avito IMV evaluation API client (3-request flow, 2026-05+).
|
||||
|
||||
Stage 2d: geocode + IMV evaluation.
|
||||
|
||||
Flow (current contract, подтверждён живым capture 2026-05 — fixtures
|
||||
tests/fixtures/avito_imv_geo_position.json + avito_imv_getdata.json):
|
||||
0) GET /evaluation/realty → warm-up (seed anti-bot cookies)
|
||||
1) GET /web/1/coords/by_address?address=<urlenc> → normalize + lat/lon
|
||||
2) POST /js/v2/geo/position → rich JWT (geoFieldsHash)
|
||||
3) POST /web/1/realty-imv/get-data → price + placementHistory? + suggestions
|
||||
|
||||
Anti-bot: zerkalim AvitoScraper (avito.py) — chrome120 TLS + document-заголовки +
|
||||
warm-up GET + XHR Sec-Fetch заголовки. Bare-session XHR ловят 403 с server-IP.
|
||||
|
||||
Таблицы:
|
||||
avito_imv_evaluations (018_avito_imv_evaluations.sql)
|
||||
house_placement_history (017_house_placement_history.sql)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urlencode
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.services.scrapers.avito_shared import _unix_to_date
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Импорт под TYPE_CHECKING (как в avito_detail.py/avito_houses.py) — избегаем
|
||||
# hard import cycle browser_fetcher → config → ... и лишней зависимости в
|
||||
# offline-тестах парсинга, которым browser не нужен.
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
AVITO_BASE = "https://www.avito.ru"
|
||||
COORDS_ENDPOINT = "/web/1/coords/by_address"
|
||||
POSITION_ENDPOINT = "/js/v2/geo/position"
|
||||
IMV_ENDPOINT = "/web/1/realty-imv/get-data"
|
||||
# Тёплая страница для seed anti-bot cookies (srv_id/_avisc/u/...) перед XHR.
|
||||
WARMUP_URL = f"{AVITO_BASE}/evaluation/realty"
|
||||
|
||||
# Заголовки document-навигации (warm-up GET) — зеркалят production-набор
|
||||
# из avito.py (AvitoScraper.__aenter__). Нужны чтобы пройти TLS+header anti-bot.
|
||||
_DOC_HEADERS = {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
"Cache-Control": "max-age=0",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
# Заголовки XHR/fetch для API-шагов (coords/position/get-data). Имитируют
|
||||
# fetch() со страницы /evaluation/realty: JSON Accept + same-origin Sec-Fetch.
|
||||
_COMMON_HEADERS = {
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
"Referer": "https://www.avito.ru/evaluation/realty",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-origin",
|
||||
}
|
||||
|
||||
# Timeout для всех Avito-вызовов (без него curl_cffi висит на anti-bot stall).
|
||||
_HTTP_TIMEOUT_SEC = 25
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class IMVAddressNotFoundError(Exception):
|
||||
"""Raised when /coords/by_address returns no geoHash for given address."""
|
||||
|
||||
|
||||
class IMVCityMismatchError(Exception):
|
||||
"""Raised when locationId не входит в ожидаемый город (sanity check)."""
|
||||
|
||||
|
||||
class IMVAuthError(Exception):
|
||||
"""Avito API rejected auth / quota exceeded (HTTP 401 / 403)."""
|
||||
|
||||
|
||||
class IMVTransientError(Exception):
|
||||
"""Network / 5xx / timeout — retryable."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class IMVGeo:
|
||||
geo_hash: str # JWT — не сохраняем как отдельную колонку, только в raw_response
|
||||
lat: float | None = None
|
||||
lon: float | None = None
|
||||
avito_address_id: int | None = None
|
||||
avito_location_id: int | None = None
|
||||
avito_metro_id: int | None = None
|
||||
avito_district_id: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IMVPlacementHistoryItem:
|
||||
ext_item_id: str # str(item['id'])
|
||||
title: str | None = None
|
||||
rooms: int | None = None # парсится из title (опционально)
|
||||
area_m2: float | None = None # парсится из title (опционально)
|
||||
floor: int | None = None # парсится из title (опционально)
|
||||
total_floors: int | None = None # парсится из title (опционально)
|
||||
start_price: int | None = None
|
||||
start_price_date: date | None = None # unix → date
|
||||
last_price: int | None = None
|
||||
last_price_date: date | None = None
|
||||
removed_date: date | None = None # только IMV API (в widget housePlacementHistory нет)
|
||||
exposure_days: int | None = None
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IMVSuggestion:
|
||||
ext_item_id: str
|
||||
title: str | None = None
|
||||
address: str | None = None
|
||||
price_rub: int | None = None
|
||||
exposure_days: int | None = None
|
||||
publish_date: date | None = None
|
||||
item_url: str | None = None
|
||||
metro_name: str | None = None
|
||||
metro_distance: str | None = None
|
||||
metro_color: str | None = None
|
||||
has_good_price_badge: bool = False
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IMVEvaluation:
|
||||
cache_key: str # sha256
|
||||
address: str
|
||||
rooms: int
|
||||
area_m2: float
|
||||
floor: int
|
||||
floor_at_home: int
|
||||
house_type: str
|
||||
renovation_type: str
|
||||
has_balcony: bool
|
||||
has_loggia: bool
|
||||
|
||||
geo: IMVGeo
|
||||
recommended_price: int
|
||||
lower_price: int
|
||||
higher_price: int
|
||||
market_count: int | None = None
|
||||
|
||||
placement_history: list[IMVPlacementHistoryItem] = field(default_factory=list)
|
||||
suggestions: list[IMVSuggestion] = field(default_factory=list)
|
||||
search_similar_url: str | None = None
|
||||
raw_response: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cache key
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_imv_cache_key(
|
||||
address: str,
|
||||
rooms: int,
|
||||
area_m2: float,
|
||||
floor: int,
|
||||
floor_at_home: int,
|
||||
house_type: str,
|
||||
renovation_type: str,
|
||||
has_balcony: bool,
|
||||
has_loggia: bool,
|
||||
) -> str:
|
||||
"""Детерминированный sha256 ключ для 24h-кэша IMV оценки."""
|
||||
key_src = (
|
||||
f"{address}|{rooms}|{area_m2}|{floor}|{floor_at_home}"
|
||||
f"|{house_type}|{renovation_type}|{int(has_balcony)}|{int(has_loggia)}"
|
||||
)
|
||||
return hashlib.sha256(key_src.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Паттерн для заголовка вида "2-к. квартира, 42 м², 4/5 эт."
|
||||
_TITLE_RE = re.compile(
|
||||
r"^(?P<rooms>\d+)-к[.\s].*?(?P<area>[\d,]+)\s*м².*?(?P<floor>\d+)/(?P<total>\d+)\s*эт",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_title(title: str | None) -> dict[str, int | float | None]:
|
||||
"""Парсит rooms, area_m2, floor, total_floors из Avito-заголовка объявления."""
|
||||
result: dict[str, int | float | None] = {
|
||||
"rooms": None,
|
||||
"area_m2": None,
|
||||
"floor": None,
|
||||
"total_floors": None,
|
||||
}
|
||||
if not title:
|
||||
return result
|
||||
m = _TITLE_RE.match(title.strip())
|
||||
if not m:
|
||||
return result
|
||||
try:
|
||||
result["rooms"] = int(m.group("rooms"))
|
||||
result["area_m2"] = float(m.group("area").replace(",", "."))
|
||||
result["floor"] = int(m.group("floor"))
|
||||
result["total_floors"] = int(m.group("total"))
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def _parse_placement_item(raw: dict[str, Any]) -> IMVPlacementHistoryItem:
|
||||
"""Парсит один элемент из placementHistory.items."""
|
||||
title = raw.get("title")
|
||||
parsed = _parse_title(title)
|
||||
|
||||
# Фильтруем itemImage из raw_payload (не нужны)
|
||||
clean_raw = {k: v for k, v in raw.items() if k not in ("itemImage",)}
|
||||
|
||||
return IMVPlacementHistoryItem(
|
||||
ext_item_id=str(raw["id"]),
|
||||
title=title,
|
||||
rooms=parsed["rooms"], # type: ignore[arg-type]
|
||||
area_m2=parsed["area_m2"], # type: ignore[arg-type]
|
||||
floor=parsed["floor"], # type: ignore[arg-type]
|
||||
total_floors=parsed["total_floors"], # type: ignore[arg-type]
|
||||
start_price=raw.get("startPrice"),
|
||||
start_price_date=_unix_to_date(raw.get("startPriceDate")),
|
||||
last_price=raw.get("lastPrice"),
|
||||
last_price_date=_unix_to_date(raw.get("lastPriceDate")),
|
||||
removed_date=_unix_to_date(raw.get("removedDate")),
|
||||
exposure_days=raw.get("exposure"),
|
||||
raw_payload=clean_raw,
|
||||
)
|
||||
|
||||
|
||||
def _parse_suggestion(raw: dict[str, Any]) -> IMVSuggestion:
|
||||
"""Парсит один элемент из suggestions.items."""
|
||||
metro = raw.get("metro") or {}
|
||||
colors: list[str] = metro.get("colors") or []
|
||||
|
||||
# Фильтруем imageLink из raw_payload
|
||||
clean_raw = {k: v for k, v in raw.items() if k not in ("imageLink",)}
|
||||
|
||||
return IMVSuggestion(
|
||||
ext_item_id=str(raw["id"]),
|
||||
title=raw.get("title"),
|
||||
address=raw.get("address"),
|
||||
price_rub=raw.get("price"),
|
||||
exposure_days=raw.get("exposure"),
|
||||
publish_date=_unix_to_date(raw.get("publishDate")),
|
||||
item_url=raw.get("itemLink"),
|
||||
metro_name=metro.get("name"),
|
||||
metro_distance=metro.get("distance"),
|
||||
metro_color=colors[0] if colors else None,
|
||||
has_good_price_badge=bool(raw.get("hasGoodPriceBadge", False)),
|
||||
raw_payload=clean_raw,
|
||||
)
|
||||
|
||||
|
||||
def _parse_geo_position(data_b: dict[str, Any], *, lat: float, lon: float) -> IMVGeo:
|
||||
"""Парсит ответ POST /js/v2/geo/position (step B) → IMVGeo.
|
||||
|
||||
Контракт (подтверждён живым capture 2026-05, fixture avito_imv_geo_position.json):
|
||||
{address, addressId, districtId, geoFieldsHash, latitude, longitude,
|
||||
locationId, parentLocationId, metroId?, errorText, ...}
|
||||
|
||||
`geoFieldsHash` — JWT, payload которого кодирует {latitude, longitude,
|
||||
address, addressId, locationId, districtId}; это то, что step C ожидает как
|
||||
`geoHash`. metroId присутствует только для метро-городов (None для ЕКБ-окраин).
|
||||
|
||||
Raises:
|
||||
IMVAddressNotFoundError если geoFieldsHash пустой/отсутствует.
|
||||
"""
|
||||
jwt = data_b.get("geoFieldsHash")
|
||||
if not jwt:
|
||||
raise IMVAddressNotFoundError(
|
||||
f"Avito /js/v2/geo/position не вернул geoFieldsHash для "
|
||||
f"{str(data_b.get('address'))[:60]!r}: errorText={data_b.get('errorText')!r}"
|
||||
)
|
||||
return IMVGeo(
|
||||
geo_hash=jwt,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
avito_address_id=data_b.get("addressId"),
|
||||
avito_location_id=data_b.get("locationId"),
|
||||
avito_metro_id=data_b.get("metroId"),
|
||||
avito_district_id=data_b.get("districtId"),
|
||||
)
|
||||
|
||||
|
||||
def _parse_price(data: dict[str, Any]) -> tuple[int, int, int, int | None]:
|
||||
"""Парсит top-level `price` блок из realty-imv/get-data (step C).
|
||||
|
||||
Контракт (fixture avito_imv_getdata.json):
|
||||
price = {recommendedPrice, lowerPrice, higherPrice, count, addItemUrl}
|
||||
|
||||
Returns: (recommended_price, lower_price, higher_price, market_count).
|
||||
Отсутствующие цены → 0 (caller трактует как "нет оценки"); count → None.
|
||||
"""
|
||||
price_block: dict[str, Any] = data.get("price") or {}
|
||||
recommended_price: int = price_block.get("recommendedPrice") or 0
|
||||
lower_price: int = price_block.get("lowerPrice") or 0
|
||||
higher_price: int = price_block.get("higherPrice") or 0
|
||||
market_count: int | None = price_block.get("count")
|
||||
return recommended_price, lower_price, higher_price, market_count
|
||||
|
||||
|
||||
def _parse_placement_history(data: dict[str, Any]) -> list[IMVPlacementHistoryItem]:
|
||||
"""Парсит опциональный `placementHistory.items` блок (step C).
|
||||
|
||||
placementHistory отсутствует для части адресов (живой capture: проспект
|
||||
Ленина вернул только itemParams/price/suggestions) — тогда возвращаем [].
|
||||
Битые элементы пропускаем с warning, не роняя весь ответ.
|
||||
"""
|
||||
history_block: dict[str, Any] = data.get("placementHistory") or {}
|
||||
items: list[IMVPlacementHistoryItem] = []
|
||||
for raw_item in history_block.get("items") or []:
|
||||
try:
|
||||
items.append(_parse_placement_item(raw_item))
|
||||
except (KeyError, TypeError) as exc:
|
||||
logger.warning("IMV: не удалось распарсить placementHistory item: %s", exc)
|
||||
return items
|
||||
|
||||
|
||||
def _parse_suggestions(data: dict[str, Any]) -> tuple[list[IMVSuggestion], str | None]:
|
||||
"""Парсит `suggestions` блок (step C) → (items, searchSimilarUrl)."""
|
||||
suggestions_block: dict[str, Any] = data.get("suggestions") or {}
|
||||
suggestions: list[IMVSuggestion] = []
|
||||
for raw_sugg in suggestions_block.get("items") or []:
|
||||
try:
|
||||
suggestions.append(_parse_suggestion(raw_sugg))
|
||||
except (KeyError, TypeError) as exc:
|
||||
logger.warning("IMV: не удалось распарсить suggestion: %s", exc)
|
||||
return suggestions, suggestions_block.get("searchSimilarUrl")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Browser-transport adapter (#915 Stage 3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _BrowserResponse:
|
||||
"""Мимикрия под curl_cffi Response поверх ответа BrowserFetcher.fetch_json.
|
||||
|
||||
IMV-флоу читает только .status_code / .text / .json() / .raise_for_status() —
|
||||
адаптируем ровно эти четыре (подтверждено по avito_imv.py: warm-up, _geocode,
|
||||
_imv_evaluate, _raise_for_status_categorized).
|
||||
"""
|
||||
|
||||
def __init__(self, status: int, body: str) -> None:
|
||||
self.status_code = status
|
||||
self.text = body
|
||||
|
||||
def json(self) -> Any:
|
||||
return json.loads(self.text)
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
if self.status_code >= 400:
|
||||
# _raise_for_status_categorized уже обрабатывает 401/403/5xx ДО вызова
|
||||
# raise_for_status(); сюда попадают только "прочие 4xx" (fall-through).
|
||||
# Зеркалим curl_cffi/httpx-семантику ровно настолько, чтобы caller
|
||||
# получил исключение.
|
||||
raise RuntimeError(f"HTTP {self.status_code}")
|
||||
|
||||
|
||||
class _BrowserSessionAdapter:
|
||||
"""Адаптер: IMV-флоу думает что это curl_cffi-сессия, а ходит через камуфокс
|
||||
in-page fetch (same-origin avito.ru). Куки/JWT живут в контексте браузера
|
||||
провайдера, поэтому 4 запроса IMV переиспользуют warmed-сессию. Прокси и
|
||||
fingerprint — из sidecar (обходит datacenter-403, #562/#853)."""
|
||||
|
||||
def __init__(self, browser_fetcher: BrowserFetcher, *, origin: str) -> None:
|
||||
self._bf = browser_fetcher
|
||||
self._origin = origin
|
||||
|
||||
async def get(self, url: str, *, headers: dict | None = None) -> _BrowserResponse:
|
||||
r = await self._bf.fetch_json(url, method="GET", headers=headers, origin=self._origin)
|
||||
return _BrowserResponse(r["status"], r["body"])
|
||||
|
||||
async def post(
|
||||
self, url: str, *, headers: dict | None = None, json: Any = None
|
||||
) -> _BrowserResponse:
|
||||
import json as _json
|
||||
|
||||
body = _json.dumps(json) if json is not None else None
|
||||
r = await self._bf.fetch_json(
|
||||
url, method="POST", headers=headers, body=body, origin=self._origin
|
||||
)
|
||||
return _BrowserResponse(r["status"], r["body"])
|
||||
|
||||
async def close(self) -> None:
|
||||
# browser принадлежит вызывающему (backfill открывает один на батч) — no-op.
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main async function
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def evaluate_via_imv(
|
||||
address: str,
|
||||
rooms: int,
|
||||
area_m2: float,
|
||||
floor: int,
|
||||
floor_at_home: int,
|
||||
house_type: str, # panel/brick/monolith/monolith_brick/block/wood
|
||||
renovation_type: str, # required/cosmetic/euro/designer
|
||||
has_balcony: bool,
|
||||
has_loggia: bool,
|
||||
*,
|
||||
cffi_session: Any | None = None,
|
||||
browser_fetcher: BrowserFetcher | None = None,
|
||||
) -> IMVEvaluation:
|
||||
"""Avito IMV — 3 HTTP requests (current contract 2026-05+):
|
||||
|
||||
1) GET /web/1/coords/by_address?address=<urlenc> → normalize + lat/lon
|
||||
2) POST /js/v2/geo/position → rich JWT (geoFieldsHash)
|
||||
3) POST /web/1/realty-imv/get-data → price + placementHistory + suggestions
|
||||
|
||||
Returns IMVEvaluation с уже вычисленным cache_key.
|
||||
|
||||
Raises:
|
||||
IMVAddressNotFoundError если point/normalizedAddress пустые (step 1)
|
||||
или geoFieldsHash не вернулся (step 2)
|
||||
IMVAuthError на 401/403
|
||||
IMVTransientError на 5xx / network errors
|
||||
"""
|
||||
# #915 Stage 3: browser-транспорт. Если передан browser_fetcher — ходим через
|
||||
# камуфокс in-page fetch (same-origin avito.ru), а не curl_cffi. Прокси +
|
||||
# warmed-cookies + реальный fingerprint из sidecar обходят datacenter-403
|
||||
# (#562/#853). _own_session=True → finally дёрнет adapter.close() (no-op,
|
||||
# браузер принадлежит caller'у — backfill открывает один на батч). Этот путь
|
||||
# НЕ требует curl_cffi, поэтому импорт пакета остаётся только в else-ветке.
|
||||
if browser_fetcher is not None:
|
||||
cffi_session = _BrowserSessionAdapter(browser_fetcher, origin=WARMUP_URL)
|
||||
_own_session = True
|
||||
else:
|
||||
# Импортируем здесь чтобы избежать циклических зависимостей при тестах без сети
|
||||
try:
|
||||
from curl_cffi.requests import AsyncSession as CffiAsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
_own_session = False
|
||||
if cffi_session is None:
|
||||
# Зеркалим production-набор из AvitoScraper.__aenter__ (avito.py:150-162):
|
||||
# chrome120 TLS + document-заголовки + timeout. Затем warm-up GET для
|
||||
# seed anti-bot cookies — bare-session XHR Avito банит на server-IP.
|
||||
# Mobile proxy wiring (#806 follow-up): только для own-session (переданная
|
||||
# сессия уже с прокси от AvitoScraper). proxy=None → прямое подключение.
|
||||
_proxy_url = settings.scraper_proxy_url
|
||||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||
cffi_session = CffiAsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=_HTTP_TIMEOUT_SEC,
|
||||
proxies=_proxies,
|
||||
headers=_DOC_HEADERS,
|
||||
)
|
||||
_own_session = True
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"curl_cffi не установлен. Добавь 'curl-cffi>=0.7.0' в pyproject.toml."
|
||||
) from exc
|
||||
|
||||
try:
|
||||
if _own_session:
|
||||
await _warmup(cffi_session)
|
||||
geo = await _geocode(cffi_session, address)
|
||||
evaluation = await _imv_evaluate(
|
||||
cffi_session,
|
||||
geo=geo,
|
||||
address=address,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
floor_at_home=floor_at_home,
|
||||
house_type=house_type,
|
||||
renovation_type=renovation_type,
|
||||
has_balcony=has_balcony,
|
||||
has_loggia=has_loggia,
|
||||
)
|
||||
finally:
|
||||
if _own_session:
|
||||
await cffi_session.close()
|
||||
|
||||
return evaluation
|
||||
|
||||
|
||||
async def _warmup(session: Any) -> None:
|
||||
"""GET /evaluation/realty чтобы seed anti-bot cookies (srv_id/_avisc/u/...).
|
||||
|
||||
Без warm-up XHR-шаги (coords/position/get-data) c server-IP ловят 403 anti-bot.
|
||||
Best-effort: не роняем оценку если страница недоступна — последующие шаги
|
||||
всё равно попробуют и дадут типизированную ошибку.
|
||||
"""
|
||||
try:
|
||||
resp = await session.get(WARMUP_URL, headers=_DOC_HEADERS)
|
||||
logger.info("IMV warm-up GET /evaluation/realty → HTTP %d", resp.status_code)
|
||||
except Exception as exc:
|
||||
# warm-up опционален: логируем и продолжаем — шаги ниже дадут типизированную
|
||||
# ошибку, если cookies реально нужны. Не re-raise намеренно.
|
||||
logger.warning("IMV warm-up failed (продолжаем без cookies): %s", exc)
|
||||
|
||||
|
||||
def _raise_for_status_categorized(resp: Any, context: str) -> None:
|
||||
"""Raise typed IMV exception based on HTTP status code.
|
||||
|
||||
Используется вместо bare raise_for_status() чтобы caller мог различать
|
||||
auth-failure (требует human attention) от transient (retryable).
|
||||
"""
|
||||
status = resp.status_code
|
||||
if status in (401, 403):
|
||||
raise IMVAuthError(f"Avito {context}: HTTP {status} — auth rejected / quota exceeded")
|
||||
if status >= 500:
|
||||
raise IMVTransientError(f"Avito {context}: HTTP {status} — transient server error")
|
||||
if status >= 400:
|
||||
body_preview = (resp.text or "")[:200]
|
||||
logger.warning("Avito %s HTTP %d: body=%r", context, status, body_preview)
|
||||
resp.raise_for_status() # other 4xx → standard HTTPStatusError
|
||||
|
||||
|
||||
async def _geocode(session: Any, address: str) -> IMVGeo:
|
||||
"""Two-step Avito IMV geocoder (current contract 2026-05+).
|
||||
|
||||
Step A: GET /web/1/coords/by_address → normalize + lat/lon
|
||||
Step B: POST /js/v2/geo/position → rich JWT (geoFieldsHash) with
|
||||
addressId/locationId/metroId/districtId
|
||||
|
||||
JWT from step B is what /realty-imv/get-data expects as `geoHash`.
|
||||
"""
|
||||
# Step A — normalize + coords
|
||||
params = urlencode({"address": address})
|
||||
url_a = f"{AVITO_BASE}{COORDS_ENDPOINT}?{params}"
|
||||
logger.info("IMV geocode A: address=%r", address)
|
||||
|
||||
try:
|
||||
resp_a = await session.get(url_a, headers=_COMMON_HEADERS)
|
||||
except Exception as exc:
|
||||
raise IMVTransientError(f"Avito geocode A network error: {exc}") from exc
|
||||
_raise_for_status_categorized(resp_a, "geocode-A")
|
||||
data_a: dict[str, Any] = resp_a.json()
|
||||
|
||||
point = data_a.get("point") or {}
|
||||
lat = point.get("latitude")
|
||||
lon = point.get("longitude")
|
||||
normalized = data_a.get("normalizedAddress")
|
||||
|
||||
if not lat or not lon or not normalized:
|
||||
raise IMVAddressNotFoundError(f"Avito /coords/by_address не вернул point для {address!r}")
|
||||
|
||||
# Step B — rich JWT
|
||||
url_b = f"{AVITO_BASE}{POSITION_ENDPOINT}"
|
||||
payload_b = {
|
||||
"address": normalized,
|
||||
"addressId": "",
|
||||
"categoryId": 24, # 24 = недвижимость / квартиры
|
||||
"isRadius": False,
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
}
|
||||
logger.info("IMV geocode B: position for %r", normalized[:60])
|
||||
|
||||
try:
|
||||
resp_b = await session.post(
|
||||
url_b,
|
||||
headers={**_COMMON_HEADERS, "Content-Type": "application/json"},
|
||||
json=payload_b,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise IMVTransientError(f"Avito geocode B network error: {exc}") from exc
|
||||
_raise_for_status_categorized(resp_b, "geocode-B")
|
||||
data_b: dict[str, Any] = resp_b.json()
|
||||
|
||||
geo = _parse_geo_position(data_b, lat=lat, lon=lon) # raises IMVAddressNotFoundError
|
||||
|
||||
logger.info(
|
||||
"IMV geocode OK: addressId=%s locationId=%s metroId=%s lat=%s lon=%s",
|
||||
geo.avito_address_id,
|
||||
geo.avito_location_id,
|
||||
geo.avito_metro_id,
|
||||
lat,
|
||||
lon,
|
||||
)
|
||||
return geo
|
||||
|
||||
|
||||
async def _imv_evaluate(
|
||||
session: Any,
|
||||
*,
|
||||
geo: IMVGeo,
|
||||
address: str,
|
||||
rooms: int,
|
||||
area_m2: float,
|
||||
floor: int,
|
||||
floor_at_home: int,
|
||||
house_type: str,
|
||||
renovation_type: str,
|
||||
has_balcony: bool,
|
||||
has_loggia: bool,
|
||||
) -> IMVEvaluation:
|
||||
"""Request 2: POST /web/1/realty-imv/get-data → IMVEvaluation."""
|
||||
url = f"{AVITO_BASE}{IMV_ENDPOINT}"
|
||||
body = {
|
||||
"geoHash": geo.geo_hash,
|
||||
"floor": floor,
|
||||
"floorAtHome": floor_at_home,
|
||||
"rooms": rooms,
|
||||
"area": area_m2,
|
||||
"houseType": house_type,
|
||||
"renovationType": renovation_type,
|
||||
"hasLoggia": has_loggia,
|
||||
"hasBalcony": has_balcony,
|
||||
}
|
||||
headers = {**_COMMON_HEADERS, "Content-Type": "application/json"}
|
||||
|
||||
logger.info(
|
||||
"IMV evaluate: address=%r rooms=%d area=%.1f floor=%d/%d",
|
||||
address,
|
||||
rooms,
|
||||
area_m2,
|
||||
floor,
|
||||
floor_at_home,
|
||||
)
|
||||
|
||||
try:
|
||||
resp = await session.post(url, json=body, headers=headers)
|
||||
except Exception as exc:
|
||||
raise IMVTransientError(f"Avito IMV network error: {exc}") from exc
|
||||
_raise_for_status_categorized(resp, "imv-evaluate")
|
||||
data: dict[str, Any] = resp.json()
|
||||
|
||||
recommended_price, lower_price, higher_price, market_count = _parse_price(data)
|
||||
placement_history = _parse_placement_history(data)
|
||||
suggestions, search_similar_url = _parse_suggestions(data)
|
||||
|
||||
logger.info(
|
||||
"IMV evaluate OK: recommended=%d range=(%d, %d) count=%s history=%d suggestions=%d",
|
||||
recommended_price,
|
||||
lower_price,
|
||||
higher_price,
|
||||
market_count,
|
||||
len(placement_history),
|
||||
len(suggestions),
|
||||
)
|
||||
|
||||
cache_key = compute_imv_cache_key(
|
||||
address=address,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
floor_at_home=floor_at_home,
|
||||
house_type=house_type,
|
||||
renovation_type=renovation_type,
|
||||
has_balcony=has_balcony,
|
||||
has_loggia=has_loggia,
|
||||
)
|
||||
|
||||
return IMVEvaluation(
|
||||
cache_key=cache_key,
|
||||
address=address,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
floor_at_home=floor_at_home,
|
||||
house_type=house_type,
|
||||
renovation_type=renovation_type,
|
||||
has_balcony=has_balcony,
|
||||
has_loggia=has_loggia,
|
||||
geo=geo,
|
||||
recommended_price=recommended_price,
|
||||
lower_price=lower_price,
|
||||
higher_price=higher_price,
|
||||
market_count=market_count,
|
||||
placement_history=placement_history,
|
||||
suggestions=suggestions,
|
||||
search_similar_url=search_similar_url,
|
||||
raw_response=data,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DB save functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def save_imv_evaluation(
|
||||
db: Session,
|
||||
e: IMVEvaluation,
|
||||
*,
|
||||
estimate_id: UUID | None = None,
|
||||
) -> int:
|
||||
"""INSERT в avito_imv_evaluations ON CONFLICT(cache_key) DO UPDATE.
|
||||
|
||||
Колонки строго по 018_avito_imv_evaluations.sql DDL.
|
||||
estimate_id — nullable FK на trade_in_estimates.
|
||||
|
||||
Returns: id вставленной/обновлённой строки.
|
||||
"""
|
||||
row = db.execute(
|
||||
text("""
|
||||
INSERT INTO avito_imv_evaluations (
|
||||
cache_key, estimate_id,
|
||||
address, rooms, area_m2, floor, floor_at_home,
|
||||
house_type, renovation_type, has_balcony, has_loggia,
|
||||
geo_hash,
|
||||
lat, lon, avito_address_id, avito_location_id,
|
||||
avito_metro_id, avito_district_id,
|
||||
recommended_price, lower_price, higher_price, market_count,
|
||||
raw_response, fetched_at
|
||||
) VALUES (
|
||||
:cache_key, CAST(:estimate_id AS uuid),
|
||||
:address, :rooms, :area_m2, :floor, :floor_at_home,
|
||||
:house_type, :renovation_type, :has_balcony, :has_loggia,
|
||||
:geo_hash,
|
||||
:lat, :lon, :avito_address_id, :avito_location_id,
|
||||
:avito_metro_id, :avito_district_id,
|
||||
:recommended_price, :lower_price, :higher_price, :market_count,
|
||||
CAST(:raw_response AS jsonb), NOW()
|
||||
)
|
||||
ON CONFLICT (cache_key) DO UPDATE SET
|
||||
estimate_id = COALESCE(EXCLUDED.estimate_id, avito_imv_evaluations.estimate_id),
|
||||
recommended_price = EXCLUDED.recommended_price,
|
||||
lower_price = EXCLUDED.lower_price,
|
||||
higher_price = EXCLUDED.higher_price,
|
||||
market_count = EXCLUDED.market_count,
|
||||
raw_response = EXCLUDED.raw_response,
|
||||
fetched_at = NOW()
|
||||
RETURNING id
|
||||
"""),
|
||||
{
|
||||
"cache_key": e.cache_key,
|
||||
"estimate_id": str(estimate_id) if estimate_id else None,
|
||||
"address": e.address,
|
||||
"rooms": e.rooms,
|
||||
"area_m2": e.area_m2,
|
||||
"floor": e.floor,
|
||||
"floor_at_home": e.floor_at_home,
|
||||
"house_type": e.house_type,
|
||||
"renovation_type": e.renovation_type,
|
||||
"has_balcony": e.has_balcony,
|
||||
"has_loggia": e.has_loggia,
|
||||
"geo_hash": e.geo.geo_hash,
|
||||
"lat": e.geo.lat,
|
||||
"lon": e.geo.lon,
|
||||
"avito_address_id": e.geo.avito_address_id,
|
||||
"avito_location_id": e.geo.avito_location_id,
|
||||
"avito_metro_id": e.geo.avito_metro_id,
|
||||
"avito_district_id": e.geo.avito_district_id,
|
||||
"recommended_price": e.recommended_price,
|
||||
"lower_price": e.lower_price,
|
||||
"higher_price": e.higher_price,
|
||||
"market_count": e.market_count,
|
||||
"raw_response": (
|
||||
json.dumps(e.raw_response, ensure_ascii=False) if e.raw_response else None
|
||||
),
|
||||
},
|
||||
).fetchone()
|
||||
db.commit()
|
||||
return row.id if row else 0
|
||||
|
||||
|
||||
def save_imv_placement_history(
|
||||
db: Session,
|
||||
eval_id: int,
|
||||
items: list[IMVPlacementHistoryItem],
|
||||
) -> int:
|
||||
"""INSERT в house_placement_history (source='avito_imv') с removed_date.
|
||||
|
||||
Колонки строго по 017_house_placement_history.sql DDL.
|
||||
Колонка scraped_at (не fetched_at) — по DDL таблицы.
|
||||
house_id — NULL (backfill через отдельный процесс).
|
||||
eval_id принимается для логирования (не сохраняется — нет FK в 017).
|
||||
|
||||
ON CONFLICT (source, ext_item_id) DO UPDATE — refresh dates.
|
||||
Returns: количество вставленных/обновлённых строк.
|
||||
"""
|
||||
count = 0
|
||||
for item in items:
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO house_placement_history (
|
||||
source, ext_item_id,
|
||||
title, rooms, area_m2, floor, total_floors,
|
||||
start_price, start_price_date,
|
||||
last_price, last_price_date,
|
||||
removed_date, exposure_days,
|
||||
raw_payload, scraped_at
|
||||
) VALUES (
|
||||
'avito_imv', :ext_item_id,
|
||||
:title, :rooms, :area_m2, :floor, :total_floors,
|
||||
:start_price, :start_price_date,
|
||||
:last_price, :last_price_date,
|
||||
:removed_date, :exposure_days,
|
||||
CAST(:raw_payload AS jsonb), NOW()
|
||||
)
|
||||
ON CONFLICT (source, ext_item_id) DO UPDATE SET
|
||||
last_price = EXCLUDED.last_price,
|
||||
last_price_date = EXCLUDED.last_price_date,
|
||||
removed_date = EXCLUDED.removed_date,
|
||||
exposure_days = EXCLUDED.exposure_days,
|
||||
raw_payload = EXCLUDED.raw_payload,
|
||||
scraped_at = NOW()
|
||||
"""),
|
||||
{
|
||||
"ext_item_id": item.ext_item_id,
|
||||
"title": item.title,
|
||||
"rooms": item.rooms,
|
||||
"area_m2": item.area_m2,
|
||||
"floor": item.floor,
|
||||
"total_floors": item.total_floors,
|
||||
"start_price": item.start_price,
|
||||
"start_price_date": item.start_price_date,
|
||||
"last_price": item.last_price,
|
||||
"last_price_date": item.last_price_date,
|
||||
"removed_date": item.removed_date,
|
||||
"exposure_days": item.exposure_days,
|
||||
"raw_payload": (
|
||||
json.dumps(item.raw_payload, ensure_ascii=False) if item.raw_payload else None
|
||||
),
|
||||
},
|
||||
)
|
||||
count += 1
|
||||
|
||||
if count:
|
||||
db.commit()
|
||||
logger.info("IMV placement history saved: eval_id=%d items=%d", eval_id, count)
|
||||
|
||||
return count
|
||||
|
|
@ -1,584 +0,0 @@
|
|||
"""DomClick.ru Layer B — detail-card enrichment (#1846 follow-up).
|
||||
|
||||
Layer A (domclick.py) собирает базовые поля через BFF JSON API. Layer B дочитывает
|
||||
страницу карточки (``https://ekaterinburg.domclick.ru/card/sale__flat__<id>``) и
|
||||
обогащает listings: renovation/repair_state, living/kitchen area, balconies,
|
||||
sale_type, owners_count, encumbrances, year_built, views и priceHistory.
|
||||
|
||||
Транспорт — BrowserFetcher через QRATOR-clean прокси (source="cian"): card-хосты
|
||||
DomClick заблокированы QRATOR через domclick/avito-прокси, но чисты через cian
|
||||
(verified live 2026-06-27). curl_cffi на card-хосте отдаёт 401/стаб. fetch_detail
|
||||
принимает уже открытый browser_fetcher; жизненным циклом сессии и выбором source
|
||||
владеет оркестратор.
|
||||
|
||||
SSR-quirk
|
||||
---------
|
||||
HTML карточки встраивает ``window.__SSR_STATE__ = {...}`` — это JS object literal,
|
||||
а НЕ чистый JSON: в value-позиции встречается bare ``undefined`` (json.loads падает).
|
||||
_extract_ssr_state делает balanced-brace scan (с пропуском скобок внутри строк) и
|
||||
заменяет bare ``undefined`` → ``null`` строго в value-позиции, не трогая строки.
|
||||
|
||||
Форма SSR-стейта подтверждена на живой карточке 2075729321 (2026-06-27):
|
||||
productCard.objectInfo.{renovation,livingArea,kitchenArea}, productCard.priceInfo.
|
||||
priceHistory ({date ISO8601+tz, price, diff, state}), productCard.egrnData
|
||||
(snake_case owners_count/collateral/collateral_sber), productCard.legalOptions.
|
||||
saleType, productCard.viewsCount/callsCount, houseInfo.info.{wallType,floorType},
|
||||
и ТОП-УРОВНЕМ pricePrediction (DomClick AVM → raw_extra.avm).
|
||||
Schedule для Layer B ships DORMANT — включается вручную после prod-smoke.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError
|
||||
from app.services.scrapers.repair_state_normalizer import (
|
||||
infer_repair_state_from_text,
|
||||
normalize_repair_state,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Маркеры anti-bot challenge (DataDome / QRATOR) ───────────────────────────
|
||||
# Зеркало идеи Layer A: если __SSR_STATE__ отсутствует И страница похожа на
|
||||
# challenge — это блок, а не parse-failure.
|
||||
_BLOCK_MARKERS: tuple[str, ...] = ("qrator", "captcha", "datadome", "access denied")
|
||||
|
||||
# ── item_id из URL карточки: .../card/sale__flat__2075729321 ─────────────────
|
||||
_ITEM_ID_RE = re.compile(r"sale__flat__(\d+)")
|
||||
|
||||
# ── SSR assignment marker ────────────────────────────────────────────────────
|
||||
_SSR_ASSIGN_RE = re.compile(r"window\.__SSR_STATE__\s*=\s*")
|
||||
|
||||
# ── bare `undefined` (value-позиция) → null, не трогая строки ─────────────────
|
||||
_UNDEFINED_RE = re.compile(r"(?<=[:\[,\s])undefined(?=[\s,\]\}])")
|
||||
|
||||
# ── renovation (RU) → каноничный repair_state enum ───────────────────────────
|
||||
# ЛОКАЛЬНАЯ карта (не трогаем shared _RAW_TO_ENUM в repair_state_normalizer).
|
||||
# Значения уже каноничны; прогоняются через normalize_repair_state для валидации.
|
||||
_RENOVATION_MAP: dict[str, str] = {
|
||||
"евроремонт": "good",
|
||||
"косметический": "standard",
|
||||
"дизайнерский": "excellent",
|
||||
"черновая": "needs_repair",
|
||||
"без отделки": "needs_repair",
|
||||
"требует ремонта": "needs_repair",
|
||||
"предчистовая": "needs_repair",
|
||||
}
|
||||
|
||||
|
||||
# ── DomClickDetailEnrichment ──────────────────────────────────────────────────
|
||||
@dataclass
|
||||
class DomClickDetailEnrichment:
|
||||
"""Результат парсинга detail-карточки DomClick (Layer B).
|
||||
|
||||
save_detail_enrichment() пишет подмножество в listings (+ offer_price_history).
|
||||
Все поля None-safe: отсутствующее → None (не крэш).
|
||||
"""
|
||||
|
||||
item_id: str | None = None
|
||||
source_url: str | None = None
|
||||
|
||||
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
||||
repair_type: str | None = None # сырое RU-значение renovation
|
||||
living_area_m2: float | None = None
|
||||
kitchen_area_m2: float | None = None
|
||||
balconies_count: int | None = None
|
||||
has_balcony: bool | None = None
|
||||
sale_type: str | None = None
|
||||
owners_count: int | None = None
|
||||
encumbrances_clean: bool | None = None # True=чисто, False=есть обременение
|
||||
year_built: int | None = None
|
||||
views_total: int | None = None
|
||||
|
||||
price_changes: list[dict[str, Any]] = field(default_factory=list)
|
||||
raw_extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ── SSR state extraction ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _scan_balanced_object(s: str, start: int) -> str:
|
||||
"""Сканирует от '{' по индексу ``start`` до парной '}'.
|
||||
|
||||
Отслеживает глубину по {/}, ИГНОРИРУЯ скобки внутри JS-строк (одинарные и
|
||||
двойные кавычки с ``\\``-эскейпами). Возвращает литерал ``{...}`` включительно.
|
||||
|
||||
Raises:
|
||||
DomClickParseError: если парная скобка не найдена (truncated HTML).
|
||||
"""
|
||||
depth = 0
|
||||
in_string = False
|
||||
quote_char = ""
|
||||
escaped = False
|
||||
for i in range(start, len(s)):
|
||||
ch = s[i]
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif ch == "\\":
|
||||
escaped = True
|
||||
elif ch == quote_char:
|
||||
in_string = False
|
||||
continue
|
||||
if ch in ('"', "'"):
|
||||
in_string = True
|
||||
quote_char = ch
|
||||
elif ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return s[start : i + 1]
|
||||
raise DomClickParseError("__SSR_STATE__ object not balanced (no matching brace)")
|
||||
|
||||
|
||||
def _extract_ssr_state(html: str) -> dict[str, Any]:
|
||||
"""Извлекает ``window.__SSR_STATE__`` object-литерал из HTML карточки.
|
||||
|
||||
Шаги:
|
||||
1. Находим присваивание ``window.__SSR_STATE__ =``. Если нет — различаем
|
||||
challenge-страницу (→ DomClickBlockedError) и просто отсутствие
|
||||
стейта (→ DomClickParseError).
|
||||
2. Balanced-brace scan от первого '{' → литерал.
|
||||
3. Санитайз bare ``undefined`` → ``null`` (только value-позиция).
|
||||
4. json.loads.
|
||||
|
||||
Raises:
|
||||
DomClickBlockedError: страница без стейта + маркеры anti-bot.
|
||||
DomClickParseError: стейт не найден / не сбалансирован / json.loads упал.
|
||||
"""
|
||||
match = _SSR_ASSIGN_RE.search(html)
|
||||
if match is None:
|
||||
html_lower = html.lower()
|
||||
if any(marker in html_lower for marker in _BLOCK_MARKERS):
|
||||
raise DomClickBlockedError(
|
||||
"DomClick detail: challenge page detected (no __SSR_STATE__, "
|
||||
f"markers checked: {_BLOCK_MARKERS})"
|
||||
)
|
||||
raise DomClickParseError("__SSR_STATE__ not found")
|
||||
|
||||
brace_start = html.find("{", match.end())
|
||||
if brace_start == -1:
|
||||
raise DomClickParseError("__SSR_STATE__ assignment has no opening '{'")
|
||||
|
||||
literal = _scan_balanced_object(html, brace_start)
|
||||
sanitized = _UNDEFINED_RE.sub("null", literal)
|
||||
try:
|
||||
data = json.loads(sanitized)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DomClickParseError(
|
||||
f"__SSR_STATE__ json.loads failed: {exc} (head={sanitized[:200]!r})"
|
||||
) from exc
|
||||
if not isinstance(data, dict):
|
||||
raise DomClickParseError(f"__SSR_STATE__ is not an object: {type(data)}")
|
||||
return data
|
||||
|
||||
|
||||
# ── Value coercion helpers ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _to_float(value: Any) -> float | None:
|
||||
"""None-safe float. bool/неконвертируемое → None."""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _to_int(value: Any) -> int | None:
|
||||
"""None-safe int. bool/неконвертируемое → None."""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_change_time(value: Any) -> datetime | None:
|
||||
"""Нормализует дату изменения цены → timezone-aware datetime.
|
||||
|
||||
Адаптация Layer A ``_parse_publish_date`` (которая возвращала date): здесь
|
||||
поддерживаем epoch (сек/мс), числовую строку и ISO8601. Real priceHistory.date
|
||||
приходит как ``2026-04-15T09:02:26.548728+03:00`` — fromisoformat СОХРАНЯЕТ
|
||||
исходный offset (+03:00), НЕ конвертирует в UTC; naive-значения получают UTC.
|
||||
epoch трактуем как UTC. Невалидное → None (caller пропускает запись).
|
||||
"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
# epoch (число): >1e12 трактуем как миллисекунды
|
||||
if isinstance(value, int | float):
|
||||
ts = float(value)
|
||||
if ts > 1e12:
|
||||
ts /= 1000.0
|
||||
try:
|
||||
return datetime.fromtimestamp(ts, tz=UTC)
|
||||
except (ValueError, OSError, OverflowError):
|
||||
return None
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
return None
|
||||
if s.isdigit():
|
||||
ts = float(s)
|
||||
if ts > 1e12:
|
||||
ts /= 1000.0
|
||||
try:
|
||||
return datetime.fromtimestamp(ts, tz=UTC)
|
||||
except (ValueError, OSError, OverflowError):
|
||||
return None
|
||||
# ISO8601 (поддержим trailing 'Z')
|
||||
try:
|
||||
dt = datetime.fromisoformat(s.replace("Z", "+00:00"))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=UTC)
|
||||
return dt
|
||||
|
||||
|
||||
def _map_repair_state(renovation: str | None) -> str | None:
|
||||
"""RU renovation → каноничный enum. Нет совпадения в карте → инференс из текста."""
|
||||
if not renovation:
|
||||
return None
|
||||
mapped = _RENOVATION_MAP.get(renovation.strip().lower())
|
||||
if mapped is not None:
|
||||
return normalize_repair_state(mapped)
|
||||
return infer_repair_state_from_text(renovation)
|
||||
|
||||
|
||||
def _compact(d: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Выкидывает None-значения, чтобы raw_extra оставался компактным."""
|
||||
return {k: v for k, v in d.items() if v is not None}
|
||||
|
||||
|
||||
def _pick(d: dict[str, Any], *keys: str) -> Any:
|
||||
"""Первое не-None значение среди ключей (defensive snake/camel spellings)."""
|
||||
for k in keys:
|
||||
v = d.get(k)
|
||||
if v is not None:
|
||||
return v
|
||||
return None
|
||||
|
||||
|
||||
# ── item_id ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _extract_item_id(url: str) -> str | None:
|
||||
"""sale__flat__<digits> из URL карточки."""
|
||||
m = _ITEM_ID_RE.search(url or "")
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
# ── parse_detail_html ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def parse_detail_html(html: str, source_url: str) -> DomClickDetailEnrichment:
|
||||
"""Pure parser detail-карточки DomClick (без сети/БД).
|
||||
|
||||
_extract_ssr_state может бросить DomClickBlockedError/DomClickParseError —
|
||||
они propagate (оркестратор различает блок и parse-failure). Дальнейшая
|
||||
навигация по полям обёрнута: per-field сюрприз формы → минимальный
|
||||
enrichment (item_id+url), а не крэш.
|
||||
"""
|
||||
state = _extract_ssr_state(html)
|
||||
item_id = _extract_item_id(source_url)
|
||||
try:
|
||||
pc = state.get("productCard") or {}
|
||||
oi = pc.get("objectInfo") or {}
|
||||
hi = (state.get("houseInfo") or {}).get("info") or {}
|
||||
egrn = pc.get("egrnData") or {}
|
||||
|
||||
# repair: enum + сырое значение
|
||||
renovation = oi.get("renovation")
|
||||
repair_type = renovation if isinstance(renovation, str) else None
|
||||
repair_state = _map_repair_state(repair_type)
|
||||
|
||||
# площади
|
||||
living_area_m2 = _to_float(oi.get("livingArea"))
|
||||
kitchen_area_m2 = _to_float(oi.get("kitchenArea"))
|
||||
|
||||
# балконы: int → count+флаг; иное (строка/др.) → raw_extra, counts None
|
||||
balconies = oi.get("balconies")
|
||||
balconies_count: int | None = None
|
||||
has_balcony: bool | None = None
|
||||
balconies_raw: Any = None
|
||||
if isinstance(balconies, bool):
|
||||
balconies_raw = balconies
|
||||
elif isinstance(balconies, int):
|
||||
balconies_count = balconies
|
||||
has_balcony = balconies > 0
|
||||
elif balconies is not None:
|
||||
balconies_raw = balconies
|
||||
|
||||
sale_type = (pc.get("legalOptions") or {}).get("saleType")
|
||||
|
||||
# owners_count — защищаемся от двух написаний ключа
|
||||
owners_count = _to_int(egrn.get("owners_count"))
|
||||
if owners_count is None:
|
||||
owners_count = _to_int(egrn.get("ownersCount"))
|
||||
|
||||
# encumbrances_clean (ИНВЕРСИЯ): collateral/collateral_sber обозначают
|
||||
# НАЛИЧИЕ обременения. Любой truthy → clean=False; оба отсутствуют/false
|
||||
# при наличии egrnData → True; egrnData нет → None.
|
||||
collateral = egrn.get("collateral")
|
||||
collateral_sber = egrn.get("collateral_sber")
|
||||
if egrn:
|
||||
encumbrances_clean: bool | None = not bool(collateral or collateral_sber)
|
||||
else:
|
||||
encumbrances_clean = None
|
||||
|
||||
year_built = _to_int(hi.get("buildYear"))
|
||||
views_total = _to_int(pc.get("viewsCount"))
|
||||
|
||||
# priceHistory → нормализованные записи (skip при битой дате / без цены)
|
||||
raw_history = (pc.get("priceInfo") or {}).get("priceHistory") or []
|
||||
price_changes: list[dict[str, Any]] = []
|
||||
if isinstance(raw_history, list):
|
||||
for entry in raw_history:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
change_time = _parse_change_time(entry.get("date"))
|
||||
price_rub = _to_int(entry.get("price"))
|
||||
# Skip при нераспарсенной дате ИЛИ отсутствующей/невалидной цене.
|
||||
if change_time is None or price_rub is None:
|
||||
continue
|
||||
diff = entry.get("diff")
|
||||
diff_percent = (
|
||||
diff if isinstance(diff, int | float) and not isinstance(diff, bool) else None
|
||||
)
|
||||
price_changes.append(
|
||||
{
|
||||
"change_time": change_time,
|
||||
"price_rub": price_rub,
|
||||
"diff_percent": diff_percent,
|
||||
}
|
||||
)
|
||||
|
||||
# AVM (Layer C, тот же fetch) — DomClick price prediction живёт ТОП-УРОВНЕМ
|
||||
# в SSR-стейте. Может быть плоским или обёрнут в .result/.data — defensively
|
||||
# разворачиваем. В raw_extra пишем только присутствующие поля.
|
||||
avm_root = state.get("pricePrediction") or {}
|
||||
avm_src = avm_root
|
||||
if isinstance(avm_root, dict):
|
||||
avm_src = avm_root.get("result") or avm_root.get("data") or avm_root
|
||||
if not isinstance(avm_src, dict):
|
||||
avm_src = {}
|
||||
avm = _compact(
|
||||
{
|
||||
"market_price": _pick(avm_src, "market_price", "marketPrice"),
|
||||
"min": _pick(avm_src, "min_market_price", "minMarketPrice"),
|
||||
"max": _pick(avm_src, "max_market_price", "maxMarketPrice"),
|
||||
"repair_quality": _pick(avm_src, "repair_quality", "repairQuality"),
|
||||
}
|
||||
)
|
||||
|
||||
# raw_extra → merge в listings.raw_payload. wall/floor type ОСТАЮТСЯ тут,
|
||||
# НЕ пишем в listings.house_type (кросс-источниковый словарь грязный).
|
||||
# egrn area key — точное написание не подтверждено; пробуем несколько.
|
||||
egrn_area = egrn.get("area") or egrn.get("rosreestrArea") or egrn.get("object_area")
|
||||
demand = _compact(
|
||||
{
|
||||
"calls": pc.get("callsCount"),
|
||||
"favorites": pc.get("favoriteOfferUsersCount"),
|
||||
"duplicates": pc.get("duplicatesOfferCount"),
|
||||
}
|
||||
)
|
||||
raw_extra = _compact(
|
||||
{
|
||||
"wall_type": hi.get("wallType") or (pc.get("house") or {}).get("wallType"),
|
||||
"floor_type": hi.get("floorType"),
|
||||
"entrance_count": hi.get("entranceCount"),
|
||||
"quarters_count": hi.get("quartersCount"),
|
||||
"energy_efficiency": hi.get("energyEfficiency"),
|
||||
"building_series": hi.get("buildingSeries"),
|
||||
"domclick_building_guid": (pc.get("address") or {}).get("guid"),
|
||||
"egrn_area": egrn_area,
|
||||
"demand": demand or None,
|
||||
"collateral": collateral,
|
||||
"collateral_sber": collateral_sber,
|
||||
"balconies_raw": balconies_raw,
|
||||
"avm": avm or None,
|
||||
}
|
||||
)
|
||||
|
||||
return DomClickDetailEnrichment(
|
||||
item_id=item_id,
|
||||
source_url=source_url,
|
||||
repair_state=repair_state,
|
||||
repair_type=repair_type,
|
||||
living_area_m2=living_area_m2,
|
||||
kitchen_area_m2=kitchen_area_m2,
|
||||
balconies_count=balconies_count,
|
||||
has_balcony=has_balcony,
|
||||
sale_type=sale_type if isinstance(sale_type, str) else None,
|
||||
owners_count=owners_count,
|
||||
encumbrances_clean=encumbrances_clean,
|
||||
year_built=year_built,
|
||||
views_total=views_total,
|
||||
price_changes=price_changes,
|
||||
raw_extra=raw_extra,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"domclick_detail: field navigation failed for %s — minimal enrichment",
|
||||
source_url,
|
||||
exc_info=True,
|
||||
)
|
||||
return DomClickDetailEnrichment(item_id=item_id, source_url=source_url)
|
||||
|
||||
|
||||
# ── fetch_detail ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def fetch_detail(
|
||||
card_url: str,
|
||||
*,
|
||||
browser_fetcher: BrowserFetcher,
|
||||
) -> DomClickDetailEnrichment:
|
||||
"""Fetch абсолютного URL карточки через BrowserFetcher → parse_detail_html.
|
||||
|
||||
card_url уже абсолютный (listings.source_url из Layer A), URL не строим.
|
||||
Сессией владеет оркестратор (передаёт уже открытый browser_fetcher).
|
||||
|
||||
Raises:
|
||||
DomClickBlockedError: anti-bot challenge ИЛИ ошибка браузерного fetch
|
||||
(нет curl-fallback — DataDome даёт 401 на curl). Оркестратор считает
|
||||
это блоком и помечает listing failed.
|
||||
DomClickParseError: HTTP 200, но SSR-стейт не парсится (дрейф схемы).
|
||||
"""
|
||||
try:
|
||||
html = await browser_fetcher.fetch(card_url)
|
||||
except DomClickBlockedError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DomClickBlockedError(
|
||||
f"DomClick detail browser fetch failed for {card_url}: {exc}"
|
||||
) from exc
|
||||
return parse_detail_html(html, card_url)
|
||||
|
||||
|
||||
# ── save_detail_enrichment ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def save_detail_enrichment(
|
||||
db: Session,
|
||||
listing_id: int,
|
||||
e: DomClickDetailEnrichment,
|
||||
) -> bool:
|
||||
"""Persist enrichment в одной транзакции (commit в конце; rollback — за caller).
|
||||
|
||||
1. UPDATE listings SET <cols> (COALESCE — None не затирает существующее),
|
||||
raw_payload || raw_extra, detail_enriched_at=NOW() WHERE id=listing_id.
|
||||
2. Для каждой записи priceHistory — INSERT INTO offer_price_history
|
||||
(ON CONFLICT DO NOTHING → идемпотентно). source='domklik'.
|
||||
|
||||
Snapshot (upsert_listing_snapshot) НЕ пишем: enrichment не несёт текущую цену,
|
||||
а detail-backfill цену не меняет — snapshot остаётся за SERP-путём (note).
|
||||
|
||||
Returns:
|
||||
True если UPDATE задел строку (listing найден), иначе False.
|
||||
"""
|
||||
raw_extra_json = json.dumps(e.raw_extra or {}, ensure_ascii=False, default=str)
|
||||
|
||||
result = db.execute(
|
||||
text("""
|
||||
UPDATE listings SET
|
||||
repair_state = COALESCE(:repair_state, repair_state),
|
||||
repair_type = COALESCE(:repair_type, repair_type),
|
||||
living_area_m2 = COALESCE(:living_area_m2, living_area_m2),
|
||||
kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2),
|
||||
balconies_count = COALESCE(:balconies_count, balconies_count),
|
||||
has_balcony = COALESCE(:has_balcony, has_balcony),
|
||||
sale_type = COALESCE(:sale_type, sale_type),
|
||||
owners_count = COALESCE(:owners_count, owners_count),
|
||||
encumbrances_clean = COALESCE(:encumbrances_clean, encumbrances_clean),
|
||||
year_built = COALESCE(:year_built, year_built),
|
||||
views_total = COALESCE(:views_total, views_total),
|
||||
raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_extra AS jsonb),
|
||||
detail_enriched_at = NOW()
|
||||
WHERE id = CAST(:lid AS bigint)
|
||||
"""),
|
||||
{
|
||||
"lid": listing_id,
|
||||
"repair_state": e.repair_state,
|
||||
"repair_type": e.repair_type,
|
||||
"living_area_m2": e.living_area_m2,
|
||||
"kitchen_area_m2": e.kitchen_area_m2,
|
||||
"balconies_count": e.balconies_count,
|
||||
"has_balcony": e.has_balcony,
|
||||
"sale_type": e.sale_type,
|
||||
"owners_count": e.owners_count,
|
||||
"encumbrances_clean": e.encumbrances_clean,
|
||||
"year_built": e.year_built,
|
||||
"views_total": e.views_total,
|
||||
"raw_extra": raw_extra_json,
|
||||
},
|
||||
)
|
||||
|
||||
# priceHistory — INSERT, де-дуп по (listing_id, change_time). SAVEPOINT per-row
|
||||
# (backend.md): битый cast одной записи не валит весь UPDATE+commit.
|
||||
for change in e.price_changes:
|
||||
ct = change.get("change_time")
|
||||
price = change.get("price_rub")
|
||||
if not ct or not price:
|
||||
continue
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO offer_price_history
|
||||
(listing_id, change_time, price_rub, diff_percent, source)
|
||||
VALUES (
|
||||
CAST(:lid AS bigint),
|
||||
CAST(:ct AS timestamptz),
|
||||
CAST(:price AS numeric),
|
||||
CAST(:diff AS numeric),
|
||||
'domklik'
|
||||
)
|
||||
ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq
|
||||
DO NOTHING
|
||||
"""),
|
||||
{
|
||||
"lid": listing_id,
|
||||
"ct": ct,
|
||||
"price": price,
|
||||
"diff": change.get("diff_percent"),
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"domclick_detail: price-history insert failed listing_id=%s ct=%s: %s",
|
||||
listing_id,
|
||||
ct,
|
||||
exc,
|
||||
)
|
||||
|
||||
db.commit()
|
||||
found = result.rowcount > 0
|
||||
if not found:
|
||||
logger.warning(
|
||||
"domclick_detail save: listing not found id=%s (item_id=%s)",
|
||||
listing_id,
|
||||
e.item_id,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"domclick_detail save: listing id=%s enriched (price_changes=%d)",
|
||||
listing_id,
|
||||
len(e.price_changes),
|
||||
)
|
||||
return found
|
||||
|
|
@ -1,238 +0,0 @@
|
|||
"""Клиент городского геопортала Екатеринбурга (геопортал.екатеринбург.рф).
|
||||
|
||||
Открытый GeoServer WFS, без авторизации. Слой `portal:portal_geo_topo_build` —
|
||||
футпринты зданий ЕКБ (намного полнее чем NSPD cad_buildings, который пропускает
|
||||
~70% зданий города).
|
||||
|
||||
ВАЖНО: сервер отдаёт 403 на дефолтный User-Agent curl/httpx. Нужны браузерные
|
||||
заголовки (UA + Referer + Accept) — тогда 200.
|
||||
|
||||
Геометрия `geoloc` в EPSG:4326 (MultiPolygon footprint). BBOX axis order =
|
||||
lon_min,lat_min,lon_max,lat_max. Сервер режет каждый ответ на 20000 features —
|
||||
лоадер тайлит мелко, чтобы не упереться.
|
||||
|
||||
Центроид считается чистым Python'ом (area-weighted, без shapely — shapely не входит
|
||||
в зависимости tradein-mvp backend).
|
||||
|
||||
Используется лоадером app.tasks.ekb_geoportal_ingest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Punycode-хост геопортал.екатеринбург.рф
|
||||
GEOPORTAL_HOST = "https://xn--80afgznagjs.xn--80acgfbsl1azdqr.xn--p1ai"
|
||||
WFS_PATH = "/api/gis/ows/3/portal/wfs"
|
||||
BUILD_LAYER = "portal:portal_geo_topo_build"
|
||||
|
||||
# Браузерные заголовки — без них хост отдаёт 403.
|
||||
_BROWSER_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/148.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Referer": f"{GEOPORTAL_HOST}/",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TopoBuilding:
|
||||
"""Одна часть-фича здания из слоя portal_geo_topo_build.
|
||||
|
||||
Здание может быть разбито на несколько part-features (одинаковые street/house,
|
||||
разные key) — лоадер группирует их по (street_norm, house_norm).
|
||||
"""
|
||||
|
||||
street: str
|
||||
house: str
|
||||
floors: int | None
|
||||
purpose: str | None
|
||||
material: str | None
|
||||
key: int | None
|
||||
lat: float
|
||||
lon: float
|
||||
|
||||
|
||||
def _parse_int(value: object) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_str(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
s = str(value).strip()
|
||||
return s or None
|
||||
|
||||
|
||||
def _ring_centroid(ring: list) -> tuple[float, float, float] | None:
|
||||
"""Area-weighted centroid of one linear ring → (cx, cy, signed_area2).
|
||||
|
||||
signed_area2 = 2× signed area (используется как вес при усреднении колец/полигонов).
|
||||
Возвращает None если кольцо вырождено (нулевая площадь).
|
||||
"""
|
||||
n = len(ring)
|
||||
if n < 3:
|
||||
return None
|
||||
a2 = 0.0 # 2× signed area
|
||||
cx = 0.0
|
||||
cy = 0.0
|
||||
for i in range(n):
|
||||
x0, y0 = ring[i][0], ring[i][1]
|
||||
x1, y1 = ring[(i + 1) % n][0], ring[(i + 1) % n][1]
|
||||
cross = x0 * y1 - x1 * y0
|
||||
a2 += cross
|
||||
cx += (x0 + x1) * cross
|
||||
cy += (y0 + y1) * cross
|
||||
if a2 == 0.0:
|
||||
return None
|
||||
cx /= 3.0 * a2
|
||||
cy /= 3.0 * a2
|
||||
return (cx, cy, a2)
|
||||
|
||||
|
||||
def _multipolygon_centroid(coords: list) -> tuple[float, float] | None:
|
||||
"""Центроид GeoJSON MultiPolygon/Polygon-частей: усреднение по |площади| внешних колец.
|
||||
|
||||
coords — список полигонов, каждый = [outer_ring, hole1, ...]. Берём только
|
||||
внешнее кольцо каждого полигона (дыр у footprint'ов зданий практически нет,
|
||||
их вклад в положение центра пренебрежим). Возвращает (lon, lat) или None.
|
||||
"""
|
||||
sum_w = 0.0
|
||||
sum_x = 0.0
|
||||
sum_y = 0.0
|
||||
for polygon in coords:
|
||||
if not polygon:
|
||||
continue
|
||||
outer = polygon[0]
|
||||
c = _ring_centroid(outer)
|
||||
if c is None:
|
||||
continue
|
||||
cx, cy, a2 = c
|
||||
w = abs(a2)
|
||||
sum_w += w
|
||||
sum_x += cx * w
|
||||
sum_y += cy * w
|
||||
if sum_w == 0.0:
|
||||
return None
|
||||
return (sum_x / sum_w, sum_y / sum_w)
|
||||
|
||||
|
||||
def _geometry_centroid(geom: dict) -> tuple[float, float] | None:
|
||||
"""Центроид GeoJSON geometry (MultiPolygon или Polygon) → (lon, lat) или None."""
|
||||
gtype = geom.get("type")
|
||||
coords = geom.get("coordinates")
|
||||
if not coords:
|
||||
return None
|
||||
if gtype == "MultiPolygon":
|
||||
return _multipolygon_centroid(coords)
|
||||
if gtype == "Polygon":
|
||||
# Polygon coordinates = [outer_ring, hole, ...] → обернём в один полигон.
|
||||
return _multipolygon_centroid([coords])
|
||||
return None
|
||||
|
||||
|
||||
def _feature_to_building(feature: dict) -> TopoBuilding | None:
|
||||
"""GeoJSON Feature → TopoBuilding. None если нет улицы/дома или геометрии."""
|
||||
props = feature.get("properties") or {}
|
||||
street = _parse_str(props.get("topo_street"))
|
||||
house = _parse_str(props.get("topo_num_house"))
|
||||
if not street or not house:
|
||||
return None
|
||||
|
||||
geom = feature.get("geometry")
|
||||
if not isinstance(geom, dict):
|
||||
return None
|
||||
try:
|
||||
centroid = _geometry_centroid(geom)
|
||||
except Exception:
|
||||
logger.warning("geoportal: bad geometry for street=%r house=%r", street, house)
|
||||
return None
|
||||
if centroid is None:
|
||||
return None
|
||||
lon, lat = centroid
|
||||
|
||||
return TopoBuilding(
|
||||
street=street,
|
||||
house=house,
|
||||
floors=_parse_int(props.get("topo_floor")),
|
||||
purpose=_parse_str(props.get("topo_purpose")),
|
||||
material=_parse_str(props.get("topo_material")),
|
||||
key=_parse_int(props.get("key")),
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
)
|
||||
|
||||
|
||||
def parse_feature_collection(payload: dict) -> list[TopoBuilding]:
|
||||
"""GeoJSON FeatureCollection → list[TopoBuilding]. Пропускает невалидные."""
|
||||
features = payload.get("features")
|
||||
if not isinstance(features, list):
|
||||
return []
|
||||
out: list[TopoBuilding] = []
|
||||
for feature in features:
|
||||
if not isinstance(feature, dict):
|
||||
continue
|
||||
building = _feature_to_building(feature)
|
||||
if building is not None:
|
||||
out.append(building)
|
||||
return out
|
||||
|
||||
|
||||
class EkbGeoportalClient:
|
||||
"""Async WFS-клиент геопортала ЕКБ. Один клиент переиспользуется лоадером."""
|
||||
|
||||
def __init__(self, *, timeout: float = 30.0) -> None:
|
||||
self._timeout = timeout
|
||||
|
||||
async def fetch_topo_buildings(
|
||||
self,
|
||||
bbox: tuple[float, float, float, float],
|
||||
*,
|
||||
count: int = 20000,
|
||||
) -> list[TopoBuilding]:
|
||||
"""Запрашивает здания в bbox (lon_min, lat_min, lon_max, lat_max).
|
||||
|
||||
Graceful: на non-200/parse-error логирует warning и возвращает [].
|
||||
"""
|
||||
lon_min, lat_min, lon_max, lat_max = bbox
|
||||
cql = f"BBOX(geoloc,{lon_min},{lat_min},{lon_max},{lat_max})"
|
||||
params = {
|
||||
"service": "WFS",
|
||||
"version": "2.0.0",
|
||||
"request": "GetFeature",
|
||||
"typeNames": BUILD_LAYER,
|
||||
"outputFormat": "application/json",
|
||||
"CQL_FILTER": cql,
|
||||
"count": str(count),
|
||||
}
|
||||
url = f"{GEOPORTAL_HOST}{WFS_PATH}"
|
||||
try:
|
||||
# Геопортал ЕКБ отдаёт сертификат российского CA (Минцифры), которого нет в
|
||||
# стандартном trust store httpx → TLS verify падает. Данные публичные,
|
||||
# read-only → verify=False (тот же подход, что curl -k при разведке).
|
||||
async with httpx.AsyncClient(
|
||||
timeout=self._timeout, headers=_BROWSER_HEADERS, verify=False
|
||||
) as client:
|
||||
resp = await client.get(url, params=params)
|
||||
if resp.status_code != 200:
|
||||
logger.warning("geoportal WFS → HTTP %d for bbox=%s", resp.status_code, bbox)
|
||||
return []
|
||||
payload = resp.json()
|
||||
except Exception as exc:
|
||||
logger.warning("geoportal WFS fetch/parse failed for bbox=%s: %s", bbox, exc)
|
||||
return []
|
||||
|
||||
return parse_feature_collection(payload)
|
||||
|
|
@ -1,662 +0,0 @@
|
|||
"""Yandex Realty detail page scraper.
|
||||
|
||||
Fetches /offer/<id>/ and extracts Product JSON-LD + DOM sections into
|
||||
a DetailEnrichment Pydantic model. Used by enrichment pipeline
|
||||
(Wave 5+ matching / Wave 6 estimator).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper
|
||||
from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text
|
||||
from app.services.scrapers.yandex_helpers import (
|
||||
RE_AGENCY_FOUNDED,
|
||||
RE_AGENCY_OBJECTS,
|
||||
RE_METRO_WALK,
|
||||
RE_VIEWS,
|
||||
RE_YEAR,
|
||||
find_ld_by_type,
|
||||
parse_house_type,
|
||||
parse_ru_date,
|
||||
parse_rub,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Pydantic models ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class MetroStation(BaseModel):
|
||||
name: str
|
||||
walk_min: int | None = None
|
||||
|
||||
|
||||
class DetailEnrichment(BaseModel):
|
||||
"""Enrichment payload from a Yandex detail page."""
|
||||
|
||||
offer_id: str
|
||||
source_url: str
|
||||
|
||||
# Pricing — Product JSON-LD `offers.price` is the exact int
|
||||
price_rub: int | None = None
|
||||
price_per_m2: int | None = None
|
||||
|
||||
# Title + basic params
|
||||
title: str | None = None
|
||||
rooms: int | None = None
|
||||
area_m2: float | None = None
|
||||
living_area_m2: float | None = None
|
||||
kitchen_area_m2: float | None = None
|
||||
ceiling_height: float | None = None # meters, e.g. 2.55
|
||||
floor: int | None = None
|
||||
total_floors: int | None = None
|
||||
|
||||
# Address (full)
|
||||
address: str | None = None
|
||||
|
||||
# Description (full text)
|
||||
description: str | None = None
|
||||
|
||||
# Repair state — enum, inferred from description text (#622).
|
||||
# Yandex не отдаёт структурного поля ремонта, поэтому только инференс.
|
||||
repair_state: str | None = None
|
||||
|
||||
# Stats
|
||||
views_total: int | None = None
|
||||
publish_date: date | None = None
|
||||
publish_date_relative: str | None = None
|
||||
|
||||
# Agency block (OfferCardAuthorInfo)
|
||||
agency_name: str | None = None
|
||||
agency_founded_year: int | None = None
|
||||
agency_objects_count: int | None = None
|
||||
seller_name: str | None = None # last text line before "Агентство «...»"
|
||||
|
||||
# Metro stations from "Расположение" section
|
||||
metro_stations: list[MetroStation] = Field(default_factory=list)
|
||||
|
||||
# Photos — 8 sizes from Product.image[]
|
||||
photo_urls: list[str] = Field(default_factory=list)
|
||||
|
||||
# Newbuilding linkage
|
||||
newbuilding_url: str | None = None
|
||||
newbuilding_id: str | None = None
|
||||
|
||||
# NLP from description (best-effort)
|
||||
house_type_nlp: str | None = None
|
||||
year_built_hint: int | None = None
|
||||
metro_walk_min: int | None = None
|
||||
|
||||
# Raw payload (trimmed)
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ── Scraper ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class YandexDetailScraper(BaseScraper):
|
||||
"""Detail page scraper for realty.yandex.ru."""
|
||||
|
||||
name = "yandex_detail"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
|
||||
# BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead."
|
||||
)
|
||||
|
||||
async def fetch_detail(self, offer_url: str) -> DetailEnrichment | None:
|
||||
try:
|
||||
response = await self._http_get(offer_url)
|
||||
except Exception:
|
||||
logger.exception("yandex detail fetch failed: %s", offer_url)
|
||||
return None
|
||||
if response.status_code != 200:
|
||||
logger.warning("yandex detail returned %d for %s", response.status_code, offer_url)
|
||||
return None
|
||||
result = self.parse(response.text, offer_url=offer_url)
|
||||
await self.sleep_between_requests()
|
||||
return result
|
||||
|
||||
def parse(self, html: str, offer_url: str) -> DetailEnrichment | None:
|
||||
offer_id_match = re.search(r"/offer/(\d+)/?", offer_url)
|
||||
if not offer_id_match:
|
||||
logger.warning("offer_url has no /offer/<id>/: %s", offer_url)
|
||||
return None
|
||||
offer_id = offer_id_match.group(1)
|
||||
|
||||
tree = HTMLParser(html)
|
||||
|
||||
# --- Product JSON-LD (authoritative price) ---
|
||||
product = find_ld_by_type(html, "Product") or {}
|
||||
offers_ld = product.get("offers") or {}
|
||||
if isinstance(offers_ld, list) and offers_ld:
|
||||
offers_ld = offers_ld[0]
|
||||
price_ld = offers_ld.get("price") if isinstance(offers_ld, dict) else None
|
||||
try:
|
||||
price_rub = int(price_ld) if price_ld else None
|
||||
except (TypeError, ValueError):
|
||||
price_rub = None
|
||||
|
||||
# Photos from JSON-LD image[] (typically 8 size variants)
|
||||
images = product.get("image") or []
|
||||
if isinstance(images, str):
|
||||
images = [images]
|
||||
photo_urls = [u for u in images if isinstance(u, str)]
|
||||
|
||||
# --- Title + summary ---
|
||||
title_node = tree.css_first("h1")
|
||||
title = title_node.text(strip=True) if title_node else None
|
||||
|
||||
rooms, area_m2, floor, total_floors = _parse_title(title or "")
|
||||
|
||||
# --- Structural offer card (window.INITIAL_STATE → offerCard.card) ---
|
||||
# Authoritative source for area/floor/ceiling/kitchen — the h1 title
|
||||
# misses non-standard layouts (студия / свободная планировка) and never
|
||||
# carries ceiling/kitchen/living. Title stays as fallback below.
|
||||
living_area_m2: float | None = None
|
||||
kitchen_area_m2: float | None = None
|
||||
ceiling_height: float | None = None
|
||||
card = _extract_offer_card(html, offer_id)
|
||||
if card is not None:
|
||||
(
|
||||
c_rooms,
|
||||
c_area,
|
||||
c_living,
|
||||
c_kitchen,
|
||||
c_ceiling,
|
||||
c_floor,
|
||||
c_total_floors,
|
||||
) = _parse_card_fields(card)
|
||||
# Structural source wins; title only fills the gaps it left.
|
||||
rooms = c_rooms if c_rooms is not None else rooms
|
||||
area_m2 = c_area if c_area is not None else area_m2
|
||||
living_area_m2 = c_living
|
||||
kitchen_area_m2 = c_kitchen
|
||||
ceiling_height = c_ceiling
|
||||
floor = c_floor if c_floor is not None else floor
|
||||
total_floors = c_total_floors if c_total_floors is not None else total_floors
|
||||
|
||||
# --- OfferCardSummary text block ---
|
||||
summary_node = tree.css_first('[data-test="OfferCardSummary"]')
|
||||
summary_text = summary_node.text(strip=True) if summary_node else ""
|
||||
|
||||
# Views + relative publish date from summary text
|
||||
views_match = RE_VIEWS.search(summary_text)
|
||||
views_total = int(views_match.group(1)) if views_match else None
|
||||
publish_date = parse_ru_date(summary_text)
|
||||
publish_date_relative = _extract_relative_date(summary_text)
|
||||
|
||||
# price_per_m2 — from summary text if absent in LD
|
||||
price_per_m2: int | None = None
|
||||
ppm2_match = re.search(r"(\d[\d\s]+)\s*₽\s*за\s*м²", summary_text)
|
||||
if ppm2_match:
|
||||
price_per_m2 = parse_rub(ppm2_match.group(1))
|
||||
|
||||
# --- OfferCardAuthorInfo (agency block) ---
|
||||
author_node = tree.css_first('[data-test="OfferCardAuthorInfo"]')
|
||||
agency_name: str | None = None
|
||||
agency_founded_year: int | None = None
|
||||
agency_objects_count: int | None = None
|
||||
seller_name: str | None = None
|
||||
if author_node is not None:
|
||||
author_text = author_node.text(strip=True)
|
||||
agency_h2 = author_node.css_first("h2")
|
||||
agency_name = agency_h2.text(strip=True) if agency_h2 else None
|
||||
founded_m = RE_AGENCY_FOUNDED.search(author_text)
|
||||
if founded_m:
|
||||
agency_founded_year = int(founded_m.group(1))
|
||||
objects_m = RE_AGENCY_OBJECTS.search(author_text)
|
||||
if objects_m:
|
||||
agency_objects_count = int(objects_m.group(1))
|
||||
# seller_name — last text line before "Агентство"
|
||||
seller_name = _extract_seller_name(summary_text, agency_name)
|
||||
|
||||
# --- Description section (after H2 "Описание") ---
|
||||
description = _find_section_text(tree, "Описание")
|
||||
|
||||
# --- Repair state: инференс из описания (#622), Yandex без структурного поля ---
|
||||
repair_state = infer_repair_state_from_text(description or summary_text)
|
||||
|
||||
# --- Address ---
|
||||
address = _extract_address(summary_text)
|
||||
|
||||
# --- Metro stations from "Расположение" section ---
|
||||
location_text = _find_section_text(tree, "Расположение") or ""
|
||||
metro_stations = _parse_metro_stations(location_text)
|
||||
|
||||
# --- Newbuilding link ---
|
||||
nb_url: str | None = None
|
||||
nb_id: str | None = None
|
||||
nb_link = tree.css_first('a[href*="/kupit/novostrojka/"]')
|
||||
if nb_link is not None:
|
||||
nb_href = nb_link.attributes.get("href", "")
|
||||
nb_match = re.search(r"/novostrojka/[\w-]+?-(\d+)/?", nb_href)
|
||||
if nb_match:
|
||||
nb_id = nb_match.group(1)
|
||||
nb_url = (
|
||||
nb_href if nb_href.startswith("http") else f"https://realty.yandex.ru{nb_href}"
|
||||
)
|
||||
|
||||
# --- NLP best-effort from description ---
|
||||
nlp_text = description or summary_text
|
||||
house_type_nlp = parse_house_type(nlp_text)
|
||||
year_hint_m = RE_YEAR.search(nlp_text or "")
|
||||
year_built_hint = int(year_hint_m.group(1)) if year_hint_m else None
|
||||
walk_m = RE_METRO_WALK.search(nlp_text or "")
|
||||
metro_walk_min = int(walk_m.group(1)) if walk_m else None
|
||||
|
||||
return DetailEnrichment(
|
||||
offer_id=offer_id,
|
||||
source_url=offer_url,
|
||||
price_rub=price_rub,
|
||||
price_per_m2=price_per_m2,
|
||||
title=title,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
living_area_m2=living_area_m2,
|
||||
kitchen_area_m2=kitchen_area_m2,
|
||||
ceiling_height=ceiling_height,
|
||||
floor=floor,
|
||||
total_floors=total_floors,
|
||||
address=address,
|
||||
description=description,
|
||||
repair_state=repair_state,
|
||||
views_total=views_total,
|
||||
publish_date=publish_date,
|
||||
publish_date_relative=publish_date_relative,
|
||||
agency_name=agency_name,
|
||||
agency_founded_year=agency_founded_year,
|
||||
agency_objects_count=agency_objects_count,
|
||||
seller_name=seller_name,
|
||||
metro_stations=metro_stations,
|
||||
photo_urls=photo_urls,
|
||||
newbuilding_url=nb_url,
|
||||
newbuilding_id=nb_id,
|
||||
house_type_nlp=house_type_nlp,
|
||||
year_built_hint=year_built_hint,
|
||||
metro_walk_min=metro_walk_min,
|
||||
raw_payload={
|
||||
"summary_text": summary_text[:1000],
|
||||
"description_len": len(description) if description else 0,
|
||||
"photo_count": len(photo_urls),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int | None]:
|
||||
"""Extract (rooms, area_m2, floor, total_floors) from h1 text."""
|
||||
rooms: int | None = None
|
||||
area_m2: float | None = None
|
||||
floor: int | None = None
|
||||
total_floors: int | None = None
|
||||
|
||||
area_m = re.search(r"(\d+[.,]?\d*)\s*м²", title)
|
||||
if area_m:
|
||||
area_m2 = float(area_m.group(1).replace(",", "."))
|
||||
|
||||
if re.search(r"студи[яюй]", title, re.IGNORECASE):
|
||||
rooms = 0
|
||||
else:
|
||||
rooms_m = re.search(r"(\d+)\s*-?\s*комнатн", title, re.IGNORECASE)
|
||||
if rooms_m:
|
||||
try:
|
||||
rooms = int(rooms_m.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
floor_m = re.search(r"(\d+)\s+этаж\s+из\s+(\d+)", title, re.IGNORECASE)
|
||||
if floor_m:
|
||||
floor = int(floor_m.group(1))
|
||||
total_floors = int(floor_m.group(2))
|
||||
|
||||
return rooms, area_m2, floor, total_floors
|
||||
|
||||
|
||||
def _extract_js_object(html: str, marker: str) -> str | None:
|
||||
"""Return the JSON object literal assigned to `marker` (e.g. window.INITIAL_STATE).
|
||||
|
||||
Brace-matches from the first ``{`` after ``marker = ...`` to its balanced
|
||||
close, respecting string literals/escapes. Returns the raw JSON text or None.
|
||||
"""
|
||||
i = html.find(marker)
|
||||
if i < 0:
|
||||
return None
|
||||
eq = html.find("=", i)
|
||||
if eq < 0:
|
||||
return None
|
||||
start = html.find("{", eq)
|
||||
if start < 0:
|
||||
return None
|
||||
bal = 0
|
||||
in_str = False
|
||||
esc = False
|
||||
k = start
|
||||
n = len(html)
|
||||
while k < n:
|
||||
c = html[k]
|
||||
if in_str:
|
||||
if esc:
|
||||
esc = False
|
||||
elif c == "\\":
|
||||
esc = True
|
||||
elif c == '"':
|
||||
in_str = False
|
||||
else:
|
||||
if c == '"':
|
||||
in_str = True
|
||||
elif c == "{":
|
||||
bal += 1
|
||||
elif c == "}":
|
||||
bal -= 1
|
||||
if bal == 0:
|
||||
return html[start : k + 1]
|
||||
k += 1
|
||||
return None
|
||||
|
||||
|
||||
def _find_card_by_offer_id(obj: Any, offer_id: str) -> dict[str, Any] | None:
|
||||
"""Recursively locate the offer dict whose offerId matches and carries `area`."""
|
||||
if isinstance(obj, dict):
|
||||
if obj.get("offerId") == offer_id and "area" in obj:
|
||||
return obj
|
||||
for v in obj.values():
|
||||
found = _find_card_by_offer_id(v, offer_id)
|
||||
if found is not None:
|
||||
return found
|
||||
elif isinstance(obj, list):
|
||||
for v in obj:
|
||||
found = _find_card_by_offer_id(v, offer_id)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _extract_offer_card(html: str, offer_id: str) -> dict[str, Any] | None:
|
||||
"""Extract the offer card object from window.INITIAL_STATE.
|
||||
|
||||
Yandex embeds full structured offer data in ``window.INITIAL_STATE`` under
|
||||
``offerCard.card``. This is the authoritative source for area / floor /
|
||||
ceiling / kitchen — unlike the h1 title which misses non-standard layouts
|
||||
and never carries ceiling/kitchen. Returns the card dict or None.
|
||||
"""
|
||||
blob = _extract_js_object(html, "window.INITIAL_STATE")
|
||||
if not blob:
|
||||
return None
|
||||
try:
|
||||
state = json.loads(blob)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
logger.warning("yandex detail: INITIAL_STATE failed to parse for offer %s", offer_id)
|
||||
return None
|
||||
# Fast path: canonical location.
|
||||
offer_card = state.get("offerCard") if isinstance(state, dict) else None
|
||||
if isinstance(offer_card, dict):
|
||||
card = offer_card.get("card")
|
||||
if isinstance(card, dict) and card.get("offerId") == offer_id:
|
||||
return card
|
||||
# Fallback: deep search (page structure may differ).
|
||||
return _find_card_by_offer_id(state, offer_id)
|
||||
|
||||
|
||||
def _space_value(node: Any) -> float | None:
|
||||
"""Yandex area fields are ``{"value": N, "unit": "SQUARE_METER"}`` → float."""
|
||||
if isinstance(node, dict):
|
||||
val = node.get("value")
|
||||
if isinstance(val, int | float):
|
||||
return float(val)
|
||||
elif isinstance(node, int | float):
|
||||
return float(node)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_card_fields(
|
||||
card: dict[str, Any],
|
||||
) -> tuple[
|
||||
int | None,
|
||||
float | None,
|
||||
float | None,
|
||||
float | None,
|
||||
float | None,
|
||||
int | None,
|
||||
int | None,
|
||||
]:
|
||||
"""Extract (rooms, area, living, kitchen, ceiling, floor, total_floors) from card.
|
||||
|
||||
rooms: ``roomsTotal``, or 0 when ``house.studio`` is truthy (студии не
|
||||
несут roomsTotal). floor: first element of ``floorsOffered``.
|
||||
"""
|
||||
area = _space_value(card.get("area"))
|
||||
living = _space_value(card.get("livingSpace"))
|
||||
kitchen = _space_value(card.get("kitchenSpace"))
|
||||
|
||||
ceiling_raw = card.get("ceilingHeight")
|
||||
ceiling: float | None = None
|
||||
if isinstance(ceiling_raw, int | float):
|
||||
ceiling = float(ceiling_raw)
|
||||
|
||||
rooms_raw = card.get("roomsTotal")
|
||||
rooms: int | None = None
|
||||
if isinstance(rooms_raw, int):
|
||||
rooms = rooms_raw
|
||||
house = card.get("house")
|
||||
if isinstance(house, dict) and house.get("studio"):
|
||||
rooms = 0
|
||||
|
||||
total_floors_raw = card.get("floorsTotal")
|
||||
total_floors = total_floors_raw if isinstance(total_floors_raw, int) else None
|
||||
|
||||
floor: int | None = None
|
||||
floors_offered = card.get("floorsOffered")
|
||||
if isinstance(floors_offered, list) and floors_offered:
|
||||
first = floors_offered[0]
|
||||
if isinstance(first, int):
|
||||
floor = first
|
||||
|
||||
return rooms, area, living, kitchen, ceiling, floor, total_floors
|
||||
|
||||
|
||||
def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
|
||||
"""Find the text content of a <section>/<div> whose preceding h2/h3 matches heading.
|
||||
|
||||
Yandex page structure varies; this scans h2/h3 nodes, then returns the
|
||||
concatenated text of subsequent sibling blocks until the next heading.
|
||||
"""
|
||||
for h in tree.css("h2, h3"):
|
||||
if heading.lower() in (h.text(strip=True) or "").lower():
|
||||
# collect subsequent siblings until the next h2/h3
|
||||
parts: list[str] = []
|
||||
node: Node | None = h.next
|
||||
while node is not None:
|
||||
tag = (node.tag or "").lower()
|
||||
if tag in {"h2", "h3"}:
|
||||
break
|
||||
txt = node.text(strip=True) if hasattr(node, "text") else ""
|
||||
if txt:
|
||||
parts.append(txt)
|
||||
node = node.next
|
||||
return " ".join(parts).strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def _extract_address(summary_text: str) -> str | None:
|
||||
"""Best-effort address extraction from summary block."""
|
||||
# Pattern: "Россия, Свердловская область, Екатеринбург, улица Х, д. N"
|
||||
m = re.search(r"(Россия[^•]+?)(?:•|\d+\s+просмотр|$)", summary_text)
|
||||
if m:
|
||||
addr = m.group(1).strip().rstrip(",").strip()
|
||||
return addr if len(addr) > 10 else None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_metro_stations(location_text: str) -> list[MetroStation]:
|
||||
"""Parse 'Уральская 11 мин. Динамо 16 мин.' → list of MetroStation."""
|
||||
stations: list[MetroStation] = []
|
||||
# name (1+ Cyrillic words) + space + N + space + мин(.|у|ут)
|
||||
for m in re.finditer(r"([А-ЯЁ][А-Яа-яё\s-]+?)\s+(\d+)\s*мин", location_text):
|
||||
name = m.group(1).strip()
|
||||
if 2 <= len(name) <= 40:
|
||||
stations.append(MetroStation(name=name, walk_min=int(m.group(2))))
|
||||
if len(stations) >= 5:
|
||||
break
|
||||
return stations
|
||||
|
||||
|
||||
def _extract_relative_date(summary_text: str) -> str | None:
|
||||
"""Capture phrases like '6 часов назад' / 'вчера' / '3 дня назад'."""
|
||||
m = re.search(
|
||||
r"(\d+\s+(?:минут|час|часов|часа|день|дня|дней|недел[ьюи])\s+назад"
|
||||
r"|вчера|сегодня|позавчера)",
|
||||
summary_text,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
return m.group(1).strip() if m else None
|
||||
|
||||
|
||||
def _extract_seller_name(summary_text: str, agency_name: str | None) -> str | None:
|
||||
"""Heuristic: line right before 'Агентство ...' in summary text."""
|
||||
if not agency_name or agency_name not in summary_text:
|
||||
return None
|
||||
head = summary_text.split(agency_name, 1)[0]
|
||||
# last short token sequence (likely "Имя Фамилия")
|
||||
m = re.findall(r"([А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+){1,2})", head)
|
||||
return m[-1] if m else None
|
||||
|
||||
|
||||
# ── Save helper ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) -> bool:
|
||||
"""Persist Yandex DetailEnrichment to listings row.
|
||||
|
||||
UPDATE listings SET <col> = COALESCE(:val, <col>), ..., detail_enriched_at = NOW()
|
||||
WHERE id = listing_id.
|
||||
|
||||
COALESCE semantics: keeps existing non-NULL value if new value is NULL (never
|
||||
overwrites a populated column with NULL). area_m2 from detail is more accurate
|
||||
than SERP, but COALESCE preserves SERP value if detail returns NULL — acceptable.
|
||||
|
||||
Returns True if the UPDATE touched at least one row (listing_id found in DB).
|
||||
"""
|
||||
metro_json: str | None = None
|
||||
if e.metro_stations:
|
||||
metro_json = json.dumps(
|
||||
[s.model_dump(exclude_none=True) for s in e.metro_stations],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
photo_json: str | None = None
|
||||
if e.photo_urls:
|
||||
photo_json = json.dumps(e.photo_urls, ensure_ascii=False)
|
||||
|
||||
result = db.execute(
|
||||
text("""
|
||||
UPDATE listings SET
|
||||
rooms = COALESCE(CAST(:rooms AS int), rooms),
|
||||
area_m2 = COALESCE(CAST(:area_m2 AS numeric), area_m2),
|
||||
living_area_m2 = COALESCE(
|
||||
CAST(:living_area_m2 AS numeric),
|
||||
living_area_m2
|
||||
),
|
||||
kitchen_area_m2 = COALESCE(
|
||||
CAST(:kitchen_area_m2 AS numeric),
|
||||
kitchen_area_m2
|
||||
),
|
||||
ceiling_height = COALESCE(
|
||||
CAST(:ceiling_height AS numeric),
|
||||
ceiling_height
|
||||
),
|
||||
floor = COALESCE(CAST(:floor AS int), floor),
|
||||
total_floors = COALESCE(CAST(:total_floors AS int), total_floors),
|
||||
address = COALESCE(CAST(:address AS text), address),
|
||||
description = COALESCE(CAST(:description AS text), description),
|
||||
repair_state = COALESCE(CAST(:repair_state AS text),repair_state),
|
||||
publish_date = COALESCE(CAST(:publish_date AS date),publish_date),
|
||||
views_total_yandex = COALESCE(CAST(:views_total AS int), views_total_yandex),
|
||||
publish_date_relative= COALESCE(
|
||||
CAST(:pub_date_rel AS text),
|
||||
publish_date_relative
|
||||
),
|
||||
agency_name = COALESCE(CAST(:agency_name AS text), agency_name),
|
||||
agency_founded_year = COALESCE(
|
||||
CAST(:agency_founded_year AS int),
|
||||
agency_founded_year
|
||||
),
|
||||
agency_objects_count = COALESCE(
|
||||
CAST(:agency_objects_count AS int),
|
||||
agency_objects_count
|
||||
),
|
||||
metro_stations = COALESCE(
|
||||
CAST(:metro_stations AS jsonb),
|
||||
metro_stations
|
||||
),
|
||||
photo_urls = COALESCE(
|
||||
CAST(:photo_urls AS jsonb),
|
||||
photo_urls
|
||||
),
|
||||
newbuilding_url = COALESCE(
|
||||
CAST(:newbuilding_url AS text),
|
||||
newbuilding_url
|
||||
),
|
||||
newbuilding_id = COALESCE(
|
||||
CAST(:newbuilding_id AS text),
|
||||
newbuilding_id
|
||||
),
|
||||
detail_enriched_at = NOW()
|
||||
WHERE id = CAST(:listing_id AS bigint)
|
||||
"""),
|
||||
{
|
||||
"listing_id": listing_id,
|
||||
"rooms": e.rooms,
|
||||
"area_m2": e.area_m2,
|
||||
"living_area_m2": e.living_area_m2,
|
||||
"kitchen_area_m2": e.kitchen_area_m2,
|
||||
"ceiling_height": e.ceiling_height,
|
||||
"floor": e.floor,
|
||||
"total_floors": e.total_floors,
|
||||
"address": e.address,
|
||||
"description": e.description,
|
||||
"repair_state": e.repair_state,
|
||||
"publish_date": e.publish_date,
|
||||
"views_total": e.views_total,
|
||||
"pub_date_rel": e.publish_date_relative,
|
||||
"agency_name": e.agency_name,
|
||||
"agency_founded_year": e.agency_founded_year,
|
||||
"agency_objects_count": e.agency_objects_count,
|
||||
"metro_stations": metro_json,
|
||||
"photo_urls": photo_json,
|
||||
"newbuilding_url": e.newbuilding_url,
|
||||
"newbuilding_id": e.newbuilding_id,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
saved = (result.rowcount or 0) > 0
|
||||
logger.info(
|
||||
"yandex detail enrichment saved listing_id=%s (metro=%d photos=%d saved=%s)",
|
||||
listing_id,
|
||||
len(e.metro_stations),
|
||||
len(e.photo_urls),
|
||||
saved,
|
||||
)
|
||||
return saved
|
||||
|
|
@ -1,526 +0,0 @@
|
|||
"""Yandex Realty anonymous valuation/history scraper.
|
||||
|
||||
URL pattern:
|
||||
GET /otsenka-kvartiry-po-adresu-onlayn/?address=...&offerCategory=...&offerType=...&page=N
|
||||
|
||||
No cookies, no auth required (unlike Cian Calculator or Avito IMV).
|
||||
|
||||
Extracts:
|
||||
- House metadata: year_built, total_floors, house_type, ceiling_height, has_lift, total_objects
|
||||
- Historical offers: area, rooms, floor, start/last price + per-m2, publish_date DMY, exposure
|
||||
|
||||
Two-strategy history extraction:
|
||||
1. data-test container selectors (CSS, if Yandex adds them)
|
||||
2. Fallback: text chunks around DD.MM.YYYY date matches with dedup by (date, area, floor)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper
|
||||
from app.services.scrapers.yandex_helpers import (
|
||||
parse_dmy,
|
||||
parse_house_type,
|
||||
parse_rub,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ValuationHouseMeta(BaseModel):
|
||||
"""House metadata extracted from Yandex valuation page."""
|
||||
|
||||
year_built: int | None = None
|
||||
total_floors: int | None = None
|
||||
house_type: str | None = None # panel/brick/monolith/...
|
||||
ceiling_height: float | None = None # in meters, e.g. 2.5
|
||||
has_lift: bool | None = None
|
||||
total_objects: int | None = None # 'N объектов' (full archive count)
|
||||
has_panorama: bool = False # 'Панорама' label present
|
||||
|
||||
def validate_match(
|
||||
self,
|
||||
expected_year_built: int | None = None,
|
||||
expected_total_floors: int | None = None,
|
||||
) -> float:
|
||||
"""Return confidence 0..1 that this house meta matches expected values.
|
||||
|
||||
Used after fetching valuation by address to detect when Yandex returned a
|
||||
different house (ambiguous address geocoding). Tolerance ±1 year, ±1 floor.
|
||||
|
||||
Both expected=None → 1.0 (no check). Mismatch on any dimension → 0.0 for it.
|
||||
"""
|
||||
score = 0.0
|
||||
checks = 0
|
||||
if expected_year_built is not None:
|
||||
checks += 1
|
||||
if self.year_built is not None and abs(self.year_built - expected_year_built) <= 1:
|
||||
score += 1.0
|
||||
if expected_total_floors is not None:
|
||||
checks += 1
|
||||
if (
|
||||
self.total_floors is not None
|
||||
and abs(self.total_floors - expected_total_floors) <= 1
|
||||
):
|
||||
score += 1.0
|
||||
if checks == 0:
|
||||
return 1.0
|
||||
return score / checks
|
||||
|
||||
|
||||
class ValuationHistoryItem(BaseModel):
|
||||
"""One historical offer entry from the valuation page."""
|
||||
|
||||
area_m2: float | None = None
|
||||
rooms: int | None = None
|
||||
floor: int | None = None
|
||||
start_price: int | None = None
|
||||
start_price_per_m2: int | None = None
|
||||
last_price: int | None = None
|
||||
last_price_per_m2: int | None = None
|
||||
publish_date: date | None = None
|
||||
removed_date: date | None = None # ← NEW
|
||||
exposure_days: int | None = None
|
||||
status: str | None = None # 'В продаже' / 'Снято'
|
||||
|
||||
|
||||
class YandexValuationResult(BaseModel):
|
||||
"""Full result of one /otsenka-... GET."""
|
||||
|
||||
address: str
|
||||
offer_category: str
|
||||
offer_type: str
|
||||
page: int
|
||||
source_url: str
|
||||
house: ValuationHouseMeta
|
||||
history_items: list[ValuationHistoryItem] = Field(default_factory=list)
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Regex constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RE_YEAR_BUILT = re.compile(r"Дом\s+(\d{4})\s+года", re.IGNORECASE)
|
||||
RE_FLOORS = re.compile(r"(\d+)\s+этажей", re.IGNORECASE)
|
||||
RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE)
|
||||
RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
|
||||
|
||||
# Lookbehind blocks digit-only adjacency (rejects year-concat like '20244,6')
|
||||
# while still allowing tokens preceded by punctuation/letter. Min 2 digits
|
||||
# rejects sub-fragments like '2,2' that come from inside '52,2'. Max 4 digits
|
||||
# whole part catches obvious junk (penthouse extremes are <500 m² in practice;
|
||||
# sanity cap from PR #538 backs this up).
|
||||
RE_ITEM_AREA = re.compile(r"(?<!\d)(\d{2,4}(?:[.,]\d{1,2})?)\s*м²")
|
||||
RE_ITEM_ROOMS = re.compile(r"(\d+)\s*-\s*комнатн", re.IGNORECASE)
|
||||
RE_ITEM_STUDIO = re.compile(r"студи[яюй]", re.IGNORECASE)
|
||||
RE_ITEM_FLOOR = re.compile(r"(\d+)\s*этаж", re.IGNORECASE)
|
||||
RE_ITEM_EXPOSURE = re.compile(r"экспозиции\s+(\d+)\s+дн", re.IGNORECASE)
|
||||
RE_ITEM_STATUS = re.compile(r"(В\s+продаже|Снят[оа])", re.IGNORECASE)
|
||||
|
||||
# Matches total-price tokens (rubles) — excludes per-m2 tokens by negative lookahead
|
||||
_RE_PRICE_TOKEN = re.compile(r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)")
|
||||
_RE_PPM2_TOKEN = re.compile(r"\d[\d\s]*\s*₽\s*за\s*м²")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scraper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class YandexValuationScraper(BaseScraper):
|
||||
"""Anonymous Yandex valuation/history scraper.
|
||||
|
||||
Fetches https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/ by address.
|
||||
Pagination via ?page=N. Use fetch_house_history() directly;
|
||||
fetch_around() is not applicable to this tool.
|
||||
"""
|
||||
|
||||
name = "yandex_valuation"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
valuation_path = "/otsenka-kvartiry-po-adresu-onlayn/"
|
||||
request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
self._cffi_session: _CurlCffiSession | None = None
|
||||
|
||||
async def __aenter__(self) -> YandexValuationScraper: # type: ignore[override]
|
||||
# Override: Yandex valuation endpoint gates SSR data on Chrome TLS
|
||||
# fingerprint. Plain httpx returns shell HTML (CSR-only).
|
||||
# Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py
|
||||
# Mobile proxy wiring (#806 follow-up): route via mobile proxy to avoid
|
||||
# datacenter-IP blocks. proxy=None → прямое подключение (dev).
|
||||
from app.core.config import settings
|
||||
|
||||
_proxy_url = settings.scraper_proxy_url
|
||||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||
self._cffi_session = _CurlCffiSession(impersonate="chrome120", proxies=_proxies)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
||||
if self._cffi_session is not None:
|
||||
await self._cffi_session.close()
|
||||
self._cffi_session = None
|
||||
|
||||
async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override]
|
||||
"""curl_cffi-based GET with Chrome120 impersonation.
|
||||
|
||||
Returns curl_cffi Response (compatible API: .status_code, .text).
|
||||
Caller must check status_code; no automatic retry (BaseScraper.retry
|
||||
decorator is per-method and not inherited cleanly when overridden).
|
||||
"""
|
||||
if self._cffi_session is None:
|
||||
raise RuntimeError("YandexValuationScraper must be used as async context manager")
|
||||
kwargs.setdefault("timeout", 30)
|
||||
return await self._cffi_session.get(url, **kwargs)
|
||||
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexValuationScraper is address-based; use fetch_house_history() instead."
|
||||
)
|
||||
|
||||
async def fetch_house_history(
|
||||
self,
|
||||
address: str,
|
||||
offer_category: str = "APARTMENT",
|
||||
offer_type: str = "SELL",
|
||||
page: int = 1,
|
||||
) -> YandexValuationResult | None:
|
||||
"""Fetch and parse one page of house history for the given address.
|
||||
|
||||
Args:
|
||||
address: Postal address string (will be URL-encoded).
|
||||
offer_category: 'APARTMENT' / 'ROOMS' / etc.
|
||||
offer_type: 'SELL' / 'RENT'.
|
||||
page: 1-based pagination index.
|
||||
|
||||
Returns:
|
||||
YandexValuationResult or None on HTTP error / network failure.
|
||||
"""
|
||||
params = {
|
||||
"address": address,
|
||||
"offerCategory": offer_category,
|
||||
"offerType": offer_type,
|
||||
"page": page,
|
||||
}
|
||||
url = f"{self.base_url}{self.valuation_path}?{urlencode(params)}"
|
||||
try:
|
||||
response = await self._http_get(url)
|
||||
except Exception:
|
||||
logger.exception("yandex valuation fetch failed: %s", url)
|
||||
return None
|
||||
if response.status_code != 200:
|
||||
logger.warning("yandex valuation returned %d for %s", response.status_code, url)
|
||||
return None
|
||||
result = self.parse(
|
||||
response.text,
|
||||
address=address,
|
||||
offer_category=offer_category,
|
||||
offer_type=offer_type,
|
||||
page=page,
|
||||
source_url=url,
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
return result
|
||||
|
||||
def parse(
|
||||
self,
|
||||
html: str,
|
||||
address: str,
|
||||
offer_category: str,
|
||||
offer_type: str,
|
||||
page: int,
|
||||
source_url: str,
|
||||
) -> YandexValuationResult:
|
||||
"""Parse raw HTML into YandexValuationResult. Pure function — usable in unit tests."""
|
||||
html_normalized = html.replace("\xa0", " ")
|
||||
tree = HTMLParser(html_normalized)
|
||||
body = tree.body
|
||||
body_text = (body.text(strip=True) if body else "").replace("\xa0", " ")
|
||||
|
||||
house = self._parse_house_meta(body_text)
|
||||
history_items = self._parse_history_items(tree, body_text)
|
||||
|
||||
return YandexValuationResult(
|
||||
address=address,
|
||||
offer_category=offer_category,
|
||||
offer_type=offer_type,
|
||||
page=page,
|
||||
source_url=source_url,
|
||||
house=house,
|
||||
history_items=history_items,
|
||||
raw_payload={
|
||||
"body_len": len(body_text),
|
||||
"items_count": len(history_items),
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_house_meta(body_text: str) -> ValuationHouseMeta:
|
||||
"""Extract house-level metadata from page body text."""
|
||||
year_m = RE_YEAR_BUILT.search(body_text)
|
||||
floors_m = RE_FLOORS.search(body_text)
|
||||
ceiling_m = RE_CEILING.search(body_text)
|
||||
objects_m = RE_TOTAL_OBJECTS.search(body_text)
|
||||
return ValuationHouseMeta(
|
||||
year_built=int(year_m.group(1)) if year_m else None,
|
||||
total_floors=int(floors_m.group(1)) if floors_m else None,
|
||||
house_type=parse_house_type(body_text),
|
||||
ceiling_height=(float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None),
|
||||
has_lift="Лифт" in body_text,
|
||||
total_objects=int(objects_m.group(1)) if objects_m else None,
|
||||
has_panorama="Панорама" in body_text,
|
||||
)
|
||||
|
||||
def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]:
|
||||
"""Extract historical offer items.
|
||||
|
||||
Strategy 1 (preferred, structured): each row at .OffersArchiveSearchOffers__row,
|
||||
with 6 cells (.OffersArchiveSearchOffers__cell) in fixed order:
|
||||
[0] area+rooms, [1] floor, [2] start_price+ppm2, [3] last_price+ppm2,
|
||||
[4] publish_date+exposure, [5] status/removed_date.
|
||||
Yields per-cell parsing — no chunked-text ambiguity.
|
||||
|
||||
Strategy 2 (last-resort fallback): legacy chunked-text scan around
|
||||
DD.MM.YYYY anchors. Kept for the rare case Yandex changes class names;
|
||||
triggers a logger.warning so we notice.
|
||||
"""
|
||||
items: list[ValuationHistoryItem] = []
|
||||
for row in tree.css(".OffersArchiveSearchOffers__row"):
|
||||
item = self._parse_row_cells(row)
|
||||
if item:
|
||||
items.append(item)
|
||||
if items:
|
||||
return items
|
||||
# Fallback (only if structured selector failed)
|
||||
logger.warning(
|
||||
"yandex_valuation: .OffersArchiveSearchOffers__row matched 0 items — "
|
||||
"falling back to chunked-text scan"
|
||||
)
|
||||
return self._parse_items_from_chunked_text(body_text)
|
||||
|
||||
@classmethod
|
||||
def _parse_row_cells(cls, row) -> ValuationHistoryItem | None:
|
||||
"""Parse one .OffersArchiveSearchOffers__row into a ValuationHistoryItem.
|
||||
|
||||
Cell layout is positional (no labels). Returns None if the row doesn't
|
||||
match the expected 6-cell structure (defensive — Yandex may A/B-test).
|
||||
"""
|
||||
cells = row.css(".OffersArchiveSearchOffers__cell")
|
||||
if len(cells) < 5:
|
||||
return None
|
||||
cell_texts = [(c.text(strip=True) or "").replace("\xa0", " ") for c in cells]
|
||||
|
||||
# [0] photo (skipped — empty cell)
|
||||
# [1] title — single cell contains area + rooms + floor jammed together,
|
||||
# e.g. "48,5 м², 1-комнатная19 этаж" (real Yandex layout, 2026-05-24)
|
||||
title_text = cell_texts[1] if len(cell_texts) > 1 else ""
|
||||
|
||||
area_m = RE_ITEM_AREA.search(title_text)
|
||||
area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
# Sanity drop (belt-and-suspenders from PR #538)
|
||||
if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0):
|
||||
area_m2 = None
|
||||
|
||||
rooms: int | None = None
|
||||
if RE_ITEM_STUDIO.search(title_text):
|
||||
rooms = 0
|
||||
else:
|
||||
rooms_m = RE_ITEM_ROOMS.search(title_text)
|
||||
if rooms_m:
|
||||
rooms = int(rooms_m.group(1))
|
||||
|
||||
floor_m = RE_ITEM_FLOOR.search(title_text)
|
||||
floor: int | None = int(floor_m.group(1)) if floor_m else None
|
||||
|
||||
# [2] start price + ppm2 — "5,1 млн ₽105 155 ₽ за м²" (no separator)
|
||||
start_price: int | None = None
|
||||
start_ppm2: int | None = None
|
||||
if len(cell_texts) > 2:
|
||||
tokens = _RE_PRICE_TOKEN.findall(cell_texts[2])
|
||||
if tokens:
|
||||
start_price = parse_rub(tokens[0])
|
||||
ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[2])
|
||||
if ppm2_tokens:
|
||||
start_ppm2 = parse_rub(ppm2_tokens[0])
|
||||
|
||||
# [3] last price + ppm2
|
||||
last_price: int | None = None
|
||||
last_ppm2: int | None = None
|
||||
if len(cell_texts) > 3:
|
||||
tokens = _RE_PRICE_TOKEN.findall(cell_texts[3])
|
||||
if tokens:
|
||||
last_price = parse_rub(tokens[0])
|
||||
ppm2_tokens = _RE_PPM2_TOKEN.findall(cell_texts[3])
|
||||
if ppm2_tokens:
|
||||
last_ppm2 = parse_rub(ppm2_tokens[0])
|
||||
|
||||
# [4] publish_date + exposure ("23.10.2023В экспозиции 945 дней")
|
||||
publish_date = None
|
||||
exposure_days: int | None = None
|
||||
if len(cell_texts) > 4:
|
||||
publish_date = parse_dmy(cell_texts[4])
|
||||
expo_m = RE_ITEM_EXPOSURE.search(cell_texts[4])
|
||||
if expo_m:
|
||||
exposure_days = int(expo_m.group(1))
|
||||
|
||||
# [5] status or removed_date
|
||||
removed_date = None
|
||||
status: str | None = None
|
||||
if len(cell_texts) > 5:
|
||||
last_cell = cell_texts[5]
|
||||
status_m = RE_ITEM_STATUS.search(last_cell)
|
||||
if status_m:
|
||||
status = status_m.group(1)
|
||||
removed_date = parse_dmy(last_cell)
|
||||
|
||||
# Sort dates chronologically (in case Yandex flips them — same fix as PR #541)
|
||||
if publish_date and removed_date and removed_date < publish_date:
|
||||
publish_date, removed_date = removed_date, publish_date
|
||||
|
||||
# Row is valid only if we got area OR start_price
|
||||
if area_m2 is None and start_price is None:
|
||||
return None
|
||||
|
||||
return ValuationHistoryItem(
|
||||
area_m2=area_m2,
|
||||
rooms=rooms,
|
||||
floor=floor,
|
||||
start_price=start_price,
|
||||
start_price_per_m2=start_ppm2,
|
||||
last_price=last_price,
|
||||
last_price_per_m2=last_ppm2,
|
||||
publish_date=publish_date,
|
||||
removed_date=removed_date,
|
||||
exposure_days=exposure_days,
|
||||
status=status,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_item_text(text: str) -> ValuationHistoryItem | None:
|
||||
"""Parse a single text block into a ValuationHistoryItem.
|
||||
|
||||
Returns None if neither area nor start_price can be extracted
|
||||
(item is considered invalid/noise).
|
||||
"""
|
||||
if not text or len(text) < 20:
|
||||
return None
|
||||
|
||||
# area + rooms
|
||||
area_m = RE_ITEM_AREA.search(text)
|
||||
area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
# Sanity cap — Yandex page text sometimes concatenates digits across DOM
|
||||
# boundaries (e.g. "2025\xa0106,7 м²" → 2_025_106.7). Flats over 10_000 m²
|
||||
# are impossible; DB column is NUMERIC(8,2) which overflows at 10^6.
|
||||
# Drop the value rather than block the whole save batch.
|
||||
if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0):
|
||||
logger.warning(
|
||||
"yandex_valuation: dropping nonsensical area_m2=%s from item chunk",
|
||||
area_m2,
|
||||
)
|
||||
area_m2 = None
|
||||
|
||||
rooms: int | None = None
|
||||
if RE_ITEM_STUDIO.search(text):
|
||||
rooms = 0
|
||||
else:
|
||||
rooms_m = RE_ITEM_ROOMS.search(text)
|
||||
if rooms_m:
|
||||
rooms = int(rooms_m.group(1))
|
||||
|
||||
floor_m = RE_ITEM_FLOOR.search(text)
|
||||
floor = int(floor_m.group(1)) if floor_m else None
|
||||
|
||||
# Prices — find first 2 price tokens (start, last)
|
||||
price_tokens = _RE_PRICE_TOKEN.findall(text)
|
||||
start_price = parse_rub(price_tokens[0]) if len(price_tokens) >= 1 else None
|
||||
last_price = parse_rub(price_tokens[1]) if len(price_tokens) >= 2 else None
|
||||
|
||||
ppm2_tokens = _RE_PPM2_TOKEN.findall(text)
|
||||
start_ppm2 = parse_rub(ppm2_tokens[0]) if len(ppm2_tokens) >= 1 else None
|
||||
last_ppm2 = parse_rub(ppm2_tokens[1]) if len(ppm2_tokens) >= 2 else None
|
||||
|
||||
# Extract ALL DD.MM.YYYY dates: first → publish_date, second → removed_date
|
||||
date_matches = list(re.finditer(r"\d{2}\.\d{2}\.\d{4}", text))
|
||||
dates_parsed: list[date] = []
|
||||
for m in date_matches:
|
||||
d = parse_dmy(m.group(0))
|
||||
if d is not None:
|
||||
dates_parsed.append(d)
|
||||
# Sort dates chronologically — chunked-text scan returns them in page-text
|
||||
# order, but semantically publish_date is earliest and removed_date is
|
||||
# latest. Without this, ~1% of rows on prod show removed < publish.
|
||||
if dates_parsed:
|
||||
dates_sorted = sorted(d for d in dates_parsed if d is not None)
|
||||
publish_date = dates_sorted[0] if dates_sorted else None
|
||||
removed_date = dates_sorted[1] if len(dates_sorted) >= 2 else None
|
||||
else:
|
||||
publish_date = None
|
||||
removed_date = None
|
||||
expo_m = RE_ITEM_EXPOSURE.search(text)
|
||||
exposure_days = int(expo_m.group(1)) if expo_m else None
|
||||
|
||||
status_m = RE_ITEM_STATUS.search(text)
|
||||
status = status_m.group(1) if status_m else None
|
||||
|
||||
# Item is valid only if we got at least area or price
|
||||
if area_m2 is None and start_price is None:
|
||||
return None
|
||||
|
||||
return ValuationHistoryItem(
|
||||
area_m2=area_m2,
|
||||
rooms=rooms,
|
||||
floor=floor,
|
||||
start_price=start_price,
|
||||
start_price_per_m2=start_ppm2,
|
||||
last_price=last_price,
|
||||
last_price_per_m2=last_ppm2,
|
||||
publish_date=publish_date,
|
||||
removed_date=removed_date,
|
||||
exposure_days=exposure_days,
|
||||
status=status,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _parse_items_from_chunked_text(cls, body_text: str) -> list[ValuationHistoryItem]:
|
||||
"""Fallback: split body text into chunks around DD.MM.YYYY dates and parse each chunk.
|
||||
|
||||
Window: 200 chars before + 100 chars after each date match.
|
||||
Deduplicates by (publish_date, area_m2, floor).
|
||||
Caps output at 30 items per page to avoid runaway extraction.
|
||||
"""
|
||||
items: list[ValuationHistoryItem] = []
|
||||
for m in re.finditer(r"\d{2}\.\d{2}\.\d{4}", body_text):
|
||||
start = max(0, m.start() - 200)
|
||||
end = min(len(body_text), m.end() + 100)
|
||||
chunk = body_text[start:end]
|
||||
item = cls._parse_item_text(chunk)
|
||||
if item:
|
||||
items.append(item)
|
||||
|
||||
# Dedup by (publish_date, area_m2, floor)
|
||||
seen: set[tuple[Any, Any, Any]] = set()
|
||||
deduped: list[ValuationHistoryItem] = []
|
||||
for item in items:
|
||||
key = (item.publish_date, item.area_m2, item.floor)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(item)
|
||||
return deduped[:30]
|
||||
|
|
@ -1,18 +1,22 @@
|
|||
"""Parity-proof: admin.py Avito debug-роуты (#2305, Group A) — legacy vs scraper_kit.
|
||||
|
||||
`app/api/v1/admin.py::scrape_avito_house` / `scrape_avito_imv` теперь зовут
|
||||
`scraper_kit.providers.avito.{houses,imv}` вместо `app.services.scrapers.avito_*`
|
||||
(тот же `/scrape` root-роут — `scraper_kit.providers.avito.serp.AvitoScraper`).
|
||||
Каждая пара функций ниже — чистое, offline-тестируемое parsing-ядро, которое
|
||||
реально используется соответствующим debug-эндпоинтом (не сам HTTP-fetch —
|
||||
см. `tests/support/README.md`: "Live network/DB в parity-тестах избегайте").
|
||||
`app/api/v1/admin.py::scrape_avito_house` теперь зовёт `scraper_kit.providers.avito.houses`
|
||||
вместо `app.services.scrapers.avito_houses` (тот же `/scrape` root-роут —
|
||||
`scraper_kit.providers.avito.serp.AvitoScraper`). Каждая пара функций ниже — чистое,
|
||||
offline-тестируемое parsing-ядро, которое реально используется соответствующим
|
||||
debug-эндпоинтом (не сам HTTP-fetch — см. `tests/support/README.md`: "Live network/DB
|
||||
в parity-тестах избегайте").
|
||||
|
||||
- `_parse_house_page` — `scrape_avito_house` → `fetch_house_catalog` парсит виджет
|
||||
`housePage` этой функцией.
|
||||
- `_parse_price` — `scrape_avito_imv` → `evaluate_via_imv` парсит top-level `price`
|
||||
блок ответа `realty-imv/get-data` этой функцией.
|
||||
- `_extract_rooms_from_title` / `_extract_area_from_title` — использует
|
||||
`AvitoScraper` (SERP-парсинг `/scrape` root-роута) для title-fallback полей.
|
||||
|
||||
Легаси `app.services.scrapers.avito_imv` (`scrape_avito_imv` → `evaluate_via_imv` →
|
||||
`_parse_price`) удалён (#2277 финальный шаг scraper_kit-миграции, 0 runtime
|
||||
importers) — parity-проверка `_parse_price` убрана вместе с ним; kit-сторона
|
||||
`_parse_price` покрыта отдельно (`tests/scrapers/test_avito_imv_kit_parity.py` /
|
||||
`tests/test_avito_imv_parse.py`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -25,7 +29,6 @@ import os
|
|||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from scraper_kit.providers.avito.houses import _parse_house_page as kit_parse_house_page
|
||||
from scraper_kit.providers.avito.imv import _parse_price as kit_parse_price
|
||||
from scraper_kit.providers.avito.serp import (
|
||||
_extract_area_from_title as kit_extract_area_from_title,
|
||||
)
|
||||
|
|
@ -38,7 +41,6 @@ from app.services.scrapers.avito import (
|
|||
_extract_rooms_from_title as legacy_extract_rooms_from_title,
|
||||
)
|
||||
from app.services.scrapers.avito_houses import _parse_house_page as legacy_parse_house_page
|
||||
from app.services.scrapers.avito_imv import _parse_price as legacy_parse_price
|
||||
from tests.support.parity import assert_parity
|
||||
|
||||
_HOUSE_WIDGET = {
|
||||
|
|
@ -71,16 +73,6 @@ _HOUSE_WIDGET = {
|
|||
}
|
||||
}
|
||||
|
||||
_IMV_GETDATA = {
|
||||
"price": {
|
||||
"recommendedPrice": 8500000,
|
||||
"lowerPrice": 7800000,
|
||||
"higherPrice": 9200000,
|
||||
"count": 42,
|
||||
"addItemUrl": "/add",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_avito_house_page_parse_are_class_distinct() -> None:
|
||||
"""Sanity: legacy/kit HouseInfo — разные классы, обычный `==` даёт False."""
|
||||
|
|
@ -101,15 +93,6 @@ def test_parity_avito_house_page() -> None:
|
|||
)
|
||||
|
||||
|
||||
def test_parity_avito_imv_price() -> None:
|
||||
"""`scrape_avito_imv` (admin.py) → evaluate_via_imv → _parse_price."""
|
||||
assert_parity(
|
||||
legacy_fn=legacy_parse_price,
|
||||
kit_fn=kit_parse_price,
|
||||
fixtures=[(_IMV_GETDATA,)],
|
||||
)
|
||||
|
||||
|
||||
def test_parity_avito_serp_title_extract() -> None:
|
||||
"""`/scrape` root-роут (admin.py) → AvitoScraper SERP title-fallback fields."""
|
||||
titles = [
|
||||
|
|
|
|||
|
|
@ -12,12 +12,13 @@ Issue #2307 (Group D, PR #2318) портировал `domclick_detail.py` в kit
|
|||
с этой задачей и явно оставил `scripts/ingest_domclick_jsonl.py` untouched
|
||||
("the only other domclick_detail caller is scripts/ingest_domclick_jsonl.py,
|
||||
also untouched") — поэтому переключение этого caller'а сделано ЗДЕСЬ, в рамках
|
||||
#2305, теперь когда kit-эквивалент существует. `parse_detail_html`/exception-
|
||||
parity для domclick_detail уже исчерпывающе доказаны в
|
||||
`tests/scrapers/test_domclick_detail_kit_parity.py` (issue #2307) — этот файл
|
||||
НЕ дублирует то доказательство, а фокусируется на конкретной точке потребления:
|
||||
`ScrapedLot`/`DomClickDetailEnrichment`, сконструированные ИЗ ТЕХ ЖЕ kwargs, что
|
||||
и `_build_lot()`/`_build_enrichment()` в самом скрипте.
|
||||
#2305, теперь когда kit-эквивалент существует.
|
||||
|
||||
Легаси `app.services.scrapers.domclick_detail` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции, 0 runtime importers, `scripts/ingest_domclick_jsonl.py` уже
|
||||
100% на kit-стороне для `DomClickDetailEnrichment`) — legacy-vs-kit parity для
|
||||
`DomClickDetailEnrichment`-конструирования убрана вместе с ним; `ScrapedLot`
|
||||
parity (`app.services.scrapers.base`, всё ещё легаси/живой модуль) остаётся.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -27,14 +28,8 @@ import os
|
|||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from scraper_kit.base import ScrapedLot as KitScrapedLot
|
||||
from scraper_kit.providers.domclick.detail import (
|
||||
DomClickDetailEnrichment as KitDomClickDetailEnrichment,
|
||||
)
|
||||
|
||||
from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot
|
||||
from app.services.scrapers.domclick_detail import (
|
||||
DomClickDetailEnrichment as LegacyDomClickDetailEnrichment,
|
||||
)
|
||||
from tests.support.parity import assert_parity
|
||||
|
||||
# Аналог одной ok-записи JSONL-раннера (см. ingest_domclick_jsonl._build_lot).
|
||||
|
|
@ -56,23 +51,6 @@ _LOT_KWARGS = {
|
|||
"raw_payload": {"is_sber_collateral": False, "square_price": 156538, "avm": {}},
|
||||
}
|
||||
|
||||
# Аналог одной ok-записи JSONL-раннера (см. ingest_domclick_jsonl._build_enrichment).
|
||||
_ENRICHMENT_KWARGS = {
|
||||
"item_id": "2075729321",
|
||||
"source_url": "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321",
|
||||
"repair_state": "euro",
|
||||
"repair_type": "евроремонт",
|
||||
"living_area_m2": 32.1,
|
||||
"kitchen_area_m2": 9.5,
|
||||
"sale_type": "free",
|
||||
"owners_count": 1,
|
||||
"encumbrances_clean": True,
|
||||
"year_built": 2015,
|
||||
"views_total": 338,
|
||||
"price_changes": [{"change_time": "2026-01-15T00:00:00", "price_rub": 8500000}],
|
||||
"raw_extra": {"wall_type": "кирпично-монолитный", "calls": 5},
|
||||
}
|
||||
|
||||
|
||||
def _build_legacy_lot(rec: dict[str, object]) -> LegacyScrapedLot:
|
||||
return LegacyScrapedLot(**rec) # type: ignore[arg-type]
|
||||
|
|
@ -82,14 +60,6 @@ def _build_kit_lot(rec: dict[str, object]) -> KitScrapedLot:
|
|||
return KitScrapedLot(**rec) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _build_legacy_enrichment(rec: dict[str, object]) -> LegacyDomClickDetailEnrichment:
|
||||
return LegacyDomClickDetailEnrichment(**rec) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _build_kit_enrichment(rec: dict[str, object]) -> KitDomClickDetailEnrichment:
|
||||
return KitDomClickDetailEnrichment(**rec) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_scraped_lot_are_class_distinct() -> None:
|
||||
"""Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False."""
|
||||
legacy_result = _build_legacy_lot(_LOT_KWARGS)
|
||||
|
|
@ -106,14 +76,3 @@ def test_parity_ingest_domclick_scraped_lot() -> None:
|
|||
kit_fn=_build_kit_lot,
|
||||
fixtures=[_LOT_KWARGS],
|
||||
)
|
||||
|
||||
|
||||
def test_parity_ingest_domclick_enrichment() -> None:
|
||||
"""`ingest_domclick_jsonl._build_enrichment()` kwargs →
|
||||
DomClickDetailEnrichment(legacy) vs DomClickDetailEnrichment(kit) —
|
||||
см. module docstring про переключение этого caller'а в рамках #2305."""
|
||||
assert_parity(
|
||||
legacy_fn=_build_legacy_enrichment,
|
||||
kit_fn=_build_kit_enrichment,
|
||||
fixtures=[_ENRICHMENT_KWARGS],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,13 +7,16 @@
|
|||
чистое, offline-тестируемое parsing-ядро (см. `tests/support/README.md`:
|
||||
"Live network/DB в parity-тестах избегайте").
|
||||
|
||||
- `_parse_card_fields` — `scrape_yandex_detail` → `YandexDetailScraper.fetch_detail`
|
||||
извлекает (rooms, area, living, kitchen, ceiling, floor, total_floors) этой функцией.
|
||||
- `YandexValuationScraper._parse_house_meta` (@staticmethod) — `scrape_yandex_valuation`
|
||||
→ `fetch_house_history` извлекает house-level метаданные этой функцией.
|
||||
- `_entity_to_lot` — `/scrape` root-роут → `YandexRealtyScraper.fetch_around`
|
||||
конвертирует gate-API entity → ScrapedLot этой функцией (также доказывает
|
||||
ScrapedLot-контракт, общий с `save_listings`/`scripts/ingest_domclick_jsonl.py`).
|
||||
|
||||
Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
|
||||
финальный шаг scraper_kit-миграции, 0 runtime importers) — `_parse_card_fields` /
|
||||
`_parse_house_meta` parity-проверки убраны отсюда вместе с ними; kit-side coverage
|
||||
для обеих функций сохранена standalone в `tests/test_yandex_detail_structural.py` /
|
||||
`tests/test_yandex_valuation.py`. `yandex_realty` остаётся легаси (не мигрирован) —
|
||||
его `_entity_to_lot` parity-проверка не тронута.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -22,39 +25,11 @@ import os
|
|||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from scraper_kit.providers.yandex.detail import _parse_card_fields as kit_parse_card_fields
|
||||
from scraper_kit.providers.yandex.serp import _entity_to_lot as kit_entity_to_lot
|
||||
from scraper_kit.providers.yandex.valuation import (
|
||||
YandexValuationScraper as KitYandexValuationScraper,
|
||||
)
|
||||
|
||||
from app.services.scrapers.yandex_detail import _parse_card_fields as legacy_parse_card_fields
|
||||
from app.services.scrapers.yandex_realty import _entity_to_lot as legacy_entity_to_lot
|
||||
from app.services.scrapers.yandex_valuation import (
|
||||
YandexValuationScraper as LegacyYandexValuationScraper,
|
||||
)
|
||||
from tests.support.parity import assert_parity
|
||||
|
||||
_CARD = {
|
||||
"roomsTotal": 2,
|
||||
"area": {"value": 54.3},
|
||||
"livingSpace": {"value": 32.1},
|
||||
"kitchenSpace": {"value": 9.5},
|
||||
"ceilingHeight": 2.7,
|
||||
"floorsOffered": [5],
|
||||
"floorsTotal": 9,
|
||||
}
|
||||
_CARD_STUDIO = {
|
||||
"house": {"studio": True},
|
||||
"area": {"value": 25.0},
|
||||
"floorsOffered": [1],
|
||||
"floorsTotal": 5,
|
||||
}
|
||||
|
||||
_VALUATION_BODY_TEXT = (
|
||||
"Дом 2015 года. 9 этажей. 2,7 м потолки. Лифт есть. 120 объектов продажи. Панорама двора."
|
||||
)
|
||||
|
||||
# gate-API entity (realty.yandex.ru), без creationDate/author — исключает
|
||||
# time-dependent поля (days_on_market) из фикстуры (deterministic offline).
|
||||
_YANDEX_ENTITY = {
|
||||
|
|
@ -77,33 +52,6 @@ _YANDEX_ENTITY = {
|
|||
}
|
||||
|
||||
|
||||
def test_parity_yandex_detail_card_fields() -> None:
|
||||
"""`scrape_yandex_detail` (admin.py) → fetch_detail → _parse_card_fields."""
|
||||
assert_parity(
|
||||
legacy_fn=legacy_parse_card_fields,
|
||||
kit_fn=kit_parse_card_fields,
|
||||
fixtures=[(_CARD,), (_CARD_STUDIO,)],
|
||||
)
|
||||
|
||||
|
||||
def test_yandex_valuation_house_meta_are_class_distinct() -> None:
|
||||
"""Sanity: legacy/kit ValuationHouseMeta — разные классы, обычный `==` даёт False."""
|
||||
legacy_result = LegacyYandexValuationScraper._parse_house_meta(_VALUATION_BODY_TEXT)
|
||||
kit_result = KitYandexValuationScraper._parse_house_meta(_VALUATION_BODY_TEXT)
|
||||
|
||||
assert type(legacy_result) is not type(kit_result)
|
||||
assert legacy_result.model_dump() == kit_result.model_dump()
|
||||
|
||||
|
||||
def test_parity_yandex_valuation_house_meta() -> None:
|
||||
"""`scrape_yandex_valuation` (admin.py) → fetch_house_history → _parse_house_meta."""
|
||||
assert_parity(
|
||||
legacy_fn=LegacyYandexValuationScraper._parse_house_meta,
|
||||
kit_fn=KitYandexValuationScraper._parse_house_meta,
|
||||
fixtures=[_VALUATION_BODY_TEXT],
|
||||
)
|
||||
|
||||
|
||||
def test_yandex_entity_to_lot_are_class_distinct() -> None:
|
||||
"""Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False."""
|
||||
legacy_result = legacy_entity_to_lot(_YANDEX_ENTITY)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@
|
|||
403 на get-data → IMVAuthError.
|
||||
|
||||
browser_fetcher мокается AsyncMock'ом — реальный HTTP/камуфокс не поднимается.
|
||||
|
||||
Легаси `app.services.scrapers.avito_imv` удалён (#2277 финальный шаг scraper_kit-
|
||||
миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
|
||||
`scraper_kit.providers.avito.imv` без изменения покрытия.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -20,8 +24,7 @@ from typing import Any
|
|||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.avito_imv import (
|
||||
from scraper_kit.providers.avito.imv import (
|
||||
AVITO_BASE,
|
||||
WARMUP_URL,
|
||||
IMVAuthError,
|
||||
|
|
|
|||
|
|
@ -1,46 +1,31 @@
|
|||
"""Golden-parity + config-footgun proof for `evaluate_via_imv` — issue #2334.
|
||||
"""Config-footgun regression for kit `evaluate_via_imv` — issue #2334.
|
||||
|
||||
Group E1 of the scraper_kit migration epic (#2277 → #2308 → #2334): migrates
|
||||
`app/services/scrapers/avito_imv.py` readiness onto its
|
||||
`scraper_kit.providers.avito.imv` equivalent. Unlike prior groups (#2305/#2306),
|
||||
`evaluate_via_imv` had ZERO parity coverage before this file — `_parse_price` alone
|
||||
is covered by `tests/scrapers/test_admin_avito_kit_parity.py` (Group A, admin debug
|
||||
route), but the full 3-request async flow (geocode A/B + evaluate C) that
|
||||
`app/services/estimator.py` and `app/services/house_imv_backfill.py` depend on was
|
||||
never proven byte-for-byte identical between legacy and kit.
|
||||
Group E1 of the scraper_kit migration epic (#2277 → #2308 → #2334 → final deletion
|
||||
step). Legacy `app/services/scrapers/avito_imv.py` was deleted once the kit provider
|
||||
(`scraper_kit.providers.avito.imv`) reached golden parity and no runtime caller
|
||||
referenced the legacy module anymore — the legacy-vs-kit comparison tests that used
|
||||
to live here were removed together with the legacy import.
|
||||
|
||||
Two independent things are proven here:
|
||||
What remains is a standalone kit-only regression, still worth keeping on its own:
|
||||
|
||||
1. **Golden parity** — legacy `evaluate_via_imv` and kit `evaluate_via_imv` give the
|
||||
IDENTICAL `IMVEvaluation` on the SAME input, using REAL Avito API responses
|
||||
recorded live (`tests/fixtures/avito_imv_geo_position.json` +
|
||||
`avito_imv_getdata.json`, see `tests/test_avito_imv_parse.py` for provenance —
|
||||
issue #565, 2026-05 capture, ЕКБ ул. Малышева 125). Both sides run through the
|
||||
`browser_fetcher=` transport (see `tests/scrapers/test_avito_imv_browser_transport.py`)
|
||||
so the test is offline/deterministic and never touches `curl_cffi`/proxy config —
|
||||
that's a SEPARATE axis, covered below.
|
||||
|
||||
2. **Config-footgun regression** — CONFIRMED by recon before this issue started:
|
||||
kit `evaluate_via_imv(config: ScraperConfig | None = None, ...)` only reads
|
||||
`config.scraper_proxy_url` when `config is not None`; legacy unconditionally reads
|
||||
`settings.scraper_proxy_url` itself. Call the kit fn WITHOUT `config=` → own
|
||||
curl_cffi session gets built with NO proxy (`proxies=None`) even though a real
|
||||
proxy is configured — a silent behaviour change vs legacy. WITH
|
||||
`config=RealScraperConfig()` (the established DI pattern, see
|
||||
`app/services/cian_price_history.py` / `tests/test_scraper_kit_pricehistory_session_parity.py::
|
||||
test_fetch_detail_own_session_proxy_wiring_parity` for precedent) the proxy is
|
||||
used identically to legacy. NOTE: neither `estimator.py` nor
|
||||
`house_imv_backfill.py` is switched to the kit import in this issue (see report in
|
||||
PR/issue #2334) — these tests exist to prove the KIT SIDE is safe to wire up
|
||||
whenever that switch happens (#2337 for estimator.py).
|
||||
**Config-footgun regression** — CONFIRMED by recon before issue #2334 started:
|
||||
kit `evaluate_via_imv(config: ScraperConfig | None = None, ...)` only reads
|
||||
`config.scraper_proxy_url` when `config is not None`; the deleted legacy module
|
||||
unconditionally read `settings.scraper_proxy_url` itself. Call the kit fn WITHOUT
|
||||
`config=` → own curl_cffi session gets built with NO proxy (`proxies=None`) even
|
||||
though a real proxy is configured — a silent behaviour change vs the old code. WITH
|
||||
`config=RealScraperConfig()` (the established DI pattern, see
|
||||
`app/services/cian_price_history.py` / `tests/test_scraper_kit_pricehistory_session_parity.py::
|
||||
test_fetch_detail_own_session_proxy_wiring_parity` for precedent) the proxy is wired
|
||||
correctly. NOTE: neither `estimator.py` nor `house_imv_backfill.py` needs this
|
||||
config= kwarg wired for `evaluate_via_imv` specifically — these tests exist to prove
|
||||
the KIT SIDE stays safe against a "simplify away the config kwarg" regression.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -54,173 +39,10 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
|
|||
from scraper_kit.providers.avito.imv import (
|
||||
IMVAddressNotFoundError as KitIMVAddressNotFoundError,
|
||||
)
|
||||
from scraper_kit.providers.avito.imv import _parse_geo_position as kit_parse_geo_position
|
||||
from scraper_kit.providers.avito.imv import _parse_placement_history as kit_parse_history
|
||||
from scraper_kit.providers.avito.imv import _parse_suggestions as kit_parse_suggestions
|
||||
from scraper_kit.providers.avito.imv import compute_imv_cache_key as kit_cache_key
|
||||
from scraper_kit.providers.avito.imv import evaluate_via_imv as kit_evaluate_via_imv
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.scraper_adapters import RealScraperConfig
|
||||
from app.services.scrapers.avito_imv import _parse_geo_position as legacy_parse_geo_position
|
||||
from app.services.scrapers.avito_imv import _parse_placement_history as legacy_parse_history
|
||||
from app.services.scrapers.avito_imv import _parse_suggestions as legacy_parse_suggestions
|
||||
from app.services.scrapers.avito_imv import compute_imv_cache_key as legacy_cache_key
|
||||
from app.services.scrapers.avito_imv import evaluate_via_imv as legacy_evaluate_via_imv
|
||||
from tests.support.parity import ParityMismatchError, assert_parity
|
||||
|
||||
_FIXTURES = Path(__file__).parent.parent / "fixtures"
|
||||
|
||||
|
||||
def _load_fixture(name: str) -> dict[str, Any]:
|
||||
with (_FIXTURES / name).open(encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cheap pure-function parity (closes the gap beyond `_parse_price`, already
|
||||
# covered by tests/scrapers/test_admin_avito_kit_parity.py Group A).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_compute_imv_cache_key_parity() -> None:
|
||||
assert_parity(
|
||||
legacy_fn=legacy_cache_key,
|
||||
kit_fn=kit_cache_key,
|
||||
fixtures=[
|
||||
("ЕКБ ул. Малышева, 125", 2, 54.0, 4, 9, "panel", "cosmetic", True, False),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_parse_geo_position_parity_real_fixture() -> None:
|
||||
data_b = _load_fixture("avito_imv_geo_position.json")
|
||||
assert_parity(
|
||||
legacy_fn=lambda: legacy_parse_geo_position(data_b, lat=56.840872, lon=60.654077),
|
||||
kit_fn=lambda: kit_parse_geo_position(data_b, lat=56.840872, lon=60.654077),
|
||||
fixtures=[()],
|
||||
)
|
||||
|
||||
|
||||
def test_parse_placement_history_parity_real_fixture() -> None:
|
||||
data_c = _load_fixture("avito_imv_getdata.json")
|
||||
assert_parity(
|
||||
legacy_fn=legacy_parse_history,
|
||||
kit_fn=kit_parse_history,
|
||||
fixtures=[(data_c,)],
|
||||
)
|
||||
|
||||
|
||||
def test_parse_suggestions_parity_real_fixture() -> None:
|
||||
data_c = _load_fixture("avito_imv_getdata.json")
|
||||
assert_parity(
|
||||
legacy_fn=legacy_parse_suggestions,
|
||||
kit_fn=kit_parse_suggestions,
|
||||
fixtures=[(data_c,)],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Golden parity — full evaluate_via_imv() flow on REAL captured API fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Same address/lat/lon/lot-params as tests/test_avito_imv_parse.py::
|
||||
# test_fixtures_feed_full_evaluation_shape — keeps this test consistent with the
|
||||
# established "reference realistic case" derived from the live 2026-05 capture
|
||||
# (issue #565), rather than inventing a new ad-hoc fixture.
|
||||
_ADDRESS = "Свердловская область, Екатеринбург, улица Малышева, 125"
|
||||
_LAT = 56.840872
|
||||
_LON = 60.654077
|
||||
_LOT_PARAMS: dict[str, Any] = {
|
||||
"rooms": 2,
|
||||
"area_m2": 54.0,
|
||||
"floor": 4,
|
||||
"floor_at_home": 9,
|
||||
"house_type": "panel",
|
||||
"renovation_type": "cosmetic",
|
||||
"has_balcony": True,
|
||||
"has_loggia": False,
|
||||
}
|
||||
|
||||
|
||||
def _canned_browser_responses() -> list[dict[str, Any]]:
|
||||
"""4 responses in call order: warm-up GET, coords A, position B, get-data C.
|
||||
|
||||
B and C bodies are the REAL live-captured Avito responses (fixtures/*.json);
|
||||
A is synthesized with the SAME lat/lon as the fixture B (both refer to the same
|
||||
ул. Малышева, 125 lookup) so the flow is internally consistent end-to-end.
|
||||
"""
|
||||
warmup = {"status": 200, "body": "<html>warm</html>"}
|
||||
coords_a_body = {
|
||||
"normalizedAddress": _ADDRESS,
|
||||
"point": {"latitude": _LAT, "longitude": _LON},
|
||||
}
|
||||
coords_a = {"status": 200, "body": json.dumps(coords_a_body)}
|
||||
position_b = {"status": 200, "body": json.dumps(_load_fixture("avito_imv_geo_position.json"))}
|
||||
getdata_c = {"status": 200, "body": json.dumps(_load_fixture("avito_imv_getdata.json"))}
|
||||
return [warmup, coords_a, position_b, getdata_c]
|
||||
|
||||
|
||||
def _run_legacy_evaluation() -> Any:
|
||||
bf = AsyncMock()
|
||||
bf.fetch_json = AsyncMock(side_effect=_canned_browser_responses())
|
||||
return asyncio.run(legacy_evaluate_via_imv(address=_ADDRESS, browser_fetcher=bf, **_LOT_PARAMS))
|
||||
|
||||
|
||||
def _run_kit_evaluation() -> Any:
|
||||
bf = AsyncMock()
|
||||
bf.fetch_json = AsyncMock(side_effect=_canned_browser_responses())
|
||||
return asyncio.run(kit_evaluate_via_imv(address=_ADDRESS, browser_fetcher=bf, **_LOT_PARAMS))
|
||||
|
||||
|
||||
def test_legacy_and_kit_imv_evaluation_are_class_distinct() -> None:
|
||||
"""Sanity-check ДО harness'а — same reason as test_avito_detail_kit_parity.py:
|
||||
legacy/kit IMVEvaluation are different classes (different modules), so plain
|
||||
`==` is always False even on identical field values.
|
||||
"""
|
||||
legacy_result = _run_legacy_evaluation()
|
||||
kit_result = _run_kit_evaluation()
|
||||
|
||||
assert type(legacy_result) is not type(kit_result)
|
||||
assert legacy_result != kit_result
|
||||
assert legacy_result.recommended_price == kit_result.recommended_price
|
||||
assert legacy_result.geo.geo_hash == kit_result.geo.geo_hash
|
||||
|
||||
|
||||
def test_evaluate_via_imv_golden_parity_real_fixtures() -> None:
|
||||
"""Proves kit evaluate_via_imv gives BYTE-IDENTICAL IMVEvaluation to legacy on a
|
||||
real recorded Avito API response (no synthetic мокы для price/history/suggestions —
|
||||
только step A синтетика для координат, зеркалящих ту же самую фикстуру B).
|
||||
"""
|
||||
assert_parity(
|
||||
legacy_fn=_run_legacy_evaluation,
|
||||
kit_fn=_run_kit_evaluation,
|
||||
fixtures=[()],
|
||||
)
|
||||
|
||||
|
||||
def test_parity_harness_catches_imv_field_divergence() -> None:
|
||||
"""Доказывает, что harness реально ЛОВИТ расхождение на этой dataclass-форме —
|
||||
мутируем kit-результат (recommended_price) и убеждаемся в ParityMismatchError,
|
||||
называющем именно это поле (mirrors test_avito_detail_kit_parity.py pattern).
|
||||
"""
|
||||
import dataclasses
|
||||
|
||||
legacy_result = _run_legacy_evaluation()
|
||||
kit_result = _run_kit_evaluation()
|
||||
mutated_kit_result = dataclasses.replace(
|
||||
kit_result, recommended_price=kit_result.recommended_price + 1
|
||||
)
|
||||
|
||||
with pytest.raises(ParityMismatchError) as exc_info:
|
||||
assert_parity(
|
||||
legacy_fn=lambda: legacy_result,
|
||||
kit_fn=lambda: mutated_kit_result,
|
||||
fixtures=[()],
|
||||
)
|
||||
|
||||
assert "recommended_price" in str(exc_info.value)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config-footgun regression (#2334 CONFIRMED recon finding)
|
||||
|
|
@ -229,7 +51,8 @@ def test_parity_harness_catches_imv_field_divergence() -> None:
|
|||
# evaluate_via_imv's own-curl_cffi-session path (browser_fetcher=None, the default)
|
||||
# is the one that reads proxy config. Mirrors
|
||||
# tests/test_scraper_proxy.py::test_avito_imv_own_session_receives_proxies (legacy
|
||||
# reference) and tests/test_scraper_kit_pricehistory_session_parity.py::
|
||||
# reference, now gone with the legacy module) and
|
||||
# tests/test_scraper_kit_pricehistory_session_parity.py::
|
||||
# test_fetch_detail_own_session_proxy_wiring_parity (kit config= precedent for a
|
||||
# sibling provider, #2306).
|
||||
|
||||
|
|
@ -287,10 +110,10 @@ def test_kit_evaluate_via_imv_without_config_drops_proxy(monkeypatch: pytest.Mon
|
|||
|
||||
def test_kit_evaluate_via_imv_with_config_uses_proxy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""With config=RealScraperConfig() (the established DI pattern for this epic) the
|
||||
kit path reads the SAME settings.scraper_proxy_url as legacy and wires it into the
|
||||
curl_cffi session identically — this is the regression guard: if someone later
|
||||
"simplifies away" the config kwarg threading in evaluate_via_imv, this test fails
|
||||
on `captured.get("proxies") is None` instead of the expected proxy dict.
|
||||
kit path reads the SAME settings.scraper_proxy_url as legacy used to and wires it
|
||||
into the curl_cffi session identically — this is the regression guard: if someone
|
||||
later "simplifies away" the config kwarg threading in evaluate_via_imv, this test
|
||||
fails on `captured.get("proxies") is None` instead of the expected proxy dict.
|
||||
"""
|
||||
monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://test-proxy.local:8080")
|
||||
mock_session = _mock_own_session_flow()
|
||||
|
|
|
|||
|
|
@ -4,12 +4,19 @@
|
|||
До фикса было 3 независимые копии с разными timezone:
|
||||
- avito.py (inline в `_build_sort_timestamp_map`) — MSK (документировано #726)
|
||||
- avito_houses.py — UTC
|
||||
- avito_imv.py — naive/system-local (`datetime.fromtimestamp(ts)` без tz)
|
||||
- avito_imv.py (легаси, удалён #2277) — naive/system-local
|
||||
(`datetime.fromtimestamp(ts)` без tz)
|
||||
|
||||
Один и тот же epoch мог давать РАЗНЫЕ `date` в разных путях (пример из живой fixture
|
||||
avito_imv_getdata.json: removedDate=1756077885 → UTC 2025-08-24, MSK 2025-08-25).
|
||||
|
||||
Офлайн: без сети/curl_cffi/БД, только импорт + чистые функции.
|
||||
|
||||
Легаси `app.services.scrapers.avito_imv` удалён (#2277 финальный шаг scraper_kit-
|
||||
миграции, 0 runtime importers) — IMV-ветка consolidation-проверки переведена на
|
||||
kit-эквивалент `scraper_kit.providers.avito.imv`, который использует ТОТ ЖЕ паттерн
|
||||
(`from scraper_kit.providers.avito.shared import _unix_to_date`) — регрессия
|
||||
"3 независимые копии с разными TZ" остаётся под наблюдением и на kit-стороне.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -19,7 +26,10 @@ from datetime import date
|
|||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from app.services.scrapers import avito, avito_houses, avito_imv, avito_shared
|
||||
from scraper_kit.providers.avito import imv as kit_avito_imv
|
||||
from scraper_kit.providers.avito import shared as kit_avito_shared
|
||||
|
||||
from app.services.scrapers import avito, avito_houses, avito_shared
|
||||
|
||||
# Живой пример из fixtures/avito_imv_getdata.json (placementHistory.items[0].removedDate) —
|
||||
# именно тот timestamp, на котором UTC и MSK расходились в date() ДО фикса
|
||||
|
|
@ -39,27 +49,29 @@ def test_avito_houses_reexports_shared_unix_to_date() -> None:
|
|||
assert avito_houses._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
|
||||
|
||||
|
||||
def test_avito_imv_reexports_shared_unix_to_date() -> None:
|
||||
"""avito_imv.py больше не держит свою копию (была naive/system-local) — общая функция."""
|
||||
assert avito_imv._unix_to_date is avito_shared._unix_to_date
|
||||
assert avito_imv._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
|
||||
def test_kit_avito_imv_reexports_shared_unix_to_date() -> None:
|
||||
"""kit imv.py не держит свою копию — импортирует общую функцию (тот же паттерн,
|
||||
что и легаси avito_houses.py/avito_shared.py после фикса #2129)."""
|
||||
assert kit_avito_imv._unix_to_date is kit_avito_shared._unix_to_date
|
||||
assert kit_avito_imv._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
|
||||
|
||||
|
||||
def test_avito_serp_sort_timestamp_matches_shared_date() -> None:
|
||||
"""avito.py путь (SERP `sortTimeStamp`, epoch-МС) для того же момента времени
|
||||
даёт ТУ ЖЕ дату, что и houses/IMV путь (epoch-секунды) — единственная разница
|
||||
в единицах измерения epoch, TZ теперь общая (MSK) во всех трёх путях."""
|
||||
в единицах измерения epoch, TZ теперь общая (MSK) во всех путях (легаси и kit)."""
|
||||
html = (
|
||||
f'<div>{{"id":8043936560,"sortTimeStamp":{_EPOCH_SECONDS * 1000},'
|
||||
f'"addressDetailed":{{"foo":"bar"}}}}</div>'
|
||||
)
|
||||
ts_map = avito._build_sort_timestamp_map(html)
|
||||
assert ts_map["8043936560"] == _EXPECTED_MSK_DATE
|
||||
# Все три пути согласованы на одном моменте времени:
|
||||
# Легаси avito/avito_houses/avito_shared + kit avito_imv все согласованы на одном
|
||||
# моменте времени:
|
||||
assert (
|
||||
ts_map["8043936560"]
|
||||
== avito_houses._unix_to_date(_EPOCH_SECONDS)
|
||||
== avito_imv._unix_to_date(_EPOCH_SECONDS)
|
||||
== kit_avito_imv._unix_to_date(_EPOCH_SECONDS)
|
||||
== avito_shared._unix_to_date(_EPOCH_SECONDS)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Unit-тесты для DomClick Layer B detail-scraper (domclick_detail.py).
|
||||
"""Unit-тесты для DomClick Layer B detail-scraper (scraper_kit.providers.domclick.detail).
|
||||
|
||||
Offline: parse/SSR-тесты без сети/БД; save-тест через MagicMock (зеркало
|
||||
test_cian_detail.py — в проекте нет live-DB фикстуры, идемпотентность
|
||||
|
|
@ -19,6 +19,14 @@ test_cian_detail.py — в проекте нет live-DB фикстуры, ид
|
|||
item_id из URL.
|
||||
- fetch_detail: browser-branch + проброс ошибок.
|
||||
- save_detail_enrichment: UPDATE-параметры, detail_enriched_at, price-history INSERT.
|
||||
|
||||
Легаси `app.services.scrapers.domclick_detail` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
|
||||
`scraper_kit.providers.domclick.detail` без изменения покрытия. Exceptions тоже
|
||||
переведены на `scraper_kit.domclick_exceptions` (kit держит свою независимую копию,
|
||||
`parse_detail_html`(kit) поднимает именно её — legacy `app.services.scrapers.
|
||||
domclick_exceptions` остаётся живым модулем для `domclick.py`, но здесь больше не
|
||||
подходящий класс для `pytest.raises`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -28,15 +36,14 @@ from datetime import UTC, datetime, timedelta
|
|||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.domclick_detail import (
|
||||
from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError
|
||||
from scraper_kit.providers.domclick.detail import (
|
||||
DomClickDetailEnrichment,
|
||||
_extract_ssr_state,
|
||||
fetch_detail,
|
||||
parse_detail_html,
|
||||
save_detail_enrichment,
|
||||
)
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError
|
||||
|
||||
_CARD_URL = "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,189 +0,0 @@
|
|||
"""Parity-proof: `app.services.scrapers.domclick_detail` vs
|
||||
`scraper_kit.providers.domclick.detail` (issue #2307, Group D).
|
||||
|
||||
`parse_detail_html` — parsing-ядро без сети/БД (SSR-стейт → dataclass), поэтому
|
||||
сравнивается offline на фиксированной HTML-фикстуре через
|
||||
`tests.support.parity.assert_parity` (harness #2304). Legacy и kit тела функции
|
||||
на момент написания теста ИДЕНТИЧНЫ (см. audit
|
||||
`Scraper_Kit_Legacy_Dependency_Audit_0703` в vault) — различаются только
|
||||
import-пути зависимостей (`app.services.scrapers.*` vs `scraper_kit.*`).
|
||||
|
||||
HTML-фикстура — независимая копия литерала из
|
||||
`tests/scrapers/test_domclick_detail.py::_HTML` (тот же live-card ground truth
|
||||
2075729321, 2026-06-27), чтобы этот тест не зависел от другого test-модуля.
|
||||
"""
|
||||
|
||||
# SSR-JSON фикстура содержит длинные строки
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
# app.services.scrapers.domclick_detail не читает settings напрямую, но соседние
|
||||
# модули пакета app.services.scrapers делают это при импорте app.core.config —
|
||||
# фиктивный DSN мостит офлайн-запуск без CI-уровневого DATABASE_URL (mirror
|
||||
# tests/scrapers/test_avito_detail_kit_parity.py).
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from scraper_kit.domclick_exceptions import (
|
||||
DomClickBlockedError as KitDomClickBlockedError,
|
||||
)
|
||||
from scraper_kit.domclick_exceptions import (
|
||||
DomClickParseError as KitDomClickParseError,
|
||||
)
|
||||
from scraper_kit.providers.domclick.detail import (
|
||||
parse_detail_html as kit_parse_detail_html,
|
||||
)
|
||||
|
||||
from app.services.scrapers.domclick_detail import (
|
||||
parse_detail_html as legacy_parse_detail_html,
|
||||
)
|
||||
from app.services.scrapers.domclick_exceptions import (
|
||||
DomClickBlockedError as LegacyDomClickBlockedError,
|
||||
)
|
||||
from app.services.scrapers.domclick_exceptions import (
|
||||
DomClickParseError as LegacyDomClickParseError,
|
||||
)
|
||||
from tests.support.parity import ParityMismatchError, assert_parity
|
||||
|
||||
_CARD_URL = "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321"
|
||||
|
||||
# Real __SSR_STATE__ shape (ground truth, live card 2075729321, 2026-06-27).
|
||||
# bare `undefined` (value-position), in-string "undefined", '{' внутри строки и
|
||||
# эскейп-кавычки — написан вручную (json.dumps использовать нельзя, нужен
|
||||
# именно НЕвалидный-для-json undefined).
|
||||
_SSR_LITERAL = """{
|
||||
"productCard": {
|
||||
"objectInfo": {
|
||||
"area": 38.0,
|
||||
"rooms": 1,
|
||||
"floor": 5,
|
||||
"isApartment": false,
|
||||
"renovation": "евроремонт",
|
||||
"livingArea": 22.37,
|
||||
"kitchenArea": 10.13,
|
||||
"description": "Состояние пока undefined.",
|
||||
"note": "угол {da}",
|
||||
"quote": "сказал \\"привет\\" сосед"
|
||||
},
|
||||
"legalOptions": {"saleType": "Свободная продажа"},
|
||||
"egrnData": {
|
||||
"area": 38.2,
|
||||
"floor": 5,
|
||||
"owners_count": 1,
|
||||
"collateral": true,
|
||||
"collateral_sber": false
|
||||
},
|
||||
"priceInfo": {
|
||||
"priceHistory": [
|
||||
{"date": "2026-04-15T09:02:26.548728+03:00", "price": 13400000, "diff": -400000, "state": "less"},
|
||||
{"date": "not-a-date", "price": 4800000, "diff": -1.0, "state": "less"},
|
||||
{"date": "2026-05-01T10:00:00Z", "price": null, "diff": 0, "state": "more"}
|
||||
]
|
||||
},
|
||||
"viewsCount": 338,
|
||||
"callsCount": 5,
|
||||
"favoriteOfferUsersCount": 12,
|
||||
"duplicatesOfferCount": 0,
|
||||
"address": {"guid": "abc-guid-123"},
|
||||
"unknownField": undefined
|
||||
},
|
||||
"houseInfo": {
|
||||
"info": {
|
||||
"buildYear": 2015,
|
||||
"wallType": "Кирпично-монолитный",
|
||||
"floorType": "Железобетонный",
|
||||
"entranceCount": 3,
|
||||
"quartersCount": 120,
|
||||
"energyEfficiency": "B",
|
||||
"buildingSeries": "индивидуальный"
|
||||
}
|
||||
},
|
||||
"pricePrediction": {
|
||||
"market_price": 13000000,
|
||||
"min_market_price": 12000000,
|
||||
"max_market_price": 14000000,
|
||||
"rent_short": 50000,
|
||||
"rent_long": 35000,
|
||||
"repair_quality": "good"
|
||||
}
|
||||
}"""
|
||||
|
||||
_HTML = (
|
||||
"<html><head></head><body><script>"
|
||||
f"window.__SSR_STATE__ = {_SSR_LITERAL};"
|
||||
"</script></body></html>"
|
||||
)
|
||||
|
||||
|
||||
def test_legacy_and_kit_parse_detail_html_are_class_distinct() -> None:
|
||||
"""Sanity-check ДО harness'а: legacy/kit DomClickDetailEnrichment — РАЗНЫЕ
|
||||
классы (разные модули) → обычный `==` даёт False, хотя данные идентичны.
|
||||
Именно это harness обходит через структурное сравнение (dataclasses.fields).
|
||||
"""
|
||||
legacy_result = legacy_parse_detail_html(_HTML, _CARD_URL)
|
||||
kit_result = kit_parse_detail_html(_HTML, _CARD_URL)
|
||||
|
||||
assert type(legacy_result) is not type(kit_result)
|
||||
assert legacy_result != kit_result # dataclass __eq__ проверяет class identity первым
|
||||
assert dataclasses.asdict(legacy_result) == dataclasses.asdict(kit_result)
|
||||
|
||||
|
||||
def test_parity_harness_confirms_domclick_detail_parse_equivalence() -> None:
|
||||
"""Реальное использование harness'а для DomClick Layer B (issue #2307)."""
|
||||
assert_parity(
|
||||
legacy_fn=legacy_parse_detail_html,
|
||||
kit_fn=kit_parse_detail_html,
|
||||
fixtures=[(_HTML, _CARD_URL)],
|
||||
)
|
||||
|
||||
|
||||
def test_parity_harness_catches_real_field_divergence() -> None:
|
||||
"""Harness реально ЛОВИТ расхождение на реальной форме — мутируем один
|
||||
field (``living_area_m2``) в kit-результате и убеждаемся, что harness
|
||||
падает с ``ParityMismatchError``, называющим именно этот field.
|
||||
"""
|
||||
legacy_result = legacy_parse_detail_html(_HTML, _CARD_URL)
|
||||
kit_result = kit_parse_detail_html(_HTML, _CARD_URL)
|
||||
assert kit_result.living_area_m2 is not None # sanity: поле реально распарсилось
|
||||
|
||||
mutated_kit_result = dataclasses.replace(
|
||||
kit_result, living_area_m2=kit_result.living_area_m2 + 1
|
||||
)
|
||||
|
||||
with pytest.raises(ParityMismatchError) as exc_info:
|
||||
assert_parity(
|
||||
legacy_fn=lambda: legacy_result,
|
||||
kit_fn=lambda: mutated_kit_result,
|
||||
fixtures=[()],
|
||||
)
|
||||
|
||||
assert "living_area_m2" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_legacy_and_kit_exceptions_behave_identically_on_challenge_page() -> None:
|
||||
"""Anti-bot challenge page (no __SSR_STATE__, block-маркеры) → оба пути
|
||||
поднимают семантически эквивалентный DomClickBlockedError (разные классы,
|
||||
те же условия детекции — зеркало `test_scraper_kit_domclick_golden_parity.py`
|
||||
для Layer A exceptions).
|
||||
"""
|
||||
challenge_html = "<html><body>Access denied by DataDome captcha</body></html>"
|
||||
|
||||
with pytest.raises(LegacyDomClickBlockedError):
|
||||
legacy_parse_detail_html(challenge_html, _CARD_URL)
|
||||
with pytest.raises(KitDomClickBlockedError):
|
||||
kit_parse_detail_html(challenge_html, _CARD_URL)
|
||||
|
||||
|
||||
def test_legacy_and_kit_exceptions_behave_identically_on_missing_state() -> None:
|
||||
"""HTML без __SSR_STATE__ и без block-маркеров → оба пути поднимают
|
||||
DomClickParseError (дрейф схемы / повреждённый ответ, не блок)."""
|
||||
no_state_html = "<html><body>no ssr state here</body></html>"
|
||||
|
||||
with pytest.raises(LegacyDomClickParseError):
|
||||
legacy_parse_detail_html(no_state_html, _CARD_URL)
|
||||
with pytest.raises(KitDomClickParseError):
|
||||
kit_parse_detail_html(no_state_html, _CARD_URL)
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
"""Parity-proof: `app.services.scrapers.ekb_geoportal_client` vs
|
||||
`scraper_kit.providers.ekb_geoportal.client` (issue #2307, Group D).
|
||||
|
||||
`parse_feature_collection` (GeoJSON FeatureCollection → list[TopoBuilding],
|
||||
area-weighted centroid) — чистая функция без сети/БД, поэтому сравнивается
|
||||
offline через `tests.support.parity.assert_parity` (harness #2304). Legacy и
|
||||
kit тела функции на момент написания теста ИДЕНТИЧНЫ (см. audit
|
||||
`Scraper_Kit_Legacy_Dependency_Audit_0703` в vault) — единственное отличие —
|
||||
import-путь модуля.
|
||||
|
||||
GeoJSON-фикстуры — независимая копия из
|
||||
`tests/test_ekb_geoportal_ingest.py::test_parse_feature_collection_basic`,
|
||||
чтобы этот тест не зависел от другого test-модуля.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
from scraper_kit.providers.ekb_geoportal.client import (
|
||||
TopoBuilding as KitTopoBuilding,
|
||||
)
|
||||
from scraper_kit.providers.ekb_geoportal.client import (
|
||||
parse_feature_collection as kit_parse_feature_collection,
|
||||
)
|
||||
|
||||
from app.services.scrapers.ekb_geoportal_client import (
|
||||
TopoBuilding as LegacyTopoBuilding,
|
||||
)
|
||||
from app.services.scrapers.ekb_geoportal_client import (
|
||||
parse_feature_collection as legacy_parse_feature_collection,
|
||||
)
|
||||
from tests.support.parity import ParityMismatchError, assert_parity
|
||||
|
||||
# A unit square [0,0]-[2,2] → centroid (1, 1) in (lon, lat).
|
||||
_SQUARE = [[[[0.0, 0.0], [2.0, 0.0], [2.0, 2.0], [0.0, 2.0], [0.0, 0.0]]]]
|
||||
|
||||
|
||||
def _multipolygon(coords: list) -> dict:
|
||||
return {"type": "MultiPolygon", "coordinates": coords}
|
||||
|
||||
|
||||
_FEATURE_COLLECTION = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": _multipolygon(_SQUARE),
|
||||
"properties": {
|
||||
"topo_street": "Космонавтов",
|
||||
"topo_num_house": "7б",
|
||||
"topo_floor": 9,
|
||||
"topo_purpose": "жилое",
|
||||
"topo_material": "3",
|
||||
"key": 12345,
|
||||
},
|
||||
},
|
||||
{
|
||||
# без street/house — должен быть пропущен обоими путями
|
||||
"type": "Feature",
|
||||
"geometry": _multipolygon(_SQUARE),
|
||||
"properties": {"topo_street": "", "topo_num_house": "5"},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_legacy_and_kit_topo_building_are_class_distinct() -> None:
|
||||
"""Sanity-check ДО harness'а: legacy/kit TopoBuilding — РАЗНЫЕ классы
|
||||
(разные модули) → обычный `==` даёт False, хотя данные идентичны."""
|
||||
legacy_result = legacy_parse_feature_collection(_FEATURE_COLLECTION)
|
||||
kit_result = kit_parse_feature_collection(_FEATURE_COLLECTION)
|
||||
|
||||
assert len(legacy_result) == len(kit_result) == 1
|
||||
assert type(legacy_result[0]) is not type(kit_result[0])
|
||||
assert legacy_result[0] != kit_result[0] # dataclass __eq__ проверяет class identity первым
|
||||
assert dataclasses.asdict(legacy_result[0]) == dataclasses.asdict(kit_result[0])
|
||||
|
||||
|
||||
def test_parity_harness_confirms_ekb_geoportal_parse_equivalence() -> None:
|
||||
"""Реальное использование harness'а для EKB Geoportal client (issue #2307)."""
|
||||
assert_parity(
|
||||
legacy_fn=legacy_parse_feature_collection,
|
||||
kit_fn=kit_parse_feature_collection,
|
||||
fixtures=[(_FEATURE_COLLECTION,)],
|
||||
)
|
||||
|
||||
|
||||
def test_parity_harness_catches_real_field_divergence() -> None:
|
||||
"""Harness реально ЛОВИТ расхождение — мутируем ``floors`` в kit-результате
|
||||
и убеждаемся, что harness падает с ``ParityMismatchError``, называющим
|
||||
именно этот field."""
|
||||
legacy_result = legacy_parse_feature_collection(_FEATURE_COLLECTION)
|
||||
kit_result = kit_parse_feature_collection(_FEATURE_COLLECTION)
|
||||
assert kit_result[0].floors == 9 # sanity: поле реально распарсилось
|
||||
|
||||
mutated_kit_building = dataclasses.replace(kit_result[0], floors=1)
|
||||
assert isinstance(mutated_kit_building, KitTopoBuilding)
|
||||
assert isinstance(legacy_result[0], LegacyTopoBuilding)
|
||||
|
||||
with pytest.raises(ParityMismatchError) as exc_info:
|
||||
assert_parity(
|
||||
legacy_fn=lambda: legacy_result,
|
||||
kit_fn=lambda: [mutated_kit_building],
|
||||
fixtures=[()],
|
||||
)
|
||||
|
||||
assert "floors" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_legacy_and_kit_empty_and_malformed_inputs_match() -> None:
|
||||
"""Graceful-degradation ветки (пустой/битый payload) дают одинаковый []
|
||||
на обоих путях."""
|
||||
for payload in ({}, {"features": "nope"}, {"features": [None, 42]}):
|
||||
assert legacy_parse_feature_collection(payload) == []
|
||||
assert kit_parse_feature_collection(payload) == []
|
||||
|
|
@ -1,24 +1,18 @@
|
|||
"""Issue #2336 (Group E3 of scraper_kit migration epic #2277/#2308).
|
||||
|
||||
`scraper_kit.providers.yandex.valuation.YandexValuationScraper` is already a
|
||||
strangler-copy (#2133) of `app.services.scrapers.yandex_valuation`, and already used
|
||||
via the kit path in `admin.py::scrape_yandex_valuation` (#2305, Group A) with the
|
||||
established DI convention: `config=RealScraperConfig()`,
|
||||
`delay_provider=get_scraper_delay`, `proxy_provider=_kit_proxy_provider()`.
|
||||
strangler-copy (#2133) of the (now deleted, #2277 final step)
|
||||
`app.services.scrapers.yandex_valuation`, and already used via the kit path in
|
||||
`admin.py::scrape_yandex_valuation` (#2305, Group A) with the established DI
|
||||
convention: `config=RealScraperConfig()`, `delay_provider=get_scraper_delay`,
|
||||
`proxy_provider=_kit_proxy_provider()`.
|
||||
|
||||
`app/services/estimator.py` (the real, paying-user-facing caller) still imports the
|
||||
LEGACY module directly and instantiates `YandexValuationScraper()` with NO arguments
|
||||
(`estimator.py:598`). Switching that call site to the kit class is #2337's job — out
|
||||
of scope here. This file exists to prove #2336's two confirmed risk patterns for
|
||||
whoever does that switch, so a regression is caught in CI rather than silently in prod:
|
||||
This file proves #2336's two confirmed risk patterns for any future call site that
|
||||
constructs the kit scraper directly, so a regression is caught in CI rather than
|
||||
silently in prod:
|
||||
|
||||
1. **Mandatory `config: ScraperConfig`** (same shape as the cian_valuation footgun,
|
||||
#2335) — omitting it raises `TypeError`. `estimator.py`'s call site sits inside a
|
||||
blanket `except Exception` (see `_get_or_fetch_yandex_valuation_cached`, lines
|
||||
~596-606) that logs via `logger.warning(..., e)` — no `exc_info`/traceback, and at
|
||||
the SAME log severity as ordinary transient fetch failures. A `TypeError` from a
|
||||
forgotten `config=` would read like "Yandex source unavailable" in prod logs, not
|
||||
"wiring bug", even though the message text itself does end up in the log line.
|
||||
#2335) — omitting it raises `TypeError`.
|
||||
|
||||
2. **Optional `delay_provider`** — if omitted, silently falls back to the hardcoded
|
||||
class-default `request_delay_sec = 5.0`, discarding whatever an admin configured via
|
||||
|
|
@ -26,11 +20,10 @@ whoever does that switch, so a regression is caught in CI rather than silently i
|
|||
`max(per_source, global)`). No exception, no log — just quietly weaker anti-ban
|
||||
throttling if an admin had raised it after ban trouble.
|
||||
|
||||
Also extends `tests/test_scraper_kit_yandex_golden_parity.py::test_valuation_parse_parity`
|
||||
(which already proves `.parse()` output identity using a duck-typed `SimpleNamespace`
|
||||
config) to the REAL production construction path — `RealScraperConfig()` +
|
||||
`get_scraper_delay` — via `tests/support/parity.assert_parity` (issue #2304 harness),
|
||||
per the epic's parity-gate convention.
|
||||
Legacy-vs-kit parity coverage (previously here, using
|
||||
`app.services.scrapers.yandex_valuation`) was removed together with the legacy module
|
||||
deletion — `.parse()` correctness itself is still covered kit-only by
|
||||
`tests/test_yandex_valuation.py` / `tests/test_yandex_valuation_save.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -40,10 +33,9 @@ from unittest.mock import patch
|
|||
|
||||
import pytest
|
||||
|
||||
# Old app.services.scrapers.yandex_valuation and app.services.scraper_adapters both
|
||||
# import app.core.config.settings=Settings(), which requires DATABASE_URL. Offline
|
||||
# parsing/construction never touches a real DB — a fake DSN is enough (same trick as
|
||||
# tests/test_scraper_kit_yandex_golden_parity.py).
|
||||
# app.services.scraper_adapters imports app.core.config.settings=Settings(), which
|
||||
# requires DATABASE_URL. Offline construction never touches a real DB — a fake DSN
|
||||
# is enough.
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from scraper_kit.providers.yandex.valuation import (
|
||||
|
|
@ -52,29 +44,6 @@ from scraper_kit.providers.yandex.valuation import (
|
|||
|
||||
from app.services import scraper_settings
|
||||
from app.services.scraper_adapters import RealScraperConfig
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.yandex_valuation import (
|
||||
YandexValuationScraper as LegacyYandexValuationScraper,
|
||||
)
|
||||
from tests.support.parity import assert_parity
|
||||
|
||||
_VALUATION_HTML = """
|
||||
<html><body>
|
||||
Дом 2016 года 25 этажей 2,7 м потолки Лифт 42 объекта Панорама
|
||||
48,5 м², 1-комнатная, 5 этаж 5,1 млн ₽ 105 155 ₽ за м² 23.10.2023
|
||||
В экспозиции 945 дней Снято 24.05.2024
|
||||
32 м², студия, 2 этаж 3,2 млн ₽ 100 000 ₽ за м² 01.02.2024 В продаже
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
_PARSE_KWARGS = {
|
||||
"address": "Екатеринбург, ул. Ленина, 5",
|
||||
"offer_category": "APARTMENT",
|
||||
"offer_type": "SELL",
|
||||
"page": 1,
|
||||
"source_url": "https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address=test",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Footgun #1 — mandatory `config`
|
||||
|
|
@ -149,62 +118,3 @@ def test_kit_valuation_delay_provider_receives_scraper_name() -> None:
|
|||
)
|
||||
assert calls == ["yandex_valuation"]
|
||||
assert scraper.request_delay_sec == 9.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extended parity: full instantiation/config-wiring path, not just parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_full_construction_and_parse_parity_with_real_config_wiring() -> None:
|
||||
"""Legacy no-arg ctor vs kit ctor with the REAL production DI convention.
|
||||
|
||||
`tests/test_scraper_kit_yandex_golden_parity.py::test_valuation_parse_parity`
|
||||
already proves `.parse()` output identity using a duck-typed `SimpleNamespace`
|
||||
config. This test additionally proves the REAL wiring convention used in
|
||||
production (`config=RealScraperConfig()`, `delay_provider=get_scraper_delay` --
|
||||
same as `admin.py::scrape_yandex_valuation`, #2305 Group A) constructs without
|
||||
error and does not change parsing output vs. the legacy no-arg constructor.
|
||||
"""
|
||||
|
||||
def _legacy_parse(html: str) -> object:
|
||||
return LegacyYandexValuationScraper().parse(html, **_PARSE_KWARGS)
|
||||
|
||||
def _kit_parse(html: str) -> object:
|
||||
scraper = KitYandexValuationScraper(
|
||||
config=RealScraperConfig(), delay_provider=get_scraper_delay
|
||||
)
|
||||
return scraper.parse(html, **_PARSE_KWARGS)
|
||||
|
||||
assert_parity(
|
||||
legacy_fn=_legacy_parse,
|
||||
kit_fn=_kit_parse,
|
||||
fixtures=[_VALUATION_HTML],
|
||||
)
|
||||
|
||||
|
||||
def test_parity_harness_catches_valuation_construction_divergence() -> None:
|
||||
"""Sanity: assert_parity actually fails if kit construction changes parsed output.
|
||||
|
||||
Proves the extended-coverage test above is not vacuously green -- mutates the
|
||||
kit-side house metadata and confirms `assert_parity` reports the exact field.
|
||||
"""
|
||||
from tests.support.parity import ParityMismatchError
|
||||
|
||||
scraper = KitYandexValuationScraper(
|
||||
config=RealScraperConfig(), delay_provider=get_scraper_delay
|
||||
)
|
||||
kit_result = scraper.parse(_VALUATION_HTML, **_PARSE_KWARGS)
|
||||
mutated = kit_result.model_copy(
|
||||
update={"house": kit_result.house.model_copy(update={"year_built": 1900})}
|
||||
)
|
||||
|
||||
with pytest.raises(ParityMismatchError) as exc_info:
|
||||
assert_parity(
|
||||
legacy_fn=lambda: LegacyYandexValuationScraper().parse(
|
||||
_VALUATION_HTML, **_PARSE_KWARGS
|
||||
),
|
||||
kit_fn=lambda: mutated,
|
||||
fixtures=[()],
|
||||
)
|
||||
assert "year_built" in str(exc_info.value)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
"""Tests for app.tasks.yandex_detail_backfill.
|
||||
"""Tests for app.tasks.yandex_detail_backfill (the KIT task — already imports
|
||||
`scraper_kit.providers.yandex.detail.{YandexDetailScraper,save_detail_enrichment}`).
|
||||
|
||||
Fetch mechanism changed: curl_cffi AsyncSession(chrome120+proxy) + YandexDetailScraper.parse
|
||||
(instead of old fetch_detail path). Mocks target session.get and scraper.parse.
|
||||
|
||||
Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции) — `save_detail_enrichment` coverage-тесты внизу файла
|
||||
переведены на kit-эквивалент (тот же, что реально вызывает эта задача).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -337,13 +342,14 @@ async def test_backfill_no_proxy_when_settings_none() -> None:
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: save_detail_enrichment (unchanged — kept for coverage)
|
||||
# Tests: save_detail_enrichment (kit — same function the task above actually calls;
|
||||
# легаси `app.services.scrapers.yandex_detail` удалён #2277 финальный шаг миграции)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_save_detail_enrichment_maps_fields_to_update() -> None:
|
||||
"""save_detail_enrichment calls db.execute with UPDATE and commits."""
|
||||
from app.services.scrapers.yandex_detail import (
|
||||
from scraper_kit.providers.yandex.detail import (
|
||||
DetailEnrichment,
|
||||
MetroStation,
|
||||
save_detail_enrichment,
|
||||
|
|
@ -403,7 +409,7 @@ def test_save_detail_enrichment_maps_fields_to_update() -> None:
|
|||
|
||||
def test_save_detail_enrichment_empty_metro_and_photos() -> None:
|
||||
"""Empty metro_stations and photo_urls -> NULL passed for both jsonb columns."""
|
||||
from app.services.scrapers.yandex_detail import DetailEnrichment, save_detail_enrichment
|
||||
from scraper_kit.providers.yandex.detail import DetailEnrichment, save_detail_enrichment
|
||||
|
||||
enrichment = DetailEnrichment(
|
||||
offer_id="99",
|
||||
|
|
@ -424,7 +430,7 @@ def test_save_detail_enrichment_empty_metro_and_photos() -> None:
|
|||
|
||||
def test_save_detail_enrichment_rowcount_zero_returns_false() -> None:
|
||||
"""rowcount=0 (listing_id not found) -> returns False."""
|
||||
from app.services.scrapers.yandex_detail import DetailEnrichment, save_detail_enrichment
|
||||
from scraper_kit.providers.yandex.detail import DetailEnrichment, save_detail_enrichment
|
||||
|
||||
enrichment = DetailEnrichment(
|
||||
offer_id="404",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
"""Offline smoke tests для avito_imv модуля.
|
||||
"""Offline smoke tests для scraper_kit avito IMV провайдера.
|
||||
|
||||
Без сети — только cache_key вычисление, парсинг заголовков, unix→date, dataclass конструирование.
|
||||
Без сети — только cache_key вычисление, парсинг заголовков, unix→date, dataclass
|
||||
конструирование. Легаси `app.services.scrapers.avito_imv` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
|
||||
`scraper_kit.providers.avito.imv`, регрессия по контракту step B/C (issue #565)
|
||||
сохранена без изменений.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
|
@ -8,8 +12,7 @@ from datetime import date
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.avito_imv import (
|
||||
from scraper_kit.providers.avito.imv import (
|
||||
IMVAddressNotFoundError,
|
||||
IMVEvaluation,
|
||||
IMVGeo,
|
||||
|
|
@ -60,12 +63,8 @@ def test_cache_key_differs_by_rooms():
|
|||
|
||||
|
||||
def test_cache_key_differs_by_balcony():
|
||||
k_with = compute_imv_cache_key(
|
||||
"ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", True, False
|
||||
)
|
||||
k_without = compute_imv_cache_key(
|
||||
"ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", False, False
|
||||
)
|
||||
k_with = compute_imv_cache_key("ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", True, False)
|
||||
k_without = compute_imv_cache_key("ул. Ленина, 1", 1, 30.0, 1, 9, "brick", "euro", False, False)
|
||||
assert k_with != k_without
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,11 @@ Covers:
|
|||
(provider="cache", confidence="exact"); parse-fail / no-row → falls through.
|
||||
|
||||
All network + DB mocked.
|
||||
|
||||
Легаси `app.services.scrapers.ekb_geoportal_client` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции, 0 runtime importers) — client-тесты переведены на kit-
|
||||
эквивалент `scraper_kit.providers.ekb_geoportal.client` (уже используемый живым
|
||||
`app.tasks.ekb_geoportal_ingest`), покрытие не изменилось.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -25,10 +30,11 @@ _wp_mock = MagicMock()
|
|||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||||
|
||||
|
||||
from app.services.scrapers.ekb_geoportal_client import ( # noqa: E402
|
||||
from scraper_kit.providers.ekb_geoportal.client import ( # noqa: E402
|
||||
TopoBuilding,
|
||||
parse_feature_collection,
|
||||
)
|
||||
|
||||
from app.tasks.ekb_geoportal_ingest import ( # noqa: E402
|
||||
_group_buildings,
|
||||
_tile_grid,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,12 @@ Best-effort: house_id=None → колонка NULL, сохранение НЕ п
|
|||
|
||||
DB — MagicMock, записываем параметры db.execute (никакого реального Postgres,
|
||||
как в test_yandex_valuation_save / test_backfill_house_coords).
|
||||
|
||||
Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции) — `_yandex_result()` переведён на kit-эквивалент
|
||||
`scraper_kit.providers.yandex.valuation` (тот же тип, что `estimator.py` реально
|
||||
использует через `YandexValuationScraper`, который здесь пропатчен). `cian_valuation`
|
||||
остаётся легаси (не мигрирован) — его импорт не тронут.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -88,7 +94,7 @@ def _make_db_cache_miss() -> MagicMock:
|
|||
|
||||
|
||||
def _yandex_result():
|
||||
from app.services.scrapers.yandex_valuation import (
|
||||
from scraper_kit.providers.yandex.valuation import (
|
||||
ValuationHouseMeta,
|
||||
YandexValuationResult,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ Migrates legacy scraper imports of these files to their `scraper_kit` equivalent
|
|||
|
||||
- app/services/house_imv_backfill.py → BrowserFetcher now from
|
||||
`scraper_kit.browser_fetcher` (was `app.services.scrapers.browser_fetcher`).
|
||||
avito_imv itself stays legacy (valuation dependency, issue #2308 — separate).
|
||||
avito_imv itself was still legacy at the time (valuation dependency, issue
|
||||
#2308 — separate); migrated to `scraper_kit.providers.avito.imv` in #2334,
|
||||
then the legacy module was deleted entirely in #2277's final step.
|
||||
- app/tasks/avito_detail_backfill.py → AvitoScraper/BrowserFetcher/avito
|
||||
exceptions/fetch_detail/research_in_session/save_detail_enrichment now from
|
||||
`scraper_kit.*`. `build_warmed_session`/`_AVITO_WARM_SEARCH_URL` stay legacy —
|
||||
|
|
|
|||
|
|
@ -10,20 +10,25 @@ house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→
|
|||
- SERP `_entity_to_lot` (gate-API entity → ScrapedLot) на репрезентативных entity;
|
||||
- SERP `_parse_gate_json` (payload → list[ScrapedLot]) с новостройкой/вторичкой;
|
||||
- `normalize_house_type` (SCREAMING/camelCase → канон);
|
||||
- detail `parse` (INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment) на
|
||||
реальных offer-HTML фикстурах;
|
||||
- valuation `parse` (house-meta + history → YandexValuationResult) на синтетике;
|
||||
- newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) на синтетике;
|
||||
- `_build_url` (gate-API URL builder) — детерминированность.
|
||||
|
||||
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
|
||||
|
||||
Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
|
||||
финальный шаг scraper_kit-миграции, 0 runtime importers) — detail/valuation
|
||||
parity-проверки убраны отсюда вместе с ними. Kit-side coverage для detail/valuation
|
||||
`.parse()` сохранена без изменений в `tests/test_yandex_detail.py` /
|
||||
`tests/test_yandex_detail_structural.py` / `tests/test_yandex_valuation.py` /
|
||||
`tests/test_yandex_valuation_save.py` (переведены на kit-эквиваленты тем же PR).
|
||||
`yandex_realty` / `yandex_newbuilding` / `house_type_normalizer` остаются легаси
|
||||
(не мигрированы) — их parity-проверки не тронуты.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -33,7 +38,6 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
|
|||
|
||||
# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном
|
||||
# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов.
|
||||
from scraper_kit.providers.yandex.detail import YandexDetailScraper as NewDetailScraper
|
||||
from scraper_kit.providers.yandex.newbuilding import (
|
||||
YandexNewbuildingScraper as NewNewbuildingScraper,
|
||||
)
|
||||
|
|
@ -41,25 +45,16 @@ from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewRealtySc
|
|||
from scraper_kit.providers.yandex.serp import _entity_to_lot as new_entity_to_lot
|
||||
from scraper_kit.providers.yandex.serp import _parse_gate_json as new_parse_gate_json
|
||||
from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type
|
||||
from scraper_kit.providers.yandex.valuation import (
|
||||
YandexValuationScraper as NewValuationScraper,
|
||||
)
|
||||
|
||||
from app.services.scrapers.house_type_normalizer import (
|
||||
normalize_house_type as old_normalize_house_type,
|
||||
)
|
||||
from app.services.scrapers.yandex_detail import YandexDetailScraper as OldDetailScraper
|
||||
from app.services.scrapers.yandex_newbuilding import (
|
||||
YandexNewbuildingScraper as OldNewbuildingScraper,
|
||||
)
|
||||
from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldRealtyScraper
|
||||
from app.services.scrapers.yandex_realty import _entity_to_lot as old_entity_to_lot
|
||||
from app.services.scrapers.yandex_realty import _parse_gate_json as old_parse_gate_json
|
||||
from app.services.scrapers.yandex_valuation import (
|
||||
YandexValuationScraper as OldValuationScraper,
|
||||
)
|
||||
|
||||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
# Инжектируемый конфиг для kit-скрапперов (duck-typed namespace, structurally
|
||||
# удовлетворяет scraper_kit.contracts.ScraperConfig в объёме читаемых полей).
|
||||
|
|
@ -268,61 +263,6 @@ def test_normalize_house_type_parity() -> None:
|
|||
assert old_normalize_house_type(raw) == new_normalize_house_type(raw)
|
||||
|
||||
|
||||
# ── detail parse parity (реальные offer-HTML фикстуры) ───────────────────────
|
||||
|
||||
_SECONDARY_ID = "3402418396407468801"
|
||||
_STUDIO_ID = "7811691822126445498"
|
||||
|
||||
|
||||
def _load_offer(offer_id: str) -> str:
|
||||
return (_FIXTURES / f"yandex_offer_{offer_id}.html").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _offer_url(offer_id: str) -> str:
|
||||
return f"https://realty.yandex.ru/offer/{offer_id}/"
|
||||
|
||||
|
||||
def test_detail_parse_parity() -> None:
|
||||
"""INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment идентичен."""
|
||||
old_scraper = OldDetailScraper()
|
||||
new_scraper = NewDetailScraper()
|
||||
for offer_id in (_SECONDARY_ID, _STUDIO_ID):
|
||||
html = _load_offer(offer_id)
|
||||
url = _offer_url(offer_id)
|
||||
old_res = old_scraper.parse(html, offer_url=url)
|
||||
new_res = new_scraper.parse(html, offer_url=url)
|
||||
assert old_res is not None and new_res is not None
|
||||
assert old_res.model_dump() == new_res.model_dump()
|
||||
|
||||
|
||||
# ── valuation parse parity (синтетика) ───────────────────────────────────────
|
||||
|
||||
_VALUATION_HTML = """
|
||||
<html><body>
|
||||
Дом 2016 года 25 этажей 2,7 м потолки Лифт 42 объекта Панорама
|
||||
48,5 м², 1-комнатная, 5 этаж 5,1 млн ₽ 105 155 ₽ за м² 23.10.2023
|
||||
В экспозиции 945 дней Снято 24.05.2024
|
||||
32 м², студия, 2 этаж 3,2 млн ₽ 100 000 ₽ за м² 01.02.2024 В продаже
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
def test_valuation_parse_parity() -> None:
|
||||
"""house-meta + history (chunked-text fallback) → идентичный результат."""
|
||||
old_scraper = OldValuationScraper()
|
||||
new_scraper = NewValuationScraper(config=_KIT_CONFIG)
|
||||
kwargs = dict(
|
||||
address="Екатеринбург, ул. Ленина, 5",
|
||||
offer_category="APARTMENT",
|
||||
offer_type="SELL",
|
||||
page=1,
|
||||
source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address=test",
|
||||
)
|
||||
old_res = old_scraper.parse(_VALUATION_HTML, **kwargs)
|
||||
new_res = new_scraper.parse(_VALUATION_HTML, **kwargs)
|
||||
assert old_res.model_dump() == new_res.model_dump()
|
||||
|
||||
|
||||
# ── newbuilding parse parity (синтетика) ─────────────────────────────────────
|
||||
|
||||
_NEWBUILDING_HTML = """
|
||||
|
|
|
|||
|
|
@ -5,9 +5,17 @@ Covers:
|
|||
- AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
|
||||
- None/empty → direct connection (no proxies dict)
|
||||
- _avito_proxies() helper returns correct dict shape
|
||||
- per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation,
|
||||
- per-scraper proxy wiring: YandexRealty, YandexValuation (kit), CianValuation,
|
||||
CianNewbuilding (fetch_newbuilding — now BrowserFetcher, proxy is browser-level),
|
||||
AvitoIMV, AvitoDetail
|
||||
AvitoDetail
|
||||
|
||||
Легаси `app.services.scrapers.yandex_valuation` / `avito_imv` удалены (#2277
|
||||
финальный шаг scraper_kit-миграции). YandexValuation proxy-wiring тесты переведены
|
||||
на kit-эквивалент (`config=` DI вместо патча `app.core.config.settings`).
|
||||
`test_avito_imv_own_session_receives_proxies` (легаси, patch-based) удалён —
|
||||
эквивалентная и более точная kit-регрессия (config footgun: с/без `config=`) уже
|
||||
покрыта в `tests/scrapers/test_avito_imv_kit_parity.py::
|
||||
test_kit_evaluate_via_imv_{with,without}_config_{uses,drops}_proxy`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -202,8 +210,8 @@ async def test_yandex_realty_opens_browser_fetcher(monkeypatch):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yandex_valuation_session_receives_proxies():
|
||||
"""YandexValuationScraper.__aenter__ forwards proxies= kwarg."""
|
||||
from app.services.scrapers.yandex_valuation import YandexValuationScraper
|
||||
"""Kit YandexValuationScraper.__aenter__ forwards proxies= kwarg from config."""
|
||||
from scraper_kit.providers.yandex.valuation import YandexValuationScraper
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_session.close = AsyncMock()
|
||||
|
|
@ -213,18 +221,18 @@ async def test_yandex_valuation_session_receives_proxies():
|
|||
captured_kwargs.update(kwargs)
|
||||
return mock_session
|
||||
|
||||
with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session):
|
||||
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
|
||||
scraper = YandexValuationScraper()
|
||||
await scraper.__aenter__()
|
||||
cfg = SimpleNamespace(use_proxy_pool_curl=False, scraper_proxy_url=_PROXY_URL)
|
||||
with patch("scraper_kit.providers.yandex.valuation._CurlCffiSession", _fake_session):
|
||||
async with YandexValuationScraper(cfg):
|
||||
pass
|
||||
|
||||
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yandex_valuation_session_no_proxies_when_none():
|
||||
"""YandexValuationScraper.__aenter__: proxies=None when no proxy configured."""
|
||||
from app.services.scrapers.yandex_valuation import YandexValuationScraper
|
||||
"""Kit YandexValuationScraper.__aenter__: proxies=None when no proxy configured."""
|
||||
from scraper_kit.providers.yandex.valuation import YandexValuationScraper
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_session.close = AsyncMock()
|
||||
|
|
@ -234,10 +242,10 @@ async def test_yandex_valuation_session_no_proxies_when_none():
|
|||
captured_kwargs.update(kwargs)
|
||||
return mock_session
|
||||
|
||||
with patch("app.services.scrapers.yandex_valuation._CurlCffiSession", _fake_session):
|
||||
with patch("app.core.config.settings", _mock_settings(None)):
|
||||
scraper = YandexValuationScraper()
|
||||
await scraper.__aenter__()
|
||||
cfg = SimpleNamespace(use_proxy_pool_curl=False, scraper_proxy_url=None)
|
||||
with patch("scraper_kit.providers.yandex.valuation._CurlCffiSession", _fake_session):
|
||||
async with YandexValuationScraper(cfg):
|
||||
pass
|
||||
|
||||
assert captured_kwargs.get("proxies") is None
|
||||
|
||||
|
|
@ -384,56 +392,6 @@ async def test_avito_detail_shared_session_not_patched():
|
|||
assert ctor_calls == []
|
||||
|
||||
|
||||
# ── AvitoIMV — evaluate_via_imv own-session ──────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_avito_imv_own_session_receives_proxies():
|
||||
"""evaluate_via_imv creates own session with proxies= when none passed.
|
||||
|
||||
`CffiAsyncSession` is imported locally inside evaluate_via_imv as
|
||||
`from curl_cffi.requests import AsyncSession as CffiAsyncSession`, so we
|
||||
patch `curl_cffi.requests.AsyncSession` (the source) to intercept it.
|
||||
"""
|
||||
from app.services.scrapers import avito_imv
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_warmup_resp = MagicMock()
|
||||
mock_warmup_resp.status_code = 200
|
||||
mock_geo_resp = MagicMock()
|
||||
mock_geo_resp.status_code = 200
|
||||
mock_geo_resp.json = MagicMock(return_value={"result": {"point": None}})
|
||||
mock_session.get = AsyncMock(return_value=mock_warmup_resp)
|
||||
mock_session.post = AsyncMock(return_value=mock_geo_resp)
|
||||
mock_session.close = AsyncMock()
|
||||
|
||||
captured_kwargs: dict = {}
|
||||
|
||||
def _fake_session(*args, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return mock_session
|
||||
|
||||
# Patch the source so the local `from curl_cffi.requests import AsyncSession` picks it up
|
||||
with patch("curl_cffi.requests.AsyncSession", _fake_session):
|
||||
with patch("app.core.config.settings", _mock_settings(_PROXY_URL)):
|
||||
try:
|
||||
await avito_imv.evaluate_via_imv(
|
||||
address="ул. Тургенева, 4",
|
||||
rooms=2,
|
||||
area_m2=50.0,
|
||||
floor=3,
|
||||
floor_at_home=9,
|
||||
house_type="panel",
|
||||
renovation_type="cosmetic",
|
||||
has_balcony=False,
|
||||
has_loggia=False,
|
||||
)
|
||||
except Exception:
|
||||
pass # network/parse failures expected in unit test
|
||||
|
||||
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||
|
||||
|
||||
# ── Settings.cian_proxy_url property ─────────────────────────────────────────
|
||||
# cian_proxy_url: CIAN_PROXY_URL > scraper_proxy_url (fallback) > None.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
"""Unit tests for YandexDetailScraper — Product JSON-LD + DOM detail parser.
|
||||
|
||||
All tests run against hand-crafted HTML fixtures. No live network access.
|
||||
|
||||
Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
|
||||
`scraper_kit.providers.yandex.detail` без изменения покрытия.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -8,8 +12,7 @@ from __future__ import annotations
|
|||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.yandex_detail import (
|
||||
from scraper_kit.providers.yandex.detail import (
|
||||
DetailEnrichment,
|
||||
MetroStation,
|
||||
YandexDetailScraper,
|
||||
|
|
@ -213,7 +216,7 @@ class TestAgencyBlock:
|
|||
<h2>АН Городской риелтор</h2>
|
||||
<p>Год основания 1998. Продали 150 объектов.</p>
|
||||
</div>"""
|
||||
summary = "Иван Петров АН Городской риелтор • 50 просмотров" " • опубликовано 3 марта 2025"
|
||||
summary = "Иван Петров АН Городской риелтор • 50 просмотров • опубликовано 3 марта 2025"
|
||||
html = _make_html(author_block=author_html, summary_text=summary)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
|
|
|
|||
|
|
@ -8,6 +8,10 @@ never carries ceiling/kitchen — which is why DB area coverage sat at ~63%.
|
|||
Fixtures are trimmed real offer pages fetched via the headless browser:
|
||||
- yandex_offer_3402418396407468801.html — 1-комн. вторичка (full chars)
|
||||
- yandex_offer_7811691822126445498.html — студия / новостройка
|
||||
|
||||
Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
|
||||
`scraper_kit.providers.yandex.detail` без изменения покрытия.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -15,8 +19,7 @@ from __future__ import annotations
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.yandex_detail import (
|
||||
from scraper_kit.providers.yandex.detail import (
|
||||
YandexDetailScraper,
|
||||
_extract_offer_card,
|
||||
_parse_card_fields,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,10 @@ _save_yandex_history_items должен:
|
|||
- сохранить валидные items (area_m2 > 0) без потерь
|
||||
- не ронять батч из-за одного битого item
|
||||
- лого кол-во отброшенных items
|
||||
|
||||
Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции) — `result` типы переведены на kit-эквивалент
|
||||
`scraper_kit.providers.yandex.valuation`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -15,13 +19,14 @@ from unittest.mock import MagicMock, patch
|
|||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
from app.services.estimator import _save_yandex_history_items
|
||||
from app.services.scrapers.yandex_valuation import (
|
||||
from scraper_kit.providers.yandex.valuation import (
|
||||
ValuationHistoryItem,
|
||||
ValuationHouseMeta,
|
||||
YandexValuationResult,
|
||||
)
|
||||
|
||||
from app.services.estimator import _save_yandex_history_items
|
||||
|
||||
|
||||
def _make_result(items: list[ValuationHistoryItem]) -> YandexValuationResult:
|
||||
return YandexValuationResult(
|
||||
|
|
|
|||
|
|
@ -1,10 +1,27 @@
|
|||
"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings."""
|
||||
"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings.
|
||||
|
||||
Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
|
||||
финальный шаг scraper_kit-миграции). `yandex_realty` / `yandex_newbuilding` пока
|
||||
остаются легаси (не мигрированы) — их delay-wiring тесты не тронуты.
|
||||
`yandex_detail`/`yandex_valuation` переведены на kit-эквиваленты, которые используют
|
||||
инжектируемый `delay_provider=` (strangler DI, #2133) вместо прямого импорта
|
||||
`get_scraper_delay` — тест адаптирован под реальный контракт kit-класса.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.scrapers.yandex_detail import YandexDetailScraper
|
||||
from scraper_kit.providers.yandex.detail import YandexDetailScraper
|
||||
from scraper_kit.providers.yandex.valuation import YandexValuationScraper
|
||||
|
||||
from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
|
||||
from app.services.scrapers.yandex_realty import YandexRealtyScraper
|
||||
from app.services.scrapers.yandex_valuation import YandexValuationScraper
|
||||
|
||||
_KIT_CONFIG = SimpleNamespace(
|
||||
scraper_proxy_url=None,
|
||||
yandex_proxy_rotate_url=None,
|
||||
avito_proxy_rotate_url=None,
|
||||
)
|
||||
|
||||
|
||||
def test_yandex_realty_picks_up_delay():
|
||||
|
|
@ -14,22 +31,23 @@ def test_yandex_realty_picks_up_delay():
|
|||
|
||||
|
||||
def test_yandex_detail_picks_up_delay():
|
||||
with patch("app.services.scrapers.yandex_detail.get_scraper_delay", return_value=9.5):
|
||||
s = YandexDetailScraper()
|
||||
assert s.request_delay_sec == 9.5
|
||||
"""Kit YandexDetailScraper: delay_provider callable вызывается с self.name."""
|
||||
s = YandexDetailScraper(delay_provider=lambda _name: 9.5)
|
||||
assert s.request_delay_sec == 9.5
|
||||
|
||||
|
||||
def test_yandex_newbuilding_picks_up_delay():
|
||||
with patch(
|
||||
"app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5
|
||||
):
|
||||
with patch("app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5):
|
||||
s = YandexNewbuildingScraper()
|
||||
assert s.request_delay_sec == 9.5
|
||||
|
||||
|
||||
def test_yandex_valuation_picks_up_delay():
|
||||
with patch(
|
||||
"app.services.scrapers.yandex_valuation.get_scraper_delay", return_value=9.5
|
||||
):
|
||||
s = YandexValuationScraper()
|
||||
assert s.request_delay_sec == 9.5
|
||||
"""Kit YandexValuationScraper: delay_provider callable вызывается с self.name.
|
||||
|
||||
Mandatory-config footgun и полное покрытие delay_provider wiring уже проверены
|
||||
в tests/scrapers/test_yandex_valuation_kit_migration.py — здесь просто smoke,
|
||||
зеркалящий остальные Yandex-скрапперы в этом файле.
|
||||
"""
|
||||
s = YandexValuationScraper(config=_KIT_CONFIG, delay_provider=lambda _name: 9.5)
|
||||
assert s.request_delay_sec == 9.5
|
||||
|
|
|
|||
|
|
@ -3,19 +3,32 @@
|
|||
Fixture HTML simulates the Yandex valuation page body text containing:
|
||||
- House meta block (year, floors, type, ceiling, lift, total objects, panorama)
|
||||
- 2-3 historical offer entries with full structure
|
||||
|
||||
Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
|
||||
`scraper_kit.providers.yandex.valuation` без изменения покрытия. Kit-конструктор
|
||||
требует обязательный `config:` (см. `tests/scrapers/test_yandex_valuation_kit_migration.py`
|
||||
для footgun-регрессии) — используем тот же duck-typed `SimpleNamespace`, что и
|
||||
`tests/test_scraper_kit_yandex_golden_parity.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.yandex_valuation import (
|
||||
from scraper_kit.providers.yandex.valuation import (
|
||||
YandexValuationResult,
|
||||
YandexValuationScraper,
|
||||
)
|
||||
|
||||
_KIT_CONFIG = SimpleNamespace(
|
||||
scraper_proxy_url=None,
|
||||
yandex_proxy_rotate_url=None,
|
||||
avito_proxy_rotate_url=None,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixture helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -59,7 +72,7 @@ FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\
|
|||
|
||||
|
||||
def test_parse_house_meta_full():
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
meta = scraper._parse_house_meta(_HOUSE_META_BLOCK)
|
||||
assert meta.year_built == 1981
|
||||
assert meta.total_floors == 9
|
||||
|
|
@ -176,7 +189,7 @@ def test_parse_items_from_chunked_text_multiple_items():
|
|||
|
||||
|
||||
def test_parse_full_result_address_propagated():
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
result = scraper.parse(
|
||||
FULL_FIXTURE_HTML,
|
||||
address="Екатеринбург, ул. Ленина, 1",
|
||||
|
|
@ -197,7 +210,7 @@ def test_parse_full_result_address_propagated():
|
|||
|
||||
|
||||
def test_parse_full_result_house_meta_populated():
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
result = scraper.parse(
|
||||
FULL_FIXTURE_HTML,
|
||||
address="test",
|
||||
|
|
@ -219,7 +232,7 @@ def test_parse_full_result_house_meta_populated():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_raises_not_implemented():
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
with pytest.raises(NotImplementedError):
|
||||
await scraper.fetch_around(lat=56.8, lon=60.6)
|
||||
|
||||
|
|
@ -287,7 +300,7 @@ def test_parse_normalizes_nbsp_for_price_per_m2():
|
|||
"4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² "
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="test",
|
||||
|
|
@ -303,21 +316,21 @@ def test_parse_normalizes_nbsp_for_price_per_m2():
|
|||
|
||||
|
||||
def test_validate_match_full_match():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2025, total_floors=25)
|
||||
assert meta.validate_match(2025, 25) == 1.0
|
||||
|
||||
|
||||
def test_validate_match_year_mismatch():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=1984, total_floors=9)
|
||||
assert meta.validate_match(2025, 25) == 0.0
|
||||
|
||||
|
||||
def test_validate_match_partial():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2025, total_floors=10)
|
||||
# year matches, floors mismatch (>1 tolerance) → 0.5
|
||||
|
|
@ -325,7 +338,7 @@ def test_validate_match_partial():
|
|||
|
||||
|
||||
def test_validate_match_tolerance_one_year():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2024, total_floors=25)
|
||||
# ±1 year tolerance
|
||||
|
|
@ -333,7 +346,7 @@ def test_validate_match_tolerance_one_year():
|
|||
|
||||
|
||||
def test_validate_match_no_expected_returns_one():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=None, total_floors=None)
|
||||
assert meta.validate_match(None, None) == 1.0
|
||||
|
|
@ -400,9 +413,9 @@ def test_area_regex_rejects_year_concat():
|
|||
item = YandexValuationScraper._parse_item_text(text)
|
||||
# The "202452,2" token has digits jammed before it (no separator), so the
|
||||
# tightened regex must NOT match it.
|
||||
assert item is None or item.area_m2 is None, (
|
||||
f"expected no area match for concat token, got {item.area_m2 if item else 'None item'}"
|
||||
)
|
||||
assert (
|
||||
item is None or item.area_m2 is None
|
||||
), f"expected no area match for concat token, got {item.area_m2 if item else 'None item'}"
|
||||
|
||||
|
||||
def test_area_regex_accepts_isolated_token():
|
||||
|
|
@ -458,9 +471,9 @@ def test_area_regex_two_komn_chunk():
|
|||
text = "8 000 000 ₽ за м²2-комнатная 52,2 м² 5 этаж 10.05.2024 8 000 000 ₽ В продаже"
|
||||
item = YandexValuationScraper._parse_item_text(text)
|
||||
assert item is not None
|
||||
assert item.area_m2 == 52.2, (
|
||||
f"2-комн area must parse despite preceding tokens, got {item.area_m2}"
|
||||
)
|
||||
assert (
|
||||
item.area_m2 == 52.2
|
||||
), f"2-комн area must parse despite preceding tokens, got {item.area_m2}"
|
||||
|
||||
|
||||
def test_area_regex_still_blocks_year_concat():
|
||||
|
|
@ -511,7 +524,7 @@ _DOM_FIXTURE_ROW_HTML = """
|
|||
|
||||
|
||||
def test_dom_parser_extracts_two_rows():
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
house_meta = "Дом 2020 года 25 этажей Монолитное здание "
|
||||
result = scraper.parse(
|
||||
f"<html><body>{house_meta}{_DOM_FIXTURE_ROW_HTML}</body></html>",
|
||||
|
|
@ -561,7 +574,7 @@ def test_dom_parser_studio_row():
|
|||
"</div>"
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="t",
|
||||
|
|
@ -586,7 +599,7 @@ def test_dom_parser_fallback_when_no_rows():
|
|||
"10.05.2024 9 000 000 ₽ В продаже"
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="t",
|
||||
|
|
@ -610,7 +623,7 @@ def test_dom_parser_skips_malformed_row():
|
|||
"</div>"
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="t",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
"""Tests for _save_yandex_history_items — house_id linking + new columns."""
|
||||
"""Tests for _save_yandex_history_items — house_id linking + new columns.
|
||||
|
||||
Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг
|
||||
scraper_kit-миграции) — `result` типы переведены на kit-эквивалент
|
||||
`scraper_kit.providers.yandex.valuation`, тот же тип, что `estimator.py` реально
|
||||
принимает в `_save_yandex_history_items`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
@ -11,14 +17,14 @@ from datetime import date
|
|||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.estimator import _save_yandex_history_items
|
||||
from app.services.scrapers.yandex_valuation import (
|
||||
from scraper_kit.providers.yandex.valuation import (
|
||||
ValuationHistoryItem,
|
||||
ValuationHouseMeta,
|
||||
YandexValuationResult,
|
||||
)
|
||||
|
||||
from app.services.estimator import _save_yandex_history_items
|
||||
|
||||
|
||||
def _make_result(items: list[ValuationHistoryItem]) -> YandexValuationResult:
|
||||
return YandexValuationResult(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue