From 879f7612140db7bfff9f3dffe3ba83111d85e0d5 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 15:27:22 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20avito=5Fhouses.py=20=E2=80=94?= =?UTF-8?q?=20Houses=20Catalog=20parser=20(state=20+=20reviews=20+=20histo?= =?UTF-8?q?ry=20+=20recs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 2c of AvitoScraper_v2. - fetch_house_catalog(house_url) — async HTTP via curl_cffi (chrome120) - parse_houses_state(state, house_url) — pure parser (testable без сети) - 10-widget JSON state structure (housePage, reviews, miniSerp, housePlacementHistory, recommendations) - Dataclasses: * HouseInfo (28 fields: identity, geo, expandParams flatten, developer, rating, map_pins, raw_characteristics) * HouseReview (textSections split: main/pros/cons; ru date parse) * PlacementHistoryItem (removed_date ALWAYS None — widget не отдаёт; точная дата только в IMV API) * SellerInfo (ext_seller_id из URL last segment, без logos) * MiniSerpListing (active listings в доме + metro + seller) * RecommendationItem (lightweight: title/price_text/address/url/image) * HouseCatalogEnrichment (orchestrator dataclass) - Save flow (5 ops): * _upsert_house_with_url → houses (UNIQUE source='avito', ext_house_id) + raw_characteristics jsonb * save_house_reviews → house_reviews (UNIQUE source, ext_review_id), без images * save_sellers → sellers (UNIQUE source, ext_seller_id), без logos * link_listings_to_seller → UPDATE listings.seller_id_fk by source_id match * save_placement_history → source='avito_widget', removed_date=NULL - pytest offline smoke: 18 tests, 5 widget types, 25+ field assertions - Recommendations save skipped — Stage 3 решит --- .../app/services/scrapers/avito_houses.py | 1141 +++++++++++++++++ .../backend/tests/test_avito_houses_parse.py | 397 ++++++ 2 files changed, 1538 insertions(+) create mode 100644 tradein-mvp/backend/app/services/scrapers/avito_houses.py create mode 100644 tradein-mvp/backend/tests/test_avito_houses_parse.py diff --git a/tradein-mvp/backend/app/services/scrapers/avito_houses.py b/tradein-mvp/backend/app/services/scrapers/avito_houses.py new file mode 100644 index 00000000..6bed1445 --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/avito_houses.py @@ -0,0 +1,1141 @@ +"""avito_houses.py — Avito Houses Catalog parser (Stage 2c). + +Парсит страницу /catalog/houses//, извлекает window.__preloadedState__ +и разбирает 10 виджетов: housePage, reviews, miniSerp, housePlacementHistory, +recommendations. + +Flow: + fetch_house_catalog(house_url) + → HTTP GET (curl_cffi chrome120) + → regex extract __preloadedState__ + → JS-unescape + json.loads + → parse_houses_state(state, house_url) + → HouseCatalogEnrichment + +Таблицы: + houses (009_houses.sql + 010_houses_alter.sql) + house_reviews (014_house_reviews.sql) + sellers (012_sellers.sql) + listings.seller_id_fk (013_listings_alter_seller.sql) + house_placement_history (017_house_placement_history.sql) +""" + +from __future__ import annotations + +import json +import logging +import re +from dataclasses import dataclass, field +from datetime import UTC, date, datetime +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + +AVITO_BASE = "https://www.avito.ru" + +# Regex для извлечения __preloadedState__ из HTML +PRELOADED_RE = re.compile( + r'window\.__preloadedState__\s*=\s*JSON\.parse\("(.*?)"\)\s*;', + re.DOTALL, +) + +# Русские месяцы для парсинга дат отзывов +RUS_MONTHS = { + "января": 1, + "февраля": 2, + "марта": 3, + "апреля": 4, + "мая": 5, + "июня": 6, + "июля": 7, + "августа": 8, + "сентября": 9, + "октября": 10, + "ноября": 11, + "декабря": 12, +} + +# Маппинг expandParams type → (field_name, cast_type) +EXPAND_PARAMS_MAP: dict[str, tuple[str, str]] = { + "Год постройки": ("year_built", "int"), + "Этажей": ("total_floors", "int"), + "Тип дома": ("house_type", "house_type"), + "Горячее водоснабжение": ("hot_water", "str"), + "Пассажирский лифт": ("passenger_elevators", "int"), + "Грузовой лифт": ("cargo_elevators", "int"), + "Консьерж": ("has_concierge", "bool_da"), + "Перекрытия": ("material_floors", "str"), + "Парковка": ("parking_type", "str"), + "Детская площадка": ("has_playground", "bool_da"), + "Закрытый двор": ("closed_yard", "bool_da"), +} + +# Нормализация типа дома из Avito текста в DB enum +HOUSE_TYPE_MAP: dict[str, str] = { + "монолитный": "monolith", + "монолит": "monolith", + "панельный": "panel", + "панель": "panel", + "кирпичный": "brick", + "кирпич": "brick", + "монолитно-кирпичный": "monolith_brick", + "блочный": "block", + "деревянный": "wood", +} + + +# --------------------------------------------------------------------------- +# Dataclasses +# --------------------------------------------------------------------------- + + +@dataclass +class HouseInfo: + ext_id: int # avitoId (int) + ext_id_hash: str | None = None # base64 internal ID + title: str | None = None + short_address: str | None = None # краткий адрес + full_address: str | None = None # полный адрес + lat: float | None = None + lon: float | None = None + year_built: int | None = None + total_floors: int | None = None + house_type: str | None = None # нормализован: monolith/panel/brick/... + material_floors: str | None = None + hot_water: str | None = None + passenger_elevators: int | None = None + cargo_elevators: int | None = None + has_concierge: bool | None = None + parking_type: str | None = None + has_playground: bool | None = None + closed_yard: bool | None = None + developer_name: str | None = None + developer_key: str | None = None + infrastructure_summary: str | None = None + infrastructure_walk_distance: str | None = None + rating_score: float | None = None + rating_string: str | None = None + reviews_count: int | None = None + rating_distribution: list[dict[str, Any]] = field(default_factory=list) + map_pins: list[dict[str, Any]] = field(default_factory=list) + raw_characteristics: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass +class HouseReview: + ext_review_id: int + author_name: str | None = None + review_title: str | None = None + score: int | None = None + model_experience: str | None = None + rated_date: date | None = None + text_main: str | None = None + text_pros: str | None = None + text_cons: str | None = None + raw_payload: dict[str, Any] | None = None + + +@dataclass +class PlacementHistoryItem: + ext_item_id: str + title: str | None = None + start_price: int | None = None + start_price_date: date | None = None + last_price: int | None = None + last_price_date: date | None = None + exposure_days: int | None = None + removed_date: date | None = None # ВСЕГДА None для source='avito_widget' + raw_payload: dict[str, Any] | None = None + + +@dataclass +class SellerInfo: + ext_seller_id: str + name: str + seller_type: str | None = None + on_platform_since: str | None = None + profile_url: str | None = None + + +@dataclass +class MiniSerpListing: + ext_item_id: int + title: str | None = None + price_rub: int | None = None + description: str | None = None + url: str = "" + metro_text: str | None = None + metro_color: str | None = None + seller: SellerInfo | None = None + + +@dataclass +class RecommendationItem: + title: str | None = None + price_text: str | None = None + address: str | None = None + url: str = "" + image_url: str | None = None + + +@dataclass +class HouseCatalogEnrichment: + house_url: str + house: HouseInfo + reviews: list[HouseReview] = field(default_factory=list) + placement_history: list[PlacementHistoryItem] = field(default_factory=list) + mini_serp: list[MiniSerpListing] = field(default_factory=list) + recommendations: list[RecommendationItem] = field(default_factory=list) + reviews_next_page_url: str | None = None + + +# --------------------------------------------------------------------------- +# Helper functions +# --------------------------------------------------------------------------- + + +def _parse_ru_date(text_val: str | None) -> date | None: + """Парсит дату вида "9 октября 2025" → date(2025, 10, 9).""" + if not text_val: + return None + m = re.match(r"(\d+)\s+([а-я]+)\s+(\d{4})", text_val.strip(), re.IGNORECASE) + if not m: + return None + day = int(m.group(1)) + month_word = m.group(2).lower() + year = int(m.group(3)) + month = RUS_MONTHS.get(month_word) + return date(year, month, day) if month else None + + +def _unix_to_date(ts: int | None) -> date | None: + """Конвертирует unix timestamp (секунды) → date.""" + if ts is None: + return None + try: + return datetime.fromtimestamp(ts, tz=UTC).date() + except (OSError, OverflowError, ValueError): + logger.warning("Не удалось конвертировать unix timestamp: %r", ts) + return None + + +def _strip_price(price_str: str | None) -> int | None: + """Извлекает целое число из строки "11 990 000 ₽" → 11990000.""" + if not price_str: + return None + digits = re.sub(r"[^\d]", "", price_str) + return int(digits) if digits else None + + +def _normalize_house_type(raw: str | None) -> str | None: + """Нормализует тип дома: "Монолитный" → "monolith".""" + if not raw: + return None + return HOUSE_TYPE_MAP.get(raw.lower(), "other") + + +def _cast_expand_param(value: str, cast_type: str) -> Any: + """Кастует строковое значение expandParams к нужному типу.""" + if cast_type == "int": + try: + return int(value) + except (ValueError, TypeError): + return None + if cast_type == "bool_da": + return value.strip().lower() == "да" + if cast_type == "house_type": + return _normalize_house_type(value) + # str + return value + + +def _parse_expand_params(items: list[dict[str, Any]]) -> dict[str, Any]: + """Извлекает поля из expandParams['items'] по EXPAND_PARAMS_MAP. + + Возвращает dict с распарсенными полями + raw_characteristics (полный список). + """ + result: dict[str, Any] = {} + for category in items: + for param in category.get("params", []): + param_type = param.get("type", "") + if param_type in EXPAND_PARAMS_MAP: + field_name, cast_type = EXPAND_PARAMS_MAP[param_type] + raw_value = param.get("value", "") + result[field_name] = _cast_expand_param(str(raw_value), cast_type) + return result + + +def _get_widget(placeholders: list[dict[str, Any]], widget_type: str) -> dict[str, Any] | None: + """Ищет виджет по type в массиве placeholders.""" + for w in placeholders: + if w.get("type") == widget_type: + return w + return None + + +# --------------------------------------------------------------------------- +# Widget parsers +# --------------------------------------------------------------------------- + + +def _parse_house_page(widget: dict[str, Any]) -> HouseInfo: + """Парсит виджет housePage → HouseInfo. + + Путь: widget['props']['developmentData'] — основной объект с данными дома. + """ + props = widget.get("props", {}) + dd = props.get("developmentData", {}) + + # expandParams → характеристики дома + about = dd.get("aboutDevelopment", {}) + expand_params_items: list[dict[str, Any]] = ( + about.get("expandParams", {}).get("items", []) + ) + parsed_params = _parse_expand_params(expand_params_items) + + # Рейтинг + rating_badge = dd.get("ratingBadge", {}) + rating_info = rating_badge.get("info", {}) + + rating_summary = dd.get("ratingSummaryStat", {}) + rating_stat: list[dict[str, Any]] = rating_summary.get("ratingStat", []) + + # Карта/инфраструктура + map_preview = dd.get("mapPreview", {}) + + # Разработчик + developer = dd.get("developer", {}) + + # Координаты + coords = dd.get("coords", {}) + + return HouseInfo( + ext_id=dd.get("avitoId", 0), + ext_id_hash=dd.get("id"), + title=dd.get("title"), + short_address=dd.get("address"), + full_address=dd.get("fullAddress"), + lat=coords.get("lat"), + lon=coords.get("lng"), + year_built=parsed_params.get("year_built"), + total_floors=parsed_params.get("total_floors"), + house_type=parsed_params.get("house_type"), + material_floors=parsed_params.get("material_floors"), + hot_water=parsed_params.get("hot_water"), + passenger_elevators=parsed_params.get("passenger_elevators"), + cargo_elevators=parsed_params.get("cargo_elevators"), + has_concierge=parsed_params.get("has_concierge"), + parking_type=parsed_params.get("parking_type"), + has_playground=parsed_params.get("has_playground"), + closed_yard=parsed_params.get("closed_yard"), + developer_name=developer.get("name"), + developer_key=developer.get("key"), + infrastructure_summary=map_preview.get("objects"), + infrastructure_walk_distance=map_preview.get("distance"), + rating_score=rating_info.get("score"), + rating_string=rating_info.get("scoreString"), + reviews_count=rating_summary.get("reviewCount"), + rating_distribution=rating_stat, + map_pins=map_preview.get("pins", []), + raw_characteristics=expand_params_items, + ) + + +def _parse_single_review(r: dict[str, Any]) -> HouseReview: + """Парсит один объект отзыва из entries.""" + text_main: str | None = None + text_pros: str | None = None + text_cons: str | None = None + + for section in r.get("textSections", []): + title = section.get("title", "") + section_text = section.get("text") + if title == "": + text_main = section_text + elif title == "Преимущества": + text_pros = section_text + elif title == "Недостатки": + text_cons = section_text + + author = r.get("author", {}) + return HouseReview( + ext_review_id=r["id"], + author_name=author.get("title"), + review_title=r.get("reviewTitle"), + score=r.get("score"), + model_experience=r.get("modelExperience"), + rated_date=_parse_ru_date(r.get("rated")), + text_main=text_main, + text_pros=text_pros, + text_cons=text_cons, + raw_payload=r, + ) + + +def _parse_reviews(widget: dict[str, Any]) -> tuple[list[HouseReview], str | None]: + """Парсит виджет reviews → (список HouseReview, следующая страница URL). + + Entries — смешанный список с type='rating' (отзывы) и type='pages' (пагинация). + """ + reviews: list[HouseReview] = [] + next_page_url: str | None = None + + entries = widget.get("props", {}).get("entries", []) + for entry in entries: + entry_type = entry.get("type") + value = entry.get("value", {}) + + if entry_type == "rating": + try: + reviews.append(_parse_single_review(value)) + except (KeyError, TypeError) as exc: + logger.warning("Не удалось распарсить отзыв: %r — %s", value.get("id"), exc) + elif entry_type == "pages": + next_page_url = value.get("nextPageUrl") + + return reviews, next_page_url + + +def _parse_seller(seller_raw: dict[str, Any]) -> SellerInfo | None: + """Парсит seller из miniSerp item → SellerInfo.""" + if not seller_raw: + return None + name = seller_raw.get("name", "") + if not name: + return None + + profile_url = seller_raw.get("url", "") + # ext_seller_id — последний сегмент URL: "/brands/domrf66" → "domrf66" + ext_seller_id = profile_url.strip("/").split("/")[-1] if profile_url else "" + if not ext_seller_id: + logger.warning("Не удалось извлечь ext_seller_id из seller URL: %r", profile_url) + return None + + return SellerInfo( + ext_seller_id=ext_seller_id, + name=name, + seller_type=seller_raw.get("type"), + on_platform_since=seller_raw.get("from"), + profile_url=profile_url or None, + ) + + +def _parse_mini_serp(widget: dict[str, Any]) -> list[MiniSerpListing]: + """Парсит виджет miniSerp → список активных объявлений в доме.""" + listings: list[MiniSerpListing] = [] + + items = widget.get("props", {}).get("items", []) + for item in items: + geo = item.get("geo", {}) + colors: list[str] = geo.get("colors", []) + + seller_raw = item.get("seller") + seller = _parse_seller(seller_raw) if seller_raw else None + + price_raw = item.get("price") + price_rub = _strip_price(price_raw) if isinstance(price_raw, str) else price_raw + + listings.append(MiniSerpListing( + ext_item_id=item["id"], + title=item.get("title"), + price_rub=price_rub, + description=item.get("description"), + url=item.get("url", ""), + metro_text=geo.get("content"), + metro_color=colors[0] if colors else None, + seller=seller, + )) + + return listings + + +def _parse_placement_history(widget: dict[str, Any]) -> list[PlacementHistoryItem]: + """Парсит виджет housePlacementHistory → список PlacementHistoryItem. + + ВАЖНО: removed_date ВСЕГДА None — виджет не отдаёт эту дату. + Точная дата только из IMV API (Stage 2d). + """ + result: list[PlacementHistoryItem] = [] + + items = widget.get("props", {}).get("items", []) + for item in items: + item_copy = {k: v for k, v in item.items() if k != "itemImage"} + result.append(PlacementHistoryItem( + ext_item_id=str(item["id"]), + title=item.get("title"), + start_price=item.get("startPrice"), + start_price_date=_unix_to_date(item.get("startPriceDate")), + last_price=item.get("lastPrice"), + last_price_date=_unix_to_date(item.get("lastPriceDate")), + exposure_days=item.get("exposure"), + removed_date=None, # widget не отдаёт! IMV API Stage 2d + raw_payload=item_copy, + )) + + return result + + +def _parse_recommendations(widget: dict[str, Any]) -> list[RecommendationItem]: + """Парсит виджет recommendations → lightweight список похожих объявлений.""" + result: list[RecommendationItem] = [] + + items = widget.get("props", {}).get("items", []) + for item in items: + additional_info: list[str] = item.get("additionalInfo", []) + address = additional_info[0] if additional_info else None + + images: list[dict[str, Any]] = item.get("images", []) + image_url: str | None = None + if images: + image_url = images[0].get("208x156") + + result.append(RecommendationItem( + title=item.get("title"), + price_text=item.get("priceRange"), + address=address, + url=item.get("url", ""), + image_url=image_url, + )) + + return result + + +# --------------------------------------------------------------------------- +# Core parser (pure, no network) +# --------------------------------------------------------------------------- + + +def parse_houses_state(state: dict[str, Any], house_url: str) -> HouseCatalogEnrichment: + """Парсит уже разобранный __preloadedState__ dict → HouseCatalogEnrichment. + + Pure функция: нет HTTP, нет I/O. Принимает state dict и house_url (для поля + HouseCatalogEnrichment.house_url). Используется в тестах и из fetch_house_catalog. + + Raises: + ValueError если placeholders не массив или housePage виджет не найден. + """ + try: + placeholders: list[dict[str, Any]] = ( + state["data"]["data"]["page"]["placeholders"] + ) + except (KeyError, TypeError) as exc: + raise ValueError( + f"Не удалось найти placeholders в __preloadedState__: {exc}" + ) from exc + + if not isinstance(placeholders, list): + raise ValueError( + f"placeholders должен быть массивом, получен: {type(placeholders).__name__}" + ) + + # housePage — обязательный виджет + house_page_widget = _get_widget(placeholders, "housePage") + if house_page_widget is None: + raise ValueError("Виджет 'housePage' не найден в placeholders") + + house_info = _parse_house_page(house_page_widget) + + # reviews — опциональный + reviews: list[HouseReview] = [] + next_page_url: str | None = None + reviews_widget = _get_widget(placeholders, "reviews") + if reviews_widget is not None: + reviews, next_page_url = _parse_reviews(reviews_widget) + else: + logger.debug("Виджет 'reviews' не найден — пропускаем") + + # miniSerp — опциональный + mini_serp: list[MiniSerpListing] = [] + mini_serp_widget = _get_widget(placeholders, "miniSerp") + if mini_serp_widget is not None: + mini_serp = _parse_mini_serp(mini_serp_widget) + else: + logger.debug("Виджет 'miniSerp' не найден — пропускаем") + + # housePlacementHistory — опциональный + placement_history: list[PlacementHistoryItem] = [] + history_widget = _get_widget(placeholders, "housePlacementHistory") + if history_widget is not None: + placement_history = _parse_placement_history(history_widget) + else: + logger.debug("Виджет 'housePlacementHistory' не найден — пропускаем") + + # recommendations — опциональный + recommendations: list[RecommendationItem] = [] + recs_widget = _get_widget(placeholders, "recommendations") + if recs_widget is not None: + recommendations = _parse_recommendations(recs_widget) + else: + logger.debug("Виджет 'recommendations' не найден — пропускаем") + + return HouseCatalogEnrichment( + house_url=house_url, + house=house_info, + reviews=reviews, + placement_history=placement_history, + mini_serp=mini_serp, + recommendations=recommendations, + reviews_next_page_url=next_page_url, + ) + + +# --------------------------------------------------------------------------- +# HTTP fetch (thin wrapper, requires curl_cffi) +# --------------------------------------------------------------------------- + + +async def fetch_house_catalog( + house_url: str, + *, + cffi_session: Any | None = None, +) -> HouseCatalogEnrichment: + """GET {AVITO_BASE}{house_url} → extract window.__preloadedState__ → parse 10 виджетов. + + Параметры: + house_url — относительный путь, напр. /catalog/houses/ekaterinburg/ul_postovskogo/3171365 + cffi_session — опциональная curl_cffi AsyncSession (для переиспользования пула соединений). + Если None — создаётся временная сессия. + + Raises: + RuntimeError если curl_cffi не установлен. + httpx.HTTPStatusError / curl_cffi аналог если status != 200. + ValueError если __preloadedState__ не найден или placeholders не массив. + """ + try: + from curl_cffi.requests import AsyncSession as CffiAsyncSession + except ImportError as exc: + raise RuntimeError( + "curl_cffi не установлен. Добавь 'curl-cffi>=0.7.0' в pyproject.toml." + ) from exc + + _own_session = False + if cffi_session is None: + cffi_session = CffiAsyncSession(impersonate="chrome120") + _own_session = True + + try: + url = f"{AVITO_BASE}{house_url}" + logger.info("Houses Catalog fetch: %s", url) + + resp = await cffi_session.get(url) + resp.raise_for_status() + html: str = resp.text + + # Извлекаем экранированный JSON из window.__preloadedState__ = JSON.parse("...") + match = PRELOADED_RE.search(html) + if match is None: + raise ValueError( + f"window.__preloadedState__ не найден в HTML страницы: {url}" + ) + + raw = match.group(1) + # JS-стиль unescaping: \\" → " , \\\\ → \ , \\/ → / + # НЕ используем .decode("unicode_escape") — ломает многобайтный UTF-8 (кириллица). + raw = raw.replace('\\"', '"').replace("\\\\", "\\").replace("\\/", "/") + + try: + state: dict[str, Any] = json.loads(raw) + except json.JSONDecodeError: + logger.warning("json.loads с replace-unescape не удался, пробуем unicode_escape") + try: + state = json.loads(bytes(raw, "utf-8").decode("unicode_escape")) + except (json.JSONDecodeError, UnicodeDecodeError) as exc2: + raise ValueError( + f"Не удалось распарсить __preloadedState__ JSON: {exc2}" + ) from exc2 + + logger.info("Houses Catalog state extracted, parsing widgets") + return parse_houses_state(state, house_url) + + finally: + if _own_session: + await cffi_session.close() + + +# --------------------------------------------------------------------------- +# Save functions +# --------------------------------------------------------------------------- + + +def upsert_house(db: Session, h: HouseInfo) -> int: + """INSERT INTO houses ON CONFLICT (source, ext_house_id) DO UPDATE. + + Source='avito'. Возвращает houses.id. + + Маппинг полей HouseInfo → DDL колонки (009 + 010): + ext_id → ext_house_id (text) + ext_id_hash → avito_id_hash + short_address → short_address (010) + full_address → full_address (010) + address (DDL) → используем short_address как fallback + rating_score → rating_score (010, numeric(4,2)) + rating (DDL) → округлённый numeric(2,1) из rating_score + """ + row = db.execute( + text(""" + INSERT INTO houses ( + source, ext_house_id, url, + address, short_address, full_address, + lat, lon, + avito_id_hash, + year_built, house_type, total_floors, + passenger_elevators, cargo_elevators, + has_concierge, closed_yard, + material_floors, hot_water, + parking_type, has_playground, + developer_name, developer_key, + infrastructure_summary, infrastructure_walk_distance, + rating, rating_score, rating_string, reviews_count, + rating_distribution, + map_pins, + raw_characteristics, + last_scraped_at + ) VALUES ( + 'avito', CAST(:ext_house_id AS text), CAST(:url AS text), + CAST(:address AS text), CAST(:short_address AS text), CAST(:full_address AS text), + CAST(:lat AS double precision), CAST(:lon AS double precision), + CAST(:avito_id_hash AS text), + CAST(:year_built AS int), CAST(:house_type AS text), CAST(:total_floors AS int), + CAST(:passenger_elevators AS int), CAST(:cargo_elevators AS int), + CAST(:has_concierge AS boolean), CAST(:closed_yard AS boolean), + CAST(:material_floors AS text), CAST(:hot_water AS text), + CAST(:parking_type AS text), CAST(:has_playground AS boolean), + CAST(:developer_name AS text), CAST(:developer_key AS text), + CAST(:infrastructure_summary AS text), CAST(:infrastructure_walk_distance AS text), + CAST(:rating AS numeric), CAST(:rating_score AS numeric), + CAST(:rating_string AS text), CAST(:reviews_count AS int), + CAST(:rating_distribution AS jsonb), + CAST(:map_pins AS jsonb), + CAST(:raw_characteristics AS jsonb), + NOW() + ) + ON CONFLICT (source, ext_house_id) DO UPDATE SET + url = EXCLUDED.url, + address = EXCLUDED.address, + short_address = EXCLUDED.short_address, + full_address = EXCLUDED.full_address, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + avito_id_hash = EXCLUDED.avito_id_hash, + year_built = EXCLUDED.year_built, + house_type = EXCLUDED.house_type, + total_floors = EXCLUDED.total_floors, + passenger_elevators = EXCLUDED.passenger_elevators, + cargo_elevators = EXCLUDED.cargo_elevators, + has_concierge = EXCLUDED.has_concierge, + closed_yard = EXCLUDED.closed_yard, + material_floors = EXCLUDED.material_floors, + hot_water = EXCLUDED.hot_water, + parking_type = EXCLUDED.parking_type, + has_playground = EXCLUDED.has_playground, + developer_name = EXCLUDED.developer_name, + developer_key = EXCLUDED.developer_key, + infrastructure_summary = EXCLUDED.infrastructure_summary, + infrastructure_walk_distance = EXCLUDED.infrastructure_walk_distance, + rating = EXCLUDED.rating, + rating_score = EXCLUDED.rating_score, + rating_string = EXCLUDED.rating_string, + reviews_count = EXCLUDED.reviews_count, + rating_distribution = EXCLUDED.rating_distribution, + map_pins = EXCLUDED.map_pins, + raw_characteristics = EXCLUDED.raw_characteristics, + last_scraped_at = NOW() + RETURNING id + """), + { + "ext_house_id": str(h.ext_id), + "url": "", # caller должен передавать house_url через enrichment + "address": h.short_address, + "short_address": h.short_address, + "full_address": h.full_address, + "lat": h.lat, + "lon": h.lon, + "avito_id_hash": h.ext_id_hash, + "year_built": h.year_built, + "house_type": h.house_type, + "total_floors": h.total_floors, + "passenger_elevators": h.passenger_elevators, + "cargo_elevators": h.cargo_elevators, + "has_concierge": h.has_concierge, + "closed_yard": h.closed_yard, + "material_floors": h.material_floors, + "hot_water": h.hot_water, + "parking_type": h.parking_type, + "has_playground": h.has_playground, + "developer_name": h.developer_name, + "developer_key": h.developer_key, + "infrastructure_summary": h.infrastructure_summary, + "infrastructure_walk_distance": h.infrastructure_walk_distance, + # rating (numeric(2,1)) — округлённое значение для совместимости с 009 DDL + "rating": round(h.rating_score, 1) if h.rating_score is not None else None, + "rating_score": h.rating_score, + "rating_string": h.rating_string, + "reviews_count": h.reviews_count, + "rating_distribution": json.dumps(h.rating_distribution, ensure_ascii=False), + "map_pins": json.dumps(h.map_pins, ensure_ascii=False), + "raw_characteristics": json.dumps(h.raw_characteristics, ensure_ascii=False), + }, + ) + house_id: int = row.scalar_one() + logger.info("upsert_house: ext_id=%s → houses.id=%s", h.ext_id, house_id) + return house_id + + +def _upsert_house_with_url(db: Session, h: HouseInfo, house_url: str) -> int: + """Внутренний вариант upsert_house с правильным URL.""" + row = db.execute( + text(""" + INSERT INTO houses ( + source, ext_house_id, url, + address, short_address, full_address, + lat, lon, + avito_id_hash, + year_built, house_type, total_floors, + passenger_elevators, cargo_elevators, + has_concierge, closed_yard, + material_floors, hot_water, + parking_type, has_playground, + developer_name, developer_key, + infrastructure_summary, infrastructure_walk_distance, + rating, rating_score, rating_string, reviews_count, + rating_distribution, + map_pins, + raw_characteristics, + last_scraped_at + ) VALUES ( + 'avito', CAST(:ext_house_id AS text), CAST(:url AS text), + CAST(:address AS text), CAST(:short_address AS text), CAST(:full_address AS text), + CAST(:lat AS double precision), CAST(:lon AS double precision), + CAST(:avito_id_hash AS text), + CAST(:year_built AS int), CAST(:house_type AS text), CAST(:total_floors AS int), + CAST(:passenger_elevators AS int), CAST(:cargo_elevators AS int), + CAST(:has_concierge AS boolean), CAST(:closed_yard AS boolean), + CAST(:material_floors AS text), CAST(:hot_water AS text), + CAST(:parking_type AS text), CAST(:has_playground AS boolean), + CAST(:developer_name AS text), CAST(:developer_key AS text), + CAST(:infrastructure_summary AS text), CAST(:infrastructure_walk_distance AS text), + CAST(:rating AS numeric), CAST(:rating_score AS numeric), + CAST(:rating_string AS text), CAST(:reviews_count AS int), + CAST(:rating_distribution AS jsonb), + CAST(:map_pins AS jsonb), + CAST(:raw_characteristics AS jsonb), + NOW() + ) + ON CONFLICT (source, ext_house_id) DO UPDATE SET + url = EXCLUDED.url, + address = EXCLUDED.address, + short_address = EXCLUDED.short_address, + full_address = EXCLUDED.full_address, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + avito_id_hash = EXCLUDED.avito_id_hash, + year_built = EXCLUDED.year_built, + house_type = EXCLUDED.house_type, + total_floors = EXCLUDED.total_floors, + passenger_elevators = EXCLUDED.passenger_elevators, + cargo_elevators = EXCLUDED.cargo_elevators, + has_concierge = EXCLUDED.has_concierge, + closed_yard = EXCLUDED.closed_yard, + material_floors = EXCLUDED.material_floors, + hot_water = EXCLUDED.hot_water, + parking_type = EXCLUDED.parking_type, + has_playground = EXCLUDED.has_playground, + developer_name = EXCLUDED.developer_name, + developer_key = EXCLUDED.developer_key, + infrastructure_summary = EXCLUDED.infrastructure_summary, + infrastructure_walk_distance = EXCLUDED.infrastructure_walk_distance, + rating = EXCLUDED.rating, + rating_score = EXCLUDED.rating_score, + rating_string = EXCLUDED.rating_string, + reviews_count = EXCLUDED.reviews_count, + rating_distribution = EXCLUDED.rating_distribution, + map_pins = EXCLUDED.map_pins, + raw_characteristics = EXCLUDED.raw_characteristics, + last_scraped_at = NOW() + RETURNING id + """), + { + "ext_house_id": str(h.ext_id), + "url": house_url, + "address": h.short_address, + "short_address": h.short_address, + "full_address": h.full_address, + "lat": h.lat, + "lon": h.lon, + "avito_id_hash": h.ext_id_hash, + "year_built": h.year_built, + "house_type": h.house_type, + "total_floors": h.total_floors, + "passenger_elevators": h.passenger_elevators, + "cargo_elevators": h.cargo_elevators, + "has_concierge": h.has_concierge, + "closed_yard": h.closed_yard, + "material_floors": h.material_floors, + "hot_water": h.hot_water, + "parking_type": h.parking_type, + "has_playground": h.has_playground, + "developer_name": h.developer_name, + "developer_key": h.developer_key, + "infrastructure_summary": h.infrastructure_summary, + "infrastructure_walk_distance": h.infrastructure_walk_distance, + "rating": round(h.rating_score, 1) if h.rating_score is not None else None, + "rating_score": h.rating_score, + "rating_string": h.rating_string, + "reviews_count": h.reviews_count, + "rating_distribution": json.dumps(h.rating_distribution, ensure_ascii=False), + "map_pins": json.dumps(h.map_pins, ensure_ascii=False), + "raw_characteristics": json.dumps(h.raw_characteristics, ensure_ascii=False), + }, + ) + house_id: int = row.scalar_one() + logger.info( + "_upsert_house_with_url: ext_id=%s url=%s → houses.id=%s", h.ext_id, house_url, house_id + ) + return house_id + + +def save_house_reviews(db: Session, house_id: int, reviews: list[HouseReview]) -> int: + """INSERT INTO house_reviews ON CONFLICT (source, ext_review_id) DO UPDATE. + + Source='avito'. Возвращает количество вставленных/обновлённых строк. + """ + if not reviews: + return 0 + + count = 0 + for r in reviews: + db.execute( + text(""" + INSERT INTO house_reviews ( + house_id, source, ext_review_id, + author_name, review_title, score, + model_experience, rated_date, + text_main, text_pros, text_cons, + raw_payload, scraped_at + ) VALUES ( + CAST(:house_id AS bigint), 'avito', CAST(:ext_review_id AS bigint), + CAST(:author_name AS text), CAST(:review_title AS text), CAST(:score AS int), + CAST(:model_experience AS text), CAST(:rated_date AS date), + CAST(:text_main AS text), CAST(:text_pros AS text), CAST(:text_cons AS text), + CAST(:raw_payload AS jsonb), NOW() + ) + ON CONFLICT (source, ext_review_id) DO UPDATE SET + author_name = EXCLUDED.author_name, + review_title = EXCLUDED.review_title, + score = EXCLUDED.score, + model_experience = EXCLUDED.model_experience, + rated_date = EXCLUDED.rated_date, + text_main = EXCLUDED.text_main, + text_pros = EXCLUDED.text_pros, + text_cons = EXCLUDED.text_cons, + raw_payload = EXCLUDED.raw_payload, + scraped_at = NOW() + """), + { + "house_id": house_id, + "ext_review_id": r.ext_review_id, + "author_name": r.author_name, + "review_title": r.review_title, + "score": r.score, + "model_experience": r.model_experience, + "rated_date": r.rated_date, + "text_main": r.text_main, + "text_pros": r.text_pros, + "text_cons": r.text_cons, + "raw_payload": ( + json.dumps(r.raw_payload, ensure_ascii=False) if r.raw_payload else None + ), + }, + ) + count += 1 + + logger.info("save_house_reviews: house_id=%s, saved=%d", house_id, count) + return count + + +def save_sellers(db: Session, sellers: list[SellerInfo]) -> dict[str, int]: + """INSERT INTO sellers ON CONFLICT (source, ext_seller_id) DO UPDATE. + + Source='avito'. Возвращает {ext_seller_id: sellers.id}. + """ + result: dict[str, int] = {} + for s in sellers: + row = db.execute( + text(""" + INSERT INTO sellers ( + source, ext_seller_id, + name, seller_type, on_platform_since, profile_url, + last_seen_at + ) VALUES ( + 'avito', CAST(:ext_seller_id AS text), + CAST(:name AS text), CAST(:seller_type AS text), + CAST(:on_platform_since AS text), CAST(:profile_url AS text), + NOW() + ) + ON CONFLICT (source, ext_seller_id) DO UPDATE SET + name = EXCLUDED.name, + seller_type = EXCLUDED.seller_type, + on_platform_since = EXCLUDED.on_platform_since, + profile_url = EXCLUDED.profile_url, + last_seen_at = NOW() + RETURNING id + """), + { + "ext_seller_id": s.ext_seller_id, + "name": s.name, + "seller_type": s.seller_type, + "on_platform_since": s.on_platform_since, + "profile_url": s.profile_url, + }, + ) + seller_db_id: int = row.scalar_one() + result[s.ext_seller_id] = seller_db_id + + logger.info("save_sellers: saved=%d", len(result)) + return result + + +def link_listings_to_seller(db: Session, ext_item_id_to_seller_db_id: dict[int, int]) -> int: + """UPDATE listings SET seller_id_fk = :seller_id WHERE source='avito' AND source_id = :item_id. + + Возвращает количество обновлённых строк. + """ + if not ext_item_id_to_seller_db_id: + return 0 + + total = 0 + for item_id, seller_db_id in ext_item_id_to_seller_db_id.items(): + result = db.execute( + text(""" + UPDATE listings + SET seller_id_fk = CAST(:seller_id AS bigint) + WHERE source = 'avito' + AND source_id = CAST(:item_id AS text) + AND seller_id_fk IS DISTINCT FROM CAST(:seller_id AS bigint) + """), + {"seller_id": seller_db_id, "item_id": str(item_id)}, + ) + total += result.rowcount + + logger.info("link_listings_to_seller: updated=%d listings", total) + return total + + +def save_placement_history(db: Session, items: list[PlacementHistoryItem]) -> int: + """INSERT INTO house_placement_history с source='avito_widget' и removed_date=NULL. + + ON CONFLICT (source, ext_item_id) DO UPDATE last_price/last_price_date/exposure_days. + Возвращает количество строк. + """ + if not items: + return 0 + + count = 0 + for item in items: + db.execute( + text(""" + INSERT INTO house_placement_history ( + source, ext_item_id, + title, + start_price, start_price_date, + last_price, last_price_date, + exposure_days, + removed_date, + raw_payload, scraped_at + ) VALUES ( + 'avito_widget', CAST(:ext_item_id AS text), + CAST(:title AS text), + CAST(:start_price AS bigint), CAST(:start_price_date AS date), + CAST(:last_price AS bigint), CAST(:last_price_date AS date), + CAST(:exposure_days AS int), + NULL, + CAST(:raw_payload AS jsonb), NOW() + ) + ON CONFLICT (source, ext_item_id) DO UPDATE SET + title = EXCLUDED.title, + last_price = EXCLUDED.last_price, + last_price_date = EXCLUDED.last_price_date, + exposure_days = EXCLUDED.exposure_days, + raw_payload = EXCLUDED.raw_payload, + scraped_at = NOW() + """), + { + "ext_item_id": item.ext_item_id, + "title": item.title, + "start_price": item.start_price, + "start_price_date": item.start_price_date, + "last_price": item.last_price, + "last_price_date": item.last_price_date, + "exposure_days": item.exposure_days, + "raw_payload": ( + json.dumps(item.raw_payload, ensure_ascii=False) + if item.raw_payload + else None + ), + }, + ) + count += 1 + + logger.info("save_placement_history: saved=%d items (source=avito_widget)", count) + return count + + +def save_house_catalog_enrichment( + db: Session, + e: HouseCatalogEnrichment, +) -> dict[str, int]: + """Orchestrator — полный pipeline сохранения одного обогащения. + + Операции (в порядке): + 1. upsert_house → houses.id + 2. save_house_reviews(house_id, reviews) + 3. save_sellers([m.seller for m in mini_serp if m.seller]) → {ext_seller_id: db_id} + 4. link_listings_to_seller({item_id: seller_db_id}) + 5. save_placement_history(placement_history) + + Возвращает: + {'house_id': N, 'reviews': N, 'sellers': N, 'listings_linked': N, 'placement_history': N} + """ + # 1. Upsert дом с правильным URL из enrichment + house_id = _upsert_house_with_url(db, e.house, e.house_url) + + # 2. Сохранить отзывы + reviews_count = save_house_reviews(db, house_id, e.reviews) + + # 3. Сохранить продавцов (дедуп по ext_seller_id) + sellers_to_save = [m.seller for m in e.mini_serp if m.seller is not None] + seen_seller_ids: set[str] = set() + unique_sellers: list[SellerInfo] = [] + for s in sellers_to_save: + if s.ext_seller_id not in seen_seller_ids: + unique_sellers.append(s) + seen_seller_ids.add(s.ext_seller_id) + + seller_id_map = save_sellers(db, unique_sellers) + + # 4. Слинковать объявления с продавцами + # Маппинг: ext_item_id (int) → seller db_id + item_to_seller: dict[int, int] = {} + for listing in e.mini_serp: + if listing.seller and listing.seller.ext_seller_id in seller_id_map: + item_to_seller[listing.ext_item_id] = seller_id_map[listing.seller.ext_seller_id] + + listings_linked = link_listings_to_seller(db, item_to_seller) + + # 5. Сохранить историю размещений + history_count = save_placement_history(db, e.placement_history) + + db.commit() + + result = { + "house_id": house_id, + "reviews": reviews_count, + "sellers": len(seller_id_map), + "listings_linked": listings_linked, + "placement_history": history_count, + } + logger.info("save_house_catalog_enrichment: %s", result) + return result diff --git a/tradein-mvp/backend/tests/test_avito_houses_parse.py b/tradein-mvp/backend/tests/test_avito_houses_parse.py new file mode 100644 index 00000000..a684f6c8 --- /dev/null +++ b/tradein-mvp/backend/tests/test_avito_houses_parse.py @@ -0,0 +1,397 @@ +"""Offline smoke for avito_houses parser. + +Uses minimal JSON fixture mimicking Avito __preloadedState__ structure. +Не требует сети, БД или curl_cffi — тестирует только чистый парсер. +""" + +from datetime import date + +import pytest + +from app.services.scrapers.avito_houses import ( + HouseCatalogEnrichment, + HouseInfo, + HouseReview, + MiniSerpListing, + PlacementHistoryItem, + RecommendationItem, + SellerInfo, + _parse_ru_date, + parse_houses_state, +) + + +# --------------------------------------------------------------------------- +# Minimal __preloadedState__ fixture (Python dict — не raw HTML) +# --------------------------------------------------------------------------- + +MINIMAL_STATE: dict = { + "data": {"data": {"page": {"placeholders": [ + {"type": "breadcrumbs"}, + {"type": "title"}, + {"type": "gallery", "props": {}}, + { + "type": "housePage", + "props": {"developmentData": { + "avitoId": 3171365, + "id": "MTE3LjM3MDg5MA", + "title": "Akademika Postovskogo 17a", + "address": "ул. Постовского, 17а", + "fullAddress": "Свердловская обл., Екатеринбург, ул. Постовского, 17а", + "coords": {"lat": 56.790699, "lng": 60.580191}, + "aboutDevelopment": {"expandParams": {"items": [ + {"title": "Параметры дома", "params": [ + {"type": "Год постройки", "value": "2019"}, + {"type": "Этажей", "value": "25"}, + {"type": "Тип дома", "value": "Монолитный"}, + {"type": "Пассажирский лифт", "value": "2"}, + {"type": "Консьерж", "value": "Да"}, + {"type": "Закрытый двор", "value": "Да"}, + {"type": "Детская площадка", "value": "Да"}, + ]}, + ]}}, + "developer": {"name": "Атомстройкомплекс", "key": "atomstroy"}, + "mapPreview": { + "distance": "в 5 минутах ходьбы", + "objects": "школа, парк, аптека", + "pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}], + }, + "ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}}, + "ratingSummaryStat": { + "ratingStat": [{"score": 5, "count": 4}, {"score": 4, "count": 2}], + "reviewCount": 6, + }, + }}, + }, + { + "type": "reviews", + "props": {"entries": [ + {"type": "rating", "value": { + "id": 12345, + "author": {"title": "Иван"}, + "reviewTitle": "Хорошо", + "score": 5, + "modelExperience": "Живу в своей квартире", + "rated": "9 октября 2025", + "textSections": [ + {"title": "", "text": "Общий отзыв"}, + {"title": "Преимущества", "text": "Парк рядом"}, + {"title": "Недостатки", "text": "Парковки мало"}, + ], + }}, + {"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}}, + ]}, + }, + { + "type": "miniSerp", + "props": {"items": [ + { + "id": 7986882804, + "title": "3-к. квартира", + "description": "Описание", + "url": "/ekaterinburg/kvartiry/test_7986882804", + "price": "11 990 000 ₽", + "geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]}, + "seller": { + "name": "DOMRF66", + "type": "Компания", + "from": "На Авито с июля 2012", + "url": "/brands/domrf66", + "logos": [], + }, + }, + ]}, + }, + { + "type": "housePlacementHistory", + "props": {"items": [ + { + "id": 7000000001, + "title": "1-к. квартира", + "startPrice": 5000000, + "startPriceDate": 1697000000, + "lastPrice": 4900000, + "lastPriceDate": 1715000000, + "exposure": 210, + "itemImage": {}, + }, + ]}, + }, + {"type": "uxFeedback"}, + { + "type": "recommendations", + "props": {"items": [ + { + "title": "Похожая", + "priceRange": "от 6 млн", + "additionalInfo": ["Адрес 1"], + "url": "/rec1", + "images": [{"208x156": "https://avito.ru/img1.jpg"}], + }, + ]}, + }, + {"type": "padding"}, + ]}}} +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def test_ru_date_parse() -> None: + assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9) + assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31) + assert _parse_ru_date("1 января 2000") == date(2000, 1, 1) + assert _parse_ru_date(None) is None + assert _parse_ru_date("bogus") is None + assert _parse_ru_date("") is None + + +# --------------------------------------------------------------------------- +# Main parse test +# --------------------------------------------------------------------------- + + +def test_parse_minimal_state() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + + assert isinstance(e, HouseCatalogEnrichment) + assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365" + + +def test_house_info() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + h = e.house + + assert isinstance(h, HouseInfo) + assert h.ext_id == 3171365 + assert h.ext_id_hash == "MTE3LjM3MDg5MA" + assert h.title == "Akademika Postovskogo 17a" + assert h.short_address == "ул. Постовского, 17а" + assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а" + assert h.lat == 56.790699 + assert h.lon == 60.580191 + + +def test_house_expand_params() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + h = e.house + + assert h.year_built == 2019 + assert h.total_floors == 25 + assert h.house_type == "monolith" + assert h.passenger_elevators == 2 + assert h.has_concierge is True + assert h.closed_yard is True + assert h.has_playground is True + + +def test_house_developer_and_rating() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + h = e.house + + assert h.developer_name == "Атомстройкомплекс" + assert h.developer_key == "atomstroy" + assert h.rating_score == 4.666 + assert h.rating_string == "4,7" + assert h.reviews_count == 6 + assert len(h.rating_distribution) == 2 + + +def test_house_map_and_infra() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + h = e.house + + assert h.infrastructure_walk_distance == "в 5 минутах ходьбы" + assert h.infrastructure_summary == "школа, парк, аптека" + assert len(h.map_pins) == 1 + assert h.map_pins[0]["type"] == "school" + + +def test_house_raw_characteristics() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + # raw_characteristics = полный expandParams items (1 категория) + assert len(e.house.raw_characteristics) == 1 + assert e.house.raw_characteristics[0]["title"] == "Параметры дома" + + +def test_reviews() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + + assert len(e.reviews) == 1 + assert e.reviews_next_page_url == "/reviews?page=2" + + r = e.reviews[0] + assert isinstance(r, HouseReview) + assert r.ext_review_id == 12345 + assert r.author_name == "Иван" + assert r.review_title == "Хорошо" + assert r.score == 5 + assert r.model_experience == "Живу в своей квартире" + assert r.rated_date == date(2025, 10, 9) + assert r.text_main == "Общий отзыв" + assert r.text_pros == "Парк рядом" + assert r.text_cons == "Парковки мало" + assert r.raw_payload is not None + + +def test_mini_serp() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + + assert len(e.mini_serp) == 1 + m = e.mini_serp[0] + assert isinstance(m, MiniSerpListing) + assert m.ext_item_id == 7986882804 + assert m.price_rub == 11990000 + assert m.metro_text == "Чкаловская, 2,7 км" + assert m.metro_color == "#cf2734" + assert m.description == "Описание" + + +def test_seller_extraction() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + m = e.mini_serp[0] + + assert m.seller is not None + s = m.seller + assert isinstance(s, SellerInfo) + assert s.ext_seller_id == "domrf66" + assert s.name == "DOMRF66" + assert s.seller_type == "Компания" + assert s.on_platform_since == "На Авито с июля 2012" + assert s.profile_url == "/brands/domrf66" + + +def test_placement_history() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + + assert len(e.placement_history) == 1 + p = e.placement_history[0] + assert isinstance(p, PlacementHistoryItem) + assert p.ext_item_id == "7000000001" + assert p.title == "1-к. квартира" + assert p.start_price == 5000000 + assert p.last_price == 4900000 + assert p.exposure_days == 210 + # removed_date ВСЕГДА None — виджет не отдаёт! + assert p.removed_date is None + # raw_payload не содержит itemImage (скипается) + assert p.raw_payload is not None + assert "itemImage" not in p.raw_payload + # start_price_date и last_price_date — из unix timestamp + assert isinstance(p.start_price_date, date) + assert isinstance(p.last_price_date, date) + + +def test_recommendations() -> None: + e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365") + + assert len(e.recommendations) == 1 + rec = e.recommendations[0] + assert isinstance(rec, RecommendationItem) + assert rec.title == "Похожая" + assert rec.price_text == "от 6 млн" + assert rec.address == "Адрес 1" + assert rec.url == "/rec1" + assert rec.image_url == "https://avito.ru/img1.jpg" + + +def test_missing_optional_widgets() -> None: + """Парсер не падает если опциональные виджеты отсутствуют.""" + state_minimal: dict = { + "data": {"data": {"page": {"placeholders": [ + { + "type": "housePage", + "props": {"developmentData": { + "avitoId": 9999, + "coords": {"lat": 56.0, "lng": 60.0}, + "aboutDevelopment": {"expandParams": {"items": []}}, + "developer": {}, + "mapPreview": {}, + "ratingBadge": {"info": {}}, + "ratingSummaryStat": {}, + }}, + }, + ]}}} + } + e = parse_houses_state(state_minimal, "/catalog/houses/test/9999") + assert e.house.ext_id == 9999 + assert e.reviews == [] + assert e.mini_serp == [] + assert e.placement_history == [] + assert e.recommendations == [] + assert e.reviews_next_page_url is None + + +def test_invalid_state_raises() -> None: + """ValueError если placeholders не найден.""" + with pytest.raises(ValueError, match="placeholders"): + parse_houses_state({}, "/catalog/houses/test/1") + + +def test_missing_house_page_raises() -> None: + """ValueError если housePage виджет отсутствует.""" + state: dict = { + "data": {"data": {"page": {"placeholders": [ + {"type": "breadcrumbs"}, + ]}}} + } + with pytest.raises(ValueError, match="housePage"): + parse_houses_state(state, "/catalog/houses/test/1") + + +def test_house_type_normalization() -> None: + """Все варианты типа дома нормализуются корректно.""" + from app.services.scrapers.avito_houses import _normalize_house_type + + assert _normalize_house_type("Монолитный") == "monolith" + assert _normalize_house_type("Панельный") == "panel" + assert _normalize_house_type("Кирпичный") == "brick" + assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick" + assert _normalize_house_type("Блочный") == "block" + assert _normalize_house_type("Деревянный") == "wood" + assert _normalize_house_type("Неизвестный") == "other" + assert _normalize_house_type(None) is None + + +def test_strip_price() -> None: + """_strip_price корректно извлекает числа из строк Avito.""" + from app.services.scrapers.avito_houses import _strip_price + + assert _strip_price("11 990 000 ₽") == 11990000 + assert _strip_price("5 000 000 ₽") == 5000000 + assert _strip_price(None) is None + assert _strip_price("") is None + + +def test_mini_serp_no_seller() -> None: + """Объявление без продавца не вызывает ошибку.""" + state: dict = { + "data": {"data": {"page": {"placeholders": [ + { + "type": "housePage", + "props": {"developmentData": { + "avitoId": 111, + "coords": {"lat": 56.0, "lng": 60.0}, + "aboutDevelopment": {"expandParams": {"items": []}}, + "developer": {}, + "mapPreview": {}, + "ratingBadge": {"info": {}}, + "ratingSummaryStat": {}, + }}, + }, + { + "type": "miniSerp", + "props": {"items": [ + {"id": 12345, "title": "1-к.", "url": "/test", "price": "3 000 000 ₽", + "geo": {"content": "Центр", "colors": []}}, + ]}, + }, + ]}}} + } + e = parse_houses_state(state, "/catalog/houses/test/111") + assert len(e.mini_serp) == 1 + assert e.mini_serp[0].seller is None + assert e.mini_serp[0].price_rub == 3000000