From dbef242716f013a1e2db1dbbb80aa062f2df8e1f Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 27 Jun 2026 13:46:51 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein/scrapers):=20domclick=20=D0=B2?= =?UTF-8?q?=D1=82=D0=BE=D1=80=D0=B8=D1=87=D0=BA=D0=B0=20=D1=87=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=B7=20bff-search-web=20JSON=20API=20(Layer=20A)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DomClickScraper переписан на GET bff-search-web.domclick.ru/api/offers/v1 с рекурсивным бинарным price-band bucketing (OFFSET_CAP=2000, open-band split через _HIGH_ANCHOR=30M); geo-guard по offerRegionName==ЕКБ / bbox - scrape_pipeline.py: rooms=[0..5] (добавлен 5+), pages default=100, _DOMCLICK_PER_FETCH_S=4.0, честный статус mark_failed при 0 лотов + errors - config.py: bff-search-web.domclick.ru добавлен в scrape_allowed_hosts - data/sql/132_update_domclick_sweep_params.sql: UPDATE default_params (DORMANT) --- tradein-mvp/backend/app/core/config.py | 1 + .../backend/app/services/scrape_pipeline.py | 48 +- .../backend/app/services/scrapers/domclick.py | 810 ++++++++++++------ .../sql/132_update_domclick_sweep_params.sql | 39 + .../backend/tests/scrapers/test_domclick.py | 630 +++++++++----- .../backend/tests/test_domclick_sweep.py | 147 ++-- 6 files changed, 1105 insertions(+), 570 deletions(-) create mode 100644 tradein-mvp/backend/data/sql/132_update_domclick_sweep_params.sql diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index 766959fb..9bea0fb6 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -322,6 +322,7 @@ class Settings(BaseSettings): "n1.ru", "domclick.ru", "www.domclick.ru", + "bff-search-web.domclick.ru", # JSON API endpoint (Layer A rewrite) } # ── Scraper mobile proxy (#806) ────────────────────────────────────────── diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index aa38b5ff..f153b9d9 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -3105,10 +3105,11 @@ class DomClickCitySweepCounters: # Дефолтные параметры sweep'а (EKB city_id=4, все комнатности). DOMCLICK_DEFAULT_CITY_ID: int = 4 -DOMCLICK_DEFAULT_ROOMS: list[int] = [0, 1, 2, 3, 4] -# Оценка времени одного fetch'а (network + camoufox render + parse) для watchdog. -_DOMCLICK_PER_FETCH_S: float = 12.0 -# Буфер сверху расчётного бюджета (cold browser start, save-фаза). +# rooms=5 добавлен (5+ квартиры); JSON API поддерживает токен "5+". +DOMCLICK_DEFAULT_ROOMS: list[int] = [0, 1, 2, 3, 4, 5] +# Оценка времени одного JSON-fetch'а (network + parse, без camoufox render). +_DOMCLICK_PER_FETCH_S: float = 4.0 +# Буфер сверху расчётного бюджета (cold start, save-фаза). _DOMCLICK_SWEEP_BUDGET_S: float = 300.0 @@ -3118,40 +3119,40 @@ async def run_domclick_city_sweep( run_id: int, city_id: int = DOMCLICK_DEFAULT_CITY_ID, rooms: list[int] | None = None, - pages: int = 5, + pages: int = 100, request_delay_sec: float | None = None, ) -> DomClickCitySweepCounters: - """DomClick citywide sweep: SERP по city_id × rooms × pages → save → house-match. + """DomClick citywide sweep: fetch_city (JSON API) → save → house-match. Структурно зеркалит run_cian_city_sweep / run_yandex_city_sweep, но CITYWIDE: - DomClick SERP не поддерживает geo-radius (fetch_around → NotImplementedError), - поэтому anchor-loop отсутствует. DomClickScraper.fetch_city уже перебирает - rooms × pages через внутренний BrowserFetcher (camoufox headless) и - дедуплицирует по source_id. + DomClick не поддерживает geo-radius (fetch_around → NotImplementedError), + anchor-loop отсутствует. DomClickScraper.fetch_city перебирает rooms × price-bands + через bff-search-web JSON API с recursive offset<2000 bucketing и дедуплицирует + по source_id. Координаты присутствуют в JSON (100% coverage) → geom заполнена. Фазы (единственный citywide проход): 1. SERP: DomClickScraper().fetch_city(city_id, rooms, pages). 2. SAVE: save_listings(db, lots, run_id=run_id) — BaseScraper save path, - триггерит существующий house-match hook (адрес-fingerprint tier, т.к. - DomClick SERP не отдаёт lat/lon). + триггерит house-match hook (адрес + coords). Watchdog: весь fetch_city оборачивается в asyncio.wait_for. Таймаут считается из rooms × pages × per-fetch + buffer (минимум ANCHOR_TIMEOUT_SEC). Cooperative cancel: is_cancelled(db, run_id) проверяется перед SERP-фазой. - mark_done вызывается ВСЕГДА (кроме cancel / fatal). lat=lon=None у всех lots → - house-match использует только address_fingerprint tier. + Honest-status: если lots_fetched==0 И errors_count>0 → mark_failed (blocked / + fetch error). Иначе → mark_done. Кроме cancel / fatal. Возвращает DomClickCitySweepCounters. """ from app.services.scrapers.domclick import DomClickScraper + from app.services.scrapers.domclick_exceptions import DomClickBlockedError _rooms = rooms if rooms is not None else list(DOMCLICK_DEFAULT_ROOMS) _resolved_delay = request_delay_sec if request_delay_sec is not None else 8.0 counters = DomClickCitySweepCounters() - # Watchdog-таймаут: число fetch'ей = len(rooms) × pages (worst-case, без - # early-break на пустой странице). Каждый fetch ≈ delay + render/parse overhead. + # Watchdog-таймаут: worst-case = rooms × pages × (delay + per-fetch) + buffer. + # JSON endpoint быстрее camoufox render: _DOMCLICK_PER_FETCH_S снижен до 4.0s. _num_fetches = max(1, len(_rooms)) * max(1, pages) _sweep_timeout = max( ANCHOR_TIMEOUT_SEC, @@ -3177,7 +3178,7 @@ async def run_domclick_city_sweep( lots: list[ScrapedLot] = [] async def _domclick_phase() -> None: - """Единственная citywide-фаза: fetch_city + save.""" + """Единственная citywide-фаза: fetch_city (JSON API) + save.""" nonlocal lots async with DomClickScraper() as scraper: if request_delay_sec is not None: @@ -3198,6 +3199,9 @@ async def run_domclick_city_sweep( _sweep_timeout, ) counters.errors_count += 1 + except DomClickBlockedError as e: + logger.error("domclick-sweep run_id=%d: BLOCKED — %s", run_id, e) + counters.errors_count += 1 except Exception: logger.exception("domclick-sweep run_id=%d: SERP phase failed", run_id) counters.errors_count += 1 @@ -3205,7 +3209,15 @@ async def run_domclick_city_sweep( # pages_fetched: worst-case число запрошенных страниц (rooms × pages). counters.pages_fetched = _num_fetches scrape_runs.update_heartbeat(db, run_id, counters.to_dict()) - scrape_runs.mark_done(db, run_id, counters.to_dict()) + + # Honest-status: 0 лотов + ошибки → mark_failed (#1968). + if counters.lots_fetched == 0 and counters.errors_count > 0: + _reason = "domclick: 0 listings collected (blocked or fetch error)" + logger.warning("domclick-sweep run_id=%d: %s", run_id, _reason) + scrape_runs.mark_failed(db, run_id, _reason, counters.to_dict()) + else: + scrape_runs.mark_done(db, run_id, counters.to_dict()) + logger.info( "domclick-sweep run_id=%d done: lots=%d (ins=%d/upd=%d) pages=%d errors=%d", run_id, diff --git a/tradein-mvp/backend/app/services/scrapers/domclick.py b/tradein-mvp/backend/app/services/scrapers/domclick.py index ccb71cdb..ac333fdf 100644 --- a/tradein-mvp/backend/app/services/scrapers/domclick.py +++ b/tradein-mvp/backend/app/services/scrapers/domclick.py @@ -1,146 +1,140 @@ -"""DomClick.ru scraper — вторичка через headless BrowserFetcher (#796). +"""DomClick.ru scraper — вторичка через bff-search-web JSON API (Layer A). -Стратегия: HTML scrape через camoufox (AsyncCamoufox) — единственный -рабочий путь: DataDome пропускает headless Playwright, curl_cffi -> 401. +Стратегия (JSON API rewrite, replaces dead HTML/camoufox scrape): + GET https://bff-search-web.domclick.ru/api/offers/v1?... → list (up to 20 items) + GET https://bff-search-web.domclick.ru/api/offers/count/v1?... → snippetsCount -URL шаблон: - https://domclick.ru/search?deal_type=sale&category=living&offer_type=flat - &city_id={city_id}&rooms={r}&p={page} +Транспорт: BrowserFetcher(source="domclick") — существующий браузерный пул + (tradein-browser), роутит bff-search-web.domclick.ru → generic provider → + shared RU mobile proxy. fetch() возвращает raw JSON body как строку. -Оффер-карточки: `a[href*="/card/"]` с href-паттерном `/card/sale__flat__`. -Координаты SERP не отдаёт (lat = lon = None). +Bucketing: + OFFSET_CAP=2000 (max 100 страниц × 20) — hardlimit DomClick API. + Когда snippetsCount > 2000 — рекурсивная бинарная partition по цене. + EKB вторичка ≈6400 лотов; только rooms=2 превышает cap (≈2215). -Важно: selectolax `.text()` объединяет все дочерние текстовые узлы без -разделителей. Числа из адреса (номер дома) могут слипнуться с ценой. -Поэтому цену извлекаем из каждого дочернего элемента отдельно (а не из -суммарного card_text), где элемент содержит символ рубля. +Layer B (detail-backfill) и Layer C (analytics XHR) — не реализованы. """ from __future__ import annotations +import json import logging -import re -from urllib.parse import urljoin +from datetime import UTC, date, datetime +from typing import Any +from urllib.parse import urlencode, urljoin -from selectolax.parser import HTMLParser, Node +import sentry_sdk +from app.core.config import settings from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.domclick_exceptions import DomClickBlockedError logger = logging.getLogger(__name__) -# ── DataDome block detection ───────────────────────────────────────────────── -_DATADOME_MARKERS = ("datadome", "blocked", "access denied", "bot detected") +# ── Константы bucketing ────────────────────────────────────────────────────── + +# Верхняя граница первого деления открытого (hi=None) бакета (30 млн ₽) +_HIGH_ANCHOR: int = 30_000_000 +# Минимальная ширина ценового диапазона — ниже этого не бисектируем +_MIN_BAND: int = 100_000 +# Максимальная глубина рекурсии +_MAX_DEPTH: int = 12 + +# ── Маппинг комнатности int → API token ───────────────────────────────────── + +_ROOMS_TOKEN: dict[int, str] = { + 0: "st", + 1: "1", + 2: "2", + 3: "3", + 4: "4", + 5: "5+", +} + +# ── Гео-guard: ЕКБ bbox ───────────────────────────────────────────────────── + +_EKB_LAT_MIN: float = 56.7 +_EKB_LAT_MAX: float = 56.95 +_EKB_LON_MIN: float = 60.4 +_EKB_LON_MAX: float = 60.8 -def _is_blocked_page(html: str) -> bool: - head = html[:2048].lower() - return any(m in head for m in _DATADOME_MARKERS) +def _is_ekb(offer: dict[str, Any]) -> bool: + """Гео-guard: офер принадлежит ЕКБ по offerRegionName или bbox координат.""" + if offer.get("offerRegionName") == "Екатеринбург": + return True + loc: dict[str, Any] = offer.get("location") or {} + lat_raw = loc.get("lat") + lon_raw = loc.get("lon") + if lat_raw is not None and lon_raw is not None: + try: + lat = float(lat_raw) + lon = float(lon_raw) + return _EKB_LAT_MIN <= lat <= _EKB_LAT_MAX and _EKB_LON_MIN <= lon <= _EKB_LON_MAX + except (TypeError, ValueError): + pass + return False -# ── Regex helpers ──────────────────────────────────────────────────────────── -# _RE_ROOMS расширен относительно avito.py: DomClick пишет «2-комн. квартира» -# (с «комн.») — avito.py-вариант ловит только «N-к. кв.» / «N-к квартира». -_RE_ROOMS = re.compile( - r"(\d)-(?:комн\.?\s*(?:квартира|кв\.?)?|к\.?\s*(?:квартира|кв\.?))", - re.IGNORECASE, -) -_RE_STUDIO = re.compile(r"\bстуди[яиюей]\b", re.IGNORECASE) -_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[2²]", re.IGNORECASE) -_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE) - -# Цена: парсим из одного элемента DOM (не из суммарного card_text) — -# иначе число дома может слипнуться с ценой. Цена всегда в элементе с «₽». -_RE_PRICE_EL = re.compile(r"([\d\s  ]+)\s*[₽р](?:уб\.?)?", re.IGNORECASE) - -# source_id из href вида /card/sale__flat__2075671636 -_RE_SOURCE_ID = re.compile(r"sale__flat__(\d+)") - -# Адресные ключевые слова (эвристика) -_RE_ADDR_KW = re.compile( - r"ул\.|улица|пер\.|переулок|пр-т|проспект" - r"|бул\.|бульвар|шоссе|наб\.|набережная" - r"|пл\.|площадь|тракт|д\.\s*\d", - re.IGNORECASE, -) - - -# ── Вспомогательные функции ────────────────────────────────────────────────── - - -def _extract_rooms(text: str) -> int | None: - if _RE_STUDIO.search(text): - return 0 - m = _RE_ROOMS.search(text) - return int(m.group(1)) if m else None - - -def _extract_area(text: str) -> float | None: - m = _RE_AREA.search(text) - return float(m.group(1).replace(",", ".")) if m else None - - -def _extract_floor(text: str) -> tuple[int | None, int | None]: - m = _RE_FLOOR.search(text) - if m: - return int(m.group(1)), int(m.group(2)) - return None, None - - -def _extract_price_from_element(text: str) -> int | None: - """Извлекаем цену из текста ОДНОГО элемента DOM. - - Парсим из одного span/div — не из суммарного card_text. Это исключает - случай, когда адресный номер дома стоит вплотную перед ценой в - объединённом тексте карточки («Ленина, 503 100 000 руб.»). - """ - m = _RE_PRICE_EL.search(text) - if m: - # Нормализуем пробелы любого вида (обычный, неразрывный, узкий) - raw = re.sub(r"[\s  ]", "", m.group(1)) - if raw.isdigit(): - val = int(raw) - if val > 0: - return val - return None - - -def _extract_source_id(href: str) -> str | None: - """Числовой ID из href типа /card/sale__flat__2075671636.""" - m = _RE_SOURCE_ID.search(href) - return m.group(1) if m else None +def _parse_publish_date(val: Any) -> date | None: + """Парсим publishedDate: epoch int/str или ISO8601 строка. None при ошибке.""" + if val is None: + return None + try: + if isinstance(val, int | float): + return datetime.fromtimestamp(int(val), tz=UTC).date() + s = str(val).strip() + if s.isdigit(): + return datetime.fromtimestamp(int(s), tz=UTC).date() + # ISO8601 — берём первые 10 символов (YYYY-MM-DD) + return date.fromisoformat(s[:10]) + except Exception: + return None # ── DomClickScraper ────────────────────────────────────────────────────────── class DomClickScraper(BaseScraper): - """DomClick вторичка parser. Источник = 'domklik'. + """DomClick вторичка через bff-search-web JSON API. Источник = 'domklik'. - Использует BrowserFetcher (camoufox headless Firefox) — DataDome пропускает - его без дополнительной stealth-настройки. curl_cffi -> 401 (DataDome блок). - - Основной метод: fetch_city(city_id, rooms, pages). - fetch_around() не реализован: DomClick не поддерживает geo-radius в URL. + Layer A: SERP list (offers/v1) → ScrapedLot. Layer B/C не реализованы. + Транспорт: одна BrowserFetcher(source="domclick") сессия в __aenter__. + fetch_around() → NotImplementedError (DomClick не поддерживает geo-radius). """ name = "domklik" source = "domklik" - base_url = "https://domclick.ru" - # DataDome — консервативная задержка между страницами (fallback до init) - request_delay_sec = 8.0 + base_url = "https://bff-search-web.domclick.ru" + request_delay_sec = 8.0 # overwritten from get_scraper_delay in __init__ + + EKB_REGION_GUID = "0d475b79-88de-4054-818c-37d8f9d0d440" + EKB_AIDS = 20561 + OFFSET_CAP = 2000 + LIMIT = 20 def __init__(self) -> None: super().__init__() self.request_delay_sec = get_scraper_delay(self.name) - # Счётчик карточек с неожиданной DOM-структурой (для observability) self.parse_failures: int = 0 + self._browser: Any | None = None async def __aenter__(self) -> DomClickScraper: + from app.services.scrapers.browser_fetcher import BrowserFetcher + await super().__aenter__() + self._browser = BrowserFetcher(source="domclick") + await self._browser.__aenter__() return self + async def __aexit__(self, *args: Any) -> None: + if self._browser is not None: + await self._browser.__aexit__(*args) + self._browser = None + await super().__aexit__(*args) + # ── fetch_around — stub ─────────────────────────────────────────────────── async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list[ScrapedLot]: @@ -155,236 +149,502 @@ class DomClickScraper(BaseScraper): self, city_id: int, rooms: list[int] | None = None, - pages: int = 20, + pages: int = 100, ) -> list[ScrapedLot]: - """Citywide sweep: все страницы SERP для city_id. + """Citywide sweep: rooms × recursive price-bands → дедуплицированный список. Args: - city_id: числовой ID города в DomClick (например, 4 = Екатеринбург). - rooms: список значений комнатности (0=студия, 1, 2, 3, 4, ...). - None -> без фильтра комнатности (все квартиры сразу). - pages: максимальное число страниц на один sweep. Break on empty. + city_id: числовой ID города (4 = ЕКБ). Только ЕКБ поддерживается; + для других значений логируем warning, продолжаем с EKB GUID. + rooms: список комнатностей (0=студия, 1, 2, 3, 4, 5=5+). + None → [0,1,2,3,4,5] (все). + pages: максимум страниц на leaf-bucket (default 100 = OFFSET_CAP/LIMIT). Returns: Дедуплицированный по source_id список ScrapedLot. """ - from app.services.scrapers.browser_fetcher import BrowserFetcher + if city_id != 4: + logger.warning( + "domklik: city_id=%d не поддерживается — используем EKB GUID/aids", + city_id, + ) - all_lots: list[ScrapedLot] = [] - seen_ids: set[str] = set() + _rooms = rooms if rooms is not None else list(_ROOMS_TOKEN.keys()) + seen: dict[str, ScrapedLot] = {} + total_geo_dropped: int = 0 - # Формируем список room-значений для sweep'ов - room_values: list[int | None] = [None] if not rooms else list(rooms) - - async with BrowserFetcher(source="domclick") as fetcher: - for room_val in room_values: - room_label = f"rooms={room_val}" if room_val is not None else "all_rooms" - logger.info( - "domklik: city_id=%d %s sweep (max %d pages)", city_id, room_label, pages - ) - - for page_num in range(1, pages + 1): - url = self._build_url(city_id, room_val, page_num) - logger.debug("domklik: fetch %s", url) - - try: - html = await fetcher.fetch(url) - except Exception as exc: - logger.error( - "domklik: fetch failed city_id=%d page=%d: %s", - city_id, - page_num, - exc, - ) - break - - try: - lots = self._parse_html(html) - except DomClickBlockedError: - logger.warning( - "domklik: DataDome block at city_id=%d page=%d — stopping sweep", - city_id, - page_num, - ) - break - - if not lots: - logger.info( - "domklik: empty page city_id=%d %s page=%d — stopping sweep", - city_id, - room_label, - page_num, - ) - break - - # Дедупликация по source_id (кросс-sweep и кросс-страница) - new_lots: list[ScrapedLot] = [] - for lot in lots: - key = lot.source_id or lot.source_url - if key not in seen_ids: - seen_ids.add(key) - new_lots.append(lot) - - all_lots.extend(new_lots) - logger.info( - "domklik: city_id=%d %s page=%d -> %d new (total %d)", - city_id, - room_label, - page_num, - len(new_lots), - len(all_lots), - ) - - # Пауза между страницами (anti-DataDome) - if page_num < pages: - await self.sleep_between_requests() + for room_int in _rooms: + room_token = _ROOMS_TOKEN.get(room_int, str(room_int)) + logger.info( + "domklik: room=%s (%d) sweep start (max %d pages per leaf)", + room_token, + room_int, + pages, + ) + geo_dropped_ref = [0] + await self._walk_price_range( + room_token=room_token, + lo=0, + hi=None, + seen=seen, + pages=pages, + geo_dropped_ref=geo_dropped_ref, + ) + total_geo_dropped += geo_dropped_ref[0] + logger.info( + "domklik: room=%s done — unique_total=%d geo_dropped_this_room=%d", + room_token, + len(seen), + geo_dropped_ref[0], + ) + if total_geo_dropped: + logger.info("domklik: fetch_city geo_dropped_total=%d", total_geo_dropped) logger.info( "domklik: fetch_city done city_id=%d total=%d parse_failures=%d", city_id, - len(all_lots), + len(seen), self.parse_failures, ) - return all_lots + return list(seen.values()) - # ── URL builder ─────────────────────────────────────────────────────────── + # ── Рекурсивное адаптивное деление ─────────────────────────────────────── - def _build_url(self, city_id: int, rooms: int | None, page: int) -> str: - """Строит URL SERP DomClick.""" - url = ( - f"{self.base_url}/search" - f"?deal_type=sale&category=living&offer_type=flat" - f"&city_id={city_id}" - ) - if rooms is not None: - url += f"&rooms={rooms}" - url += f"&p={page}" - return url + async def _walk_price_range( + self, + *, + room_token: str, + lo: int, + hi: int | None, + seen: dict[str, ScrapedLot], + pages: int, + geo_dropped_ref: list[int], + _depth: int = 0, + ) -> None: + """Рекурсивная бинарная partition ценового диапазона [lo, hi]. - # ── HTML parsing ────────────────────────────────────────────────────────── + Open band (hi is None): + snippetsCount ≤ OFFSET_CAP → пагинируем leaf. + snippetsCount > OFFSET_CAP → разбиваем через HIGH_ANCHOR: + recurse [lo, HIGH_ANCHOR] + [HIGH_ANCHOR+1, None]. - def _parse_html(self, html: str) -> list[ScrapedLot]: - """Парсим карточки из HTML через selectolax. + Closed band (hi is int): + snippetsCount ≤ OFFSET_CAP → пагинируем leaf. + snippetsCount > OFFSET_CAP и (depth > MAX_DEPTH или band < MIN_BAND): + пагинируем как есть (tail-loss WARNING). + Иначе → mid=(lo+hi)//2, recurse [lo, mid] и [mid+1, hi]. - Селектор: `a[href*="/card/"]` с паттерном `/card/sale__flat__`. + Raises DomClickBlockedError при QRATOR-блоке (propagate к вызывающему). """ - if _is_blocked_page(html): - logger.warning("domklik: DataDome block page detected, 0 cards returned") - raise DomClickBlockedError("DomClick returned DataDome block page") - tree = HTMLParser(html) - card_links = tree.css('a[href*="/card/"]') + hi_repr = "open" if hi is None else str(hi) - seen_hrefs: set[str] = set() - lots: list[ScrapedLot] = [] + # ── 1. COUNT ───────────────────────────────────────────────────────── + count_url = self._build_count_url(room_token, lo, hi) + count_data = await self._fetch_json(count_url) # raises DomClickBlockedError on block + snippets: int = (count_data.get("result") or {}).get("snippetsCount", 0) - for link in card_links: - href = link.attributes.get("href", "") - # Только карточки вторичного жилья - if "sale__flat__" not in href: + logger.info( + "domklik: %s [%d, %s] snippetsCount=%d depth=%d", + room_token, + lo, + hi_repr, + snippets, + _depth, + ) + + if snippets == 0: + return + + await self.sleep_between_requests() + + # ── 2a. Open band — нельзя бисектировать напрямую ──────────────────── + if hi is None: + if snippets <= self.OFFSET_CAP: + await self._paginate_leaf( + room_token, lo, None, snippets, seen, pages, geo_dropped_ref + ) + else: + # Делим через HIGH_ANCHOR; хвост [HIGH_ANCHOR+1, None] обычно мал + await self._walk_price_range( + room_token=room_token, + lo=lo, + hi=_HIGH_ANCHOR, + seen=seen, + pages=pages, + geo_dropped_ref=geo_dropped_ref, + _depth=_depth + 1, + ) + await self._walk_price_range( + room_token=room_token, + lo=_HIGH_ANCHOR + 1, + hi=None, + seen=seen, + pages=pages, + geo_dropped_ref=geo_dropped_ref, + _depth=_depth + 1, + ) + return + + # ── 2b. Closed band ─────────────────────────────────────────────────── + if snippets <= self.OFFSET_CAP: + await self._paginate_leaf(room_token, lo, hi, snippets, seen, pages, geo_dropped_ref) + return + + # Нужно деление — проверяем ограничения + band = hi - lo + if _depth >= _MAX_DEPTH or band < _MIN_BAND: + tail_loss = max(0, snippets - self.OFFSET_CAP) + logger.warning( + "domklik: %s [%d, %d] snippets=%d > cap=%d band=%d depth=%d " + "— paginating as-is (tail-loss ~%d)", + room_token, + lo, + hi, + snippets, + self.OFFSET_CAP, + band, + _depth, + tail_loss, + ) + await self._paginate_leaf(room_token, lo, hi, snippets, seen, pages, geo_dropped_ref) + return + + mid = (lo + hi) // 2 + await self._walk_price_range( + room_token=room_token, + lo=lo, + hi=mid, + seen=seen, + pages=pages, + geo_dropped_ref=geo_dropped_ref, + _depth=_depth + 1, + ) + await self._walk_price_range( + room_token=room_token, + lo=mid + 1, + hi=hi, + seen=seen, + pages=pages, + geo_dropped_ref=geo_dropped_ref, + _depth=_depth + 1, + ) + + # ── Пагинация leaf-bucket ───────────────────────────────────────────────── + + async def _paginate_leaf( + self, + room_token: str, + lo: int, + hi: int | None, + snippets: int, + seen: dict[str, ScrapedLot], + pages: int, + geo_dropped_ref: list[int], + ) -> None: + """Последовательная пагинация одного leaf-bucket [lo, hi] по offset.""" + max_offset = min(snippets, self.OFFSET_CAP, pages * self.LIMIT) + offset = 0 + + while offset < max_offset: + list_url = self._build_list_url(room_token, lo, hi, offset) + list_data = await self._fetch_json(list_url) # DomClickBlockedError propagates + items: list[dict[str, Any]] = (list_data.get("result") or {}).get("items") or [] + + if not items: + logger.debug( + "domklik: empty items at %s [%s, %s] offset=%d — stopping", + room_token, + lo, + hi or "open", + offset, + ) + break + + page_lots = self._parse_page(items, room_token, geo_dropped_ref) + for lot in page_lots: + key = lot.source_id or lot.source_url + if key: + seen[key] = lot + + logger.debug( + "domklik: %s [%s, %s] offset=%d items=%d parsed=%d total_unique=%d", + room_token, + lo, + hi or "open", + offset, + len(items), + len(page_lots), + len(seen), + ) + + offset += self.LIMIT + if offset < max_offset: + await self.sleep_between_requests() + + # ── Парсинг страницы офферов ────────────────────────────────────────────── + + def _parse_page( + self, + items: list[dict[str, Any]], + room_token: str, + geo_dropped_ref: list[int], + ) -> list[ScrapedLot]: + """Парсинг списка офферов → ScrapedLot с гео-guard. + + Эмитирует Sentry-сообщение если raw_count > 0 но 0 прошло парсинг + (guard от silent schema regression). + """ + result: list[ScrapedLot] = [] + local_geo_dropped = 0 + + for offer in items: + if not _is_ekb(offer): + local_geo_dropped += 1 + geo_dropped_ref[0] += 1 continue - if href in seen_hrefs: - continue - seen_hrefs.add(href) - - lot = self._card_link_to_lot(link, href) + lot = self._offer_to_lot(offer, room_token) if lot is not None: - lots.append(lot) + result.append(lot) - return lots + raw_count = len(items) + ekb_count = raw_count - local_geo_dropped + saved = len(result) - def _card_link_to_lot(self, link: Node, href: str) -> ScrapedLot | None: - """Парсинг одной карточки-ссылки -> ScrapedLot. + # Schema regression guard + if raw_count > 0 and saved == 0 and ekb_count > 0: + logger.error( + "domklik: 0/%d offers прошли _offer_to_lot (ekb=%d) " + "— возможна schema regression", + raw_count, + ekb_count, + ) + try: + if settings.glitchtip_dsn: + sentry_sdk.capture_message( + f"domklik SERP: {raw_count} offers, 0 parsed (ekb={ekb_count})" + " — possible schema regression", + level="error", + ) + except Exception: + pass # sentry_sdk не инициализирован в dev - DomClick SERP оборачивает карточку в `` — вся - информация (title, цена, адрес) внутри этого элемента. + return result - Цену извлекаем из отдельных дочерних элементов (не из объединённого - card_text) чтобы исключить слипание числа дома с ценой. + def _offer_to_lot(self, offer: dict[str, Any], room_token: str) -> ScrapedLot | None: + """Парсинг одного DomClick bff-JSON оффера → ScrapedLot. + + Маппинг полей per спецификации Layer A. + source='domklik', listing_segment='vtorichka' всегда. """ try: - source_id = _extract_source_id(href) - source_url = urljoin(self.base_url, href) + # ── Identity ───────────────────────────────────────────────────── + offer_id = offer.get("id") + if not offer_id: + return None + source_id = str(offer_id) - # Суммарный текст — для rooms/area/floor (эти поля не подвержены - # проблеме слипания, т.к. используют специфичные маркеры: «м²», «эт.») - card_text = link.text(strip=True) - if not card_text: + path = offer.get("path") or "" + source_url = urljoin("https://domclick.ru", path) + + # ── Цена ───────────────────────────────────────────────────────── + price_raw = offer.get("price") + if price_raw is None: + return None + try: + price_rub = int(price_raw) + except (TypeError, ValueError): + return None + if price_rub <= 0: return None - rooms = _extract_rooms(card_text) - area = _extract_area(card_text) - floor, total_floors = _extract_floor(card_text) + # ── squarePrice (price_per_m2) ──────────────────────────────────── + sq_price_raw = offer.get("squarePrice") + price_per_m2: int | None = None + if sq_price_raw is not None: + try: + price_per_m2 = int(sq_price_raw) + except (TypeError, ValueError): + pass - # Цена: ищем в каждом дочернем элементе отдельно чтобы не слипались - # числа дома с ценой при конкатенации card_text - price = self._extract_price_from_children(link) - if not price or price <= 0: - return None + # ── Координаты ─────────────────────────────────────────────────── + location: dict[str, Any] = offer.get("location") or {} + lat: float | None = None + lon: float | None = None + try: + if location.get("lat") is not None: + lat = float(location["lat"]) + if location.get("lon") is not None: + lon = float(location["lon"]) + except (TypeError, ValueError): + lat = lon = None - address = self._extract_address(link) + # ── Адрес ──────────────────────────────────────────────────────── + address_obj: dict[str, Any] = offer.get("address") or {} + address: str | None = address_obj.get("displayName") or None + + # ── Параметры квартиры ──────────────────────────────────────────── + obj_info: dict[str, Any] = offer.get("objectInfo") or {} + + area_raw = obj_info.get("area") + area_m2: float | None = None + if area_raw is not None: + try: + area_m2 = float(area_raw) + except (TypeError, ValueError): + pass + + # Комнатность: студии → 0; числовые → из objectInfo.rooms или токена + if room_token == "st": + rooms: int | None = 0 + else: + rooms_raw = obj_info.get("rooms") + if rooms_raw is not None: + try: + rooms = int(rooms_raw) + except (TypeError, ValueError): + rooms = None + else: + # fallback: числовой токен + try: + rooms = int(room_token) + except ValueError: + rooms = None + + floor_raw = obj_info.get("floor") + floor: int | None = None + if floor_raw is not None: + try: + floor = int(floor_raw) + except (TypeError, ValueError): + pass + + # ── Здание ─────────────────────────────────────────────────────── + house: dict[str, Any] = offer.get("house") or {} + + total_floors_raw = house.get("floors") + total_floors: int | None = None + if total_floors_raw is not None: + try: + total_floors = int(total_floors_raw) + except (TypeError, ValueError): + pass + + year_built_raw = house.get("buildYear") + year_built: int | None = None + if year_built_raw is not None: + try: + year_built = int(year_built_raw) + except (TypeError, ValueError): + pass + + # ── Дата публикации ─────────────────────────────────────────────── + published_date = _parse_publish_date(offer.get("publishedDate")) + + # ── Raw payload (компактный) ─────────────────────────────────────── + flat_complex: dict[str, Any] = offer.get("flatComplex") or {} + raw_payload: dict[str, Any] = { + "is_apartments": obj_info.get("isApartment"), + "is_rosreestr_approved": offer.get("isRosreestrApproved"), + "offer_region_name": offer.get("offerRegionName"), + "updated_date": offer.get("updatedDate"), + "last_price_history_state": offer.get("lastPriceHistoryState"), + "flat_complex": ( + { + "id": flat_complex.get("id"), + "name": flat_complex.get("name"), + "slug": flat_complex.get("slug"), + } + if flat_complex + else None + ), + } return ScrapedLot( source="domklik", source_url=source_url, source_id=source_id, address=address, - lat=None, # DomClick SERP координаты не отдаёт - lon=None, + lat=lat, + lon=lon, rooms=rooms, - area_m2=area, + area_m2=area_m2, floor=floor, total_floors=total_floors, - price_rub=price, + year_built=year_built, + price_rub=price_rub, + price_per_m2=price_per_m2, listing_segment="vtorichka", - raw_payload={"card_text": card_text[:500]}, + listing_date=published_date, + publish_date=published_date, + raw_payload=raw_payload, ) + except Exception: self.parse_failures += 1 logger.warning( - "domclick _card_link_to_lot: parse failed href=%r (parse_failures=%d)", - href, + "domklik _offer_to_lot: parse failed offer_id=%r (failures=%d)", + offer.get("id"), self.parse_failures, exc_info=True, ) return None - def _extract_price_from_children(self, link: Node) -> int | None: - """Ищем цену в дочерних элементах карточки по символу рубля. + # ── URL builders ────────────────────────────────────────────────────────── - Итерируем по всем потомкам: первый элемент с рублёвым символом - в тексте — источник цены. + def _base_params(self, room_token: str, lo: int, hi: int | None) -> list[tuple[str, Any]]: + """Базовые query params (без offset/limit). Используется count и list URL.""" + params: list[tuple[str, Any]] = [ + ("address", self.EKB_REGION_GUID), + ("aids", self.EKB_AIDS), + ("deal_type", "sale"), + ("category", "living"), + ("offer_type", "flat"), + ("rooms", room_token), + ("sort", "qi"), + ] + if lo > 0: + params.append(("sale_price__gte", lo)) + if hi is not None: + params.append(("sale_price__lte", hi)) + return params + + def _build_count_url(self, room_token: str, lo: int, hi: int | None) -> str: + """URL для count/v1 — без offset/limit.""" + params = self._base_params(room_token, lo, hi) + return f"{self.base_url}/api/offers/count/v1?{urlencode(params)}" + + def _build_list_url(self, room_token: str, lo: int, hi: int | None, offset: int) -> str: + """URL для offers/v1 — с offset/limit.""" + params = self._base_params(room_token, lo, hi) + params.append(("offset", offset)) + params.append(("limit", self.LIMIT)) + return f"{self.base_url}/api/offers/v1?{urlencode(params)}" + + # ── JSON fetch helper ───────────────────────────────────────────────────── + + async def _fetch_json(self, url: str) -> dict[str, Any]: + """Fetch URL через BrowserFetcher → parse JSON. + + Raises: + DomClickBlockedError: ответ не JSON (QRATOR challenge HTML), + или BrowserFetcher вернул HTML-страницу. """ - for selector in ("span", "div", "p", "strong", "b"): - for el in link.css(selector): - text = el.text(strip=True) - price = _extract_price_from_element(text) - if price: - return price - # Fallback: суммарный текст карточки (если структура нестандартная) - return _extract_price_from_element(link.text(strip=True)) - - def _extract_address(self, link: Node) -> str | None: - """Эвристика извлечения адреса из DOM карточки. - - DomClick помещает адрес в отдельный текстовый блок внутри карточки. - Обходим вложенные элементы и ищем первый текст с адресными ключевыми - словами, пропуская строки с площадью/этажом/ценой. - - Возвращает None если адрес не найден. - """ - for selector in ("span", "p", "div"): - for el in link.css(selector): - text = el.text(strip=True) - if not text or len(text) < 5 or len(text) > 200: - continue - # Пропускаем блоки с площадью / этажом / ценой - if _RE_AREA.search(text) or _RE_FLOOR.search(text): - continue - if _RE_PRICE_EL.search(text): - continue - if _RE_ADDR_KW.search(text): - return text - return None + assert self._browser is not None, "Используй DomClickScraper как async context manager" + raw: str = await self._browser.fetch(url) + # QRATOR block / captcha возвращает HTML — детектируем по первому символу + stripped = raw.lstrip() + if stripped.startswith("<"): + logger.warning( + "domklik: HTML challenge url=%s (первые 300 символов): %r", + url, + raw[:300], + ) + raise DomClickBlockedError("DomClick вернул HTML вместо JSON (QRATOR block)") + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + logger.warning("domklik: JSONDecodeError url=%s: %s", url, exc) + raise DomClickBlockedError(f"DomClick: не JSON ответ: {exc}") from exc # ── Convenience runner (для Celery tasks) ──────────────────────────────────── @@ -393,7 +653,7 @@ class DomClickScraper(BaseScraper): async def scrape_domclick_city( city_id: int, rooms: list[int] | None = None, - pages: int = 20, + pages: int = 100, ) -> list[ScrapedLot]: """Удобная точка входа для вызова из Celery tasks. diff --git a/tradein-mvp/backend/data/sql/132_update_domclick_sweep_params.sql b/tradein-mvp/backend/data/sql/132_update_domclick_sweep_params.sql new file mode 100644 index 00000000..f5c56c2e --- /dev/null +++ b/tradein-mvp/backend/data/sql/132_update_domclick_sweep_params.sql @@ -0,0 +1,39 @@ +-- 132_update_domclick_sweep_params.sql +-- Обновление параметров DomClick citywide sweep после рефакторинга на JSON API. +-- +-- Контекст (2026-06-27): +-- Старый скрейпер (122_enable_domclick_city_sweep.sql) использовал HTML/camoufox +-- через domclick.ru/search → 0 листингов (URL redesigned + datacenter IP banned +-- QRATOR). Новая реализация (Layer A) использует: +-- GET https://bff-search-web.domclick.ru/api/offers/v1 +-- GET https://bff-search-web.domclick.ru/api/offers/count/v1 +-- Транспорт: BrowserFetcher(source="domclick") → tradein-browser pool → generic +-- provider → shared RU mobile proxy (не заблокировано, PoC verified 2026-06-27). +-- +-- Изменения default_params: +-- rooms: добавлен 5 (токен "5+", ранее отсутствовал). +-- pages_per_anchor: 100 (= OFFSET_CAP/LIMIT = 2000/20, exhaustive within cap). +-- request_delay_sec: 3 (JSON endpoint легче camoufox-рендера; QRATOR rate-limit +-- не обнаружен на datacenter IP с shared proxy; conservative polite delay). +-- +-- !!! DORMANT BY DESIGN !!! enabled остаётся false. +-- Включать вручную после prod-smoke: +-- UPDATE scrape_schedules SET enabled = true WHERE source = 'domclick_city_sweep'; +-- +-- Idempotent: UPDATE с WHERE source = 'domclick_city_sweep' — no-op если строка +-- отсутствует (не должна: засеяна в 122; guard на случай misapply-порядка). +-- +-- ЗАВИСИМОСТИ: 122_enable_domclick_city_sweep.sql (INSERT строки в scrape_schedules). + +BEGIN; + +UPDATE scrape_schedules +SET default_params = '{ + "city_id": 4, + "rooms": [0, 1, 2, 3, 4, 5], + "pages_per_anchor": 100, + "request_delay_sec": 3 +}'::jsonb +WHERE source = 'domclick_city_sweep'; + +COMMIT; diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick.py b/tradein-mvp/backend/tests/scrapers/test_domclick.py index 04d0557d..04813c2a 100644 --- a/tradein-mvp/backend/tests/scrapers/test_domclick.py +++ b/tradein-mvp/backend/tests/scrapers/test_domclick.py @@ -1,252 +1,432 @@ -"""Unit-тесты для DomClick scraper (#796). +"""Unit-тесты для DomClick JSON API scraper (Layer A rewrite). -Тестируем: -- Парсинг title-паттерна (rooms/area/floor) из текста карточки -- Парсинг цены из отдельного DOM-элемента -- Извлечение source_id из href -- Полный цикл _parse_html -> list[ScrapedLot] на mocked HTML +Offline, без сети и БД. Монкейпатчим BrowserFetcher / _fetch_json на canned JSON. + +Coverage: + - _offer_to_lot: поля ScrapedLot (source_id, source_url, lat/lon, address, + rooms, area, floor, total_floors, year_built, price, listing_segment, source) + - Studio bucket (room_token="st"): rooms=0 + - Гео-guard: Челябинск оффер отбрасывается, EKB оффер проходит + - Block detection: не-JSON body → DomClickBlockedError + - URL building: rooms=5%2B в URL; sale_price__gte/lte в banded URL; + offset/limit в list URL и отсутствует в count URL + - Bucketing: snippetsCount>2000 → рекурсивный split; offset не достигает 2000 """ from __future__ import annotations -import asyncio +import json +import os +from datetime import date +from urllib.parse import parse_qs, urlparse import pytest -from app.services.scrapers.domclick import ( - DomClickScraper, - _extract_area, - _extract_floor, - _extract_price_from_element, - _extract_rooms, - _extract_source_id, -) +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from app.services.scrapers.domclick import DomClickScraper, _is_ekb, _parse_publish_date from app.services.scrapers.domclick_exceptions import DomClickBlockedError -# ── Rooms / studio ─────────────────────────────────────────────────────────── +# ── Фикстуры ───────────────────────────────────────────────────────────────── + + +def _make_scraper() -> DomClickScraper: + """Создаём DomClickScraper без DB (монкейпатчим get_scraper_delay).""" + from unittest.mock import patch + + with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=0.0): + scraper = DomClickScraper() + scraper._browser = object() # не None — assertion в _fetch_json проверяет + return scraper + + +_SAMPLE_OFFER: dict = { + "id": 2075671636, + "path": "/card/sale__flat__2075671636", + "location": {"lat": 56.83, "lon": 60.61}, + "address": {"displayName": "Екатеринбург, улица Ленина, 10"}, + "objectInfo": { + "area": 52.3, + "rooms": 2, + "floor": 5, + "isApartment": False, + }, + "house": {"floors": 16, "buildYear": 2005}, + "price": 6500000, + "squarePrice": 124283, + "flatComplex": {"id": 123, "name": "ЖК Тест", "slug": "zhk-test"}, + "isRosreestrApproved": True, + "publishedDate": "2026-06-01", + "updatedDate": "2026-06-15", + "lastPriceHistoryState": None, + "offerRegionName": "Екатеринбург", +} + +_STUDIO_OFFER: dict = { + "id": 9001, + "path": "/card/sale__flat__9001", + "location": {"lat": 56.84, "lon": 60.60}, + "address": {"displayName": "Екатеринбург, пр-т Ленина, 50"}, + "objectInfo": {"area": 24.5, "rooms": None, "floor": 3, "isApartment": False}, + "house": {"floors": 10, "buildYear": 2020}, + "price": 3100000, + "squarePrice": 126531, + "flatComplex": None, + "isRosreestrApproved": False, + "publishedDate": None, + "updatedDate": None, + "lastPriceHistoryState": None, + "offerRegionName": "Екатеринбург", +} + +_CHELYABINSK_OFFER: dict = { + "id": 99999, + "path": "/card/sale__flat__99999", + "location": {"lat": 55.15, "lon": 61.39}, # вне EKB bbox + "address": {"displayName": "Челябинск, улица Ленина, 1"}, + "objectInfo": {"area": 40.0, "rooms": 1, "floor": 3, "isApartment": False}, + "house": {"floors": 9, "buildYear": 1990}, + "price": 3000000, + "squarePrice": 75000, + "flatComplex": None, + "isRosreestrApproved": False, + "publishedDate": None, + "updatedDate": None, + "lastPriceHistoryState": None, + "offerRegionName": "Челябинск", # не Екатеринбург +} + + +# ── _offer_to_lot — базовый маппинг ────────────────────────────────────────── + + +def test_offer_to_lot_basic_fields() -> None: + """Репрезентативный оффер → ScrapedLot с корректными полями.""" + scraper = _make_scraper() + lot = scraper._offer_to_lot(_SAMPLE_OFFER, room_token="2") + + assert lot is not None + assert lot.source == "domklik" + assert lot.source_id == "2075671636" + assert lot.source_url == "https://domclick.ru/card/sale__flat__2075671636" + assert lot.lat == pytest.approx(56.83) + assert lot.lon == pytest.approx(60.61) + assert lot.address == "Екатеринбург, улица Ленина, 10" + assert lot.rooms == 2 + assert lot.area_m2 == pytest.approx(52.3) + assert lot.floor == 5 + assert lot.total_floors == 16 + assert lot.year_built == 2005 + assert lot.price_rub == 6_500_000 + assert lot.price_per_m2 == 124_283 + assert lot.listing_segment == "vtorichka" + assert lot.listing_date == date(2026, 6, 1) + assert lot.publish_date == date(2026, 6, 1) + assert lot.raw_payload is not None + assert lot.raw_payload["is_rosreestr_approved"] is True + assert lot.raw_payload["flat_complex"]["name"] == "ЖК Тест" + + +def test_offer_to_lot_studio_rooms_zero() -> None: + """Studio bucket (room_token='st') → rooms=0 независимо от objectInfo.rooms.""" + scraper = _make_scraper() + lot = scraper._offer_to_lot(_STUDIO_OFFER, room_token="st") + + assert lot is not None + assert lot.rooms == 0 + assert lot.area_m2 == pytest.approx(24.5) + assert lot.price_rub == 3_100_000 + # publishedDate=None → None + assert lot.listing_date is None + + +def test_offer_to_lot_missing_price_returns_none() -> None: + """Оффер без цены → None.""" + scraper = _make_scraper() + offer = {**_SAMPLE_OFFER, "price": None} + assert scraper._offer_to_lot(offer, "2") is None + + +def test_offer_to_lot_zero_price_returns_none() -> None: + """Оффер с price=0 → None.""" + scraper = _make_scraper() + offer = {**_SAMPLE_OFFER, "price": 0} + assert scraper._offer_to_lot(offer, "2") is None + + +def test_offer_to_lot_missing_id_returns_none() -> None: + """Оффер без id → None.""" + scraper = _make_scraper() + offer = {k: v for k, v in _SAMPLE_OFFER.items() if k != "id"} + assert scraper._offer_to_lot(offer, "2") is None + + +# ── Гео-guard ───────────────────────────────────────────────────────────────── + + +def test_geo_guard_ekb_by_region_name() -> None: + """offerRegionName='Екатеринбург' → EKB.""" + assert _is_ekb({"offerRegionName": "Екатеринбург"}) is True + + +def test_geo_guard_ekb_by_bbox_coords() -> None: + """Координаты внутри EKB bbox → EKB (даже без offerRegionName).""" + assert _is_ekb({"location": {"lat": 56.83, "lon": 60.61}}) is True + + +def test_geo_guard_chelyabinsk_dropped() -> None: + """Челябинск (имя и bbox) → не EKB.""" + assert _is_ekb(_CHELYABINSK_OFFER) is False + + +def test_geo_guard_out_of_bbox_no_name() -> None: + """Координаты вне EKB bbox и нет offerRegionName → не EKB.""" + assert _is_ekb({"location": {"lat": 55.0, "lon": 73.0}}) is False + + +def test_parse_page_drops_non_ekb() -> None: + """_parse_page: Челябинск оффер отброшен, EKB проходит.""" + scraper = _make_scraper() + dropped_ref = [0] + items = [_SAMPLE_OFFER, _CHELYABINSK_OFFER] + lots = scraper._parse_page(items, "2", dropped_ref) + + assert len(lots) == 1 + assert lots[0].source_id == "2075671636" + assert dropped_ref[0] == 1 # один дроп + + +# ── Block detection ─────────────────────────────────────────────────────────── + + +async def test_fetch_json_html_body_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None: + """Если BrowserFetcher.fetch возвращает HTML → DomClickBlockedError.""" + + class _FakeBrowser: + async def fetch(self, url: str) -> str: + return "QRATOR protection" + + scraper = _make_scraper() + scraper._browser = _FakeBrowser() + + with pytest.raises(DomClickBlockedError): + await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1") + + +async def test_fetch_json_non_json_body_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None: + """Не-JSON body (не HTML, просто мусор) → DomClickBlockedError.""" + + class _FakeBrowser: + async def fetch(self, url: str) -> str: + return "not valid json at all |||" + + scraper = _make_scraper() + scraper._browser = _FakeBrowser() + + with pytest.raises(DomClickBlockedError): + await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1") + + +async def test_fetch_json_valid_json_returns_dict(monkeypatch: pytest.MonkeyPatch) -> None: + """Валидный JSON → dict без исключений.""" + + class _FakeBrowser: + async def fetch(self, url: str) -> str: + return json.dumps({"result": {"snippetsCount": 42}}) + + scraper = _make_scraper() + scraper._browser = _FakeBrowser() + + data = await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1") + assert data["result"]["snippetsCount"] == 42 + + +# ── URL building ────────────────────────────────────────────────────────────── + + +def test_build_count_url_no_offset_limit() -> None: + """count URL не содержит offset и limit.""" + scraper = _make_scraper() + url = scraper._build_count_url("2", lo=0, hi=None) + qs = parse_qs(urlparse(url).query) + assert "offset" not in qs + assert "limit" not in qs + assert "/count/v1" in url + + +def test_build_list_url_has_offset_limit() -> None: + """list URL содержит offset и limit.""" + scraper = _make_scraper() + url = scraper._build_list_url("2", lo=0, hi=None, offset=40) + qs = parse_qs(urlparse(url).query) + assert qs["offset"] == ["40"] + assert qs["limit"] == ["20"] + assert "/offers/v1" in url + + +def test_build_list_url_5plus_encoded() -> None: + """rooms=5+ корректно URL-кодируется (5%2B или эквивалент).""" + scraper = _make_scraper() + url = scraper._build_list_url("5+", lo=0, hi=None, offset=0) + # urlencode кодирует '+' как '%2B в query string + assert "rooms=5%2B" in url or "rooms=5+" in url # оба варианта корректны + # Проверяем через parse_qs + qs = parse_qs(urlparse(url).query) + assert qs.get("rooms") == ["5+"] + + +def test_build_list_url_price_band() -> None: + """Бандовый URL содержит sale_price__gte и sale_price__lte.""" + scraper = _make_scraper() + url = scraper._build_list_url("2", lo=2_000_000, hi=5_000_000, offset=0) + qs = parse_qs(urlparse(url).query) + assert qs["sale_price__gte"] == ["2000000"] + assert qs["sale_price__lte"] == ["5000000"] + + +def test_build_list_url_no_price_params_when_open() -> None: + """Открытый бакет (lo=0, hi=None) → нет ценовых параметров.""" + scraper = _make_scraper() + url = scraper._build_list_url("1", lo=0, hi=None, offset=0) + qs = parse_qs(urlparse(url).query) + assert "sale_price__gte" not in qs + assert "sale_price__lte" not in qs + + +def test_build_url_contains_ekb_params() -> None: + """URL содержит EKB region GUID и aids.""" + scraper = _make_scraper() + url = scraper._build_count_url("1", lo=0, hi=None) + assert "0d475b79-88de-4054-818c-37d8f9d0d440" in url + qs = parse_qs(urlparse(url).query) + assert qs["aids"] == ["20561"] + assert qs["deal_type"] == ["sale"] + + +# ── Bucketing — split при snippetsCount > 2000 ──────────────────────────────── + + +async def test_bucketing_split_on_overflow(monkeypatch: pytest.MonkeyPatch) -> None: + """snippetsCount>2000 для открытого бакета → split; offset never ≥ 2000.""" + call_log: list[str] = [] + + def _make_ekb_offer(offer_id: str) -> dict: + return { + "id": offer_id, + "path": f"/card/sale__flat__{offer_id}", + "location": {"lat": 56.83, "lon": 60.61}, + "address": {"displayName": "Екатеринбург, ул. Тест, 1"}, + "objectInfo": {"area": 45.0, "rooms": 1, "floor": 3, "isApartment": False}, + "house": {"floors": 9, "buildYear": 2000}, + "price": 4000000, + "squarePrice": 88888, + "flatComplex": None, + "isRosreestrApproved": False, + "publishedDate": None, + "updatedDate": None, + "lastPriceHistoryState": None, + "offerRegionName": "Екатеринбург", + } + + async def mock_fetch_json(self: DomClickScraper, url: str) -> dict: + call_log.append(url) + parsed = urlparse(url) + qs = parse_qs(parsed.query) + + if "/count/v1" in url: + has_gte = "sale_price__gte" in qs + has_lte = "sale_price__lte" in qs + if not has_gte and not has_lte: + # Открытый бакет [0, None] → overflow, trigger split + return {"result": {"snippetsCount": 2001, "offersCount": 2001}} + elif has_lte and int(qs["sale_price__lte"][0]) == 30_000_000: + # [0, 30M] → within cap + return {"result": {"snippetsCount": 20, "offersCount": 20}} + else: + # Хвост [30M+1, None] → empty + return {"result": {"snippetsCount": 0, "offersCount": 0}} + + # list URL + if "/offers/v1" in url: + offset_val = int(qs.get("offset", ["0"])[0]) + if offset_val == 0: + return {"result": {"items": [_make_ekb_offer("101")]}} + return {"result": {"items": []}} + + return {"result": {}} + + import asyncio as _asyncio + + monkeypatch.setattr(DomClickScraper, "_fetch_json", mock_fetch_json) + # sleep_between_requests is inherited from BaseScraper → shadow on subclass + monkeypatch.setattr(DomClickScraper, "sleep_between_requests", lambda self: _asyncio.sleep(0)) + + from unittest.mock import patch + + with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=0.0): + scraper = DomClickScraper() + scraper._browser = object() # не None + + seen: dict = {} + geo_ref = [0] + await scraper._walk_price_range( + room_token="1", + lo=0, + hi=None, + seen=seen, + pages=100, + geo_dropped_ref=geo_ref, + ) + + # Split произошёл: как минимум 2 count-вызова (open band + [0, 30M]) + count_urls = [u for u in call_log if "/count/v1" in u] + assert len(count_urls) >= 2, f"Expected split, got count_urls={count_urls}" + + # Ни один list-запрос не должен иметь offset >= 2000 + list_urls = [u for u in call_log if "/offers/v1" in u] + for lu in list_urls: + offset_val = int(parse_qs(urlparse(lu).query).get("offset", ["0"])[0]) + assert offset_val < 2000, f"offset={offset_val} >= OFFSET_CAP in url={lu}" + + # Лот прошёл и дедуплицирован + assert "101" in seen + + +# ── _parse_publish_date ─────────────────────────────────────────────────────── @pytest.mark.parametrize( - "text, expected", + "val, expected", [ - ("2-комн. квартира 53,5 м² 5/15 эт.", 2), - ("1-к. квартира 32,1 м² 3/9 эт.", 1), - ("3-комн. кв. 78 м² 10/12 эт.", 3), - ("4-комн. квартира 95 м² 2/5 эт.", 4), - ("студия 24 м² 1/10 эт.", 0), - ("Студия 30,5 м² 8/22 эт.", 0), - ("Нет комнат — просто текст", None), - ], -) -def test_extract_rooms(text: str, expected: int | None) -> None: - assert _extract_rooms(text) == expected - - -# ── Area ───────────────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - "text, expected", - [ - ("2-комн. квартира 53,5 м² 5/15 эт.", 53.5), - ("студия 24 м² 1/10 эт.", 24.0), - ("78м2 на этаже", 78.0), - ("площадь 101.7 м²", 101.7), - ("Нет площади", None), - ], -) -def test_extract_area(text: str, expected: float | None) -> None: - assert _extract_area(text) == expected - - -# ── Floor ───────────────────────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - "text, floor, total", - [ - ("2-комн. квартира 53,5 м² 5/15 эт.", 5, 15), - ("студия 24 м² 1/10 эт.", 1, 10), - ("12/24 эт.", 12, 24), - ("нет этажа", None, None), - ], -) -def test_extract_floor(text: str, floor: int | None, total: int | None) -> None: - assert _extract_floor(text) == (floor, total) - - -# ── Price (per-element) ─────────────────────────────────────────────────────── -# Цена парсится из отдельного DOM-элемента (не из суммарного card_text), -# поэтому числа адреса не загрязняют парсинг цены. - - -@pytest.mark.parametrize( - "text, expected", - [ - ("5 200 000 ₽", 5_200_000), - ("12 345 678 ₽", 12_345_678), - ("3 990 000 руб.", 3_990_000), - ("от 4 500 000 ₽", 4_500_000), - ("без цены", None), - ("0 ₽", None), - ], -) -def test_extract_price_from_element(text: str, expected: int | None) -> None: - assert _extract_price_from_element(text) == expected - - -# ── source_id from href ─────────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - "href, expected", - [ - ("/card/sale__flat__2075671636", "2075671636"), - ("https://domclick.ru/card/sale__flat__123456789", "123456789"), - ("/card/sale__flat__1", "1"), - ("/card/something_else", None), + ("2026-06-01", date(2026, 6, 1)), + ("2026-06-01T12:00:00Z", date(2026, 6, 1)), + (1748736000, date(2025, 6, 1)), # 2025-06-01 00:00 UTC epoch + ("1748736000", date(2025, 6, 1)), + (None, None), + ("invalid-date", None), ("", None), ], ) -def test_extract_source_id(href: str, expected: str | None) -> None: - assert _extract_source_id(href) == expected +def test_parse_publish_date(val: object, expected: date | None) -> None: + result = _parse_publish_date(val) + assert result == expected -# ── _parse_html -> list[ScrapedLot] ────────────────────────────────────────── -# Цена в каждой карточке вынесена в отдельный с символом рубля, -# чтобы не сливаться с адресным номером дома при конкатенации card_text. - -_MOCK_HTML = """\ - - - - - - 2-комн. квартира 53,5 м² 5/15 эт. - ул. Малышева, 1 - 5 200 000 ₽ - - - - - Студия 28 м² 3/10 эт. - пр-т Ленина, 50 - 3 100 000 ₽ - - - - - 1-комн. квартира 32 м² 2/9 эт. - - - - - что-то другое - - - - - 2-комн. квартира 53,5 м² 5/15 эт. - 5 200 000 ₽ - - - -""" +# ── Scraper meta ───────────────────────────────────────────────────────────── -def test_parse_html_returns_lots() -> None: - scraper = DomClickScraper() - lots = scraper._parse_html(_MOCK_HTML) - - # Карточки 1 и 2 парсятся; карточка 3 (без цены) отфильтрована; дубль — дедуплицирован - assert len(lots) == 2 - - -def test_parse_html_card1_fields() -> None: - scraper = DomClickScraper() - lots = scraper._parse_html(_MOCK_HTML) - - card1 = next(lot for lot in lots if lot.source_id == "2075671636") - assert card1.source == "domklik" - assert card1.source_url == "https://domclick.ru/card/sale__flat__2075671636" - assert card1.rooms == 2 - assert card1.area_m2 == 53.5 - assert card1.floor == 5 - assert card1.total_floors == 15 - assert card1.price_rub == 5_200_000 - assert card1.lat is None - assert card1.lon is None - assert card1.listing_segment == "vtorichka" - - -def test_parse_html_studio() -> None: - scraper = DomClickScraper() - lots = scraper._parse_html(_MOCK_HTML) - - studio = next(lot for lot in lots if lot.source_id == "1111111111") - assert studio.rooms == 0 # студия - assert studio.area_m2 == 28.0 - assert studio.price_rub == 3_100_000 - - -def test_parse_html_no_price_filtered() -> None: - scraper = DomClickScraper() - lots = scraper._parse_html(_MOCK_HTML) - - source_ids = {lot.source_id for lot in lots} - assert "9999999999" not in source_ids - - -def test_parse_html_dedup_same_href() -> None: - """Дубликат href на одной странице должен давать одну карточку.""" - scraper = DomClickScraper() - lots = scraper._parse_html(_MOCK_HTML) - - ids_2075 = [lot for lot in lots if lot.source_id == "2075671636"] - assert len(ids_2075) == 1 - - -def test_scraper_source_name() -> None: - scraper = DomClickScraper() +def test_scraper_attributes() -> None: + scraper = _make_scraper() assert scraper.name == "domklik" assert scraper.source == "domklik" - assert scraper.base_url == "https://domclick.ru" - # request_delay_sec установлен из get_scraper_delay (DB или fallback 8.0) + assert scraper.base_url == "https://bff-search-web.domclick.ru" assert scraper.request_delay_sec >= 0.0 - - -def test_fetch_around_raises() -> None: - scraper = DomClickScraper() - with pytest.raises(NotImplementedError, match="geo-radius"): - asyncio.run(scraper.fetch_around(56.8, 60.6)) - - -def test_build_url_no_rooms() -> None: - scraper = DomClickScraper() - url = scraper._build_url(city_id=4, rooms=None, page=1) - assert "city_id=4" in url - assert "rooms" not in url - assert "p=1" in url - - -def test_build_url_with_rooms() -> None: - scraper = DomClickScraper() - url = scraper._build_url(city_id=4, rooms=2, page=3) - assert "rooms=2" in url - assert "p=3" in url - - -def test_parse_failures_counter() -> None: - """parse_failures счётчик начинается с 0.""" - scraper = DomClickScraper() assert scraper.parse_failures == 0 -def test_blocked_page_raises_domclick_blocked_error() -> None: - """HTML с datadome в начале вызывает DomClickBlockedError.""" - scraper = DomClickScraper() - blocked_html = ( - "datadome protectionblocked" - ) - with pytest.raises(DomClickBlockedError): - scraper._parse_html(blocked_html) +def test_fetch_around_raises() -> None: + import asyncio + + scraper = _make_scraper() + with pytest.raises(NotImplementedError, match="geo-radius"): + asyncio.run(scraper.fetch_around(56.8, 60.6)) diff --git a/tradein-mvp/backend/tests/test_domclick_sweep.py b/tradein-mvp/backend/tests/test_domclick_sweep.py index e6f9200c..fac43463 100644 --- a/tradein-mvp/backend/tests/test_domclick_sweep.py +++ b/tradein-mvp/backend/tests/test_domclick_sweep.py @@ -1,14 +1,15 @@ -"""Offline unit tests for run_domclick_city_sweep + DomClick parser. +"""Offline unit tests for run_domclick_city_sweep orchestration (JSON API rewrite). -Pure & fast: NO live network, NO DB. Mirrors test_cian_city_sweep orchestration -semantics, but DomClick is CITYWIDE (no anchor-loop) и SERP-only (no detail/houses). +Pure & fast: NO live network, NO DB. Coverage: - - fetch_city вызывается, save_listings получает накопленные lots с source='domklik' - - DomClickCitySweepCounters корректно заполняется (lots_fetched/inserted/updated) + - fetch_city вызывается, save_listings получает lots с source='domklik' + - DomClickCitySweepCounters корректно заполняется - Cooperative cancel прерывает sweep перед SERP - - fetch_city exception не валит sweep (graceful, errors_count++, mark_done) - - _parse_html: минимальная DomClick карточка → ScrapedLot с rooms/area/floor/price + - fetch_city exception → errors_count++; 0 lots + errors → mark_failed (#1968) + - Успешный sweep (lots > 0) → mark_done + - Пустой SERP (0 lots, 0 errors) → mark_done (не ошибка) + - DomClickBlockedError в фазе → errors_count++; 0 lots → mark_failed """ from __future__ import annotations @@ -25,6 +26,7 @@ import pytest from app.services import scrape_pipeline from app.services.scrapers.base import ScrapedLot from app.services.scrapers.domclick import DomClickScraper +from app.services.scrapers.domclick_exceptions import DomClickBlockedError def _fake_lot(offer_id: str) -> ScrapedLot: @@ -36,8 +38,8 @@ def _fake_lot(offer_id: str) -> ScrapedLot: price_rub=5_000_000, area_m2=50.0, rooms=2, - lat=None, - lon=None, + lat=56.83, + lon=60.61, ) @@ -78,6 +80,7 @@ def _stub_domclick_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None: def _init(self: DomClickScraper) -> None: self.request_delay_sec = 0.0 self.parse_failures = 0 + self._browser = None async def _aenter(self: DomClickScraper) -> DomClickScraper: return self @@ -125,8 +128,8 @@ def test_domclick_sweep_counters_to_dict_all_keys() -> None: # ── Full sweep — фазы и агрегация ────────────────────────────────────────── -async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPatch) -> None: - """fetch_city → save_listings получает 2 lots с source='domklik', counters заполнены.""" +async def test_sweep_fetches_and_saves_calls_mark_done(monkeypatch: pytest.MonkeyPatch) -> None: + """fetch_city → 2 lots → save_listings → mark_done (не mark_failed).""" fetch_calls: list[tuple[int, int]] = [] save_calls: list[list[ScrapedLot]] = [] @@ -134,7 +137,7 @@ async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPat self: DomClickScraper, city_id: int, rooms: list[int] | None = None, - pages: int = 20, + pages: int = 100, ) -> list[ScrapedLot]: fetch_calls.append((city_id, pages)) return [_fake_lot("100"), _fake_lot("101")] @@ -167,20 +170,21 @@ async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPat assert counters.lots_inserted == 2 assert counters.lots_updated == 0 assert counters.errors_count == 0 + # Успешный sweep → mark_done, НЕ mark_failed assert fake.done is not None assert fake.done["lots_fetched"] == 2 assert fake.failed is None -async def test_sweep_empty_serp_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None: - """Пустой SERP → save не вызывается, но mark_done всё равно вызывается.""" +async def test_sweep_empty_serp_no_errors_mark_done(monkeypatch: pytest.MonkeyPatch) -> None: + """Пустой SERP (0 lots, 0 errors) → mark_done (не ошибка, не mark_failed).""" save_called = False async def fake_fetch_empty( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, - pages: int = 20, + pages: int = 100, ) -> list[ScrapedLot]: return [] @@ -201,7 +205,10 @@ async def test_sweep_empty_serp_still_mark_done(monkeypatch: pytest.MonkeyPatch) assert save_called is False assert counters.lots_fetched == 0 + assert counters.errors_count == 0 + # 0 lots + 0 errors → mark_done (не mark_failed) assert fake.done is not None + assert fake.failed is None async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -> None: @@ -212,7 +219,7 @@ async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) - self: DomClickScraper, city_id: int, rooms: list[int] | None = None, - pages: int = 20, + pages: int = 100, ) -> list[ScrapedLot]: nonlocal fetch_called fetch_called = True @@ -233,16 +240,18 @@ async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) - assert fake.done is None -async def test_fetch_city_exception_graceful(monkeypatch: pytest.MonkeyPatch) -> None: - """Исключение в fetch_city → errors_count++, sweep завершается mark_done (не fatal).""" +async def test_fetch_city_exception_triggers_mark_failed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Исключение в fetch_city → errors_count++; 0 lots + errors → mark_failed (#1968).""" async def fake_fetch_fail( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, - pages: int = 20, + pages: int = 100, ) -> list[ScrapedLot]: - raise RuntimeError("camoufox crashed") + raise RuntimeError("network error") monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_fail) monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0)) @@ -256,36 +265,70 @@ async def test_fetch_city_exception_graceful(monkeypatch: pytest.MonkeyPatch) -> assert counters.errors_count == 1 assert counters.lots_fetched == 0 - assert fake.done is not None, "graceful: mark_done вызывается несмотря на ошибку SERP" + # 0 lots + errors → mark_failed, НЕ mark_done + assert fake.failed is not None, "mark_failed должен быть вызван при 0 lots + errors" + assert fake.done is None + + +async def test_domclick_blocked_triggers_mark_failed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """DomClickBlockedError в fetch_city → errors_count++; 0 lots → mark_failed.""" + + async def fake_fetch_blocked( + self: DomClickScraper, + city_id: int, + rooms: list[int] | None = None, + pages: int = 100, + ) -> list[ScrapedLot]: + raise DomClickBlockedError("QRATOR blocked") + + monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_blocked) + monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0)) + + fake = _FakeRuns() + _install_runs(monkeypatch, fake) + + counters = await scrape_pipeline.run_domclick_city_sweep( + db=MagicMock(), run_id=5, request_delay_sec=0.0 + ) + + assert counters.errors_count == 1 + assert counters.lots_fetched == 0 + assert fake.failed is not None + assert fake.done is None + + +async def test_lots_but_errors_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None: + """Если лоты собраны (>0), но были partial errors → всё равно mark_done.""" + # Этот случай не тривиален — errors могут прийти при save_listings, но + # lots_fetched > 0. В нашей логике: lots_fetched>0 → mark_done. + + async def fake_fetch_partial( + self: DomClickScraper, + city_id: int, + rooms: list[int] | None = None, + pages: int = 100, + ) -> list[ScrapedLot]: + return [_fake_lot("200")] + + fake_save_calls = [0] + + def fake_save_ok(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]: + fake_save_calls[0] += 1 + return 1, 0 + + monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_partial) + monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_ok) + + fake = _FakeRuns() + _install_runs(monkeypatch, fake) + + counters = await scrape_pipeline.run_domclick_city_sweep( + db=MagicMock(), run_id=6, request_delay_sec=0.0 + ) + + assert counters.lots_fetched == 1 + # Успешно → mark_done + assert fake.done is not None assert fake.failed is None - - -# ── _parse_html parser unit test ──────────────────────────────────────────── - - -def test_parse_html_minimal_card() -> None: - """Минимальная DomClick карточка → ScrapedLot с rooms/area/floor/price/source_id.""" - # __init__ застаблен фикстурой _stub_domclick_lifecycle → нет DB/browser - scraper = DomClickScraper() - html = """ - - -
2-комн. квартира
- 52,3 м² - 5 / 16 эт. - Екатеринбург, улица Ленина, 10 - 6 500 000 ₽ - - - """ - lots = scraper._parse_html(html) - assert len(lots) == 1 - lot = lots[0] - assert lot.source == "domklik" - assert lot.source_id == "2075671636" - assert lot.rooms == 2 - assert lot.area_m2 == pytest.approx(52.3) - assert lot.floor == 5 - assert lot.total_floors == 16 - assert lot.price_rub == 6_500_000 - assert lot.lat is None and lot.lon is None