feat(tradein/scrapers): domclick вторичка через bff-search-web JSON API (Layer A)
- DomClickScraper переписан на GET bff-search-web.domclick.ru/api/offers/v1 с рекурсивным бинарным price-band bucketing (OFFSET_CAP=2000, open-band split через _HIGH_ANCHOR=30M); geo-guard по offerRegionName==ЕКБ / bbox - scrape_pipeline.py: rooms=[0..5] (добавлен 5+), pages default=100, _DOMCLICK_PER_FETCH_S=4.0, честный статус mark_failed при 0 лотов + errors - config.py: bff-search-web.domclick.ru добавлен в scrape_allowed_hosts - data/sql/132_update_domclick_sweep_params.sql: UPDATE default_params (DORMANT)
This commit is contained in:
parent
86ddf99bb6
commit
dbef242716
6 changed files with 1105 additions and 570 deletions
|
|
@ -322,6 +322,7 @@ class Settings(BaseSettings):
|
|||
"n1.ru",
|
||||
"domclick.ru",
|
||||
"www.domclick.ru",
|
||||
"bff-search-web.domclick.ru", # JSON API endpoint (Layer A rewrite)
|
||||
}
|
||||
|
||||
# ── Scraper mobile proxy (#806) ──────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -3105,10 +3105,11 @@ class DomClickCitySweepCounters:
|
|||
|
||||
# Дефолтные параметры sweep'а (EKB city_id=4, все комнатности).
|
||||
DOMCLICK_DEFAULT_CITY_ID: int = 4
|
||||
DOMCLICK_DEFAULT_ROOMS: list[int] = [0, 1, 2, 3, 4]
|
||||
# Оценка времени одного fetch'а (network + camoufox render + parse) для watchdog.
|
||||
_DOMCLICK_PER_FETCH_S: float = 12.0
|
||||
# Буфер сверху расчётного бюджета (cold browser start, save-фаза).
|
||||
# rooms=5 добавлен (5+ квартиры); JSON API поддерживает токен "5+".
|
||||
DOMCLICK_DEFAULT_ROOMS: list[int] = [0, 1, 2, 3, 4, 5]
|
||||
# Оценка времени одного JSON-fetch'а (network + parse, без camoufox render).
|
||||
_DOMCLICK_PER_FETCH_S: float = 4.0
|
||||
# Буфер сверху расчётного бюджета (cold start, save-фаза).
|
||||
_DOMCLICK_SWEEP_BUDGET_S: float = 300.0
|
||||
|
||||
|
||||
|
|
@ -3118,40 +3119,40 @@ async def run_domclick_city_sweep(
|
|||
run_id: int,
|
||||
city_id: int = DOMCLICK_DEFAULT_CITY_ID,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 5,
|
||||
pages: int = 100,
|
||||
request_delay_sec: float | None = None,
|
||||
) -> DomClickCitySweepCounters:
|
||||
"""DomClick citywide sweep: SERP по city_id × rooms × pages → save → house-match.
|
||||
"""DomClick citywide sweep: fetch_city (JSON API) → save → house-match.
|
||||
|
||||
Структурно зеркалит run_cian_city_sweep / run_yandex_city_sweep, но CITYWIDE:
|
||||
DomClick SERP не поддерживает geo-radius (fetch_around → NotImplementedError),
|
||||
поэтому anchor-loop отсутствует. DomClickScraper.fetch_city уже перебирает
|
||||
rooms × pages через внутренний BrowserFetcher (camoufox headless) и
|
||||
дедуплицирует по source_id.
|
||||
DomClick не поддерживает geo-radius (fetch_around → NotImplementedError),
|
||||
anchor-loop отсутствует. DomClickScraper.fetch_city перебирает rooms × price-bands
|
||||
через bff-search-web JSON API с recursive offset<2000 bucketing и дедуплицирует
|
||||
по source_id. Координаты присутствуют в JSON (100% coverage) → geom заполнена.
|
||||
|
||||
Фазы (единственный citywide проход):
|
||||
1. SERP: DomClickScraper().fetch_city(city_id, rooms, pages).
|
||||
2. SAVE: save_listings(db, lots, run_id=run_id) — BaseScraper save path,
|
||||
триггерит существующий house-match hook (адрес-fingerprint tier, т.к.
|
||||
DomClick SERP не отдаёт lat/lon).
|
||||
триггерит house-match hook (адрес + coords).
|
||||
|
||||
Watchdog: весь fetch_city оборачивается в asyncio.wait_for. Таймаут считается
|
||||
из rooms × pages × per-fetch + buffer (минимум ANCHOR_TIMEOUT_SEC).
|
||||
|
||||
Cooperative cancel: is_cancelled(db, run_id) проверяется перед SERP-фазой.
|
||||
mark_done вызывается ВСЕГДА (кроме cancel / fatal). lat=lon=None у всех lots →
|
||||
house-match использует только address_fingerprint tier.
|
||||
Honest-status: если lots_fetched==0 И errors_count>0 → mark_failed (blocked /
|
||||
fetch error). Иначе → mark_done. Кроме cancel / fatal.
|
||||
|
||||
Возвращает DomClickCitySweepCounters.
|
||||
"""
|
||||
from app.services.scrapers.domclick import DomClickScraper
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||||
|
||||
_rooms = rooms if rooms is not None else list(DOMCLICK_DEFAULT_ROOMS)
|
||||
_resolved_delay = request_delay_sec if request_delay_sec is not None else 8.0
|
||||
counters = DomClickCitySweepCounters()
|
||||
|
||||
# Watchdog-таймаут: число fetch'ей = len(rooms) × pages (worst-case, без
|
||||
# early-break на пустой странице). Каждый fetch ≈ delay + render/parse overhead.
|
||||
# Watchdog-таймаут: worst-case = rooms × pages × (delay + per-fetch) + buffer.
|
||||
# JSON endpoint быстрее camoufox render: _DOMCLICK_PER_FETCH_S снижен до 4.0s.
|
||||
_num_fetches = max(1, len(_rooms)) * max(1, pages)
|
||||
_sweep_timeout = max(
|
||||
ANCHOR_TIMEOUT_SEC,
|
||||
|
|
@ -3177,7 +3178,7 @@ async def run_domclick_city_sweep(
|
|||
lots: list[ScrapedLot] = []
|
||||
|
||||
async def _domclick_phase() -> None:
|
||||
"""Единственная citywide-фаза: fetch_city + save."""
|
||||
"""Единственная citywide-фаза: fetch_city (JSON API) + save."""
|
||||
nonlocal lots
|
||||
async with DomClickScraper() as scraper:
|
||||
if request_delay_sec is not None:
|
||||
|
|
@ -3198,6 +3199,9 @@ async def run_domclick_city_sweep(
|
|||
_sweep_timeout,
|
||||
)
|
||||
counters.errors_count += 1
|
||||
except DomClickBlockedError as e:
|
||||
logger.error("domclick-sweep run_id=%d: BLOCKED — %s", run_id, e)
|
||||
counters.errors_count += 1
|
||||
except Exception:
|
||||
logger.exception("domclick-sweep run_id=%d: SERP phase failed", run_id)
|
||||
counters.errors_count += 1
|
||||
|
|
@ -3205,7 +3209,15 @@ async def run_domclick_city_sweep(
|
|||
# pages_fetched: worst-case число запрошенных страниц (rooms × pages).
|
||||
counters.pages_fetched = _num_fetches
|
||||
scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
|
||||
scrape_runs.mark_done(db, run_id, counters.to_dict())
|
||||
|
||||
# Honest-status: 0 лотов + ошибки → mark_failed (#1968).
|
||||
if counters.lots_fetched == 0 and counters.errors_count > 0:
|
||||
_reason = "domclick: 0 listings collected (blocked or fetch error)"
|
||||
logger.warning("domclick-sweep run_id=%d: %s", run_id, _reason)
|
||||
scrape_runs.mark_failed(db, run_id, _reason, counters.to_dict())
|
||||
else:
|
||||
scrape_runs.mark_done(db, run_id, counters.to_dict())
|
||||
|
||||
logger.info(
|
||||
"domclick-sweep run_id=%d done: lots=%d (ins=%d/upd=%d) pages=%d errors=%d",
|
||||
run_id,
|
||||
|
|
|
|||
|
|
@ -1,146 +1,140 @@
|
|||
"""DomClick.ru scraper — вторичка через headless BrowserFetcher (#796).
|
||||
"""DomClick.ru scraper — вторичка через bff-search-web JSON API (Layer A).
|
||||
|
||||
Стратегия: HTML scrape через camoufox (AsyncCamoufox) — единственный
|
||||
рабочий путь: DataDome пропускает headless Playwright, curl_cffi -> 401.
|
||||
Стратегия (JSON API rewrite, replaces dead HTML/camoufox scrape):
|
||||
GET https://bff-search-web.domclick.ru/api/offers/v1?... → list (up to 20 items)
|
||||
GET https://bff-search-web.domclick.ru/api/offers/count/v1?... → snippetsCount
|
||||
|
||||
URL шаблон:
|
||||
https://domclick.ru/search?deal_type=sale&category=living&offer_type=flat
|
||||
&city_id={city_id}&rooms={r}&p={page}
|
||||
Транспорт: BrowserFetcher(source="domclick") — существующий браузерный пул
|
||||
(tradein-browser), роутит bff-search-web.domclick.ru → generic provider →
|
||||
shared RU mobile proxy. fetch() возвращает raw JSON body как строку.
|
||||
|
||||
Оффер-карточки: `a[href*="/card/"]` с href-паттерном `/card/sale__flat__<id>`.
|
||||
Координаты SERP не отдаёт (lat = lon = None).
|
||||
Bucketing:
|
||||
OFFSET_CAP=2000 (max 100 страниц × 20) — hardlimit DomClick API.
|
||||
Когда snippetsCount > 2000 — рекурсивная бинарная partition по цене.
|
||||
EKB вторичка ≈6400 лотов; только rooms=2 превышает cap (≈2215).
|
||||
|
||||
Важно: selectolax `.text()` объединяет все дочерние текстовые узлы без
|
||||
разделителей. Числа из адреса (номер дома) могут слипнуться с ценой.
|
||||
Поэтому цену извлекаем из каждого дочернего элемента отдельно (а не из
|
||||
суммарного card_text), где элемент содержит символ рубля.
|
||||
Layer B (detail-backfill) и Layer C (analytics XHR) — не реализованы.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urljoin
|
||||
from datetime import UTC, date, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode, urljoin
|
||||
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
import sentry_sdk
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── DataDome block detection ─────────────────────────────────────────────────
|
||||
_DATADOME_MARKERS = ("datadome", "blocked", "access denied", "bot detected")
|
||||
# ── Константы bucketing ──────────────────────────────────────────────────────
|
||||
|
||||
# Верхняя граница первого деления открытого (hi=None) бакета (30 млн ₽)
|
||||
_HIGH_ANCHOR: int = 30_000_000
|
||||
# Минимальная ширина ценового диапазона — ниже этого не бисектируем
|
||||
_MIN_BAND: int = 100_000
|
||||
# Максимальная глубина рекурсии
|
||||
_MAX_DEPTH: int = 12
|
||||
|
||||
# ── Маппинг комнатности int → API token ─────────────────────────────────────
|
||||
|
||||
_ROOMS_TOKEN: dict[int, str] = {
|
||||
0: "st",
|
||||
1: "1",
|
||||
2: "2",
|
||||
3: "3",
|
||||
4: "4",
|
||||
5: "5+",
|
||||
}
|
||||
|
||||
# ── Гео-guard: ЕКБ bbox ─────────────────────────────────────────────────────
|
||||
|
||||
_EKB_LAT_MIN: float = 56.7
|
||||
_EKB_LAT_MAX: float = 56.95
|
||||
_EKB_LON_MIN: float = 60.4
|
||||
_EKB_LON_MAX: float = 60.8
|
||||
|
||||
|
||||
def _is_blocked_page(html: str) -> bool:
|
||||
head = html[:2048].lower()
|
||||
return any(m in head for m in _DATADOME_MARKERS)
|
||||
def _is_ekb(offer: dict[str, Any]) -> bool:
|
||||
"""Гео-guard: офер принадлежит ЕКБ по offerRegionName или bbox координат."""
|
||||
if offer.get("offerRegionName") == "Екатеринбург":
|
||||
return True
|
||||
loc: dict[str, Any] = offer.get("location") or {}
|
||||
lat_raw = loc.get("lat")
|
||||
lon_raw = loc.get("lon")
|
||||
if lat_raw is not None and lon_raw is not None:
|
||||
try:
|
||||
lat = float(lat_raw)
|
||||
lon = float(lon_raw)
|
||||
return _EKB_LAT_MIN <= lat <= _EKB_LAT_MAX and _EKB_LON_MIN <= lon <= _EKB_LON_MAX
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
# ── Regex helpers ────────────────────────────────────────────────────────────
|
||||
# _RE_ROOMS расширен относительно avito.py: DomClick пишет «2-комн. квартира»
|
||||
# (с «комн.») — avito.py-вариант ловит только «N-к. кв.» / «N-к квартира».
|
||||
_RE_ROOMS = re.compile(
|
||||
r"(\d)-(?:комн\.?\s*(?:квартира|кв\.?)?|к\.?\s*(?:квартира|кв\.?))",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_RE_STUDIO = re.compile(r"\bстуди[яиюей]\b", re.IGNORECASE)
|
||||
_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м[2²]", re.IGNORECASE)
|
||||
_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт\.?", re.IGNORECASE)
|
||||
|
||||
# Цена: парсим из одного элемента DOM (не из суммарного card_text) —
|
||||
# иначе число дома может слипнуться с ценой. Цена всегда в элементе с «₽».
|
||||
_RE_PRICE_EL = re.compile(r"([\d\s ]+)\s*[₽р](?:уб\.?)?", re.IGNORECASE)
|
||||
|
||||
# source_id из href вида /card/sale__flat__2075671636
|
||||
_RE_SOURCE_ID = re.compile(r"sale__flat__(\d+)")
|
||||
|
||||
# Адресные ключевые слова (эвристика)
|
||||
_RE_ADDR_KW = re.compile(
|
||||
r"ул\.|улица|пер\.|переулок|пр-т|проспект"
|
||||
r"|бул\.|бульвар|шоссе|наб\.|набережная"
|
||||
r"|пл\.|площадь|тракт|д\.\s*\d",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
# ── Вспомогательные функции ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _extract_rooms(text: str) -> int | None:
|
||||
if _RE_STUDIO.search(text):
|
||||
return 0
|
||||
m = _RE_ROOMS.search(text)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _extract_area(text: str) -> float | None:
|
||||
m = _RE_AREA.search(text)
|
||||
return float(m.group(1).replace(",", ".")) if m else None
|
||||
|
||||
|
||||
def _extract_floor(text: str) -> tuple[int | None, int | None]:
|
||||
m = _RE_FLOOR.search(text)
|
||||
if m:
|
||||
return int(m.group(1)), int(m.group(2))
|
||||
return None, None
|
||||
|
||||
|
||||
def _extract_price_from_element(text: str) -> int | None:
|
||||
"""Извлекаем цену из текста ОДНОГО элемента DOM.
|
||||
|
||||
Парсим из одного span/div — не из суммарного card_text. Это исключает
|
||||
случай, когда адресный номер дома стоит вплотную перед ценой в
|
||||
объединённом тексте карточки («Ленина, 503 100 000 руб.»).
|
||||
"""
|
||||
m = _RE_PRICE_EL.search(text)
|
||||
if m:
|
||||
# Нормализуем пробелы любого вида (обычный, неразрывный, узкий)
|
||||
raw = re.sub(r"[\s ]", "", m.group(1))
|
||||
if raw.isdigit():
|
||||
val = int(raw)
|
||||
if val > 0:
|
||||
return val
|
||||
return None
|
||||
|
||||
|
||||
def _extract_source_id(href: str) -> str | None:
|
||||
"""Числовой ID из href типа /card/sale__flat__2075671636."""
|
||||
m = _RE_SOURCE_ID.search(href)
|
||||
return m.group(1) if m else None
|
||||
def _parse_publish_date(val: Any) -> date | None:
|
||||
"""Парсим publishedDate: epoch int/str или ISO8601 строка. None при ошибке."""
|
||||
if val is None:
|
||||
return None
|
||||
try:
|
||||
if isinstance(val, int | float):
|
||||
return datetime.fromtimestamp(int(val), tz=UTC).date()
|
||||
s = str(val).strip()
|
||||
if s.isdigit():
|
||||
return datetime.fromtimestamp(int(s), tz=UTC).date()
|
||||
# ISO8601 — берём первые 10 символов (YYYY-MM-DD)
|
||||
return date.fromisoformat(s[:10])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
# ── DomClickScraper ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class DomClickScraper(BaseScraper):
|
||||
"""DomClick вторичка parser. Источник = 'domklik'.
|
||||
"""DomClick вторичка через bff-search-web JSON API. Источник = 'domklik'.
|
||||
|
||||
Использует BrowserFetcher (camoufox headless Firefox) — DataDome пропускает
|
||||
его без дополнительной stealth-настройки. curl_cffi -> 401 (DataDome блок).
|
||||
|
||||
Основной метод: fetch_city(city_id, rooms, pages).
|
||||
fetch_around() не реализован: DomClick не поддерживает geo-radius в URL.
|
||||
Layer A: SERP list (offers/v1) → ScrapedLot. Layer B/C не реализованы.
|
||||
Транспорт: одна BrowserFetcher(source="domclick") сессия в __aenter__.
|
||||
fetch_around() → NotImplementedError (DomClick не поддерживает geo-radius).
|
||||
"""
|
||||
|
||||
name = "domklik"
|
||||
source = "domklik"
|
||||
base_url = "https://domclick.ru"
|
||||
# DataDome — консервативная задержка между страницами (fallback до init)
|
||||
request_delay_sec = 8.0
|
||||
base_url = "https://bff-search-web.domclick.ru"
|
||||
request_delay_sec = 8.0 # overwritten from get_scraper_delay in __init__
|
||||
|
||||
EKB_REGION_GUID = "0d475b79-88de-4054-818c-37d8f9d0d440"
|
||||
EKB_AIDS = 20561
|
||||
OFFSET_CAP = 2000
|
||||
LIMIT = 20
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
# Счётчик карточек с неожиданной DOM-структурой (для observability)
|
||||
self.parse_failures: int = 0
|
||||
self._browser: Any | None = None
|
||||
|
||||
async def __aenter__(self) -> DomClickScraper:
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
|
||||
await super().__aenter__()
|
||||
self._browser = BrowserFetcher(source="domclick")
|
||||
await self._browser.__aenter__()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: Any) -> None:
|
||||
if self._browser is not None:
|
||||
await self._browser.__aexit__(*args)
|
||||
self._browser = None
|
||||
await super().__aexit__(*args)
|
||||
|
||||
# ── fetch_around — stub ───────────────────────────────────────────────────
|
||||
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list[ScrapedLot]:
|
||||
|
|
@ -155,236 +149,502 @@ class DomClickScraper(BaseScraper):
|
|||
self,
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 20,
|
||||
pages: int = 100,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Citywide sweep: все страницы SERP для city_id.
|
||||
"""Citywide sweep: rooms × recursive price-bands → дедуплицированный список.
|
||||
|
||||
Args:
|
||||
city_id: числовой ID города в DomClick (например, 4 = Екатеринбург).
|
||||
rooms: список значений комнатности (0=студия, 1, 2, 3, 4, ...).
|
||||
None -> без фильтра комнатности (все квартиры сразу).
|
||||
pages: максимальное число страниц на один sweep. Break on empty.
|
||||
city_id: числовой ID города (4 = ЕКБ). Только ЕКБ поддерживается;
|
||||
для других значений логируем warning, продолжаем с EKB GUID.
|
||||
rooms: список комнатностей (0=студия, 1, 2, 3, 4, 5=5+).
|
||||
None → [0,1,2,3,4,5] (все).
|
||||
pages: максимум страниц на leaf-bucket (default 100 = OFFSET_CAP/LIMIT).
|
||||
|
||||
Returns:
|
||||
Дедуплицированный по source_id список ScrapedLot.
|
||||
"""
|
||||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||||
if city_id != 4:
|
||||
logger.warning(
|
||||
"domklik: city_id=%d не поддерживается — используем EKB GUID/aids",
|
||||
city_id,
|
||||
)
|
||||
|
||||
all_lots: list[ScrapedLot] = []
|
||||
seen_ids: set[str] = set()
|
||||
_rooms = rooms if rooms is not None else list(_ROOMS_TOKEN.keys())
|
||||
seen: dict[str, ScrapedLot] = {}
|
||||
total_geo_dropped: int = 0
|
||||
|
||||
# Формируем список room-значений для sweep'ов
|
||||
room_values: list[int | None] = [None] if not rooms else list(rooms)
|
||||
|
||||
async with BrowserFetcher(source="domclick") as fetcher:
|
||||
for room_val in room_values:
|
||||
room_label = f"rooms={room_val}" if room_val is not None else "all_rooms"
|
||||
logger.info(
|
||||
"domklik: city_id=%d %s sweep (max %d pages)", city_id, room_label, pages
|
||||
)
|
||||
|
||||
for page_num in range(1, pages + 1):
|
||||
url = self._build_url(city_id, room_val, page_num)
|
||||
logger.debug("domklik: fetch %s", url)
|
||||
|
||||
try:
|
||||
html = await fetcher.fetch(url)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"domklik: fetch failed city_id=%d page=%d: %s",
|
||||
city_id,
|
||||
page_num,
|
||||
exc,
|
||||
)
|
||||
break
|
||||
|
||||
try:
|
||||
lots = self._parse_html(html)
|
||||
except DomClickBlockedError:
|
||||
logger.warning(
|
||||
"domklik: DataDome block at city_id=%d page=%d — stopping sweep",
|
||||
city_id,
|
||||
page_num,
|
||||
)
|
||||
break
|
||||
|
||||
if not lots:
|
||||
logger.info(
|
||||
"domklik: empty page city_id=%d %s page=%d — stopping sweep",
|
||||
city_id,
|
||||
room_label,
|
||||
page_num,
|
||||
)
|
||||
break
|
||||
|
||||
# Дедупликация по source_id (кросс-sweep и кросс-страница)
|
||||
new_lots: list[ScrapedLot] = []
|
||||
for lot in lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key not in seen_ids:
|
||||
seen_ids.add(key)
|
||||
new_lots.append(lot)
|
||||
|
||||
all_lots.extend(new_lots)
|
||||
logger.info(
|
||||
"domklik: city_id=%d %s page=%d -> %d new (total %d)",
|
||||
city_id,
|
||||
room_label,
|
||||
page_num,
|
||||
len(new_lots),
|
||||
len(all_lots),
|
||||
)
|
||||
|
||||
# Пауза между страницами (anti-DataDome)
|
||||
if page_num < pages:
|
||||
await self.sleep_between_requests()
|
||||
for room_int in _rooms:
|
||||
room_token = _ROOMS_TOKEN.get(room_int, str(room_int))
|
||||
logger.info(
|
||||
"domklik: room=%s (%d) sweep start (max %d pages per leaf)",
|
||||
room_token,
|
||||
room_int,
|
||||
pages,
|
||||
)
|
||||
geo_dropped_ref = [0]
|
||||
await self._walk_price_range(
|
||||
room_token=room_token,
|
||||
lo=0,
|
||||
hi=None,
|
||||
seen=seen,
|
||||
pages=pages,
|
||||
geo_dropped_ref=geo_dropped_ref,
|
||||
)
|
||||
total_geo_dropped += geo_dropped_ref[0]
|
||||
logger.info(
|
||||
"domklik: room=%s done — unique_total=%d geo_dropped_this_room=%d",
|
||||
room_token,
|
||||
len(seen),
|
||||
geo_dropped_ref[0],
|
||||
)
|
||||
|
||||
if total_geo_dropped:
|
||||
logger.info("domklik: fetch_city geo_dropped_total=%d", total_geo_dropped)
|
||||
logger.info(
|
||||
"domklik: fetch_city done city_id=%d total=%d parse_failures=%d",
|
||||
city_id,
|
||||
len(all_lots),
|
||||
len(seen),
|
||||
self.parse_failures,
|
||||
)
|
||||
return all_lots
|
||||
return list(seen.values())
|
||||
|
||||
# ── URL builder ───────────────────────────────────────────────────────────
|
||||
# ── Рекурсивное адаптивное деление ───────────────────────────────────────
|
||||
|
||||
def _build_url(self, city_id: int, rooms: int | None, page: int) -> str:
|
||||
"""Строит URL SERP DomClick."""
|
||||
url = (
|
||||
f"{self.base_url}/search"
|
||||
f"?deal_type=sale&category=living&offer_type=flat"
|
||||
f"&city_id={city_id}"
|
||||
)
|
||||
if rooms is not None:
|
||||
url += f"&rooms={rooms}"
|
||||
url += f"&p={page}"
|
||||
return url
|
||||
async def _walk_price_range(
|
||||
self,
|
||||
*,
|
||||
room_token: str,
|
||||
lo: int,
|
||||
hi: int | None,
|
||||
seen: dict[str, ScrapedLot],
|
||||
pages: int,
|
||||
geo_dropped_ref: list[int],
|
||||
_depth: int = 0,
|
||||
) -> None:
|
||||
"""Рекурсивная бинарная partition ценового диапазона [lo, hi].
|
||||
|
||||
# ── HTML parsing ──────────────────────────────────────────────────────────
|
||||
Open band (hi is None):
|
||||
snippetsCount ≤ OFFSET_CAP → пагинируем leaf.
|
||||
snippetsCount > OFFSET_CAP → разбиваем через HIGH_ANCHOR:
|
||||
recurse [lo, HIGH_ANCHOR] + [HIGH_ANCHOR+1, None].
|
||||
|
||||
def _parse_html(self, html: str) -> list[ScrapedLot]:
|
||||
"""Парсим карточки из HTML через selectolax.
|
||||
Closed band (hi is int):
|
||||
snippetsCount ≤ OFFSET_CAP → пагинируем leaf.
|
||||
snippetsCount > OFFSET_CAP и (depth > MAX_DEPTH или band < MIN_BAND):
|
||||
пагинируем как есть (tail-loss WARNING).
|
||||
Иначе → mid=(lo+hi)//2, recurse [lo, mid] и [mid+1, hi].
|
||||
|
||||
Селектор: `a[href*="/card/"]` с паттерном `/card/sale__flat__<id>`.
|
||||
Raises DomClickBlockedError при QRATOR-блоке (propagate к вызывающему).
|
||||
"""
|
||||
if _is_blocked_page(html):
|
||||
logger.warning("domklik: DataDome block page detected, 0 cards returned")
|
||||
raise DomClickBlockedError("DomClick returned DataDome block page")
|
||||
tree = HTMLParser(html)
|
||||
card_links = tree.css('a[href*="/card/"]')
|
||||
hi_repr = "open" if hi is None else str(hi)
|
||||
|
||||
seen_hrefs: set[str] = set()
|
||||
lots: list[ScrapedLot] = []
|
||||
# ── 1. COUNT ─────────────────────────────────────────────────────────
|
||||
count_url = self._build_count_url(room_token, lo, hi)
|
||||
count_data = await self._fetch_json(count_url) # raises DomClickBlockedError on block
|
||||
snippets: int = (count_data.get("result") or {}).get("snippetsCount", 0)
|
||||
|
||||
for link in card_links:
|
||||
href = link.attributes.get("href", "")
|
||||
# Только карточки вторичного жилья
|
||||
if "sale__flat__" not in href:
|
||||
logger.info(
|
||||
"domklik: %s [%d, %s] snippetsCount=%d depth=%d",
|
||||
room_token,
|
||||
lo,
|
||||
hi_repr,
|
||||
snippets,
|
||||
_depth,
|
||||
)
|
||||
|
||||
if snippets == 0:
|
||||
return
|
||||
|
||||
await self.sleep_between_requests()
|
||||
|
||||
# ── 2a. Open band — нельзя бисектировать напрямую ────────────────────
|
||||
if hi is None:
|
||||
if snippets <= self.OFFSET_CAP:
|
||||
await self._paginate_leaf(
|
||||
room_token, lo, None, snippets, seen, pages, geo_dropped_ref
|
||||
)
|
||||
else:
|
||||
# Делим через HIGH_ANCHOR; хвост [HIGH_ANCHOR+1, None] обычно мал
|
||||
await self._walk_price_range(
|
||||
room_token=room_token,
|
||||
lo=lo,
|
||||
hi=_HIGH_ANCHOR,
|
||||
seen=seen,
|
||||
pages=pages,
|
||||
geo_dropped_ref=geo_dropped_ref,
|
||||
_depth=_depth + 1,
|
||||
)
|
||||
await self._walk_price_range(
|
||||
room_token=room_token,
|
||||
lo=_HIGH_ANCHOR + 1,
|
||||
hi=None,
|
||||
seen=seen,
|
||||
pages=pages,
|
||||
geo_dropped_ref=geo_dropped_ref,
|
||||
_depth=_depth + 1,
|
||||
)
|
||||
return
|
||||
|
||||
# ── 2b. Closed band ───────────────────────────────────────────────────
|
||||
if snippets <= self.OFFSET_CAP:
|
||||
await self._paginate_leaf(room_token, lo, hi, snippets, seen, pages, geo_dropped_ref)
|
||||
return
|
||||
|
||||
# Нужно деление — проверяем ограничения
|
||||
band = hi - lo
|
||||
if _depth >= _MAX_DEPTH or band < _MIN_BAND:
|
||||
tail_loss = max(0, snippets - self.OFFSET_CAP)
|
||||
logger.warning(
|
||||
"domklik: %s [%d, %d] snippets=%d > cap=%d band=%d depth=%d "
|
||||
"— paginating as-is (tail-loss ~%d)",
|
||||
room_token,
|
||||
lo,
|
||||
hi,
|
||||
snippets,
|
||||
self.OFFSET_CAP,
|
||||
band,
|
||||
_depth,
|
||||
tail_loss,
|
||||
)
|
||||
await self._paginate_leaf(room_token, lo, hi, snippets, seen, pages, geo_dropped_ref)
|
||||
return
|
||||
|
||||
mid = (lo + hi) // 2
|
||||
await self._walk_price_range(
|
||||
room_token=room_token,
|
||||
lo=lo,
|
||||
hi=mid,
|
||||
seen=seen,
|
||||
pages=pages,
|
||||
geo_dropped_ref=geo_dropped_ref,
|
||||
_depth=_depth + 1,
|
||||
)
|
||||
await self._walk_price_range(
|
||||
room_token=room_token,
|
||||
lo=mid + 1,
|
||||
hi=hi,
|
||||
seen=seen,
|
||||
pages=pages,
|
||||
geo_dropped_ref=geo_dropped_ref,
|
||||
_depth=_depth + 1,
|
||||
)
|
||||
|
||||
# ── Пагинация leaf-bucket ─────────────────────────────────────────────────
|
||||
|
||||
async def _paginate_leaf(
|
||||
self,
|
||||
room_token: str,
|
||||
lo: int,
|
||||
hi: int | None,
|
||||
snippets: int,
|
||||
seen: dict[str, ScrapedLot],
|
||||
pages: int,
|
||||
geo_dropped_ref: list[int],
|
||||
) -> None:
|
||||
"""Последовательная пагинация одного leaf-bucket [lo, hi] по offset."""
|
||||
max_offset = min(snippets, self.OFFSET_CAP, pages * self.LIMIT)
|
||||
offset = 0
|
||||
|
||||
while offset < max_offset:
|
||||
list_url = self._build_list_url(room_token, lo, hi, offset)
|
||||
list_data = await self._fetch_json(list_url) # DomClickBlockedError propagates
|
||||
items: list[dict[str, Any]] = (list_data.get("result") or {}).get("items") or []
|
||||
|
||||
if not items:
|
||||
logger.debug(
|
||||
"domklik: empty items at %s [%s, %s] offset=%d — stopping",
|
||||
room_token,
|
||||
lo,
|
||||
hi or "open",
|
||||
offset,
|
||||
)
|
||||
break
|
||||
|
||||
page_lots = self._parse_page(items, room_token, geo_dropped_ref)
|
||||
for lot in page_lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key:
|
||||
seen[key] = lot
|
||||
|
||||
logger.debug(
|
||||
"domklik: %s [%s, %s] offset=%d items=%d parsed=%d total_unique=%d",
|
||||
room_token,
|
||||
lo,
|
||||
hi or "open",
|
||||
offset,
|
||||
len(items),
|
||||
len(page_lots),
|
||||
len(seen),
|
||||
)
|
||||
|
||||
offset += self.LIMIT
|
||||
if offset < max_offset:
|
||||
await self.sleep_between_requests()
|
||||
|
||||
# ── Парсинг страницы офферов ──────────────────────────────────────────────
|
||||
|
||||
def _parse_page(
|
||||
self,
|
||||
items: list[dict[str, Any]],
|
||||
room_token: str,
|
||||
geo_dropped_ref: list[int],
|
||||
) -> list[ScrapedLot]:
|
||||
"""Парсинг списка офферов → ScrapedLot с гео-guard.
|
||||
|
||||
Эмитирует Sentry-сообщение если raw_count > 0 но 0 прошло парсинг
|
||||
(guard от silent schema regression).
|
||||
"""
|
||||
result: list[ScrapedLot] = []
|
||||
local_geo_dropped = 0
|
||||
|
||||
for offer in items:
|
||||
if not _is_ekb(offer):
|
||||
local_geo_dropped += 1
|
||||
geo_dropped_ref[0] += 1
|
||||
continue
|
||||
if href in seen_hrefs:
|
||||
continue
|
||||
seen_hrefs.add(href)
|
||||
|
||||
lot = self._card_link_to_lot(link, href)
|
||||
lot = self._offer_to_lot(offer, room_token)
|
||||
if lot is not None:
|
||||
lots.append(lot)
|
||||
result.append(lot)
|
||||
|
||||
return lots
|
||||
raw_count = len(items)
|
||||
ekb_count = raw_count - local_geo_dropped
|
||||
saved = len(result)
|
||||
|
||||
def _card_link_to_lot(self, link: Node, href: str) -> ScrapedLot | None:
|
||||
"""Парсинг одной карточки-ссылки -> ScrapedLot.
|
||||
# Schema regression guard
|
||||
if raw_count > 0 and saved == 0 and ekb_count > 0:
|
||||
logger.error(
|
||||
"domklik: 0/%d offers прошли _offer_to_lot (ekb=%d) "
|
||||
"— возможна schema regression",
|
||||
raw_count,
|
||||
ekb_count,
|
||||
)
|
||||
try:
|
||||
if settings.glitchtip_dsn:
|
||||
sentry_sdk.capture_message(
|
||||
f"domklik SERP: {raw_count} offers, 0 parsed (ekb={ekb_count})"
|
||||
" — possible schema regression",
|
||||
level="error",
|
||||
)
|
||||
except Exception:
|
||||
pass # sentry_sdk не инициализирован в dev
|
||||
|
||||
DomClick SERP оборачивает карточку в `<a href="/card/...">` — вся
|
||||
информация (title, цена, адрес) внутри этого элемента.
|
||||
return result
|
||||
|
||||
Цену извлекаем из отдельных дочерних элементов (не из объединённого
|
||||
card_text) чтобы исключить слипание числа дома с ценой.
|
||||
def _offer_to_lot(self, offer: dict[str, Any], room_token: str) -> ScrapedLot | None:
|
||||
"""Парсинг одного DomClick bff-JSON оффера → ScrapedLot.
|
||||
|
||||
Маппинг полей per спецификации Layer A.
|
||||
source='domklik', listing_segment='vtorichka' всегда.
|
||||
"""
|
||||
try:
|
||||
source_id = _extract_source_id(href)
|
||||
source_url = urljoin(self.base_url, href)
|
||||
# ── Identity ─────────────────────────────────────────────────────
|
||||
offer_id = offer.get("id")
|
||||
if not offer_id:
|
||||
return None
|
||||
source_id = str(offer_id)
|
||||
|
||||
# Суммарный текст — для rooms/area/floor (эти поля не подвержены
|
||||
# проблеме слипания, т.к. используют специфичные маркеры: «м²», «эт.»)
|
||||
card_text = link.text(strip=True)
|
||||
if not card_text:
|
||||
path = offer.get("path") or ""
|
||||
source_url = urljoin("https://domclick.ru", path)
|
||||
|
||||
# ── Цена ─────────────────────────────────────────────────────────
|
||||
price_raw = offer.get("price")
|
||||
if price_raw is None:
|
||||
return None
|
||||
try:
|
||||
price_rub = int(price_raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if price_rub <= 0:
|
||||
return None
|
||||
|
||||
rooms = _extract_rooms(card_text)
|
||||
area = _extract_area(card_text)
|
||||
floor, total_floors = _extract_floor(card_text)
|
||||
# ── squarePrice (price_per_m2) ────────────────────────────────────
|
||||
sq_price_raw = offer.get("squarePrice")
|
||||
price_per_m2: int | None = None
|
||||
if sq_price_raw is not None:
|
||||
try:
|
||||
price_per_m2 = int(sq_price_raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# Цена: ищем в каждом дочернем элементе отдельно чтобы не слипались
|
||||
# числа дома с ценой при конкатенации card_text
|
||||
price = self._extract_price_from_children(link)
|
||||
if not price or price <= 0:
|
||||
return None
|
||||
# ── Координаты ───────────────────────────────────────────────────
|
||||
location: dict[str, Any] = offer.get("location") or {}
|
||||
lat: float | None = None
|
||||
lon: float | None = None
|
||||
try:
|
||||
if location.get("lat") is not None:
|
||||
lat = float(location["lat"])
|
||||
if location.get("lon") is not None:
|
||||
lon = float(location["lon"])
|
||||
except (TypeError, ValueError):
|
||||
lat = lon = None
|
||||
|
||||
address = self._extract_address(link)
|
||||
# ── Адрес ────────────────────────────────────────────────────────
|
||||
address_obj: dict[str, Any] = offer.get("address") or {}
|
||||
address: str | None = address_obj.get("displayName") or None
|
||||
|
||||
# ── Параметры квартиры ────────────────────────────────────────────
|
||||
obj_info: dict[str, Any] = offer.get("objectInfo") or {}
|
||||
|
||||
area_raw = obj_info.get("area")
|
||||
area_m2: float | None = None
|
||||
if area_raw is not None:
|
||||
try:
|
||||
area_m2 = float(area_raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# Комнатность: студии → 0; числовые → из objectInfo.rooms или токена
|
||||
if room_token == "st":
|
||||
rooms: int | None = 0
|
||||
else:
|
||||
rooms_raw = obj_info.get("rooms")
|
||||
if rooms_raw is not None:
|
||||
try:
|
||||
rooms = int(rooms_raw)
|
||||
except (TypeError, ValueError):
|
||||
rooms = None
|
||||
else:
|
||||
# fallback: числовой токен
|
||||
try:
|
||||
rooms = int(room_token)
|
||||
except ValueError:
|
||||
rooms = None
|
||||
|
||||
floor_raw = obj_info.get("floor")
|
||||
floor: int | None = None
|
||||
if floor_raw is not None:
|
||||
try:
|
||||
floor = int(floor_raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# ── Здание ───────────────────────────────────────────────────────
|
||||
house: dict[str, Any] = offer.get("house") or {}
|
||||
|
||||
total_floors_raw = house.get("floors")
|
||||
total_floors: int | None = None
|
||||
if total_floors_raw is not None:
|
||||
try:
|
||||
total_floors = int(total_floors_raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
year_built_raw = house.get("buildYear")
|
||||
year_built: int | None = None
|
||||
if year_built_raw is not None:
|
||||
try:
|
||||
year_built = int(year_built_raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# ── Дата публикации ───────────────────────────────────────────────
|
||||
published_date = _parse_publish_date(offer.get("publishedDate"))
|
||||
|
||||
# ── Raw payload (компактный) ───────────────────────────────────────
|
||||
flat_complex: dict[str, Any] = offer.get("flatComplex") or {}
|
||||
raw_payload: dict[str, Any] = {
|
||||
"is_apartments": obj_info.get("isApartment"),
|
||||
"is_rosreestr_approved": offer.get("isRosreestrApproved"),
|
||||
"offer_region_name": offer.get("offerRegionName"),
|
||||
"updated_date": offer.get("updatedDate"),
|
||||
"last_price_history_state": offer.get("lastPriceHistoryState"),
|
||||
"flat_complex": (
|
||||
{
|
||||
"id": flat_complex.get("id"),
|
||||
"name": flat_complex.get("name"),
|
||||
"slug": flat_complex.get("slug"),
|
||||
}
|
||||
if flat_complex
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
return ScrapedLot(
|
||||
source="domklik",
|
||||
source_url=source_url,
|
||||
source_id=source_id,
|
||||
address=address,
|
||||
lat=None, # DomClick SERP координаты не отдаёт
|
||||
lon=None,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
rooms=rooms,
|
||||
area_m2=area,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
total_floors=total_floors,
|
||||
price_rub=price,
|
||||
year_built=year_built,
|
||||
price_rub=price_rub,
|
||||
price_per_m2=price_per_m2,
|
||||
listing_segment="vtorichka",
|
||||
raw_payload={"card_text": card_text[:500]},
|
||||
listing_date=published_date,
|
||||
publish_date=published_date,
|
||||
raw_payload=raw_payload,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
self.parse_failures += 1
|
||||
logger.warning(
|
||||
"domclick _card_link_to_lot: parse failed href=%r (parse_failures=%d)",
|
||||
href,
|
||||
"domklik _offer_to_lot: parse failed offer_id=%r (failures=%d)",
|
||||
offer.get("id"),
|
||||
self.parse_failures,
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
|
||||
def _extract_price_from_children(self, link: Node) -> int | None:
|
||||
"""Ищем цену в дочерних элементах карточки по символу рубля.
|
||||
# ── URL builders ──────────────────────────────────────────────────────────
|
||||
|
||||
Итерируем по всем потомкам: первый элемент с рублёвым символом
|
||||
в тексте — источник цены.
|
||||
def _base_params(self, room_token: str, lo: int, hi: int | None) -> list[tuple[str, Any]]:
|
||||
"""Базовые query params (без offset/limit). Используется count и list URL."""
|
||||
params: list[tuple[str, Any]] = [
|
||||
("address", self.EKB_REGION_GUID),
|
||||
("aids", self.EKB_AIDS),
|
||||
("deal_type", "sale"),
|
||||
("category", "living"),
|
||||
("offer_type", "flat"),
|
||||
("rooms", room_token),
|
||||
("sort", "qi"),
|
||||
]
|
||||
if lo > 0:
|
||||
params.append(("sale_price__gte", lo))
|
||||
if hi is not None:
|
||||
params.append(("sale_price__lte", hi))
|
||||
return params
|
||||
|
||||
def _build_count_url(self, room_token: str, lo: int, hi: int | None) -> str:
|
||||
"""URL для count/v1 — без offset/limit."""
|
||||
params = self._base_params(room_token, lo, hi)
|
||||
return f"{self.base_url}/api/offers/count/v1?{urlencode(params)}"
|
||||
|
||||
def _build_list_url(self, room_token: str, lo: int, hi: int | None, offset: int) -> str:
|
||||
"""URL для offers/v1 — с offset/limit."""
|
||||
params = self._base_params(room_token, lo, hi)
|
||||
params.append(("offset", offset))
|
||||
params.append(("limit", self.LIMIT))
|
||||
return f"{self.base_url}/api/offers/v1?{urlencode(params)}"
|
||||
|
||||
# ── JSON fetch helper ─────────────────────────────────────────────────────
|
||||
|
||||
async def _fetch_json(self, url: str) -> dict[str, Any]:
|
||||
"""Fetch URL через BrowserFetcher → parse JSON.
|
||||
|
||||
Raises:
|
||||
DomClickBlockedError: ответ не JSON (QRATOR challenge HTML),
|
||||
или BrowserFetcher вернул HTML-страницу.
|
||||
"""
|
||||
for selector in ("span", "div", "p", "strong", "b"):
|
||||
for el in link.css(selector):
|
||||
text = el.text(strip=True)
|
||||
price = _extract_price_from_element(text)
|
||||
if price:
|
||||
return price
|
||||
# Fallback: суммарный текст карточки (если структура нестандартная)
|
||||
return _extract_price_from_element(link.text(strip=True))
|
||||
|
||||
def _extract_address(self, link: Node) -> str | None:
|
||||
"""Эвристика извлечения адреса из DOM карточки.
|
||||
|
||||
DomClick помещает адрес в отдельный текстовый блок внутри карточки.
|
||||
Обходим вложенные элементы и ищем первый текст с адресными ключевыми
|
||||
словами, пропуская строки с площадью/этажом/ценой.
|
||||
|
||||
Возвращает None если адрес не найден.
|
||||
"""
|
||||
for selector in ("span", "p", "div"):
|
||||
for el in link.css(selector):
|
||||
text = el.text(strip=True)
|
||||
if not text or len(text) < 5 or len(text) > 200:
|
||||
continue
|
||||
# Пропускаем блоки с площадью / этажом / ценой
|
||||
if _RE_AREA.search(text) or _RE_FLOOR.search(text):
|
||||
continue
|
||||
if _RE_PRICE_EL.search(text):
|
||||
continue
|
||||
if _RE_ADDR_KW.search(text):
|
||||
return text
|
||||
return None
|
||||
assert self._browser is not None, "Используй DomClickScraper как async context manager"
|
||||
raw: str = await self._browser.fetch(url)
|
||||
# QRATOR block / captcha возвращает HTML — детектируем по первому символу
|
||||
stripped = raw.lstrip()
|
||||
if stripped.startswith("<"):
|
||||
logger.warning(
|
||||
"domklik: HTML challenge url=%s (первые 300 символов): %r",
|
||||
url,
|
||||
raw[:300],
|
||||
)
|
||||
raise DomClickBlockedError("DomClick вернул HTML вместо JSON (QRATOR block)")
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.warning("domklik: JSONDecodeError url=%s: %s", url, exc)
|
||||
raise DomClickBlockedError(f"DomClick: не JSON ответ: {exc}") from exc
|
||||
|
||||
|
||||
# ── Convenience runner (для Celery tasks) ────────────────────────────────────
|
||||
|
|
@ -393,7 +653,7 @@ class DomClickScraper(BaseScraper):
|
|||
async def scrape_domclick_city(
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 20,
|
||||
pages: int = 100,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Удобная точка входа для вызова из Celery tasks.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
-- 132_update_domclick_sweep_params.sql
|
||||
-- Обновление параметров DomClick citywide sweep после рефакторинга на JSON API.
|
||||
--
|
||||
-- Контекст (2026-06-27):
|
||||
-- Старый скрейпер (122_enable_domclick_city_sweep.sql) использовал HTML/camoufox
|
||||
-- через domclick.ru/search → 0 листингов (URL redesigned + datacenter IP banned
|
||||
-- QRATOR). Новая реализация (Layer A) использует:
|
||||
-- GET https://bff-search-web.domclick.ru/api/offers/v1
|
||||
-- GET https://bff-search-web.domclick.ru/api/offers/count/v1
|
||||
-- Транспорт: BrowserFetcher(source="domclick") → tradein-browser pool → generic
|
||||
-- provider → shared RU mobile proxy (не заблокировано, PoC verified 2026-06-27).
|
||||
--
|
||||
-- Изменения default_params:
|
||||
-- rooms: добавлен 5 (токен "5+", ранее отсутствовал).
|
||||
-- pages_per_anchor: 100 (= OFFSET_CAP/LIMIT = 2000/20, exhaustive within cap).
|
||||
-- request_delay_sec: 3 (JSON endpoint легче camoufox-рендера; QRATOR rate-limit
|
||||
-- не обнаружен на datacenter IP с shared proxy; conservative polite delay).
|
||||
--
|
||||
-- !!! DORMANT BY DESIGN !!! enabled остаётся false.
|
||||
-- Включать вручную после prod-smoke:
|
||||
-- UPDATE scrape_schedules SET enabled = true WHERE source = 'domclick_city_sweep';
|
||||
--
|
||||
-- Idempotent: UPDATE с WHERE source = 'domclick_city_sweep' — no-op если строка
|
||||
-- отсутствует (не должна: засеяна в 122; guard на случай misapply-порядка).
|
||||
--
|
||||
-- ЗАВИСИМОСТИ: 122_enable_domclick_city_sweep.sql (INSERT строки в scrape_schedules).
|
||||
|
||||
BEGIN;
|
||||
|
||||
UPDATE scrape_schedules
|
||||
SET default_params = '{
|
||||
"city_id": 4,
|
||||
"rooms": [0, 1, 2, 3, 4, 5],
|
||||
"pages_per_anchor": 100,
|
||||
"request_delay_sec": 3
|
||||
}'::jsonb
|
||||
WHERE source = 'domclick_city_sweep';
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -1,252 +1,432 @@
|
|||
"""Unit-тесты для DomClick scraper (#796).
|
||||
"""Unit-тесты для DomClick JSON API scraper (Layer A rewrite).
|
||||
|
||||
Тестируем:
|
||||
- Парсинг title-паттерна (rooms/area/floor) из текста карточки
|
||||
- Парсинг цены из отдельного DOM-элемента
|
||||
- Извлечение source_id из href
|
||||
- Полный цикл _parse_html -> list[ScrapedLot] на mocked HTML
|
||||
Offline, без сети и БД. Монкейпатчим BrowserFetcher / _fetch_json на canned JSON.
|
||||
|
||||
Coverage:
|
||||
- _offer_to_lot: поля ScrapedLot (source_id, source_url, lat/lon, address,
|
||||
rooms, area, floor, total_floors, year_built, price, listing_segment, source)
|
||||
- Studio bucket (room_token="st"): rooms=0
|
||||
- Гео-guard: Челябинск оффер отбрасывается, EKB оффер проходит
|
||||
- Block detection: не-JSON body → DomClickBlockedError
|
||||
- URL building: rooms=5%2B в URL; sale_price__gte/lte в banded URL;
|
||||
offset/limit в list URL и отсутствует в count URL
|
||||
- Bucketing: snippetsCount>2000 → рекурсивный split; offset не достигает 2000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from datetime import date
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.domclick import (
|
||||
DomClickScraper,
|
||||
_extract_area,
|
||||
_extract_floor,
|
||||
_extract_price_from_element,
|
||||
_extract_rooms,
|
||||
_extract_source_id,
|
||||
)
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
from app.services.scrapers.domclick import DomClickScraper, _is_ekb, _parse_publish_date
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||||
|
||||
# ── Rooms / studio ───────────────────────────────────────────────────────────
|
||||
# ── Фикстуры ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_scraper() -> DomClickScraper:
|
||||
"""Создаём DomClickScraper без DB (монкейпатчим get_scraper_delay)."""
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=0.0):
|
||||
scraper = DomClickScraper()
|
||||
scraper._browser = object() # не None — assertion в _fetch_json проверяет
|
||||
return scraper
|
||||
|
||||
|
||||
_SAMPLE_OFFER: dict = {
|
||||
"id": 2075671636,
|
||||
"path": "/card/sale__flat__2075671636",
|
||||
"location": {"lat": 56.83, "lon": 60.61},
|
||||
"address": {"displayName": "Екатеринбург, улица Ленина, 10"},
|
||||
"objectInfo": {
|
||||
"area": 52.3,
|
||||
"rooms": 2,
|
||||
"floor": 5,
|
||||
"isApartment": False,
|
||||
},
|
||||
"house": {"floors": 16, "buildYear": 2005},
|
||||
"price": 6500000,
|
||||
"squarePrice": 124283,
|
||||
"flatComplex": {"id": 123, "name": "ЖК Тест", "slug": "zhk-test"},
|
||||
"isRosreestrApproved": True,
|
||||
"publishedDate": "2026-06-01",
|
||||
"updatedDate": "2026-06-15",
|
||||
"lastPriceHistoryState": None,
|
||||
"offerRegionName": "Екатеринбург",
|
||||
}
|
||||
|
||||
_STUDIO_OFFER: dict = {
|
||||
"id": 9001,
|
||||
"path": "/card/sale__flat__9001",
|
||||
"location": {"lat": 56.84, "lon": 60.60},
|
||||
"address": {"displayName": "Екатеринбург, пр-т Ленина, 50"},
|
||||
"objectInfo": {"area": 24.5, "rooms": None, "floor": 3, "isApartment": False},
|
||||
"house": {"floors": 10, "buildYear": 2020},
|
||||
"price": 3100000,
|
||||
"squarePrice": 126531,
|
||||
"flatComplex": None,
|
||||
"isRosreestrApproved": False,
|
||||
"publishedDate": None,
|
||||
"updatedDate": None,
|
||||
"lastPriceHistoryState": None,
|
||||
"offerRegionName": "Екатеринбург",
|
||||
}
|
||||
|
||||
_CHELYABINSK_OFFER: dict = {
|
||||
"id": 99999,
|
||||
"path": "/card/sale__flat__99999",
|
||||
"location": {"lat": 55.15, "lon": 61.39}, # вне EKB bbox
|
||||
"address": {"displayName": "Челябинск, улица Ленина, 1"},
|
||||
"objectInfo": {"area": 40.0, "rooms": 1, "floor": 3, "isApartment": False},
|
||||
"house": {"floors": 9, "buildYear": 1990},
|
||||
"price": 3000000,
|
||||
"squarePrice": 75000,
|
||||
"flatComplex": None,
|
||||
"isRosreestrApproved": False,
|
||||
"publishedDate": None,
|
||||
"updatedDate": None,
|
||||
"lastPriceHistoryState": None,
|
||||
"offerRegionName": "Челябинск", # не Екатеринбург
|
||||
}
|
||||
|
||||
|
||||
# ── _offer_to_lot — базовый маппинг ──────────────────────────────────────────
|
||||
|
||||
|
||||
def test_offer_to_lot_basic_fields() -> None:
|
||||
"""Репрезентативный оффер → ScrapedLot с корректными полями."""
|
||||
scraper = _make_scraper()
|
||||
lot = scraper._offer_to_lot(_SAMPLE_OFFER, room_token="2")
|
||||
|
||||
assert lot is not None
|
||||
assert lot.source == "domklik"
|
||||
assert lot.source_id == "2075671636"
|
||||
assert lot.source_url == "https://domclick.ru/card/sale__flat__2075671636"
|
||||
assert lot.lat == pytest.approx(56.83)
|
||||
assert lot.lon == pytest.approx(60.61)
|
||||
assert lot.address == "Екатеринбург, улица Ленина, 10"
|
||||
assert lot.rooms == 2
|
||||
assert lot.area_m2 == pytest.approx(52.3)
|
||||
assert lot.floor == 5
|
||||
assert lot.total_floors == 16
|
||||
assert lot.year_built == 2005
|
||||
assert lot.price_rub == 6_500_000
|
||||
assert lot.price_per_m2 == 124_283
|
||||
assert lot.listing_segment == "vtorichka"
|
||||
assert lot.listing_date == date(2026, 6, 1)
|
||||
assert lot.publish_date == date(2026, 6, 1)
|
||||
assert lot.raw_payload is not None
|
||||
assert lot.raw_payload["is_rosreestr_approved"] is True
|
||||
assert lot.raw_payload["flat_complex"]["name"] == "ЖК Тест"
|
||||
|
||||
|
||||
def test_offer_to_lot_studio_rooms_zero() -> None:
|
||||
"""Studio bucket (room_token='st') → rooms=0 независимо от objectInfo.rooms."""
|
||||
scraper = _make_scraper()
|
||||
lot = scraper._offer_to_lot(_STUDIO_OFFER, room_token="st")
|
||||
|
||||
assert lot is not None
|
||||
assert lot.rooms == 0
|
||||
assert lot.area_m2 == pytest.approx(24.5)
|
||||
assert lot.price_rub == 3_100_000
|
||||
# publishedDate=None → None
|
||||
assert lot.listing_date is None
|
||||
|
||||
|
||||
def test_offer_to_lot_missing_price_returns_none() -> None:
|
||||
"""Оффер без цены → None."""
|
||||
scraper = _make_scraper()
|
||||
offer = {**_SAMPLE_OFFER, "price": None}
|
||||
assert scraper._offer_to_lot(offer, "2") is None
|
||||
|
||||
|
||||
def test_offer_to_lot_zero_price_returns_none() -> None:
|
||||
"""Оффер с price=0 → None."""
|
||||
scraper = _make_scraper()
|
||||
offer = {**_SAMPLE_OFFER, "price": 0}
|
||||
assert scraper._offer_to_lot(offer, "2") is None
|
||||
|
||||
|
||||
def test_offer_to_lot_missing_id_returns_none() -> None:
|
||||
"""Оффер без id → None."""
|
||||
scraper = _make_scraper()
|
||||
offer = {k: v for k, v in _SAMPLE_OFFER.items() if k != "id"}
|
||||
assert scraper._offer_to_lot(offer, "2") is None
|
||||
|
||||
|
||||
# ── Гео-guard ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_geo_guard_ekb_by_region_name() -> None:
|
||||
"""offerRegionName='Екатеринбург' → EKB."""
|
||||
assert _is_ekb({"offerRegionName": "Екатеринбург"}) is True
|
||||
|
||||
|
||||
def test_geo_guard_ekb_by_bbox_coords() -> None:
|
||||
"""Координаты внутри EKB bbox → EKB (даже без offerRegionName)."""
|
||||
assert _is_ekb({"location": {"lat": 56.83, "lon": 60.61}}) is True
|
||||
|
||||
|
||||
def test_geo_guard_chelyabinsk_dropped() -> None:
|
||||
"""Челябинск (имя и bbox) → не EKB."""
|
||||
assert _is_ekb(_CHELYABINSK_OFFER) is False
|
||||
|
||||
|
||||
def test_geo_guard_out_of_bbox_no_name() -> None:
|
||||
"""Координаты вне EKB bbox и нет offerRegionName → не EKB."""
|
||||
assert _is_ekb({"location": {"lat": 55.0, "lon": 73.0}}) is False
|
||||
|
||||
|
||||
def test_parse_page_drops_non_ekb() -> None:
|
||||
"""_parse_page: Челябинск оффер отброшен, EKB проходит."""
|
||||
scraper = _make_scraper()
|
||||
dropped_ref = [0]
|
||||
items = [_SAMPLE_OFFER, _CHELYABINSK_OFFER]
|
||||
lots = scraper._parse_page(items, "2", dropped_ref)
|
||||
|
||||
assert len(lots) == 1
|
||||
assert lots[0].source_id == "2075671636"
|
||||
assert dropped_ref[0] == 1 # один дроп
|
||||
|
||||
|
||||
# ── Block detection ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_fetch_json_html_body_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Если BrowserFetcher.fetch возвращает HTML → DomClickBlockedError."""
|
||||
|
||||
class _FakeBrowser:
|
||||
async def fetch(self, url: str) -> str:
|
||||
return "<html><head><title>QRATOR protection</title></head></html>"
|
||||
|
||||
scraper = _make_scraper()
|
||||
scraper._browser = _FakeBrowser()
|
||||
|
||||
with pytest.raises(DomClickBlockedError):
|
||||
await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1")
|
||||
|
||||
|
||||
async def test_fetch_json_non_json_body_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Не-JSON body (не HTML, просто мусор) → DomClickBlockedError."""
|
||||
|
||||
class _FakeBrowser:
|
||||
async def fetch(self, url: str) -> str:
|
||||
return "not valid json at all |||"
|
||||
|
||||
scraper = _make_scraper()
|
||||
scraper._browser = _FakeBrowser()
|
||||
|
||||
with pytest.raises(DomClickBlockedError):
|
||||
await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1")
|
||||
|
||||
|
||||
async def test_fetch_json_valid_json_returns_dict(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Валидный JSON → dict без исключений."""
|
||||
|
||||
class _FakeBrowser:
|
||||
async def fetch(self, url: str) -> str:
|
||||
return json.dumps({"result": {"snippetsCount": 42}})
|
||||
|
||||
scraper = _make_scraper()
|
||||
scraper._browser = _FakeBrowser()
|
||||
|
||||
data = await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1")
|
||||
assert data["result"]["snippetsCount"] == 42
|
||||
|
||||
|
||||
# ── URL building ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_build_count_url_no_offset_limit() -> None:
|
||||
"""count URL не содержит offset и limit."""
|
||||
scraper = _make_scraper()
|
||||
url = scraper._build_count_url("2", lo=0, hi=None)
|
||||
qs = parse_qs(urlparse(url).query)
|
||||
assert "offset" not in qs
|
||||
assert "limit" not in qs
|
||||
assert "/count/v1" in url
|
||||
|
||||
|
||||
def test_build_list_url_has_offset_limit() -> None:
|
||||
"""list URL содержит offset и limit."""
|
||||
scraper = _make_scraper()
|
||||
url = scraper._build_list_url("2", lo=0, hi=None, offset=40)
|
||||
qs = parse_qs(urlparse(url).query)
|
||||
assert qs["offset"] == ["40"]
|
||||
assert qs["limit"] == ["20"]
|
||||
assert "/offers/v1" in url
|
||||
|
||||
|
||||
def test_build_list_url_5plus_encoded() -> None:
|
||||
"""rooms=5+ корректно URL-кодируется (5%2B или эквивалент)."""
|
||||
scraper = _make_scraper()
|
||||
url = scraper._build_list_url("5+", lo=0, hi=None, offset=0)
|
||||
# urlencode кодирует '+' как '%2B в query string
|
||||
assert "rooms=5%2B" in url or "rooms=5+" in url # оба варианта корректны
|
||||
# Проверяем через parse_qs
|
||||
qs = parse_qs(urlparse(url).query)
|
||||
assert qs.get("rooms") == ["5+"]
|
||||
|
||||
|
||||
def test_build_list_url_price_band() -> None:
|
||||
"""Бандовый URL содержит sale_price__gte и sale_price__lte."""
|
||||
scraper = _make_scraper()
|
||||
url = scraper._build_list_url("2", lo=2_000_000, hi=5_000_000, offset=0)
|
||||
qs = parse_qs(urlparse(url).query)
|
||||
assert qs["sale_price__gte"] == ["2000000"]
|
||||
assert qs["sale_price__lte"] == ["5000000"]
|
||||
|
||||
|
||||
def test_build_list_url_no_price_params_when_open() -> None:
|
||||
"""Открытый бакет (lo=0, hi=None) → нет ценовых параметров."""
|
||||
scraper = _make_scraper()
|
||||
url = scraper._build_list_url("1", lo=0, hi=None, offset=0)
|
||||
qs = parse_qs(urlparse(url).query)
|
||||
assert "sale_price__gte" not in qs
|
||||
assert "sale_price__lte" not in qs
|
||||
|
||||
|
||||
def test_build_url_contains_ekb_params() -> None:
|
||||
"""URL содержит EKB region GUID и aids."""
|
||||
scraper = _make_scraper()
|
||||
url = scraper._build_count_url("1", lo=0, hi=None)
|
||||
assert "0d475b79-88de-4054-818c-37d8f9d0d440" in url
|
||||
qs = parse_qs(urlparse(url).query)
|
||||
assert qs["aids"] == ["20561"]
|
||||
assert qs["deal_type"] == ["sale"]
|
||||
|
||||
|
||||
# ── Bucketing — split при snippetsCount > 2000 ────────────────────────────────
|
||||
|
||||
|
||||
async def test_bucketing_split_on_overflow(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""snippetsCount>2000 для открытого бакета → split; offset never ≥ 2000."""
|
||||
call_log: list[str] = []
|
||||
|
||||
def _make_ekb_offer(offer_id: str) -> dict:
|
||||
return {
|
||||
"id": offer_id,
|
||||
"path": f"/card/sale__flat__{offer_id}",
|
||||
"location": {"lat": 56.83, "lon": 60.61},
|
||||
"address": {"displayName": "Екатеринбург, ул. Тест, 1"},
|
||||
"objectInfo": {"area": 45.0, "rooms": 1, "floor": 3, "isApartment": False},
|
||||
"house": {"floors": 9, "buildYear": 2000},
|
||||
"price": 4000000,
|
||||
"squarePrice": 88888,
|
||||
"flatComplex": None,
|
||||
"isRosreestrApproved": False,
|
||||
"publishedDate": None,
|
||||
"updatedDate": None,
|
||||
"lastPriceHistoryState": None,
|
||||
"offerRegionName": "Екатеринбург",
|
||||
}
|
||||
|
||||
async def mock_fetch_json(self: DomClickScraper, url: str) -> dict:
|
||||
call_log.append(url)
|
||||
parsed = urlparse(url)
|
||||
qs = parse_qs(parsed.query)
|
||||
|
||||
if "/count/v1" in url:
|
||||
has_gte = "sale_price__gte" in qs
|
||||
has_lte = "sale_price__lte" in qs
|
||||
if not has_gte and not has_lte:
|
||||
# Открытый бакет [0, None] → overflow, trigger split
|
||||
return {"result": {"snippetsCount": 2001, "offersCount": 2001}}
|
||||
elif has_lte and int(qs["sale_price__lte"][0]) == 30_000_000:
|
||||
# [0, 30M] → within cap
|
||||
return {"result": {"snippetsCount": 20, "offersCount": 20}}
|
||||
else:
|
||||
# Хвост [30M+1, None] → empty
|
||||
return {"result": {"snippetsCount": 0, "offersCount": 0}}
|
||||
|
||||
# list URL
|
||||
if "/offers/v1" in url:
|
||||
offset_val = int(qs.get("offset", ["0"])[0])
|
||||
if offset_val == 0:
|
||||
return {"result": {"items": [_make_ekb_offer("101")]}}
|
||||
return {"result": {"items": []}}
|
||||
|
||||
return {"result": {}}
|
||||
|
||||
import asyncio as _asyncio
|
||||
|
||||
monkeypatch.setattr(DomClickScraper, "_fetch_json", mock_fetch_json)
|
||||
# sleep_between_requests is inherited from BaseScraper → shadow on subclass
|
||||
monkeypatch.setattr(DomClickScraper, "sleep_between_requests", lambda self: _asyncio.sleep(0))
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=0.0):
|
||||
scraper = DomClickScraper()
|
||||
scraper._browser = object() # не None
|
||||
|
||||
seen: dict = {}
|
||||
geo_ref = [0]
|
||||
await scraper._walk_price_range(
|
||||
room_token="1",
|
||||
lo=0,
|
||||
hi=None,
|
||||
seen=seen,
|
||||
pages=100,
|
||||
geo_dropped_ref=geo_ref,
|
||||
)
|
||||
|
||||
# Split произошёл: как минимум 2 count-вызова (open band + [0, 30M])
|
||||
count_urls = [u for u in call_log if "/count/v1" in u]
|
||||
assert len(count_urls) >= 2, f"Expected split, got count_urls={count_urls}"
|
||||
|
||||
# Ни один list-запрос не должен иметь offset >= 2000
|
||||
list_urls = [u for u in call_log if "/offers/v1" in u]
|
||||
for lu in list_urls:
|
||||
offset_val = int(parse_qs(urlparse(lu).query).get("offset", ["0"])[0])
|
||||
assert offset_val < 2000, f"offset={offset_val} >= OFFSET_CAP in url={lu}"
|
||||
|
||||
# Лот прошёл и дедуплицирован
|
||||
assert "101" in seen
|
||||
|
||||
|
||||
# ── _parse_publish_date ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, expected",
|
||||
"val, expected",
|
||||
[
|
||||
("2-комн. квартира 53,5 м² 5/15 эт.", 2),
|
||||
("1-к. квартира 32,1 м² 3/9 эт.", 1),
|
||||
("3-комн. кв. 78 м² 10/12 эт.", 3),
|
||||
("4-комн. квартира 95 м² 2/5 эт.", 4),
|
||||
("студия 24 м² 1/10 эт.", 0),
|
||||
("Студия 30,5 м² 8/22 эт.", 0),
|
||||
("Нет комнат — просто текст", None),
|
||||
],
|
||||
)
|
||||
def test_extract_rooms(text: str, expected: int | None) -> None:
|
||||
assert _extract_rooms(text) == expected
|
||||
|
||||
|
||||
# ── Area ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, expected",
|
||||
[
|
||||
("2-комн. квартира 53,5 м² 5/15 эт.", 53.5),
|
||||
("студия 24 м² 1/10 эт.", 24.0),
|
||||
("78м2 на этаже", 78.0),
|
||||
("площадь 101.7 м²", 101.7),
|
||||
("Нет площади", None),
|
||||
],
|
||||
)
|
||||
def test_extract_area(text: str, expected: float | None) -> None:
|
||||
assert _extract_area(text) == expected
|
||||
|
||||
|
||||
# ── Floor ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, floor, total",
|
||||
[
|
||||
("2-комн. квартира 53,5 м² 5/15 эт.", 5, 15),
|
||||
("студия 24 м² 1/10 эт.", 1, 10),
|
||||
("12/24 эт.", 12, 24),
|
||||
("нет этажа", None, None),
|
||||
],
|
||||
)
|
||||
def test_extract_floor(text: str, floor: int | None, total: int | None) -> None:
|
||||
assert _extract_floor(text) == (floor, total)
|
||||
|
||||
|
||||
# ── Price (per-element) ───────────────────────────────────────────────────────
|
||||
# Цена парсится из отдельного DOM-элемента (не из суммарного card_text),
|
||||
# поэтому числа адреса не загрязняют парсинг цены.
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, expected",
|
||||
[
|
||||
("5 200 000 ₽", 5_200_000),
|
||||
("12 345 678 ₽", 12_345_678),
|
||||
("3 990 000 руб.", 3_990_000),
|
||||
("от 4 500 000 ₽", 4_500_000),
|
||||
("без цены", None),
|
||||
("0 ₽", None),
|
||||
],
|
||||
)
|
||||
def test_extract_price_from_element(text: str, expected: int | None) -> None:
|
||||
assert _extract_price_from_element(text) == expected
|
||||
|
||||
|
||||
# ── source_id from href ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"href, expected",
|
||||
[
|
||||
("/card/sale__flat__2075671636", "2075671636"),
|
||||
("https://domclick.ru/card/sale__flat__123456789", "123456789"),
|
||||
("/card/sale__flat__1", "1"),
|
||||
("/card/something_else", None),
|
||||
("2026-06-01", date(2026, 6, 1)),
|
||||
("2026-06-01T12:00:00Z", date(2026, 6, 1)),
|
||||
(1748736000, date(2025, 6, 1)), # 2025-06-01 00:00 UTC epoch
|
||||
("1748736000", date(2025, 6, 1)),
|
||||
(None, None),
|
||||
("invalid-date", None),
|
||||
("", None),
|
||||
],
|
||||
)
|
||||
def test_extract_source_id(href: str, expected: str | None) -> None:
|
||||
assert _extract_source_id(href) == expected
|
||||
def test_parse_publish_date(val: object, expected: date | None) -> None:
|
||||
result = _parse_publish_date(val)
|
||||
assert result == expected
|
||||
|
||||
|
||||
# ── _parse_html -> list[ScrapedLot] ──────────────────────────────────────────
|
||||
# Цена в каждой карточке вынесена в отдельный <span> с символом рубля,
|
||||
# чтобы не сливаться с адресным номером дома при конкатенации card_text.
|
||||
|
||||
_MOCK_HTML = """\
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<!-- Карточка 1: комнатность, площадь, этаж, адрес, цена в отдельных span -->
|
||||
<a href="/card/sale__flat__2075671636">
|
||||
<span>2-комн. квартира 53,5 м² 5/15 эт.</span>
|
||||
<span>ул. Малышева, 1</span>
|
||||
<span>5 200 000 ₽</span>
|
||||
</a>
|
||||
|
||||
<!-- Карточка 2: студия -->
|
||||
<a href="/card/sale__flat__1111111111">
|
||||
<span>Студия 28 м² 3/10 эт.</span>
|
||||
<span>пр-т Ленина, 50</span>
|
||||
<span>3 100 000 ₽</span>
|
||||
</a>
|
||||
|
||||
<!-- Карточка 3: без цены -> должна быть отфильтрована -->
|
||||
<a href="/card/sale__flat__9999999999">
|
||||
<span>1-комн. квартира 32 м² 2/9 эт.</span>
|
||||
</a>
|
||||
|
||||
<!-- Не карточка (без sale__flat__) -> должна быть отфильтрована -->
|
||||
<a href="/card/sale__apartment__123">
|
||||
<span>что-то другое</span>
|
||||
</a>
|
||||
|
||||
<!-- Дубликат первой карточки -> один раз на страницу -->
|
||||
<a href="/card/sale__flat__2075671636">
|
||||
<span>2-комн. квартира 53,5 м² 5/15 эт.</span>
|
||||
<span>5 200 000 ₽</span>
|
||||
</a>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
# ── Scraper meta ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_html_returns_lots() -> None:
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
# Карточки 1 и 2 парсятся; карточка 3 (без цены) отфильтрована; дубль — дедуплицирован
|
||||
assert len(lots) == 2
|
||||
|
||||
|
||||
def test_parse_html_card1_fields() -> None:
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
card1 = next(lot for lot in lots if lot.source_id == "2075671636")
|
||||
assert card1.source == "domklik"
|
||||
assert card1.source_url == "https://domclick.ru/card/sale__flat__2075671636"
|
||||
assert card1.rooms == 2
|
||||
assert card1.area_m2 == 53.5
|
||||
assert card1.floor == 5
|
||||
assert card1.total_floors == 15
|
||||
assert card1.price_rub == 5_200_000
|
||||
assert card1.lat is None
|
||||
assert card1.lon is None
|
||||
assert card1.listing_segment == "vtorichka"
|
||||
|
||||
|
||||
def test_parse_html_studio() -> None:
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
studio = next(lot for lot in lots if lot.source_id == "1111111111")
|
||||
assert studio.rooms == 0 # студия
|
||||
assert studio.area_m2 == 28.0
|
||||
assert studio.price_rub == 3_100_000
|
||||
|
||||
|
||||
def test_parse_html_no_price_filtered() -> None:
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
source_ids = {lot.source_id for lot in lots}
|
||||
assert "9999999999" not in source_ids
|
||||
|
||||
|
||||
def test_parse_html_dedup_same_href() -> None:
|
||||
"""Дубликат href на одной странице должен давать одну карточку."""
|
||||
scraper = DomClickScraper()
|
||||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
ids_2075 = [lot for lot in lots if lot.source_id == "2075671636"]
|
||||
assert len(ids_2075) == 1
|
||||
|
||||
|
||||
def test_scraper_source_name() -> None:
|
||||
scraper = DomClickScraper()
|
||||
def test_scraper_attributes() -> None:
|
||||
scraper = _make_scraper()
|
||||
assert scraper.name == "domklik"
|
||||
assert scraper.source == "domklik"
|
||||
assert scraper.base_url == "https://domclick.ru"
|
||||
# request_delay_sec установлен из get_scraper_delay (DB или fallback 8.0)
|
||||
assert scraper.base_url == "https://bff-search-web.domclick.ru"
|
||||
assert scraper.request_delay_sec >= 0.0
|
||||
|
||||
|
||||
def test_fetch_around_raises() -> None:
|
||||
scraper = DomClickScraper()
|
||||
with pytest.raises(NotImplementedError, match="geo-radius"):
|
||||
asyncio.run(scraper.fetch_around(56.8, 60.6))
|
||||
|
||||
|
||||
def test_build_url_no_rooms() -> None:
|
||||
scraper = DomClickScraper()
|
||||
url = scraper._build_url(city_id=4, rooms=None, page=1)
|
||||
assert "city_id=4" in url
|
||||
assert "rooms" not in url
|
||||
assert "p=1" in url
|
||||
|
||||
|
||||
def test_build_url_with_rooms() -> None:
|
||||
scraper = DomClickScraper()
|
||||
url = scraper._build_url(city_id=4, rooms=2, page=3)
|
||||
assert "rooms=2" in url
|
||||
assert "p=3" in url
|
||||
|
||||
|
||||
def test_parse_failures_counter() -> None:
|
||||
"""parse_failures счётчик начинается с 0."""
|
||||
scraper = DomClickScraper()
|
||||
assert scraper.parse_failures == 0
|
||||
|
||||
|
||||
def test_blocked_page_raises_domclick_blocked_error() -> None:
|
||||
"""HTML с datadome в начале вызывает DomClickBlockedError."""
|
||||
scraper = DomClickScraper()
|
||||
blocked_html = (
|
||||
"<html><head><title>datadome protection</title></head><body>blocked</body></html>"
|
||||
)
|
||||
with pytest.raises(DomClickBlockedError):
|
||||
scraper._parse_html(blocked_html)
|
||||
def test_fetch_around_raises() -> None:
|
||||
import asyncio
|
||||
|
||||
scraper = _make_scraper()
|
||||
with pytest.raises(NotImplementedError, match="geo-radius"):
|
||||
asyncio.run(scraper.fetch_around(56.8, 60.6))
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
"""Offline unit tests for run_domclick_city_sweep + DomClick parser.
|
||||
"""Offline unit tests for run_domclick_city_sweep orchestration (JSON API rewrite).
|
||||
|
||||
Pure & fast: NO live network, NO DB. Mirrors test_cian_city_sweep orchestration
|
||||
semantics, but DomClick is CITYWIDE (no anchor-loop) и SERP-only (no detail/houses).
|
||||
Pure & fast: NO live network, NO DB.
|
||||
|
||||
Coverage:
|
||||
- fetch_city вызывается, save_listings получает накопленные lots с source='domklik'
|
||||
- DomClickCitySweepCounters корректно заполняется (lots_fetched/inserted/updated)
|
||||
- fetch_city вызывается, save_listings получает lots с source='domklik'
|
||||
- DomClickCitySweepCounters корректно заполняется
|
||||
- Cooperative cancel прерывает sweep перед SERP
|
||||
- fetch_city exception не валит sweep (graceful, errors_count++, mark_done)
|
||||
- _parse_html: минимальная DomClick карточка → ScrapedLot с rooms/area/floor/price
|
||||
- fetch_city exception → errors_count++; 0 lots + errors → mark_failed (#1968)
|
||||
- Успешный sweep (lots > 0) → mark_done
|
||||
- Пустой SERP (0 lots, 0 errors) → mark_done (не ошибка)
|
||||
- DomClickBlockedError в фазе → errors_count++; 0 lots → mark_failed
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -25,6 +26,7 @@ import pytest
|
|||
from app.services import scrape_pipeline
|
||||
from app.services.scrapers.base import ScrapedLot
|
||||
from app.services.scrapers.domclick import DomClickScraper
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||||
|
||||
|
||||
def _fake_lot(offer_id: str) -> ScrapedLot:
|
||||
|
|
@ -36,8 +38,8 @@ def _fake_lot(offer_id: str) -> ScrapedLot:
|
|||
price_rub=5_000_000,
|
||||
area_m2=50.0,
|
||||
rooms=2,
|
||||
lat=None,
|
||||
lon=None,
|
||||
lat=56.83,
|
||||
lon=60.61,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -78,6 +80,7 @@ def _stub_domclick_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
def _init(self: DomClickScraper) -> None:
|
||||
self.request_delay_sec = 0.0
|
||||
self.parse_failures = 0
|
||||
self._browser = None
|
||||
|
||||
async def _aenter(self: DomClickScraper) -> DomClickScraper:
|
||||
return self
|
||||
|
|
@ -125,8 +128,8 @@ def test_domclick_sweep_counters_to_dict_all_keys() -> None:
|
|||
# ── Full sweep — фазы и агрегация ──────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""fetch_city → save_listings получает 2 lots с source='domklik', counters заполнены."""
|
||||
async def test_sweep_fetches_and_saves_calls_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""fetch_city → 2 lots → save_listings → mark_done (не mark_failed)."""
|
||||
fetch_calls: list[tuple[int, int]] = []
|
||||
save_calls: list[list[ScrapedLot]] = []
|
||||
|
||||
|
|
@ -134,7 +137,7 @@ async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPat
|
|||
self: DomClickScraper,
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 20,
|
||||
pages: int = 100,
|
||||
) -> list[ScrapedLot]:
|
||||
fetch_calls.append((city_id, pages))
|
||||
return [_fake_lot("100"), _fake_lot("101")]
|
||||
|
|
@ -167,20 +170,21 @@ async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPat
|
|||
assert counters.lots_inserted == 2
|
||||
assert counters.lots_updated == 0
|
||||
assert counters.errors_count == 0
|
||||
# Успешный sweep → mark_done, НЕ mark_failed
|
||||
assert fake.done is not None
|
||||
assert fake.done["lots_fetched"] == 2
|
||||
assert fake.failed is None
|
||||
|
||||
|
||||
async def test_sweep_empty_serp_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Пустой SERP → save не вызывается, но mark_done всё равно вызывается."""
|
||||
async def test_sweep_empty_serp_no_errors_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Пустой SERP (0 lots, 0 errors) → mark_done (не ошибка, не mark_failed)."""
|
||||
save_called = False
|
||||
|
||||
async def fake_fetch_empty(
|
||||
self: DomClickScraper,
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 20,
|
||||
pages: int = 100,
|
||||
) -> list[ScrapedLot]:
|
||||
return []
|
||||
|
||||
|
|
@ -201,7 +205,10 @@ async def test_sweep_empty_serp_still_mark_done(monkeypatch: pytest.MonkeyPatch)
|
|||
|
||||
assert save_called is False
|
||||
assert counters.lots_fetched == 0
|
||||
assert counters.errors_count == 0
|
||||
# 0 lots + 0 errors → mark_done (не mark_failed)
|
||||
assert fake.done is not None
|
||||
assert fake.failed is None
|
||||
|
||||
|
||||
async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
|
@ -212,7 +219,7 @@ async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -
|
|||
self: DomClickScraper,
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 20,
|
||||
pages: int = 100,
|
||||
) -> list[ScrapedLot]:
|
||||
nonlocal fetch_called
|
||||
fetch_called = True
|
||||
|
|
@ -233,16 +240,18 @@ async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -
|
|||
assert fake.done is None
|
||||
|
||||
|
||||
async def test_fetch_city_exception_graceful(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Исключение в fetch_city → errors_count++, sweep завершается mark_done (не fatal)."""
|
||||
async def test_fetch_city_exception_triggers_mark_failed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Исключение в fetch_city → errors_count++; 0 lots + errors → mark_failed (#1968)."""
|
||||
|
||||
async def fake_fetch_fail(
|
||||
self: DomClickScraper,
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 20,
|
||||
pages: int = 100,
|
||||
) -> list[ScrapedLot]:
|
||||
raise RuntimeError("camoufox crashed")
|
||||
raise RuntimeError("network error")
|
||||
|
||||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_fail)
|
||||
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
|
||||
|
|
@ -256,36 +265,70 @@ async def test_fetch_city_exception_graceful(monkeypatch: pytest.MonkeyPatch) ->
|
|||
|
||||
assert counters.errors_count == 1
|
||||
assert counters.lots_fetched == 0
|
||||
assert fake.done is not None, "graceful: mark_done вызывается несмотря на ошибку SERP"
|
||||
# 0 lots + errors → mark_failed, НЕ mark_done
|
||||
assert fake.failed is not None, "mark_failed должен быть вызван при 0 lots + errors"
|
||||
assert fake.done is None
|
||||
|
||||
|
||||
async def test_domclick_blocked_triggers_mark_failed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""DomClickBlockedError в fetch_city → errors_count++; 0 lots → mark_failed."""
|
||||
|
||||
async def fake_fetch_blocked(
|
||||
self: DomClickScraper,
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 100,
|
||||
) -> list[ScrapedLot]:
|
||||
raise DomClickBlockedError("QRATOR blocked")
|
||||
|
||||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_blocked)
|
||||
monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0))
|
||||
|
||||
fake = _FakeRuns()
|
||||
_install_runs(monkeypatch, fake)
|
||||
|
||||
counters = await scrape_pipeline.run_domclick_city_sweep(
|
||||
db=MagicMock(), run_id=5, request_delay_sec=0.0
|
||||
)
|
||||
|
||||
assert counters.errors_count == 1
|
||||
assert counters.lots_fetched == 0
|
||||
assert fake.failed is not None
|
||||
assert fake.done is None
|
||||
|
||||
|
||||
async def test_lots_but_errors_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Если лоты собраны (>0), но были partial errors → всё равно mark_done."""
|
||||
# Этот случай не тривиален — errors могут прийти при save_listings, но
|
||||
# lots_fetched > 0. В нашей логике: lots_fetched>0 → mark_done.
|
||||
|
||||
async def fake_fetch_partial(
|
||||
self: DomClickScraper,
|
||||
city_id: int,
|
||||
rooms: list[int] | None = None,
|
||||
pages: int = 100,
|
||||
) -> list[ScrapedLot]:
|
||||
return [_fake_lot("200")]
|
||||
|
||||
fake_save_calls = [0]
|
||||
|
||||
def fake_save_ok(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
|
||||
fake_save_calls[0] += 1
|
||||
return 1, 0
|
||||
|
||||
monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_partial)
|
||||
monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_ok)
|
||||
|
||||
fake = _FakeRuns()
|
||||
_install_runs(monkeypatch, fake)
|
||||
|
||||
counters = await scrape_pipeline.run_domclick_city_sweep(
|
||||
db=MagicMock(), run_id=6, request_delay_sec=0.0
|
||||
)
|
||||
|
||||
assert counters.lots_fetched == 1
|
||||
# Успешно → mark_done
|
||||
assert fake.done is not None
|
||||
assert fake.failed is None
|
||||
|
||||
|
||||
# ── _parse_html parser unit test ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_html_minimal_card() -> None:
|
||||
"""Минимальная DomClick карточка → ScrapedLot с rooms/area/floor/price/source_id."""
|
||||
# __init__ застаблен фикстурой _stub_domclick_lifecycle → нет DB/browser
|
||||
scraper = DomClickScraper()
|
||||
html = """
|
||||
<html><body>
|
||||
<a href="/card/sale__flat__2075671636">
|
||||
<div>2-комн. квартира</div>
|
||||
<span>52,3 м²</span>
|
||||
<span>5 / 16 эт.</span>
|
||||
<span>Екатеринбург, улица Ленина, 10</span>
|
||||
<span>6 500 000 ₽</span>
|
||||
</a>
|
||||
</body></html>
|
||||
"""
|
||||
lots = scraper._parse_html(html)
|
||||
assert len(lots) == 1
|
||||
lot = lots[0]
|
||||
assert lot.source == "domklik"
|
||||
assert lot.source_id == "2075671636"
|
||||
assert lot.rooms == 2
|
||||
assert lot.area_m2 == pytest.approx(52.3)
|
||||
assert lot.floor == 5
|
||||
assert lot.total_floors == 16
|
||||
assert lot.price_rub == 6_500_000
|
||||
assert lot.lat is None and lot.lon is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue