diff --git a/tradein-mvp/backend/app/services/scraper_settings.py b/tradein-mvp/backend/app/services/scraper_settings.py index cce374f6..42baf593 100644 --- a/tradein-mvp/backend/app/services/scraper_settings.py +++ b/tradein-mvp/backend/app/services/scraper_settings.py @@ -31,6 +31,7 @@ _CACHE_TTL_SEC = 60.0 _DEFAULT_DELAY_BY_SOURCE: dict[str, float] = { "avito": 7.0, "cian": 5.0, + "domklik": 8.0, "n1": 5.0, "yandex": 5.0, "yandex_detail": 5.0, @@ -82,9 +83,7 @@ def _get_setting_cached(key: str) -> float: try: with _open_session() as db: row = db.execute( - text( - "SELECT request_delay_sec FROM scraper_settings WHERE source = :s" - ), + text("SELECT request_delay_sec FROM scraper_settings WHERE source = :s"), {"s": key}, ).first() if row is not None: @@ -96,8 +95,8 @@ def _get_setting_cached(key: str) -> float: value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) except Exception as e: logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e) - value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get( - key, _GLOBAL_DEFAULT_DELAY + value = ( + 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) ) with _CACHE_LOCK: diff --git a/tradein-mvp/backend/app/services/scrapers/domclick.py b/tradein-mvp/backend/app/services/scrapers/domclick.py index f2156647..a135e184 100644 --- a/tradein-mvp/backend/app/services/scrapers/domclick.py +++ b/tradein-mvp/backend/app/services/scrapers/domclick.py @@ -24,10 +24,21 @@ from urllib.parse import urljoin from selectolax.parser import HTMLParser, Node +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot +from app.services.scrapers.domclick_exceptions import DomClickBlockedError logger = logging.getLogger(__name__) +# ── DataDome block detection ───────────────────────────────────────────────── +_DATADOME_MARKERS = ("datadome", "blocked", "access denied", "bot detected") + + +def _is_blocked_page(html: str) -> bool: + head = html[:2048].lower() + return any(m in head for m in _DATADOME_MARKERS) + + # ── Regex helpers ──────────────────────────────────────────────────────────── # _RE_ROOMS расширен относительно avito.py: DomClick пишет «2-комн. квартира» # (с «комн.») — avito.py-вариант ловит только «N-к. кв.» / «N-к квартира». @@ -105,7 +116,7 @@ def _extract_source_id(href: str) -> str | None: class DomClickScraper(BaseScraper): - """DomClick вторичка parser. Источник = 'domclick'. + """DomClick вторичка parser. Источник = 'domklik'. Использует BrowserFetcher (camoufox headless Firefox) — DataDome пропускает его без дополнительной stealth-настройки. curl_cffi -> 401 (DataDome блок). @@ -114,14 +125,15 @@ class DomClickScraper(BaseScraper): fetch_around() не реализован: DomClick не поддерживает geo-radius в URL. """ - name = "domclick" - source = "domclick" + name = "domklik" + source = "domklik" base_url = "https://domclick.ru" - # DataDome — консервативная задержка между страницами + # DataDome — консервативная задержка между страницами (fallback до init) request_delay_sec = 8.0 def __init__(self) -> None: super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) # Счётчик карточек с неожиданной DOM-структурой (для observability) self.parse_failures: int = 0 @@ -168,29 +180,37 @@ class DomClickScraper(BaseScraper): for room_val in room_values: room_label = f"rooms={room_val}" if room_val is not None else "all_rooms" logger.info( - "domclick: city_id=%d %s sweep (max %d pages)", city_id, room_label, pages + "domklik: city_id=%d %s sweep (max %d pages)", city_id, room_label, pages ) for page_num in range(1, pages + 1): url = self._build_url(city_id, room_val, page_num) - logger.debug("domclick: fetch %s", url) + logger.debug("domklik: fetch %s", url) try: html = await fetcher.fetch(url) except Exception as exc: logger.error( - "domclick: fetch failed city_id=%d page=%d: %s", + "domklik: fetch failed city_id=%d page=%d: %s", city_id, page_num, exc, ) break - lots = self._parse_html(html) + try: + lots = self._parse_html(html) + except DomClickBlockedError: + logger.warning( + "domklik: DataDome block at city_id=%d page=%d — stopping sweep", + city_id, + page_num, + ) + break if not lots: logger.info( - "domclick: empty page city_id=%d %s page=%d — stopping sweep", + "domklik: empty page city_id=%d %s page=%d — stopping sweep", city_id, room_label, page_num, @@ -207,7 +227,7 @@ class DomClickScraper(BaseScraper): all_lots.extend(new_lots) logger.info( - "domclick: city_id=%d %s page=%d -> %d new (total %d)", + "domklik: city_id=%d %s page=%d -> %d new (total %d)", city_id, room_label, page_num, @@ -220,7 +240,7 @@ class DomClickScraper(BaseScraper): await self.sleep_between_requests() logger.info( - "domclick: fetch_city done city_id=%d total=%d parse_failures=%d", + "domklik: fetch_city done city_id=%d total=%d parse_failures=%d", city_id, len(all_lots), self.parse_failures, @@ -248,6 +268,9 @@ class DomClickScraper(BaseScraper): Селектор: `a[href*="/card/"]` с паттерном `/card/sale__flat__`. """ + if _is_blocked_page(html): + logger.warning("domklik: DataDome block page detected, 0 cards returned") + raise DomClickBlockedError("DomClick returned DataDome block page") tree = HTMLParser(html) card_links = tree.css('a[href*="/card/"]') @@ -301,7 +324,7 @@ class DomClickScraper(BaseScraper): address = self._extract_address(link) return ScrapedLot( - source="domclick", + source="domklik", source_url=source_url, source_id=source_id, address=address, diff --git a/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py b/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py new file mode 100644 index 00000000..118adb72 --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py @@ -0,0 +1,5 @@ +"""DomClick-specific exceptions для anti-bot detection.""" + + +class DomClickBlockedError(Exception): + """DomClick вернул DataDome block-страницу (HTTP 200 + block HTML).""" diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick.py b/tradein-mvp/backend/tests/scrapers/test_domclick.py index 878452c8..04d0557d 100644 --- a/tradein-mvp/backend/tests/scrapers/test_domclick.py +++ b/tradein-mvp/backend/tests/scrapers/test_domclick.py @@ -21,6 +21,7 @@ from app.services.scrapers.domclick import ( _extract_rooms, _extract_source_id, ) +from app.services.scrapers.domclick_exceptions import DomClickBlockedError # ── Rooms / studio ─────────────────────────────────────────────────────────── @@ -166,7 +167,7 @@ def test_parse_html_card1_fields() -> None: lots = scraper._parse_html(_MOCK_HTML) card1 = next(lot for lot in lots if lot.source_id == "2075671636") - assert card1.source == "domclick" + assert card1.source == "domklik" assert card1.source_url == "https://domclick.ru/card/sale__flat__2075671636" assert card1.rooms == 2 assert card1.area_m2 == 53.5 @@ -207,10 +208,11 @@ def test_parse_html_dedup_same_href() -> None: def test_scraper_source_name() -> None: scraper = DomClickScraper() - assert scraper.name == "domclick" - assert scraper.source == "domclick" + assert scraper.name == "domklik" + assert scraper.source == "domklik" assert scraper.base_url == "https://domclick.ru" - assert scraper.request_delay_sec == 8.0 + # request_delay_sec установлен из get_scraper_delay (DB или fallback 8.0) + assert scraper.request_delay_sec >= 0.0 def test_fetch_around_raises() -> None: @@ -238,3 +240,13 @@ def test_parse_failures_counter() -> None: """parse_failures счётчик начинается с 0.""" scraper = DomClickScraper() assert scraper.parse_failures == 0 + + +def test_blocked_page_raises_domclick_blocked_error() -> None: + """HTML с datadome в начале вызывает DomClickBlockedError.""" + scraper = DomClickScraper() + blocked_html = ( + "datadome protectionblocked" + ) + with pytest.raises(DomClickBlockedError): + scraper._parse_html(blocked_html)