From 1600a6be18165ccd10755c304c98dfce9d2cf72d Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 5 Aug 2026 07:06:41 +0000 Subject: [PATCH] =?UTF-8?q?fix(tradein/domclick):=20=D0=B5=D0=B4=D0=B8?= =?UTF-8?q?=D0=BD=D1=8B=D0=B9=20=D1=81=D0=BF=D0=B8=D1=81=D0=BE=D0=BA=20QRA?= =?UTF-8?q?TOR-=D0=BC=D0=B0=D1=80=D0=BA=D0=B5=D1=80=D0=BE=D0=B2=20serp+det?= =?UTF-8?q?ail=20(#2636)=20(#2645)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/scrapers/test_domclick_detail.py | 19 +++++++++++++++ .../src/scraper_kit/domclick_exceptions.py | 20 ++++++++++++++-- .../scraper_kit/providers/domclick/detail.py | 24 ++++++++++++------- .../scraper_kit/providers/domclick/serp.py | 18 ++++---------- 4 files changed, 57 insertions(+), 24 deletions(-) diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py b/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py index d5c96c0e..1d1c805c 100644 --- a/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py +++ b/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py @@ -159,6 +159,25 @@ def test_extract_ssr_state_challenge_raises_blocked() -> None: _extract_ssr_state(html) +# ── QRATOR markers (#2636) — Layer B had only 4/7 canonical markers, so a +# QRATOR block page using bot_mitigation/система защиты/403 | домклик fell +# through to DomClickParseError instead of DomClickBlockedError (block-breaker +# never triggered, batch burned through instead of pausing). Case varied per +# marker to exercise the case-insensitive comparison. +@pytest.mark.parametrize( + "html", + [ + "BOT_MITIGATION в процессе, подождите", + "СИСТЕМА ЗАЩИТЫ от ботов активна", + "403 | ДОМКЛИК — доступ ограничен", + ], + ids=["bot_mitigation_upper", "sistema_zashchity_upper", "403_domklik_upper"], +) +def test_extract_ssr_state_qrator_markers_raise_blocked_not_parse(html: str) -> None: + with pytest.raises(DomClickBlockedError): + _extract_ssr_state(html) + + def test_extract_ssr_state_unbalanced_raises_parse() -> None: with pytest.raises(DomClickParseError): _extract_ssr_state('window.__SSR_STATE__ = {"a": 1') diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/domclick_exceptions.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/domclick_exceptions.py index 247de7fe..21a0356e 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/domclick_exceptions.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/domclick_exceptions.py @@ -1,12 +1,28 @@ """DomClick-specific exceptions для anti-bot detection.""" +# ── Канонический список anti-bot маркеров (QRATOR/DataDome) ────────────────── +# Единый источник для Layer A (providers/domclick/serp.py) и Layer B +# (providers/domclick/detail.py) — были два расходящихся списка (#2636), теперь +# оба слоя импортируют этот. Сравнение case-insensitive: caller лоуеркейзит HTML +# перед `marker in html_lower`. +DOMCLICK_BLOCK_MARKERS: tuple[str, ...] = ( + "qrator", + "bot_mitigation", + "система защиты", + "403 | домклик", + "captcha", + "access denied", + "datadome", +) + class DomClickBlockedError(Exception): """DomClick BFF вернул QRATOR block-страницу (HTTP 200 + block HTML). QRATOR (qrator.net) — WAF/DDoS-защита domclick.ru. Блокирует datacenter-IP - и возвращает HTML с маркерами: "qrator", "bot_mitigation", "система защиты", - "403 | домклик", "captcha", "access denied". + и возвращает HTML с маркерами: см. `DOMCLICK_BLOCK_MARKERS` в этом модуле + ("qrator", "bot_mitigation", "система защиты", "403 | домклик", "captcha", + "access denied", "datadome"). Обходится через shared mobile proxy (BrowserFetcher(source="domclick") → generic provider → мобильный egress). diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py index b9eeae49..c576bdaa 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py @@ -56,7 +56,11 @@ from urllib.parse import urlsplit from sqlalchemy import text from sqlalchemy.orm import Session -from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError +from scraper_kit.domclick_exceptions import ( + DOMCLICK_BLOCK_MARKERS, + DomClickBlockedError, + DomClickParseError, +) from scraper_kit.offer_price_history import clamp_diff_percent from scraper_kit.repair_state_normalizer import ( infer_repair_state_from_text, @@ -68,11 +72,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# ── Маркеры anti-bot challenge (DataDome / QRATOR) ─────────────────────────── -# Зеркало идеи Layer A: если __SSR_STATE__ отсутствует И страница похожа на -# challenge — это блок, а не parse-failure. -_BLOCK_MARKERS: tuple[str, ...] = ("qrator", "captcha", "datadome", "access denied") - # ── item_id из URL карточки: .../card/sale__flat__2075729321 ───────────────── _ITEM_ID_RE = re.compile(r"sale__flat__(\d+)") @@ -180,11 +179,20 @@ def _extract_ssr_state(html: str) -> dict[str, Any]: match = _SSR_ASSIGN_RE.search(html) if match is None: html_lower = html.lower() - if any(marker in html_lower for marker in _BLOCK_MARKERS): + if any(marker in html_lower for marker in DOMCLICK_BLOCK_MARKERS): raise DomClickBlockedError( "DomClick detail: challenge page detected (no __SSR_STATE__, " - f"markers checked: {_BLOCK_MARKERS})" + f"markers checked: {DOMCLICK_BLOCK_MARKERS})" ) + # Ни SSR-стейта, ни известного anti-bot маркера — либо честный parse-failure + # (дрейф схемы), либо новый вариант блок-страницы, которого нет в + # DOMCLICK_BLOCK_MARKERS (#2636: расхождение списков привело к misclassify + # blocked→failed). Логируем голову HTML, чтобы дрейф маркеров был виден. + logger.warning( + "domclick_detail: __SSR_STATE__ not found, no known block marker — " + "head=%r", + html[:300].replace("\n", " "), + ) raise DomClickParseError("__SSR_STATE__ not found") brace_start = html.find("{", match.end()) diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py index 33ad905f..69ac0593 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py @@ -36,7 +36,7 @@ from typing import TYPE_CHECKING, Any from urllib.parse import urlencode from scraper_kit.base import BaseScraper, ScrapedLot -from scraper_kit.domclick_exceptions import DomClickBlockedError +from scraper_kit.domclick_exceptions import DOMCLICK_BLOCK_MARKERS, DomClickBlockedError from scraper_kit.pricing import BisectionConfig, ProbeResult, walk_price_range from scraper_kit.repair_state_normalizer import infer_repair_state_from_text @@ -80,17 +80,6 @@ _DOMCLICK_BISECTION = BisectionConfig( open_split_floor=0, ) -# ── QRATOR block detection ──────────────────────────────────────────────────── - -_QRATOR_MARKERS: tuple[str, ...] = ( - "qrator", - "bot_mitigation", - "система защиты", - "403 | домклик", - "captcha", - "access denied", -) - # ── EKB geo guard ───────────────────────────────────────────────────────────── _EKB_LAT_MIN: float = 56.6 @@ -119,9 +108,10 @@ def _extract_json(html: str) -> dict[str, Any]: # Сканируем ВЕСЬ ответ (а не только первые 4096B): block-маркер может # стоять за пределами head в крупных challenge-страницах. html_lower = html.lower() - if any(m in html_lower for m in _QRATOR_MARKERS): + if any(m in html_lower for m in DOMCLICK_BLOCK_MARKERS): raise DomClickBlockedError( - f"DomClick BFF: QRATOR block page detected (markers checked: {_QRATOR_MARKERS[:2]})" + "DomClick BFF: QRATOR block page detected " + f"(markers checked: {DOMCLICK_BLOCK_MARKERS[:2]})" ) start = html.find("{")