lekss361/gendesign
1
0
Fork 0
Code Issues 141 Pull requests 3 Projects Releases Packages Wiki Activity Actions

fix(tradein/domclick): единый список QRATOR-маркеров serp+detail (#2636) (#2645)
All checks were successful
Deploy Trade-In / changes (push) Successful in 9s
Details
Deploy Trade-In / build-frontend (push) Has been skipped
Details
Deploy Trade-In / build-browser (push) Has been skipped
Details
Deploy Trade-In / test (push) Successful in 2m38s
Details
Deploy Trade-In / build-backend (push) Successful in 1m33s
Details
Deploy Trade-In / deploy (push) Successful in 1m54s
Details

Browse source
This commit is contained in:
bot-backend 2026-08-05 07:06:41 +00:00
parent 8e909faca8
commit 1600a6be18
4 changed files with 57 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
tradein-mvp/backend/tests/scrapers/test_domclick_detail.py
View file

@ -159,6 +159,25 @@ def test_extract_ssr_state_challenge_raises_blocked() -> None:
_extract_ssr_state(html) _extract_ssr_state(html)
# ── QRATOR markers (#2636) — Layer B had only 4/7 canonical markers, so a
# QRATOR block page using bot_mitigation/система защиты/403 | домклик fell
# through to DomClickParseError instead of DomClickBlockedError (block-breaker
# never triggered, batch burned through instead of pausing). Case varied per
# marker to exercise the case-insensitive comparison.
@pytest.mark.parametrize(
"html",
[
"<html><body>BOT_MITIGATION в процессе, подождите</body></html>",
"<html><body>СИСТЕМА ЗАЩИТЫ от ботов активна</body></html>",
"<html><body>403 | ДОМКЛИК — доступ ограничен</body></html>",
],
ids=["bot_mitigation_upper", "sistema_zashchity_upper", "403_domklik_upper"],
)
def test_extract_ssr_state_qrator_markers_raise_blocked_not_parse(html: str) -> None:
with pytest.raises(DomClickBlockedError):
_extract_ssr_state(html)
def test_extract_ssr_state_unbalanced_raises_parse() -> None: def test_extract_ssr_state_unbalanced_raises_parse() -> None:
with pytest.raises(DomClickParseError): with pytest.raises(DomClickParseError):
_extract_ssr_state('window.__SSR_STATE__ = {"a": 1') _extract_ssr_state('window.__SSR_STATE__ = {"a": 1')

20
tradein-mvp/packages/scraper-kit/src/scraper_kit/domclick_exceptions.py
View file

@ -1,12 +1,28 @@
"""DomClick-specific exceptions для anti-bot detection.""" """DomClick-specific exceptions для anti-bot detection."""
# ── Канонический список anti-bot маркеров (QRATOR/DataDome) ──────────────────
# Единый источник для Layer A (providers/domclick/serp.py) и Layer B
# (providers/domclick/detail.py) — были два расходящихся списка (#2636), теперь
# оба слоя импортируют этот. Сравнение case-insensitive: caller лоуеркейзит HTML
# перед `marker in html_lower`.
DOMCLICK_BLOCK_MARKERS: tuple[str, ...] = (
"qrator",
"bot_mitigation",
"система защиты",
"403 | домклик",
"captcha",
"access denied",
"datadome",
)
class DomClickBlockedError(Exception): class DomClickBlockedError(Exception):
"""DomClick BFF вернул QRATOR block-страницу (HTTP 200 + block HTML). """DomClick BFF вернул QRATOR block-страницу (HTTP 200 + block HTML).
QRATOR (qrator.net) — WAF/DDoS-защита domclick.ru. Блокирует datacenter-IP QRATOR (qrator.net) — WAF/DDoS-защита domclick.ru. Блокирует datacenter-IP
и возвращает HTML с маркерами: "qrator", "bot_mitigation", "система защиты", и возвращает HTML с маркерами: см. `DOMCLICK_BLOCK_MARKERS` в этом модуле
"403 | домклик", "captcha", "access denied". ("qrator", "bot_mitigation", "система защиты", "403 | домклик", "captcha",
"access denied", "datadome").
Обходится через shared mobile proxy (BrowserFetcher(source="domclick") → Обходится через shared mobile proxy (BrowserFetcher(source="domclick") →
generic provider → мобильный egress). generic provider → мобильный egress).

24
tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/detail.py
View file

@ -56,7 +56,11 @@ from urllib.parse import urlsplit
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError from scraper_kit.domclick_exceptions import (
DOMCLICK_BLOCK_MARKERS,
DomClickBlockedError,
DomClickParseError,
)
from scraper_kit.offer_price_history import clamp_diff_percent from scraper_kit.offer_price_history import clamp_diff_percent
from scraper_kit.repair_state_normalizer import ( from scraper_kit.repair_state_normalizer import (
infer_repair_state_from_text, infer_repair_state_from_text,
@ -68,11 +72,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# ── Маркеры anti-bot challenge (DataDome / QRATOR) ───────────────────────────
# Зеркало идеи Layer A: если __SSR_STATE__ отсутствует И страница похожа на
# challenge — это блок, а не parse-failure.
_BLOCK_MARKERS: tuple[str, ...] = ("qrator", "captcha", "datadome", "access denied")
# ── item_id из URL карточки: .../card/sale__flat__2075729321 ───────────────── # ── item_id из URL карточки: .../card/sale__flat__2075729321 ─────────────────
_ITEM_ID_RE = re.compile(r"sale__flat__(\d+)") _ITEM_ID_RE = re.compile(r"sale__flat__(\d+)")
@ -180,11 +179,20 @@ def _extract_ssr_state(html: str) -> dict[str, Any]:
match = _SSR_ASSIGN_RE.search(html) match = _SSR_ASSIGN_RE.search(html)
if match is None: if match is None:
html_lower = html.lower() html_lower = html.lower()
if any(marker in html_lower for marker in _BLOCK_MARKERS): if any(marker in html_lower for marker in DOMCLICK_BLOCK_MARKERS):
raise DomClickBlockedError( raise DomClickBlockedError(
"DomClick detail: challenge page detected (no __SSR_STATE__, " "DomClick detail: challenge page detected (no __SSR_STATE__, "
f"markers checked: {_BLOCK_MARKERS})" f"markers checked: {DOMCLICK_BLOCK_MARKERS})"
) )
# Ни SSR-стейта, ни известного anti-bot маркера — либо честный parse-failure
# (дрейф схемы), либо новый вариант блок-страницы, которого нет в
# DOMCLICK_BLOCK_MARKERS (#2636: расхождение списков привело к misclassify
# blocked→failed). Логируем голову HTML, чтобы дрейф маркеров был виден.
logger.warning(
"domclick_detail: __SSR_STATE__ not found, no known block marker — "
"head=%r",
html[:300].replace("\n", " "),
)
raise DomClickParseError("__SSR_STATE__ not found") raise DomClickParseError("__SSR_STATE__ not found")
brace_start = html.find("{", match.end()) brace_start = html.find("{", match.end())

18
tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py
View file

@ -36,7 +36,7 @@ from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode from urllib.parse import urlencode
from scraper_kit.base import BaseScraper, ScrapedLot from scraper_kit.base import BaseScraper, ScrapedLot
from scraper_kit.domclick_exceptions import DomClickBlockedError from scraper_kit.domclick_exceptions import DOMCLICK_BLOCK_MARKERS, DomClickBlockedError
from scraper_kit.pricing import BisectionConfig, ProbeResult, walk_price_range from scraper_kit.pricing import BisectionConfig, ProbeResult, walk_price_range
from scraper_kit.repair_state_normalizer import infer_repair_state_from_text from scraper_kit.repair_state_normalizer import infer_repair_state_from_text
@ -80,17 +80,6 @@ _DOMCLICK_BISECTION = BisectionConfig(
open_split_floor=0, open_split_floor=0,
) )
# ── QRATOR block detection ────────────────────────────────────────────────────
_QRATOR_MARKERS: tuple[str, ...] = (
"qrator",
"bot_mitigation",
"система защиты",
"403 | домклик",
"captcha",
"access denied",
)
# ── EKB geo guard ───────────────────────────────────────────────────────────── # ── EKB geo guard ─────────────────────────────────────────────────────────────
_EKB_LAT_MIN: float = 56.6 _EKB_LAT_MIN: float = 56.6
@ -119,9 +108,10 @@ def _extract_json(html: str) -> dict[str, Any]:
# Сканируем ВЕСЬ ответ (а не только первые 4096B): block-маркер может # Сканируем ВЕСЬ ответ (а не только первые 4096B): block-маркер может
# стоять за пределами head в крупных challenge-страницах. # стоять за пределами head в крупных challenge-страницах.
html_lower = html.lower() html_lower = html.lower()
if any(m in html_lower for m in _QRATOR_MARKERS): if any(m in html_lower for m in DOMCLICK_BLOCK_MARKERS):
raise DomClickBlockedError( raise DomClickBlockedError(
f"DomClick BFF: QRATOR block page detected (markers checked: {_QRATOR_MARKERS[:2]})" "DomClick BFF: QRATOR block page detected "
f"(markers checked: {DOMCLICK_BLOCK_MARKERS[:2]})"
) )
start = html.find("{") start = html.find("{")