fix(tradein): domklik source name, scraper delay, DataDome block detect (#796 fixup)
This commit is contained in:
parent
3c5e9ad368
commit
4c0338ce1d
4 changed files with 60 additions and 21 deletions
|
|
@ -31,6 +31,7 @@ _CACHE_TTL_SEC = 60.0
|
|||
_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
|
||||
"avito": 7.0,
|
||||
"cian": 5.0,
|
||||
"domklik": 8.0,
|
||||
"n1": 5.0,
|
||||
"yandex": 5.0,
|
||||
"yandex_detail": 5.0,
|
||||
|
|
@ -82,9 +83,7 @@ def _get_setting_cached(key: str) -> float:
|
|||
try:
|
||||
with _open_session() as db:
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT request_delay_sec FROM scraper_settings WHERE source = :s"
|
||||
),
|
||||
text("SELECT request_delay_sec FROM scraper_settings WHERE source = :s"),
|
||||
{"s": key},
|
||||
).first()
|
||||
if row is not None:
|
||||
|
|
@ -96,8 +95,8 @@ def _get_setting_cached(key: str) -> float:
|
|||
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
||||
except Exception as e:
|
||||
logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
|
||||
value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(
|
||||
key, _GLOBAL_DEFAULT_DELAY
|
||||
value = (
|
||||
0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
||||
)
|
||||
|
||||
with _CACHE_LOCK:
|
||||
|
|
|
|||
|
|
@ -24,10 +24,21 @@ from urllib.parse import urljoin
|
|||
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── DataDome block detection ─────────────────────────────────────────────────
|
||||
_DATADOME_MARKERS = ("datadome", "blocked", "access denied", "bot detected")
|
||||
|
||||
|
||||
def _is_blocked_page(html: str) -> bool:
|
||||
head = html[:2048].lower()
|
||||
return any(m in head for m in _DATADOME_MARKERS)
|
||||
|
||||
|
||||
# ── Regex helpers ────────────────────────────────────────────────────────────
|
||||
# _RE_ROOMS расширен относительно avito.py: DomClick пишет «2-комн. квартира»
|
||||
# (с «комн.») — avito.py-вариант ловит только «N-к. кв.» / «N-к квартира».
|
||||
|
|
@ -105,7 +116,7 @@ def _extract_source_id(href: str) -> str | None:
|
|||
|
||||
|
||||
class DomClickScraper(BaseScraper):
|
||||
"""DomClick вторичка parser. Источник = 'domclick'.
|
||||
"""DomClick вторичка parser. Источник = 'domklik'.
|
||||
|
||||
Использует BrowserFetcher (camoufox headless Firefox) — DataDome пропускает
|
||||
его без дополнительной stealth-настройки. curl_cffi -> 401 (DataDome блок).
|
||||
|
|
@ -114,14 +125,15 @@ class DomClickScraper(BaseScraper):
|
|||
fetch_around() не реализован: DomClick не поддерживает geo-radius в URL.
|
||||
"""
|
||||
|
||||
name = "domclick"
|
||||
source = "domclick"
|
||||
name = "domklik"
|
||||
source = "domklik"
|
||||
base_url = "https://domclick.ru"
|
||||
# DataDome — консервативная задержка между страницами
|
||||
# DataDome — консервативная задержка между страницами (fallback до init)
|
||||
request_delay_sec = 8.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
# Счётчик карточек с неожиданной DOM-структурой (для observability)
|
||||
self.parse_failures: int = 0
|
||||
|
||||
|
|
@ -168,29 +180,37 @@ class DomClickScraper(BaseScraper):
|
|||
for room_val in room_values:
|
||||
room_label = f"rooms={room_val}" if room_val is not None else "all_rooms"
|
||||
logger.info(
|
||||
"domclick: city_id=%d %s sweep (max %d pages)", city_id, room_label, pages
|
||||
"domklik: city_id=%d %s sweep (max %d pages)", city_id, room_label, pages
|
||||
)
|
||||
|
||||
for page_num in range(1, pages + 1):
|
||||
url = self._build_url(city_id, room_val, page_num)
|
||||
logger.debug("domclick: fetch %s", url)
|
||||
logger.debug("domklik: fetch %s", url)
|
||||
|
||||
try:
|
||||
html = await fetcher.fetch(url)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"domclick: fetch failed city_id=%d page=%d: %s",
|
||||
"domklik: fetch failed city_id=%d page=%d: %s",
|
||||
city_id,
|
||||
page_num,
|
||||
exc,
|
||||
)
|
||||
break
|
||||
|
||||
lots = self._parse_html(html)
|
||||
try:
|
||||
lots = self._parse_html(html)
|
||||
except DomClickBlockedError:
|
||||
logger.warning(
|
||||
"domklik: DataDome block at city_id=%d page=%d — stopping sweep",
|
||||
city_id,
|
||||
page_num,
|
||||
)
|
||||
break
|
||||
|
||||
if not lots:
|
||||
logger.info(
|
||||
"domclick: empty page city_id=%d %s page=%d — stopping sweep",
|
||||
"domklik: empty page city_id=%d %s page=%d — stopping sweep",
|
||||
city_id,
|
||||
room_label,
|
||||
page_num,
|
||||
|
|
@ -207,7 +227,7 @@ class DomClickScraper(BaseScraper):
|
|||
|
||||
all_lots.extend(new_lots)
|
||||
logger.info(
|
||||
"domclick: city_id=%d %s page=%d -> %d new (total %d)",
|
||||
"domklik: city_id=%d %s page=%d -> %d new (total %d)",
|
||||
city_id,
|
||||
room_label,
|
||||
page_num,
|
||||
|
|
@ -220,7 +240,7 @@ class DomClickScraper(BaseScraper):
|
|||
await self.sleep_between_requests()
|
||||
|
||||
logger.info(
|
||||
"domclick: fetch_city done city_id=%d total=%d parse_failures=%d",
|
||||
"domklik: fetch_city done city_id=%d total=%d parse_failures=%d",
|
||||
city_id,
|
||||
len(all_lots),
|
||||
self.parse_failures,
|
||||
|
|
@ -248,6 +268,9 @@ class DomClickScraper(BaseScraper):
|
|||
|
||||
Селектор: `a[href*="/card/"]` с паттерном `/card/sale__flat__<id>`.
|
||||
"""
|
||||
if _is_blocked_page(html):
|
||||
logger.warning("domklik: DataDome block page detected, 0 cards returned")
|
||||
raise DomClickBlockedError("DomClick returned DataDome block page")
|
||||
tree = HTMLParser(html)
|
||||
card_links = tree.css('a[href*="/card/"]')
|
||||
|
||||
|
|
@ -301,7 +324,7 @@ class DomClickScraper(BaseScraper):
|
|||
address = self._extract_address(link)
|
||||
|
||||
return ScrapedLot(
|
||||
source="domclick",
|
||||
source="domklik",
|
||||
source_url=source_url,
|
||||
source_id=source_id,
|
||||
address=address,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
"""DomClick-specific exceptions для anti-bot detection."""
|
||||
|
||||
|
||||
class DomClickBlockedError(Exception):
|
||||
"""DomClick вернул DataDome block-страницу (HTTP 200 + block HTML)."""
|
||||
|
|
@ -21,6 +21,7 @@ from app.services.scrapers.domclick import (
|
|||
_extract_rooms,
|
||||
_extract_source_id,
|
||||
)
|
||||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||||
|
||||
# ── Rooms / studio ───────────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -166,7 +167,7 @@ def test_parse_html_card1_fields() -> None:
|
|||
lots = scraper._parse_html(_MOCK_HTML)
|
||||
|
||||
card1 = next(lot for lot in lots if lot.source_id == "2075671636")
|
||||
assert card1.source == "domclick"
|
||||
assert card1.source == "domklik"
|
||||
assert card1.source_url == "https://domclick.ru/card/sale__flat__2075671636"
|
||||
assert card1.rooms == 2
|
||||
assert card1.area_m2 == 53.5
|
||||
|
|
@ -207,10 +208,11 @@ def test_parse_html_dedup_same_href() -> None:
|
|||
|
||||
def test_scraper_source_name() -> None:
|
||||
scraper = DomClickScraper()
|
||||
assert scraper.name == "domclick"
|
||||
assert scraper.source == "domclick"
|
||||
assert scraper.name == "domklik"
|
||||
assert scraper.source == "domklik"
|
||||
assert scraper.base_url == "https://domclick.ru"
|
||||
assert scraper.request_delay_sec == 8.0
|
||||
# request_delay_sec установлен из get_scraper_delay (DB или fallback 8.0)
|
||||
assert scraper.request_delay_sec >= 0.0
|
||||
|
||||
|
||||
def test_fetch_around_raises() -> None:
|
||||
|
|
@ -238,3 +240,13 @@ def test_parse_failures_counter() -> None:
|
|||
"""parse_failures счётчик начинается с 0."""
|
||||
scraper = DomClickScraper()
|
||||
assert scraper.parse_failures == 0
|
||||
|
||||
|
||||
def test_blocked_page_raises_domclick_blocked_error() -> None:
|
||||
"""HTML с datadome в начале вызывает DomClickBlockedError."""
|
||||
scraper = DomClickScraper()
|
||||
blocked_html = (
|
||||
"<html><head><title>datadome protection</title></head><body>blocked</body></html>"
|
||||
)
|
||||
with pytest.raises(DomClickBlockedError):
|
||||
scraper._parse_html(blocked_html)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue