bot-backend
52fc920507
Old HTML scraper dead: /search?city_id retired in DomClick redesign + VPS IP QRATOR-banned (prod domclick_city_sweep reported done/0 every run). New path: - GET bff-search-web.domclick.ru/api/offers/v1 + count/v1 via BrowserFetcher(source="domclick") -> generic provider -> shared mobile proxy -> bypasses QRATOR (prod-verified live: 20 items returned, no block). - Sweeps 6 room buckets (st/1/2/3/4/5+); recursive binary price-split for oversized buckets (rooms=2 ~2215 > offset cap 2000); count/v1 sizes buckets. - Maps SERP JSON -> ScrapedLot with lat/lon at ~100% (closes avito 54% NULL-lat gap); per-card EKB geo-guard. - Honest status (#1968): QRATOR block OR fetch errors with 0 lots -> mark_failed, not mark_done. Parse failures are per-page resilient (break bucket, keep partial lots) so a transient bad page never aborts the whole sweep. - Schedule shipped DORMANT (enabled=false, pages=100) via 138_domclick_bff_rewrite_schedule.sql; operator enables after prod smoke. 42 scraper tests (price-split boundaries, parse-error resilience, honest status); full backend suite green; ruff clean. Layer B (detail renovation/wallType/ priceHistory) deferred to #1846 follow-up.
526 lines
19 KiB
Python
526 lines
19 KiB
Python
"""Unit-тесты для DomClick BFF scraper (#1968).
|
||
|
||
Тестируем:
|
||
- _extract_json: bare + <pre>-wrapped → dict; QRATOR-маркер → DomClickBlockedError
|
||
- _build_offers_url / _build_count_url: корректные параметры, "5+" → "5%2B",
|
||
offers имеет offset/limit, count — нет
|
||
- _map_item: маппинг realistic offer dict → ScrapedLot; price=0 → None; studio
|
||
- _is_geo_ok: гео-гард по bbox + offerRegionName
|
||
- _sweep_bucket: price-split при count > OFFSET_CAP
|
||
- DomClickScraper: source/name, fetch_around raises NotImplementedError
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
from typing import Any
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.domclick import (
|
||
LTE_MAX,
|
||
OFFSET_CAP,
|
||
DomClickScraper,
|
||
_build_count_url,
|
||
_build_offers_url,
|
||
_extract_json,
|
||
)
|
||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||
|
||
# ── _extract_json ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_extract_json_bare_body() -> None:
|
||
"""Bare JSON body (без HTML-обёртки) корректно парсится."""
|
||
html = '{"result": {"items": [], "snippetsCount": 42}}'
|
||
data = _extract_json(html)
|
||
assert data["result"]["snippetsCount"] == 42
|
||
|
||
|
||
def test_extract_json_pre_wrapped() -> None:
|
||
"""JSON в <pre>-теге корректно парсится."""
|
||
html = (
|
||
"<html><head><title>DomClick</title></head>"
|
||
'<body><pre>{"result": {"items": [{"id": 1}]}}</pre></body></html>'
|
||
)
|
||
data = _extract_json(html)
|
||
assert data["result"]["items"][0]["id"] == 1
|
||
|
||
|
||
def test_extract_json_qrator_marker_raises() -> None:
|
||
"""HTML с QRATOR-маркером вызывает DomClickBlockedError."""
|
||
blocked_html = (
|
||
"<html><head><title>QRATOR protection</title></head>"
|
||
"<body>Access denied by bot_mitigation system</body></html>"
|
||
)
|
||
with pytest.raises(DomClickBlockedError):
|
||
_extract_json(blocked_html)
|
||
|
||
|
||
def test_extract_json_captcha_raises() -> None:
|
||
"""HTML с captcha-маркером вызывает DomClickBlockedError."""
|
||
html = "<html><body>Please solve the captcha to continue</body></html>"
|
||
with pytest.raises(DomClickBlockedError):
|
||
_extract_json(html)
|
||
|
||
|
||
def test_extract_json_no_json_raises() -> None:
|
||
"""HTML без JSON-объекта вызывает ValueError."""
|
||
with pytest.raises(ValueError, match="No JSON"):
|
||
_extract_json("<html><body>plain text</body></html>")
|
||
|
||
|
||
# ── _build_offers_url ────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_build_offers_url_basic_params() -> None:
|
||
url = _build_offers_url("1", None, None, 0)
|
||
assert "deal_type=sale" in url
|
||
assert "category=living" in url
|
||
assert "offer_type=flat" in url
|
||
assert "rooms=1" in url
|
||
assert "offset=0" in url
|
||
assert "limit=20" in url
|
||
assert "sale_price__gte" not in url
|
||
assert "sale_price__lte" not in url
|
||
|
||
|
||
def test_build_offers_url_5plus_encoded() -> None:
|
||
"""rooms='5+' должен кодироваться как '5%2B' в URL."""
|
||
url = _build_offers_url("5+", None, None, 0)
|
||
assert "5%2B" in url
|
||
# literal '+' в URL не должно быть для значения rooms
|
||
assert "rooms=5+" not in url
|
||
|
||
|
||
def test_build_offers_url_with_price_range() -> None:
|
||
url = _build_offers_url("2", 3_000_000, 6_000_000, 40)
|
||
assert "sale_price__gte=3000000" in url
|
||
assert "sale_price__lte=6000000" in url
|
||
assert "offset=40" in url
|
||
|
||
|
||
def test_build_offers_url_ekb_guid_present() -> None:
|
||
url = _build_offers_url("3", None, None, 0)
|
||
assert "0d475b79-88de-4054-818c-37d8f9d0d440" in url
|
||
assert "aids=20561" in url
|
||
|
||
|
||
def test_build_offers_url_bff_host() -> None:
|
||
url = _build_offers_url("1", None, None, 0)
|
||
assert url.startswith("https://bff-search-web.domclick.ru/api/offers/v1?")
|
||
|
||
|
||
# ── _build_count_url ─────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_build_count_url_no_offset_limit() -> None:
|
||
"""count url не должен содержать offset и limit."""
|
||
url = _build_count_url("2", None, None)
|
||
assert "offset" not in url
|
||
assert "limit" not in url
|
||
|
||
|
||
def test_build_count_url_5plus_encoded() -> None:
|
||
url = _build_count_url("5+", None, None)
|
||
assert "5%2B" in url
|
||
assert "rooms=5+" not in url
|
||
|
||
|
||
def test_build_count_url_with_price() -> None:
|
||
url = _build_count_url("1", 1_000_000, 4_000_000)
|
||
assert "sale_price__gte=1000000" in url
|
||
assert "sale_price__lte=4000000" in url
|
||
|
||
|
||
def test_build_count_url_bff_host() -> None:
|
||
url = _build_count_url("1", None, None)
|
||
assert url.startswith("https://bff-search-web.domclick.ru/api/offers/count/v1?")
|
||
|
||
|
||
# ── _map_item → ScrapedLot ────────────────────────────────────────────────────
|
||
|
||
_REALISTIC_ITEM: dict[str, Any] = {
|
||
"id": 2075671636,
|
||
"path": "https://domclick.ru/card/sale__flat__2075671636",
|
||
"location": {"lat": 56.838, "lon": 60.612},
|
||
"address": {"displayName": "Екатеринбург, улица Малышева, 1"},
|
||
"objectInfo": {"area": 53.5, "rooms": 2, "floor": 5, "isApartment": False},
|
||
"house": {"floors": 15, "buildYear": 2003},
|
||
"price": 5_200_000,
|
||
"squarePrice": 97196,
|
||
"flatComplex": {"id": 12345, "name": "ЖК Тест", "slug": "zhk-test"},
|
||
"isRosreestrApproved": True,
|
||
"publishedDate": "2026-06-01T10:00:00+05:00",
|
||
"updatedDate": "2026-06-15T12:00:00+05:00",
|
||
"lastPriceHistoryState": {"direction": "DECREASED"},
|
||
"offerRegionName": "Екатеринбург",
|
||
}
|
||
|
||
|
||
def test_map_item_basic_fields() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
lot = scraper._map_item(_REALISTIC_ITEM)
|
||
assert lot is not None
|
||
assert lot.source == "domklik"
|
||
assert lot.source_id == "2075671636"
|
||
assert lot.source_url == "https://domclick.ru/card/sale__flat__2075671636"
|
||
assert lot.lat == pytest.approx(56.838)
|
||
assert lot.lon == pytest.approx(60.612)
|
||
assert lot.address == "Екатеринбург, улица Малышева, 1"
|
||
assert lot.rooms == 2
|
||
assert lot.area_m2 == pytest.approx(53.5)
|
||
assert lot.floor == 5
|
||
assert lot.total_floors == 15
|
||
assert lot.year_built == 2003
|
||
assert lot.price_rub == 5_200_000
|
||
assert lot.price_per_m2 == 97196
|
||
assert lot.listing_segment == "vtorichka"
|
||
|
||
|
||
def test_map_item_publish_date() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
from datetime import date
|
||
|
||
lot = scraper._map_item(_REALISTIC_ITEM)
|
||
assert lot is not None
|
||
assert lot.publish_date == date(2026, 6, 1)
|
||
|
||
|
||
def test_map_item_raw_payload() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
lot = scraper._map_item(_REALISTIC_ITEM)
|
||
assert lot is not None
|
||
assert lot.raw_payload is not None
|
||
assert lot.raw_payload["isRosreestrApproved"] is True
|
||
assert lot.raw_payload["flatComplex"]["name"] == "ЖК Тест"
|
||
assert lot.raw_payload["lastPriceHistoryState"]["direction"] == "DECREASED"
|
||
|
||
|
||
def test_map_item_studio_force_rooms_zero() -> None:
|
||
"""force_rooms=0 (студия-бакет) перебивает objectInfo.rooms."""
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
item = {**_REALISTIC_ITEM, "objectInfo": {"area": 28.0, "rooms": 1, "floor": 3}}
|
||
lot = scraper._map_item(item, force_rooms=0)
|
||
assert lot is not None
|
||
assert lot.rooms == 0
|
||
|
||
|
||
def test_map_item_zero_price_returns_none() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
item = {**_REALISTIC_ITEM, "price": 0}
|
||
assert scraper._map_item(item) is None
|
||
|
||
|
||
def test_map_item_zero_sentinels_become_none() -> None:
|
||
"""buildYear/area/floor/total_floors == 0 (sentinel «неизвестно») → None.
|
||
|
||
rooms=0 НЕ затрагивается (валидная студия), но здесь force_rooms не задан и
|
||
objectInfo.rooms=2, так что rooms остаётся 2.
|
||
"""
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
item = {
|
||
**_REALISTIC_ITEM,
|
||
"objectInfo": {"area": 0, "rooms": 2, "floor": 0},
|
||
"house": {"floors": 0, "buildYear": 0},
|
||
}
|
||
lot = scraper._map_item(item)
|
||
assert lot is not None
|
||
assert lot.area_m2 is None
|
||
assert lot.floor is None
|
||
assert lot.total_floors is None
|
||
assert lot.year_built is None
|
||
assert lot.rooms == 2 # rooms 0-sentinel НЕ применяется
|
||
|
||
|
||
def test_map_item_rooms_zero_studio_preserved() -> None:
|
||
"""force_rooms=0 даёт rooms=0 (студия) — 0 здесь валиден, не sentinel."""
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
item = {**_REALISTIC_ITEM, "objectInfo": {"area": 24.0, "rooms": 0, "floor": 2}}
|
||
lot = scraper._map_item(item, force_rooms=0)
|
||
assert lot is not None
|
||
assert lot.rooms == 0
|
||
|
||
|
||
def test_map_item_price_per_m2_fallback() -> None:
|
||
"""Если squarePrice отсутствует, price_per_m2 вычисляется из price / area_m2."""
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
item = {**_REALISTIC_ITEM, "squarePrice": None}
|
||
lot = scraper._map_item(item)
|
||
assert lot is not None
|
||
assert lot.price_per_m2 == int(5_200_000 / 53.5)
|
||
|
||
|
||
# ── _is_geo_ok — гео-гард ────────────────────────────────────────────────────
|
||
|
||
|
||
def test_is_geo_ok_valid_ekb_item() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
assert scraper._is_geo_ok(_REALISTIC_ITEM) is True
|
||
|
||
|
||
def test_is_geo_ok_wrong_region_name() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
item = {**_REALISTIC_ITEM, "offerRegionName": "Москва"}
|
||
assert scraper._is_geo_ok(item) is False
|
||
|
||
|
||
def test_is_geo_ok_out_of_bbox_lat() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
item = {**_REALISTIC_ITEM, "location": {"lat": 55.5, "lon": 60.6}}
|
||
assert scraper._is_geo_ok(item) is False
|
||
|
||
|
||
def test_is_geo_ok_out_of_bbox_lon() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
item = {**_REALISTIC_ITEM, "location": {"lat": 56.8, "lon": 61.5}}
|
||
assert scraper._is_geo_ok(item) is False
|
||
|
||
|
||
def test_is_geo_ok_missing_coords() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
item = {**_REALISTIC_ITEM, "location": {"lat": None, "lon": None}}
|
||
assert scraper._is_geo_ok(item) is False
|
||
|
||
|
||
# ── _sweep_bucket price-split ────────────────────────────────────────────────
|
||
|
||
|
||
async def test_sweep_bucket_price_split_boundaries(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""_sweep_bucket: top-level unbounded split даёт contiguous non-overlapping ranges.
|
||
|
||
rooms="2" count>cap один раз, потом ≤cap → ровно 2 leaf-бакета, покрывающих
|
||
[0, LTE_MAX] без перекрытия и без зазора: [(0, mid), (mid+1, LTE_MAX)].
|
||
"""
|
||
call_count: dict[str, int] = {"n": 0}
|
||
paginate_calls: list[tuple[str, int | None, int | None]] = []
|
||
|
||
async def fake_count(
|
||
self: DomClickScraper,
|
||
fetcher: object,
|
||
rooms: str,
|
||
price_gte: int | None,
|
||
price_lte: int | None,
|
||
) -> int:
|
||
call_count["n"] += 1
|
||
# Первый вызов (unbounded) → over cap → price-split; остальные → под cap
|
||
return OFFSET_CAP + 1 if call_count["n"] == 1 else 1
|
||
|
||
async def fake_paginate(
|
||
self: DomClickScraper,
|
||
fetcher: object,
|
||
rooms: str,
|
||
price_gte: int | None,
|
||
price_lte: int | None,
|
||
seen_ids: set[str],
|
||
out_lots: list,
|
||
pages: int,
|
||
) -> None:
|
||
paginate_calls.append((rooms, price_gte, price_lte))
|
||
|
||
monkeypatch.setattr(DomClickScraper, "_count", fake_count)
|
||
monkeypatch.setattr(DomClickScraper, "_paginate", fake_paginate)
|
||
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
scraper.geo_filtered = 0
|
||
scraper.blocked = False
|
||
scraper.fetch_errors = 0
|
||
scraper.request_delay_sec = 0.0
|
||
|
||
await scraper._sweep_bucket(
|
||
fetcher=object(),
|
||
rooms="2",
|
||
price_gte=None,
|
||
price_lte=None,
|
||
seen_ids=set(),
|
||
out_lots=[],
|
||
pages=10,
|
||
)
|
||
|
||
# 1 count call (over cap) + 2 recursive count calls (≤ cap) = 3 total
|
||
assert call_count["n"] == 3
|
||
# Both sub-buckets paginated, с точными границами split'а unbounded-диапазона.
|
||
mid = (0 + LTE_MAX) // 2
|
||
assert paginate_calls == [("2", 0, mid), ("2", mid + 1, LTE_MAX)]
|
||
# Sanity: contiguous (нет зазора) + non-overlapping + покрывают весь хвост.
|
||
assert paginate_calls[0][2] + 1 == paginate_calls[1][1] # mid + 1
|
||
assert paginate_calls[0][1] == 0
|
||
assert paginate_calls[1][2] == LTE_MAX
|
||
|
||
|
||
async def test_sweep_bucket_zero_count_no_paginate(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""_sweep_bucket при count==0 не вызывает _paginate."""
|
||
paginate_called = False
|
||
|
||
async def fake_count(self: DomClickScraper, *_args: object, **_kwargs: object) -> int:
|
||
return 0
|
||
|
||
async def fake_paginate(self: DomClickScraper, *_args: object, **_kwargs: object) -> None:
|
||
nonlocal paginate_called
|
||
paginate_called = True
|
||
|
||
monkeypatch.setattr(DomClickScraper, "_count", fake_count)
|
||
monkeypatch.setattr(DomClickScraper, "_paginate", fake_paginate)
|
||
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
scraper.geo_filtered = 0
|
||
scraper.blocked = False
|
||
scraper.fetch_errors = 0
|
||
scraper.request_delay_sec = 0.0
|
||
|
||
await scraper._sweep_bucket(
|
||
fetcher=object(),
|
||
rooms="3",
|
||
price_gte=None,
|
||
price_lte=None,
|
||
seen_ids=set(),
|
||
out_lots=[],
|
||
pages=5,
|
||
)
|
||
assert paginate_called is False
|
||
|
||
|
||
# ── _paginate resilience — mid-bucket parse error ────────────────────────────
|
||
|
||
|
||
class _FakeFetcher:
|
||
"""Стаб BrowserFetcher: возвращает заранее заданные ответы по порядку вызовов."""
|
||
|
||
def __init__(self, responses: list[str]) -> None:
|
||
self._responses = responses
|
||
self._idx = 0
|
||
|
||
async def fetch(self, _url: str) -> str:
|
||
resp = self._responses[min(self._idx, len(self._responses) - 1)]
|
||
self._idx += 1
|
||
return resp
|
||
|
||
|
||
async def test_paginate_mid_bucket_parse_error_preserves_lots() -> None:
|
||
"""Garbled JSON на page 2 → bucket breaks, page-1 lots survive, fetch_errors++.
|
||
|
||
Page 1 — валидный JSON с 20 items (PAGE_SIZE) → продолжаем; page 2 — мусор
|
||
(ValueError из _extract_json) → fetch_errors++ и break, уже собранные lots целы.
|
||
"""
|
||
import json
|
||
|
||
page1_items = [
|
||
{
|
||
"id": 1000 + i,
|
||
"path": f"https://domclick.ru/card/sale__flat__{1000 + i}",
|
||
"location": {"lat": 56.84, "lon": 60.6},
|
||
"address": {"displayName": "Екатеринбург, тест"},
|
||
"objectInfo": {"area": 40.0, "rooms": 1, "floor": 3},
|
||
"house": {"floors": 9, "buildYear": 2005},
|
||
"price": 4_000_000 + i,
|
||
"squarePrice": 100000,
|
||
"offerRegionName": "Екатеринбург",
|
||
}
|
||
for i in range(20)
|
||
]
|
||
page1 = json.dumps({"result": {"items": page1_items}})
|
||
garbled = "<html><body><pre>{not valid json at all</pre></body></html>"
|
||
fetcher = _FakeFetcher([page1, garbled])
|
||
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
scraper.geo_filtered = 0
|
||
scraper.blocked = False
|
||
scraper.fetch_errors = 0
|
||
scraper.request_delay_sec = 0.0
|
||
|
||
out_lots: list = []
|
||
await scraper._paginate(
|
||
fetcher=fetcher,
|
||
rooms="1",
|
||
price_gte=None,
|
||
price_lte=None,
|
||
seen_ids=set(),
|
||
out_lots=out_lots,
|
||
pages=10,
|
||
)
|
||
|
||
assert len(out_lots) == 20 # page-1 lots сохранены
|
||
assert scraper.fetch_errors == 1 # page-2 garbled посчитан
|
||
|
||
|
||
async def test_fetch_city_bucket_error_continues_to_next_bucket(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Ошибка в одном бакете НЕ убивает весь sweep — остальные бакеты отрабатывают."""
|
||
swept: list[str] = []
|
||
|
||
async def fake_sweep_bucket(
|
||
self: DomClickScraper,
|
||
fetcher: object,
|
||
rooms: str,
|
||
price_gte: int | None,
|
||
price_lte: int | None,
|
||
seen_ids: set[str],
|
||
out_lots: list,
|
||
pages: int,
|
||
) -> None:
|
||
swept.append(rooms)
|
||
if rooms == "2":
|
||
raise ValueError("garbled bucket")
|
||
|
||
# Заглушка BrowserFetcher через async context manager
|
||
class _BFStub:
|
||
async def __aenter__(self) -> _BFStub:
|
||
return self
|
||
|
||
async def __aexit__(self, *_a: object) -> None:
|
||
return None
|
||
|
||
async def fetch(self, _url: str) -> str:
|
||
return "{}"
|
||
|
||
import app.services.scrapers.browser_fetcher as bf_mod
|
||
|
||
monkeypatch.setattr(DomClickScraper, "_sweep_bucket", fake_sweep_bucket)
|
||
monkeypatch.setattr(bf_mod, "BrowserFetcher", lambda **_k: _BFStub())
|
||
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
scraper.geo_filtered = 0
|
||
scraper.blocked = False
|
||
scraper.fetch_errors = 0
|
||
scraper.request_delay_sec = 0.0
|
||
|
||
lots = await scraper.fetch_city(city_id=4, rooms=None, pages=5)
|
||
|
||
# Все 6 бакетов посещены несмотря на ошибку в "2"
|
||
assert swept == ["st", "1", "2", "3", "4", "5+"]
|
||
assert scraper.fetch_errors == 1
|
||
assert scraper.blocked is False
|
||
assert lots == []
|
||
|
||
|
||
# ── DomClickScraper — class attrs ─────────────────────────────────────────────
|
||
|
||
|
||
def test_scraper_source_name() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
scraper.parse_failures = 0
|
||
scraper.geo_filtered = 0
|
||
scraper.blocked = False
|
||
scraper.request_delay_sec = 0.0
|
||
assert scraper.name == "domklik"
|
||
assert scraper.source == "domklik"
|
||
assert scraper.base_url == "https://domclick.ru"
|
||
|
||
|
||
def test_fetch_around_raises() -> None:
|
||
scraper = DomClickScraper.__new__(DomClickScraper)
|
||
with pytest.raises(NotImplementedError, match="geo-radius"):
|
||
asyncio.run(scraper.fetch_around(56.8, 60.6))
|