bot-backend
dbef242716
- DomClickScraper переписан на GET bff-search-web.domclick.ru/api/offers/v1 с рекурсивным бинарным price-band bucketing (OFFSET_CAP=2000, open-band split через _HIGH_ANCHOR=30M); geo-guard по offerRegionName==ЕКБ / bbox - scrape_pipeline.py: rooms=[0..5] (добавлен 5+), pages default=100, _DOMCLICK_PER_FETCH_S=4.0, честный статус mark_failed при 0 лотов + errors - config.py: bff-search-web.domclick.ru добавлен в scrape_allowed_hosts - data/sql/132_update_domclick_sweep_params.sql: UPDATE default_params (DORMANT)
432 lines
17 KiB
Python
432 lines
17 KiB
Python
"""Unit-тесты для DomClick JSON API scraper (Layer A rewrite).
|
||
|
||
Offline, без сети и БД. Монкейпатчим BrowserFetcher / _fetch_json на canned JSON.
|
||
|
||
Coverage:
|
||
- _offer_to_lot: поля ScrapedLot (source_id, source_url, lat/lon, address,
|
||
rooms, area, floor, total_floors, year_built, price, listing_segment, source)
|
||
- Studio bucket (room_token="st"): rooms=0
|
||
- Гео-guard: Челябинск оффер отбрасывается, EKB оффер проходит
|
||
- Block detection: не-JSON body → DomClickBlockedError
|
||
- URL building: rooms=5%2B в URL; sale_price__gte/lte в banded URL;
|
||
offset/limit в list URL и отсутствует в count URL
|
||
- Bucketing: snippetsCount>2000 → рекурсивный split; offset не достигает 2000
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
from datetime import date
|
||
from urllib.parse import parse_qs, urlparse
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from app.services.scrapers.domclick import DomClickScraper, _is_ekb, _parse_publish_date
|
||
from app.services.scrapers.domclick_exceptions import DomClickBlockedError
|
||
|
||
# ── Фикстуры ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _make_scraper() -> DomClickScraper:
|
||
"""Создаём DomClickScraper без DB (монкейпатчим get_scraper_delay)."""
|
||
from unittest.mock import patch
|
||
|
||
with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=0.0):
|
||
scraper = DomClickScraper()
|
||
scraper._browser = object() # не None — assertion в _fetch_json проверяет
|
||
return scraper
|
||
|
||
|
||
_SAMPLE_OFFER: dict = {
|
||
"id": 2075671636,
|
||
"path": "/card/sale__flat__2075671636",
|
||
"location": {"lat": 56.83, "lon": 60.61},
|
||
"address": {"displayName": "Екатеринбург, улица Ленина, 10"},
|
||
"objectInfo": {
|
||
"area": 52.3,
|
||
"rooms": 2,
|
||
"floor": 5,
|
||
"isApartment": False,
|
||
},
|
||
"house": {"floors": 16, "buildYear": 2005},
|
||
"price": 6500000,
|
||
"squarePrice": 124283,
|
||
"flatComplex": {"id": 123, "name": "ЖК Тест", "slug": "zhk-test"},
|
||
"isRosreestrApproved": True,
|
||
"publishedDate": "2026-06-01",
|
||
"updatedDate": "2026-06-15",
|
||
"lastPriceHistoryState": None,
|
||
"offerRegionName": "Екатеринбург",
|
||
}
|
||
|
||
_STUDIO_OFFER: dict = {
|
||
"id": 9001,
|
||
"path": "/card/sale__flat__9001",
|
||
"location": {"lat": 56.84, "lon": 60.60},
|
||
"address": {"displayName": "Екатеринбург, пр-т Ленина, 50"},
|
||
"objectInfo": {"area": 24.5, "rooms": None, "floor": 3, "isApartment": False},
|
||
"house": {"floors": 10, "buildYear": 2020},
|
||
"price": 3100000,
|
||
"squarePrice": 126531,
|
||
"flatComplex": None,
|
||
"isRosreestrApproved": False,
|
||
"publishedDate": None,
|
||
"updatedDate": None,
|
||
"lastPriceHistoryState": None,
|
||
"offerRegionName": "Екатеринбург",
|
||
}
|
||
|
||
_CHELYABINSK_OFFER: dict = {
|
||
"id": 99999,
|
||
"path": "/card/sale__flat__99999",
|
||
"location": {"lat": 55.15, "lon": 61.39}, # вне EKB bbox
|
||
"address": {"displayName": "Челябинск, улица Ленина, 1"},
|
||
"objectInfo": {"area": 40.0, "rooms": 1, "floor": 3, "isApartment": False},
|
||
"house": {"floors": 9, "buildYear": 1990},
|
||
"price": 3000000,
|
||
"squarePrice": 75000,
|
||
"flatComplex": None,
|
||
"isRosreestrApproved": False,
|
||
"publishedDate": None,
|
||
"updatedDate": None,
|
||
"lastPriceHistoryState": None,
|
||
"offerRegionName": "Челябинск", # не Екатеринбург
|
||
}
|
||
|
||
|
||
# ── _offer_to_lot — базовый маппинг ──────────────────────────────────────────
|
||
|
||
|
||
def test_offer_to_lot_basic_fields() -> None:
|
||
"""Репрезентативный оффер → ScrapedLot с корректными полями."""
|
||
scraper = _make_scraper()
|
||
lot = scraper._offer_to_lot(_SAMPLE_OFFER, room_token="2")
|
||
|
||
assert lot is not None
|
||
assert lot.source == "domklik"
|
||
assert lot.source_id == "2075671636"
|
||
assert lot.source_url == "https://domclick.ru/card/sale__flat__2075671636"
|
||
assert lot.lat == pytest.approx(56.83)
|
||
assert lot.lon == pytest.approx(60.61)
|
||
assert lot.address == "Екатеринбург, улица Ленина, 10"
|
||
assert lot.rooms == 2
|
||
assert lot.area_m2 == pytest.approx(52.3)
|
||
assert lot.floor == 5
|
||
assert lot.total_floors == 16
|
||
assert lot.year_built == 2005
|
||
assert lot.price_rub == 6_500_000
|
||
assert lot.price_per_m2 == 124_283
|
||
assert lot.listing_segment == "vtorichka"
|
||
assert lot.listing_date == date(2026, 6, 1)
|
||
assert lot.publish_date == date(2026, 6, 1)
|
||
assert lot.raw_payload is not None
|
||
assert lot.raw_payload["is_rosreestr_approved"] is True
|
||
assert lot.raw_payload["flat_complex"]["name"] == "ЖК Тест"
|
||
|
||
|
||
def test_offer_to_lot_studio_rooms_zero() -> None:
|
||
"""Studio bucket (room_token='st') → rooms=0 независимо от objectInfo.rooms."""
|
||
scraper = _make_scraper()
|
||
lot = scraper._offer_to_lot(_STUDIO_OFFER, room_token="st")
|
||
|
||
assert lot is not None
|
||
assert lot.rooms == 0
|
||
assert lot.area_m2 == pytest.approx(24.5)
|
||
assert lot.price_rub == 3_100_000
|
||
# publishedDate=None → None
|
||
assert lot.listing_date is None
|
||
|
||
|
||
def test_offer_to_lot_missing_price_returns_none() -> None:
|
||
"""Оффер без цены → None."""
|
||
scraper = _make_scraper()
|
||
offer = {**_SAMPLE_OFFER, "price": None}
|
||
assert scraper._offer_to_lot(offer, "2") is None
|
||
|
||
|
||
def test_offer_to_lot_zero_price_returns_none() -> None:
|
||
"""Оффер с price=0 → None."""
|
||
scraper = _make_scraper()
|
||
offer = {**_SAMPLE_OFFER, "price": 0}
|
||
assert scraper._offer_to_lot(offer, "2") is None
|
||
|
||
|
||
def test_offer_to_lot_missing_id_returns_none() -> None:
|
||
"""Оффер без id → None."""
|
||
scraper = _make_scraper()
|
||
offer = {k: v for k, v in _SAMPLE_OFFER.items() if k != "id"}
|
||
assert scraper._offer_to_lot(offer, "2") is None
|
||
|
||
|
||
# ── Гео-guard ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_geo_guard_ekb_by_region_name() -> None:
|
||
"""offerRegionName='Екатеринбург' → EKB."""
|
||
assert _is_ekb({"offerRegionName": "Екатеринбург"}) is True
|
||
|
||
|
||
def test_geo_guard_ekb_by_bbox_coords() -> None:
|
||
"""Координаты внутри EKB bbox → EKB (даже без offerRegionName)."""
|
||
assert _is_ekb({"location": {"lat": 56.83, "lon": 60.61}}) is True
|
||
|
||
|
||
def test_geo_guard_chelyabinsk_dropped() -> None:
|
||
"""Челябинск (имя и bbox) → не EKB."""
|
||
assert _is_ekb(_CHELYABINSK_OFFER) is False
|
||
|
||
|
||
def test_geo_guard_out_of_bbox_no_name() -> None:
|
||
"""Координаты вне EKB bbox и нет offerRegionName → не EKB."""
|
||
assert _is_ekb({"location": {"lat": 55.0, "lon": 73.0}}) is False
|
||
|
||
|
||
def test_parse_page_drops_non_ekb() -> None:
|
||
"""_parse_page: Челябинск оффер отброшен, EKB проходит."""
|
||
scraper = _make_scraper()
|
||
dropped_ref = [0]
|
||
items = [_SAMPLE_OFFER, _CHELYABINSK_OFFER]
|
||
lots = scraper._parse_page(items, "2", dropped_ref)
|
||
|
||
assert len(lots) == 1
|
||
assert lots[0].source_id == "2075671636"
|
||
assert dropped_ref[0] == 1 # один дроп
|
||
|
||
|
||
# ── Block detection ───────────────────────────────────────────────────────────
|
||
|
||
|
||
async def test_fetch_json_html_body_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Если BrowserFetcher.fetch возвращает HTML → DomClickBlockedError."""
|
||
|
||
class _FakeBrowser:
|
||
async def fetch(self, url: str) -> str:
|
||
return "<html><head><title>QRATOR protection</title></head></html>"
|
||
|
||
scraper = _make_scraper()
|
||
scraper._browser = _FakeBrowser()
|
||
|
||
with pytest.raises(DomClickBlockedError):
|
||
await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1")
|
||
|
||
|
||
async def test_fetch_json_non_json_body_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Не-JSON body (не HTML, просто мусор) → DomClickBlockedError."""
|
||
|
||
class _FakeBrowser:
|
||
async def fetch(self, url: str) -> str:
|
||
return "not valid json at all |||"
|
||
|
||
scraper = _make_scraper()
|
||
scraper._browser = _FakeBrowser()
|
||
|
||
with pytest.raises(DomClickBlockedError):
|
||
await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1")
|
||
|
||
|
||
async def test_fetch_json_valid_json_returns_dict(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Валидный JSON → dict без исключений."""
|
||
|
||
class _FakeBrowser:
|
||
async def fetch(self, url: str) -> str:
|
||
return json.dumps({"result": {"snippetsCount": 42}})
|
||
|
||
scraper = _make_scraper()
|
||
scraper._browser = _FakeBrowser()
|
||
|
||
data = await scraper._fetch_json("https://bff-search-web.domclick.ru/api/offers/count/v1?x=1")
|
||
assert data["result"]["snippetsCount"] == 42
|
||
|
||
|
||
# ── URL building ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_build_count_url_no_offset_limit() -> None:
|
||
"""count URL не содержит offset и limit."""
|
||
scraper = _make_scraper()
|
||
url = scraper._build_count_url("2", lo=0, hi=None)
|
||
qs = parse_qs(urlparse(url).query)
|
||
assert "offset" not in qs
|
||
assert "limit" not in qs
|
||
assert "/count/v1" in url
|
||
|
||
|
||
def test_build_list_url_has_offset_limit() -> None:
|
||
"""list URL содержит offset и limit."""
|
||
scraper = _make_scraper()
|
||
url = scraper._build_list_url("2", lo=0, hi=None, offset=40)
|
||
qs = parse_qs(urlparse(url).query)
|
||
assert qs["offset"] == ["40"]
|
||
assert qs["limit"] == ["20"]
|
||
assert "/offers/v1" in url
|
||
|
||
|
||
def test_build_list_url_5plus_encoded() -> None:
|
||
"""rooms=5+ корректно URL-кодируется (5%2B или эквивалент)."""
|
||
scraper = _make_scraper()
|
||
url = scraper._build_list_url("5+", lo=0, hi=None, offset=0)
|
||
# urlencode кодирует '+' как '%2B в query string
|
||
assert "rooms=5%2B" in url or "rooms=5+" in url # оба варианта корректны
|
||
# Проверяем через parse_qs
|
||
qs = parse_qs(urlparse(url).query)
|
||
assert qs.get("rooms") == ["5+"]
|
||
|
||
|
||
def test_build_list_url_price_band() -> None:
|
||
"""Бандовый URL содержит sale_price__gte и sale_price__lte."""
|
||
scraper = _make_scraper()
|
||
url = scraper._build_list_url("2", lo=2_000_000, hi=5_000_000, offset=0)
|
||
qs = parse_qs(urlparse(url).query)
|
||
assert qs["sale_price__gte"] == ["2000000"]
|
||
assert qs["sale_price__lte"] == ["5000000"]
|
||
|
||
|
||
def test_build_list_url_no_price_params_when_open() -> None:
|
||
"""Открытый бакет (lo=0, hi=None) → нет ценовых параметров."""
|
||
scraper = _make_scraper()
|
||
url = scraper._build_list_url("1", lo=0, hi=None, offset=0)
|
||
qs = parse_qs(urlparse(url).query)
|
||
assert "sale_price__gte" not in qs
|
||
assert "sale_price__lte" not in qs
|
||
|
||
|
||
def test_build_url_contains_ekb_params() -> None:
|
||
"""URL содержит EKB region GUID и aids."""
|
||
scraper = _make_scraper()
|
||
url = scraper._build_count_url("1", lo=0, hi=None)
|
||
assert "0d475b79-88de-4054-818c-37d8f9d0d440" in url
|
||
qs = parse_qs(urlparse(url).query)
|
||
assert qs["aids"] == ["20561"]
|
||
assert qs["deal_type"] == ["sale"]
|
||
|
||
|
||
# ── Bucketing — split при snippetsCount > 2000 ────────────────────────────────
|
||
|
||
|
||
async def test_bucketing_split_on_overflow(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""snippetsCount>2000 для открытого бакета → split; offset never ≥ 2000."""
|
||
call_log: list[str] = []
|
||
|
||
def _make_ekb_offer(offer_id: str) -> dict:
|
||
return {
|
||
"id": offer_id,
|
||
"path": f"/card/sale__flat__{offer_id}",
|
||
"location": {"lat": 56.83, "lon": 60.61},
|
||
"address": {"displayName": "Екатеринбург, ул. Тест, 1"},
|
||
"objectInfo": {"area": 45.0, "rooms": 1, "floor": 3, "isApartment": False},
|
||
"house": {"floors": 9, "buildYear": 2000},
|
||
"price": 4000000,
|
||
"squarePrice": 88888,
|
||
"flatComplex": None,
|
||
"isRosreestrApproved": False,
|
||
"publishedDate": None,
|
||
"updatedDate": None,
|
||
"lastPriceHistoryState": None,
|
||
"offerRegionName": "Екатеринбург",
|
||
}
|
||
|
||
async def mock_fetch_json(self: DomClickScraper, url: str) -> dict:
|
||
call_log.append(url)
|
||
parsed = urlparse(url)
|
||
qs = parse_qs(parsed.query)
|
||
|
||
if "/count/v1" in url:
|
||
has_gte = "sale_price__gte" in qs
|
||
has_lte = "sale_price__lte" in qs
|
||
if not has_gte and not has_lte:
|
||
# Открытый бакет [0, None] → overflow, trigger split
|
||
return {"result": {"snippetsCount": 2001, "offersCount": 2001}}
|
||
elif has_lte and int(qs["sale_price__lte"][0]) == 30_000_000:
|
||
# [0, 30M] → within cap
|
||
return {"result": {"snippetsCount": 20, "offersCount": 20}}
|
||
else:
|
||
# Хвост [30M+1, None] → empty
|
||
return {"result": {"snippetsCount": 0, "offersCount": 0}}
|
||
|
||
# list URL
|
||
if "/offers/v1" in url:
|
||
offset_val = int(qs.get("offset", ["0"])[0])
|
||
if offset_val == 0:
|
||
return {"result": {"items": [_make_ekb_offer("101")]}}
|
||
return {"result": {"items": []}}
|
||
|
||
return {"result": {}}
|
||
|
||
import asyncio as _asyncio
|
||
|
||
monkeypatch.setattr(DomClickScraper, "_fetch_json", mock_fetch_json)
|
||
# sleep_between_requests is inherited from BaseScraper → shadow on subclass
|
||
monkeypatch.setattr(DomClickScraper, "sleep_between_requests", lambda self: _asyncio.sleep(0))
|
||
|
||
from unittest.mock import patch
|
||
|
||
with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=0.0):
|
||
scraper = DomClickScraper()
|
||
scraper._browser = object() # не None
|
||
|
||
seen: dict = {}
|
||
geo_ref = [0]
|
||
await scraper._walk_price_range(
|
||
room_token="1",
|
||
lo=0,
|
||
hi=None,
|
||
seen=seen,
|
||
pages=100,
|
||
geo_dropped_ref=geo_ref,
|
||
)
|
||
|
||
# Split произошёл: как минимум 2 count-вызова (open band + [0, 30M])
|
||
count_urls = [u for u in call_log if "/count/v1" in u]
|
||
assert len(count_urls) >= 2, f"Expected split, got count_urls={count_urls}"
|
||
|
||
# Ни один list-запрос не должен иметь offset >= 2000
|
||
list_urls = [u for u in call_log if "/offers/v1" in u]
|
||
for lu in list_urls:
|
||
offset_val = int(parse_qs(urlparse(lu).query).get("offset", ["0"])[0])
|
||
assert offset_val < 2000, f"offset={offset_val} >= OFFSET_CAP in url={lu}"
|
||
|
||
# Лот прошёл и дедуплицирован
|
||
assert "101" in seen
|
||
|
||
|
||
# ── _parse_publish_date ───────────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"val, expected",
|
||
[
|
||
("2026-06-01", date(2026, 6, 1)),
|
||
("2026-06-01T12:00:00Z", date(2026, 6, 1)),
|
||
(1748736000, date(2025, 6, 1)), # 2025-06-01 00:00 UTC epoch
|
||
("1748736000", date(2025, 6, 1)),
|
||
(None, None),
|
||
("invalid-date", None),
|
||
("", None),
|
||
],
|
||
)
|
||
def test_parse_publish_date(val: object, expected: date | None) -> None:
|
||
result = _parse_publish_date(val)
|
||
assert result == expected
|
||
|
||
|
||
# ── Scraper meta ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_scraper_attributes() -> None:
|
||
scraper = _make_scraper()
|
||
assert scraper.name == "domklik"
|
||
assert scraper.source == "domklik"
|
||
assert scraper.base_url == "https://bff-search-web.domclick.ru"
|
||
assert scraper.request_delay_sec >= 0.0
|
||
assert scraper.parse_failures == 0
|
||
|
||
|
||
def test_fetch_around_raises() -> None:
|
||
import asyncio
|
||
|
||
scraper = _make_scraper()
|
||
with pytest.raises(NotImplementedError, match="geo-radius"):
|
||
asyncio.run(scraper.fetch_around(56.8, 60.6))
|