bot-backend
7b36f86ea6
Some checks failed
Deploy Trade-In / deploy (push) Blocked by required conditions
Deploy Trade-In / changes (push) Successful in 12s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 3m5s
Deploy Trade-In / build-backend (push) Has been cancelled
144 lines
5.6 KiB
Python
144 lines
5.6 KiB
Python
"""Offline unit tests for DomClickCitySweepCounters + DomClick parser.
|
||
|
||
Legacy `run_domclick_city_sweep` orchestration behavior (sweep phases, cooperative
|
||
cancel, honest mark_done/mark_failed/mark_banned statuses) удалён вместе с
|
||
`app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная regression-coverage
|
||
теперь в `test_scraper_kit_pipeline_parity2.py::test_domclick_city_sweep_*`
|
||
(kit `run_domclick_city_sweep`).
|
||
|
||
`app.services.scrapers.domclick` удалён (#2397 Part E2, 0 runtime-импортёров) —
|
||
`_map_item` ретаргетирован на kit `scraper_kit.providers.domclick.serp.DomClickScraper`
|
||
(единственный живой путь).
|
||
|
||
Остались:
|
||
- DomClickCitySweepCounters (dataclass, из scraper_kit) — defaults + to_dict
|
||
- DomClickScraper._map_item (kit) — BFF offer-item → ScrapedLot маппинг (не
|
||
зависит от оркестрации)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
import pytest
|
||
from scraper_kit.providers.domclick.serp import DomClickScraper
|
||
|
||
# ── DomClickCitySweepCounters ───────────────────────────────────────────────
|
||
|
||
|
||
def test_domclick_sweep_counters_defaults() -> None:
|
||
from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters
|
||
|
||
c = DomClickCitySweepCounters()
|
||
assert c.lots_fetched == 0
|
||
assert c.lots_inserted == 0
|
||
assert c.lots_updated == 0
|
||
assert c.pages_fetched == 0
|
||
assert c.errors_count == 0
|
||
assert c.blocked == 0
|
||
assert c.geo_filtered == 0
|
||
|
||
|
||
def test_domclick_sweep_counters_to_dict_all_keys() -> None:
|
||
from dataclasses import fields
|
||
|
||
from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters
|
||
|
||
c = DomClickCitySweepCounters()
|
||
d = c.to_dict()
|
||
expected = {f.name for f in fields(c)}
|
||
assert set(d.keys()) == expected
|
||
|
||
|
||
# ── _map_item ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_map_item_basic_mapping() -> None:
|
||
"""DomClickScraper._map_item маппит BFF offer-item → ScrapedLot корректно."""
|
||
config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
|
||
scraper = DomClickScraper(config, delay_provider=lambda _name: 8.0)
|
||
item = {
|
||
"id": 111222333,
|
||
"path": "https://domclick.ru/card/sale__flat__111222333",
|
||
"location": {"lat": 56.838, "lon": 60.612},
|
||
"address": {"displayName": "Екатеринбург, улица Ленина, 10"},
|
||
"objectInfo": {"area": 52.3, "rooms": 2, "floor": 5},
|
||
"house": {"floors": 16, "buildYear": 2010},
|
||
"price": 6_500_000,
|
||
"squarePrice": 124282,
|
||
"flatComplex": None,
|
||
"isRosreestrApproved": False,
|
||
"publishedDate": "2026-06-01T10:00:00+05:00",
|
||
"updatedDate": None,
|
||
"lastPriceHistoryState": None,
|
||
"offerRegionName": "Екатеринбург",
|
||
}
|
||
lot = scraper._map_item(item)
|
||
assert lot is not None
|
||
assert lot.source == "domklik"
|
||
assert lot.source_id == "111222333"
|
||
assert lot.rooms == 2
|
||
assert lot.area_m2 == pytest.approx(52.3)
|
||
assert lot.floor == 5
|
||
assert lot.total_floors == 16
|
||
assert lot.price_rub == 6_500_000
|
||
assert lot.listing_segment == "vtorichka"
|
||
assert lot.lat == pytest.approx(56.838)
|
||
assert lot.lon == pytest.approx(60.612)
|
||
|
||
|
||
# ── fetch_city: report_ban на QRATOR-блок (#2600 п.1) ───────────────────────────
|
||
|
||
|
||
class _FakeFetcher:
|
||
"""Заглушка BrowserFetcher: async context manager + report_ban recorder."""
|
||
|
||
def __init__(self) -> None:
|
||
self.banned: list[str] = []
|
||
|
||
async def __aenter__(self) -> _FakeFetcher:
|
||
return self
|
||
|
||
async def __aexit__(self, *args: object) -> None:
|
||
return None
|
||
|
||
def report_ban(self, reason: str) -> None:
|
||
self.banned.append(reason)
|
||
|
||
|
||
async def test_fetch_city_reports_ban_on_qrator_block(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""QRATOR-блок в fetch_city → fetcher.report_ban вызывается ВНУТРИ `async with
|
||
build_browser_fetcher(...) as fetcher:` (lease ещё держится) — #2600 п.1.
|
||
|
||
no-op сегодня (domclick SERP собирается БЕЗ proxy_provider — #2160 P4 wiring
|
||
gap), но сам вызов должен произойти корректно на нужном (живом) fetcher'е.
|
||
"""
|
||
from scraper_kit.domclick_exceptions import DomClickBlockedError
|
||
|
||
fake_fetcher = _FakeFetcher()
|
||
|
||
def _fake_build_browser_fetcher(config: object, source: str, **_kw: object) -> _FakeFetcher:
|
||
assert source == "domclick"
|
||
return fake_fetcher
|
||
|
||
monkeypatch.setattr(
|
||
"scraper_kit.providers._base.build_browser_fetcher", _fake_build_browser_fetcher
|
||
)
|
||
|
||
async def _raise_blocked(self: DomClickScraper, **_: object) -> None:
|
||
raise DomClickBlockedError("QRATOR block page")
|
||
|
||
monkeypatch.setattr(DomClickScraper, "_sweep_bucket", _raise_blocked)
|
||
|
||
config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
|
||
scraper = DomClickScraper(config)
|
||
|
||
lots = await scraper.fetch_city(city_id=1)
|
||
|
||
assert lots == []
|
||
assert scraper.blocked is True
|
||
assert fake_fetcher.banned # report_ban был вызван на ЖИВОМ fetcher'е
|
||
assert "QRATOR" in fake_fetcher.banned[0]
|