gendesign/tradein-mvp/backend/tests/test_domclick_sweep.py

"""Offline unit tests for DomClickCitySweepCounters + DomClick parser.

Legacy `run_domclick_city_sweep` orchestration behavior (sweep phases, cooperative
cancel, honest mark_done/mark_failed/mark_banned statuses) удалён вместе с
`app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная regression-coverage
теперь в `test_scraper_kit_pipeline_parity2.py::test_domclick_city_sweep_*`
(kit `run_domclick_city_sweep`).

`app.services.scrapers.domclick` удалён (#2397 Part E2, 0 runtime-импортёров) —
`_map_item` ретаргетирован на kit `scraper_kit.providers.domclick.serp.DomClickScraper`
(единственный живой путь).

Остались:
  - DomClickCitySweepCounters (dataclass, из scraper_kit) — defaults + to_dict
  - DomClickScraper._map_item (kit) — BFF offer-item → ScrapedLot маппинг (не
    зависит от оркестрации)
"""

from __future__ import annotations

import os
from types import SimpleNamespace

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

import pytest
from scraper_kit.providers.domclick.serp import DomClickScraper

# ── DomClickCitySweepCounters ───────────────────────────────────────────────


def test_domclick_sweep_counters_defaults() -> None:
    from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters

    c = DomClickCitySweepCounters()
    assert c.lots_fetched == 0
    assert c.lots_inserted == 0
    assert c.lots_updated == 0
    assert c.pages_fetched == 0
    assert c.errors_count == 0
    assert c.blocked == 0
    assert c.geo_filtered == 0


def test_domclick_sweep_counters_to_dict_all_keys() -> None:
    from dataclasses import fields

    from scraper_kit.orchestration.pipeline import DomClickCitySweepCounters

    c = DomClickCitySweepCounters()
    d = c.to_dict()
    expected = {f.name for f in fields(c)}
    assert set(d.keys()) == expected


# ── _map_item ─────────────────────────────────────────────────────────────────


def test_map_item_basic_mapping() -> None:
    """DomClickScraper._map_item маппит BFF offer-item → ScrapedLot корректно."""
    config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
    scraper = DomClickScraper(config, delay_provider=lambda _name: 8.0)
    item = {
        "id": 111222333,
        "path": "https://domclick.ru/card/sale__flat__111222333",
        "location": {"lat": 56.838, "lon": 60.612},
        "address": {"displayName": "Екатеринбург, улица Ленина, 10"},
        "objectInfo": {"area": 52.3, "rooms": 2, "floor": 5},
        "house": {"floors": 16, "buildYear": 2010},
        "price": 6_500_000,
        "squarePrice": 124282,
        "flatComplex": None,
        "isRosreestrApproved": False,
        "publishedDate": "2026-06-01T10:00:00+05:00",
        "updatedDate": None,
        "lastPriceHistoryState": None,
        "offerRegionName": "Екатеринбург",
    }
    lot = scraper._map_item(item)
    assert lot is not None
    assert lot.source == "domklik"
    assert lot.source_id == "111222333"
    assert lot.rooms == 2
    assert lot.area_m2 == pytest.approx(52.3)
    assert lot.floor == 5
    assert lot.total_floors == 16
    assert lot.price_rub == 6_500_000
    assert lot.listing_segment == "vtorichka"
    assert lot.lat == pytest.approx(56.838)
    assert lot.lon == pytest.approx(60.612)


# ── fetch_city: report_ban на QRATOR-блок (#2600 п.1) ───────────────────────────


class _FakeFetcher:
    """Заглушка BrowserFetcher: async context manager + report_ban recorder."""

    def __init__(self) -> None:
        self.banned: list[str] = []

    async def __aenter__(self) -> _FakeFetcher:
        return self

    async def __aexit__(self, *args: object) -> None:
        return None

    def report_ban(self, reason: str) -> None:
        self.banned.append(reason)


async def test_fetch_city_reports_ban_on_qrator_block(monkeypatch: pytest.MonkeyPatch) -> None:
    """QRATOR-блок в fetch_city → fetcher.report_ban вызывается ВНУТРИ `async with
    build_browser_fetcher(...) as fetcher:` (lease ещё держится) — #2600 п.1.

    no-op сегодня (domclick SERP собирается БЕЗ proxy_provider — #2160 P4 wiring
    gap), но сам вызов должен произойти корректно на нужном (живом) fetcher'е.
    """
    from scraper_kit.domclick_exceptions import DomClickBlockedError

    fake_fetcher = _FakeFetcher()

    def _fake_build_browser_fetcher(config: object, source: str) -> _FakeFetcher:
        assert source == "domclick"
        return fake_fetcher

    monkeypatch.setattr(
        "scraper_kit.providers._base.build_browser_fetcher", _fake_build_browser_fetcher
    )

    async def _raise_blocked(self: DomClickScraper, **_: object) -> None:
        raise DomClickBlockedError("QRATOR block page")

    monkeypatch.setattr(DomClickScraper, "_sweep_bucket", _raise_blocked)

    config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
    scraper = DomClickScraper(config)

    lots = await scraper.fetch_city(city_id=1)

    assert lots == []
    assert scraper.blocked is True
    assert fake_fetcher.banned  # report_ban был вызван на ЖИВОМ fetcher'е
    assert "QRATOR" in fake_fetcher.banned[0]