"""Avito newbuilding (novostroyka) citywide sweep. Тесты (без сети, без реального DB): 1. _build_newbuilding_url — novostroyka-slug в пути, s=104, page param, без geo. 2. _parse_html на novostroyka-SERP fixture → listing_segment='novostroyki' + newbuilding_id заполнен (DOM-маркер застройщика + zhk-якорь). 3. fetch_newbuildings break-on-empty + dedup (mirror fetch_city_wide). 4. NewbuildingSweepCounters — defaults + to_dict. 5. run_avito_newbuilding_sweep wiring — мокаем fetch_newbuildings + save_listings, assert save вызван и SERP-счётчики записаны (mark_done). """ from __future__ import annotations import os from pathlib import Path from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from app.services import scrape_pipeline from app.services.scrapers.avito import NOVOSTROYKA_SLUG, AvitoScraper from app.services.scrapers.base import ScrapedLot FIXTURE = Path(__file__).parent / "fixtures" / "avito_serp_novostroyka.html" def _make_lot(source_id: str) -> ScrapedLot: return ScrapedLot( source="avito", source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}", source_id=source_id, price_rub=6_000_000, listing_segment="novostroyki", ) # ── 1. _build_newbuilding_url ─────────────────────────────────────────────── def test_build_newbuilding_url_contains_slug_and_sort() -> None: s = AvitoScraper() url = s._build_newbuilding_url(page=1) assert NOVOSTROYKA_SLUG in url assert "novostroyka-" in url assert "geoCoords" not in url assert "radius" not in url assert "s=104" in url assert "p=1" in url assert "ekaterinburg/kvartiry/prodam/" in url def test_build_newbuilding_url_page_param() -> None: s = AvitoScraper() url = s._build_newbuilding_url(page=2) assert "novostroyka-" in url assert "p=2" in url def test_build_newbuilding_url_differs_from_citywide() -> None: s = AvitoScraper() assert s._build_newbuilding_url(page=1) != s._build_citywide_url(page=1) # ── 2. _parse_html classifies novostroyka cards ───────────────────────────── def test_parse_novostroyka_fixture_segments_and_newbuilding_id() -> None: html = FIXTURE.read_text(encoding="utf-8") s = AvitoScraper() lots = s._parse_html(html, source_url_base=s._build_newbuilding_url(page=1)) assert len(lots) == 2 for lot in lots: assert lot.listing_segment == "novostroyki" assert lot.newbuilding_id is not None assert lot.newbuilding_url is not None ids = {lot.newbuilding_id for lot in lots} assert "meridian-ekaterinburg" in ids assert "severnyy-kvartal-ekaterinburg" in ids # ── 3. fetch_newbuildings break-on-empty + dedup ──────────────────────────── @pytest.mark.asyncio async def test_fetch_newbuildings_break_on_empty() -> None: s = AvitoScraper() pages_data: list[list[ScrapedLot]] = [[_make_lot("A"), _make_lot("B")], []] call_n = 0 async def mock_html(url: str, page: int) -> str: assert "novostroyka-" in url return f"{page}" def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: nonlocal call_n idx = call_n call_n += 1 return pages_data[idx] if idx < len(pages_data) else [] with patch.object(s, "_fetch_serp_html", side_effect=mock_html): with patch.object(s, "_parse_html", side_effect=mock_parse): with patch.object(s, "sleep_between_requests", new=AsyncMock()): result = await s.fetch_newbuildings(pages=10, delay_override_sec=0) assert len(result) == 2 assert call_n == 2 # page1 + page2(empty) → stop @pytest.mark.asyncio async def test_fetch_newbuildings_dedup_by_source_id() -> None: s = AvitoScraper() dup = _make_lot("SAME") pages_data: list[list[ScrapedLot]] = [[dup], [dup], []] call_n = 0 async def mock_html(url: str, page: int) -> str: return f"{page}" def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: nonlocal call_n idx = call_n call_n += 1 return pages_data[idx] if idx < len(pages_data) else [] with patch.object(s, "_fetch_serp_html", side_effect=mock_html): with patch.object(s, "_parse_html", side_effect=mock_parse): with patch.object(s, "sleep_between_requests", new=AsyncMock()): result = await s.fetch_newbuildings(pages=10, delay_override_sec=0) assert len(result) == 1 assert result[0].source_id == "SAME" def test_fetch_city_wide_url_unchanged() -> None: """fetch_city_wide остаётся citywide-URL — рефактор не сменил его поведение.""" s = AvitoScraper() assert "novostroyka-" not in s._build_citywide_url(page=1) assert "prodam-ASgBAgICAUSSA8YQ" in s._build_citywide_url(page=1) # ── 4. NewbuildingSweepCounters ───────────────────────────────────────────── def test_newbuilding_counters_defaults_and_to_dict() -> None: from app.services.scrape_pipeline import NewbuildingSweepCounters c = NewbuildingSweepCounters() assert c.lots_fetched == 0 assert c.lots_inserted == 0 assert c.lots_updated == 0 assert c.errors_count == 0 d = c.to_dict() assert set(d.keys()) == {"lots_fetched", "lots_inserted", "lots_updated", "errors_count"} # ── 5. run_avito_newbuilding_sweep wiring ─────────────────────────────────── class _FakeRuns: def __init__(self) -> None: self.heartbeats: list[dict[str, int]] = [] self.done: dict[str, int] | None = None self.failed: tuple[str, dict[str, int]] | None = None self.banned: tuple[str, dict[str, int]] | None = None def is_cancelled(self, _db: Any, _run_id: int) -> bool: return False def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.heartbeats.append(dict(counters)) def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.done = dict(counters) def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: self.failed = (error, dict(counters)) def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: self.banned = (error, dict(counters)) def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None: monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled) monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned) @pytest.mark.asyncio async def test_run_avito_newbuilding_sweep_saves_and_counts( monkeypatch: pytest.MonkeyPatch, ) -> None: fake_runs = _FakeRuns() _install_runs(monkeypatch, fake_runs) fake_lots = [_make_lot("N1"), _make_lot("N2"), _make_lot("N3")] async def fake_fetch_newbuildings( self: AvitoScraper, pages: int = 30, *, delay_override_sec: float | None = None, ) -> list[ScrapedLot]: return list(fake_lots) monkeypatch.setattr(AvitoScraper, "fetch_newbuildings", fake_fetch_newbuildings) save_calls: list[int] = [] def fake_save_listings(_db: Any, lots: Any, **_kw: Any) -> tuple[int, int]: save_calls.append(len(lots)) return (len(lots), 0) monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save_listings) # Force cffi-mode (no browser) so we don't spin up BrowserFetcher. monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "cffi") # Stub shared AsyncSession (used by the sweep in cffi-mode) — no real session. fake_session = AsyncMock() fake_session.__aenter__ = AsyncMock(return_value=fake_session) fake_session.__aexit__ = AsyncMock(return_value=None) monkeypatch.setattr(scrape_pipeline, "AsyncSession", lambda **_kw: fake_session) counters = await scrape_pipeline.run_avito_newbuilding_sweep( MagicMock(), run_id=42, pages=2, request_delay_sec=0.0, ) assert save_calls == [3] assert counters.lots_fetched == 3 assert counters.lots_inserted == 3 assert counters.lots_updated == 0 assert fake_runs.done is not None assert fake_runs.done["lots_fetched"] == 3 assert fake_runs.failed is None assert fake_runs.banned is None