"""INCREMENTAL SERP-пагинация Avito с date early-stop (param-gated, default off). `fetch_all_secondary(since=date)` на каждый (комнатность × seed-брекет) идёт ПОСЛЕДОВАТЕЛЬНО newest-first и останавливается, как только страница уходит ниже watermark `since` (SERP s=104 newest-first). `since=None` сохраняет точный exhaustive bisection-обход (_walk_price_range) — без изменений. Тестируем через monkeypatch page-fetch seam (`_fetch_rooms_page_html`) + `_parse_html`, отдавая контролируемые страницы ScrapedLot с заданными listing_date. asyncio.sleep мокается в no-op для скорости (паттерн из test_avito_403_backconnect_reconnect.py). """ from datetime import date from unittest.mock import AsyncMock, patch import pytest from app.services.scrapers import avito as avito_mod from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.avito_exceptions import AvitoBlockedError from app.services.scrapers.base import ScrapedLot SINCE = date(2026, 6, 10) def _lot(item_id: str, listing_date: date | None, segment: str = "vtorichka") -> ScrapedLot: """Минимальный валидный ScrapedLot с заданной датой/сегментом.""" return ScrapedLot( source="avito", source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{item_id}", source_id=item_id, rooms=2, price_rub=4_500_000, listing_date=listing_date, listing_segment=segment, ) def _make_scraper() -> AvitoScraper: scraper = AvitoScraper() scraper.request_delay_sec = 0.0 return scraper def _install_pages( scraper: AvitoScraper, monkeypatch: pytest.MonkeyPatch, pages: list[list[ScrapedLot] | None], ) -> dict[str, int]: """Подменяет page-fetch seam: страница p (1-based) → pages[p-1]. pages[i] is None → _fetch_rooms_page_html вернёт None (end-of-pages). pages[i] == [] → HTML непустой, но _parse_html вернёт [] (end-of-pages). Возвращает счётчик {"fetched": N} — сколько раз _fetch_rooms_page_html вызван. """ counter = {"fetched": 0} async def fake_fetch(room_slug: str, page: int, lo: int | None, hi: int | None) -> str | None: counter["fetched"] += 1 if page > len(pages): return None canned = pages[page - 1] if canned is None: return None # Непустой HTML-маркер; реальный парс подменён ниже. return f"" def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]: # Достаём номер страницы из маркера, отдаём соответствующие лоты. marker = html.split("data-page='")[1].split("'")[0] page = int(marker) canned = pages[page - 1] return list(canned) if canned else [] monkeypatch.setattr(scraper, "_fetch_rooms_page_html", fake_fetch) monkeypatch.setattr(scraper, "_parse_html", fake_parse) return counter @pytest.mark.asyncio async def test_early_stop_on_all_older_page(monkeypatch: pytest.MonkeyPatch) -> None: """page1 fresh, page2 mixed (fresh+old), page3 all older → стоп на page3, page4 НЕ фетчится.""" scraper = _make_scraper() pages: list[list[ScrapedLot] | None] = [ [_lot("p1a", date(2026, 6, 15)), _lot("p1b", date(2026, 6, 14))], # все свежие [_lot("p2a", date(2026, 6, 11)), _lot("p2b", date(2026, 6, 5))], # mixed (один >= since) [_lot("p3a", date(2026, 6, 3)), _lot("p3b", date(2026, 6, 1))], # все старше since [_lot("p4a", date(2026, 5, 20))], # НЕ должна фетчиться ] counter = _install_pages(scraper, monkeypatch, pages) seen: dict[str, ScrapedLot] = {} captured: list[tuple[str, list[ScrapedLot]]] = [] with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): await scraper._paginate_incremental_bracket( room_slug="rooms-slug", room_label="2к", lo=0, hi=None, since=SINCE, seen=seen, max_pages_per_bucket=100, secondary_only=True, on_bucket=lambda k, lots: captured.append((k, lots)), ) # page3 обработана (она old → break ПОСЛЕ неё), page4 не фетчилась. assert counter["fetched"] == 3 # Собраны лоты страниц 1-3 (включая boundary-страницу 3), но не 4. assert set(seen.keys()) == {"p1a", "p1b", "p2a", "p2b", "p3a", "p3b"} assert "p4a" not in seen assert captured and captured[0][0] == "2к:0:open" @pytest.mark.asyncio async def test_no_early_stop_when_all_fresh(monkeypatch: pytest.MonkeyPatch) -> None: """Все страницы свежее since до естественного конца (page возвращает None) → walk до конца.""" scraper = _make_scraper() pages: list[list[ScrapedLot] | None] = [ [_lot("p1", date(2026, 6, 18))], [_lot("p2", date(2026, 6, 16))], [_lot("p3", date(2026, 6, 12))], None, # end-of-pages ] counter = _install_pages(scraper, monkeypatch, pages) seen: dict[str, ScrapedLot] = {} with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): await scraper._paginate_incremental_bracket( room_slug="rooms-slug", room_label="2к", lo=0, hi=None, since=SINCE, seen=seen, max_pages_per_bucket=100, secondary_only=True, on_bucket=None, ) # 3 страницы с лотами + 4-я (None) → 4 фетча, затем стоп. assert counter["fetched"] == 4 assert set(seen.keys()) == {"p1", "p2", "p3"} @pytest.mark.asyncio async def test_all_none_dates_grace_stop(monkeypatch: pytest.MonkeyPatch) -> None: """Страницы all-None дат: page1 не стопит сразу, но 2 подряд undated → grace-стоп на page2.""" scraper = _make_scraper() pages: list[list[ScrapedLot] | None] = [ [_lot("p1", None)], # undated → не стоп, streak=1 [_lot("p2", None)], # undated → streak=2 → grace-стоп ПОСЛЕ page2 [_lot("p3", None)], # НЕ должна фетчиться ] counter = _install_pages(scraper, monkeypatch, pages) seen: dict[str, ScrapedLot] = {} with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): await scraper._paginate_incremental_bracket( room_slug="rooms-slug", room_label="2к", lo=0, hi=None, since=SINCE, seen=seen, max_pages_per_bucket=100, secondary_only=True, on_bucket=None, ) assert counter["fetched"] == 2 # page3 не фетчилась (grace bound) assert set(seen.keys()) == {"p1", "p2"} @pytest.mark.asyncio async def test_none_date_does_not_stop_then_fresh_continues( monkeypatch: pytest.MonkeyPatch, ) -> None: """Одна undated-страница между свежими не триггерит стоп (streak сбрасывается).""" scraper = _make_scraper() pages: list[list[ScrapedLot] | None] = [ [_lot("p1", date(2026, 6, 15))], # fresh [_lot("p2", None)], # undated → streak=1, продолжаем [_lot("p3", date(2026, 6, 12))], # fresh → streak сброшен [_lot("p4", date(2026, 6, 2))], # all older → стоп после page4 [_lot("p5", date(2026, 5, 1))], # НЕ фетчится ] counter = _install_pages(scraper, monkeypatch, pages) seen: dict[str, ScrapedLot] = {} with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): await scraper._paginate_incremental_bracket( room_slug="rooms-slug", room_label="2к", lo=0, hi=None, since=SINCE, seen=seen, max_pages_per_bucket=100, secondary_only=True, on_bucket=None, ) assert counter["fetched"] == 4 assert set(seen.keys()) == {"p1", "p2", "p3", "p4"} @pytest.mark.asyncio async def test_secondary_only_filters_novostroyki(monkeypatch: pytest.MonkeyPatch) -> None: """secondary_only=True отбрасывает novostroyki до дедупа в seen (как leaf-bucket).""" scraper = _make_scraper() pages: list[list[ScrapedLot] | None] = [ [ _lot("vt1", date(2026, 6, 15), segment="vtorichka"), _lot("nb1", date(2026, 6, 15), segment="novostroyki"), ], None, ] _install_pages(scraper, monkeypatch, pages) seen: dict[str, ScrapedLot] = {} with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): await scraper._paginate_incremental_bracket( room_slug="rooms-slug", room_label="2к", lo=0, hi=None, since=SINCE, seen=seen, max_pages_per_bucket=100, secondary_only=True, on_bucket=None, ) assert set(seen.keys()) == {"vt1"} # novostroyki отфильтрована @pytest.mark.asyncio async def test_blocked_error_propagates(monkeypatch: pytest.MonkeyPatch) -> None: """AvitoBlockedError из page-фетча пробрасывается наверх (не глушится).""" scraper = _make_scraper() async def fake_fetch(room_slug: str, page: int, lo: int | None, hi: int | None) -> str | None: raise AvitoBlockedError("hard block") monkeypatch.setattr(scraper, "_fetch_rooms_page_html", fake_fetch) seen: dict[str, ScrapedLot] = {} with patch.object(avito_mod.asyncio, "sleep", AsyncMock()): with pytest.raises(AvitoBlockedError): await scraper._paginate_incremental_bracket( room_slug="rooms-slug", room_label="2к", lo=0, hi=None, since=SINCE, seen=seen, max_pages_per_bucket=100, secondary_only=True, on_bucket=None, ) @pytest.mark.asyncio async def test_since_none_uses_exhaustive_path(monkeypatch: pytest.MonkeyPatch) -> None: """since=None → НЕ зовёт _paginate_incremental_bracket; идёт бисекция (_walk_price_range).""" scraper = _make_scraper() walk_calls = {"n": 0} incr_calls = {"n": 0} async def fake_walk(**kwargs: object) -> None: walk_calls["n"] += 1 async def fake_incr(**kwargs: object) -> None: incr_calls["n"] += 1 monkeypatch.setattr(scraper, "_walk_price_range", fake_walk) monkeypatch.setattr(scraper, "_paginate_incremental_bracket", fake_incr) await scraper.fetch_all_secondary( rooms_buckets=[("2-комнатные", "rooms-slug")], since=None, ) # Один room × N seed-брекетов → _walk_price_range зовётся, incremental — нет. assert walk_calls["n"] == len(avito_mod._AVITO_PRICE_SEED_BRACKETS) assert incr_calls["n"] == 0 @pytest.mark.asyncio async def test_since_date_uses_incremental_path(monkeypatch: pytest.MonkeyPatch) -> None: """since=date → зовёт _paginate_incremental_bracket вместо _walk_price_range.""" scraper = _make_scraper() walk_calls = {"n": 0} incr_calls = {"n": 0} async def fake_walk(**kwargs: object) -> None: walk_calls["n"] += 1 async def fake_incr(**kwargs: object) -> None: incr_calls["n"] += 1 monkeypatch.setattr(scraper, "_walk_price_range", fake_walk) monkeypatch.setattr(scraper, "_paginate_incremental_bracket", fake_incr) await scraper.fetch_all_secondary( rooms_buckets=[("2-комнатные", "rooms-slug")], since=SINCE, ) assert incr_calls["n"] == len(avito_mod._AVITO_PRICE_SEED_BRACKETS) assert walk_calls["n"] == 0