bot-backend
bdedac3012
Adds since=date mode to fetch_all_secondary: per room x seed-bracket, paginate sequentially newest-first and stop once cards fall below the watermark, skipping deep pagination (and the 403 clusters it triggers). since=None keeps the exact exhaustive bisection walk. No scheduler/schedule change here — wiring follows in a separate PR.
315 lines
12 KiB
Python
315 lines
12 KiB
Python
"""INCREMENTAL SERP-пагинация Avito с date early-stop (param-gated, default off).
|
||
|
||
`fetch_all_secondary(since=date)` на каждый (комнатность × seed-брекет) идёт
|
||
ПОСЛЕДОВАТЕЛЬНО newest-first и останавливается, как только страница уходит ниже
|
||
watermark `since` (SERP s=104 newest-first). `since=None` сохраняет точный
|
||
exhaustive bisection-обход (_walk_price_range) — без изменений.
|
||
|
||
Тестируем через monkeypatch page-fetch seam (`_fetch_rooms_page_html`) +
|
||
`_parse_html`, отдавая контролируемые страницы ScrapedLot с заданными listing_date.
|
||
asyncio.sleep мокается в no-op для скорости (паттерн из
|
||
test_avito_403_backconnect_reconnect.py).
|
||
"""
|
||
|
||
from datetime import date
|
||
from unittest.mock import AsyncMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers import avito as avito_mod
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
SINCE = date(2026, 6, 10)
|
||
|
||
|
||
def _lot(item_id: str, listing_date: date | None, segment: str = "vtorichka") -> ScrapedLot:
|
||
"""Минимальный валидный ScrapedLot с заданной датой/сегментом."""
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{item_id}",
|
||
source_id=item_id,
|
||
rooms=2,
|
||
price_rub=4_500_000,
|
||
listing_date=listing_date,
|
||
listing_segment=segment,
|
||
)
|
||
|
||
|
||
def _make_scraper() -> AvitoScraper:
|
||
scraper = AvitoScraper()
|
||
scraper.request_delay_sec = 0.0
|
||
return scraper
|
||
|
||
|
||
def _install_pages(
|
||
scraper: AvitoScraper,
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
pages: list[list[ScrapedLot] | None],
|
||
) -> dict[str, int]:
|
||
"""Подменяет page-fetch seam: страница p (1-based) → pages[p-1].
|
||
|
||
pages[i] is None → _fetch_rooms_page_html вернёт None (end-of-pages).
|
||
pages[i] == [] → HTML непустой, но _parse_html вернёт [] (end-of-pages).
|
||
Возвращает счётчик {"fetched": N} — сколько раз _fetch_rooms_page_html вызван.
|
||
"""
|
||
counter = {"fetched": 0}
|
||
|
||
async def fake_fetch(room_slug: str, page: int, lo: int | None, hi: int | None) -> str | None:
|
||
counter["fetched"] += 1
|
||
if page > len(pages):
|
||
return None
|
||
canned = pages[page - 1]
|
||
if canned is None:
|
||
return None
|
||
# Непустой HTML-маркер; реальный парс подменён ниже.
|
||
return f"<html data-page='{page}'></html>"
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
# Достаём номер страницы из маркера, отдаём соответствующие лоты.
|
||
marker = html.split("data-page='")[1].split("'")[0]
|
||
page = int(marker)
|
||
canned = pages[page - 1]
|
||
return list(canned) if canned else []
|
||
|
||
monkeypatch.setattr(scraper, "_fetch_rooms_page_html", fake_fetch)
|
||
monkeypatch.setattr(scraper, "_parse_html", fake_parse)
|
||
return counter
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_early_stop_on_all_older_page(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""page1 fresh, page2 mixed (fresh+old), page3 all older → стоп на page3, page4 НЕ фетчится."""
|
||
scraper = _make_scraper()
|
||
pages: list[list[ScrapedLot] | None] = [
|
||
[_lot("p1a", date(2026, 6, 15)), _lot("p1b", date(2026, 6, 14))], # все свежие
|
||
[_lot("p2a", date(2026, 6, 11)), _lot("p2b", date(2026, 6, 5))], # mixed (один >= since)
|
||
[_lot("p3a", date(2026, 6, 3)), _lot("p3b", date(2026, 6, 1))], # все старше since
|
||
[_lot("p4a", date(2026, 5, 20))], # НЕ должна фетчиться
|
||
]
|
||
counter = _install_pages(scraper, monkeypatch, pages)
|
||
seen: dict[str, ScrapedLot] = {}
|
||
captured: list[tuple[str, list[ScrapedLot]]] = []
|
||
|
||
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
|
||
await scraper._paginate_incremental_bracket(
|
||
room_slug="rooms-slug",
|
||
room_label="2к",
|
||
lo=0,
|
||
hi=None,
|
||
since=SINCE,
|
||
seen=seen,
|
||
max_pages_per_bucket=100,
|
||
secondary_only=True,
|
||
on_bucket=lambda k, lots: captured.append((k, lots)),
|
||
)
|
||
|
||
# page3 обработана (она old → break ПОСЛЕ неё), page4 не фетчилась.
|
||
assert counter["fetched"] == 3
|
||
# Собраны лоты страниц 1-3 (включая boundary-страницу 3), но не 4.
|
||
assert set(seen.keys()) == {"p1a", "p1b", "p2a", "p2b", "p3a", "p3b"}
|
||
assert "p4a" not in seen
|
||
assert captured and captured[0][0] == "2к:0:open"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_no_early_stop_when_all_fresh(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Все страницы свежее since до естественного конца (page возвращает None) → walk до конца."""
|
||
scraper = _make_scraper()
|
||
pages: list[list[ScrapedLot] | None] = [
|
||
[_lot("p1", date(2026, 6, 18))],
|
||
[_lot("p2", date(2026, 6, 16))],
|
||
[_lot("p3", date(2026, 6, 12))],
|
||
None, # end-of-pages
|
||
]
|
||
counter = _install_pages(scraper, monkeypatch, pages)
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
|
||
await scraper._paginate_incremental_bracket(
|
||
room_slug="rooms-slug",
|
||
room_label="2к",
|
||
lo=0,
|
||
hi=None,
|
||
since=SINCE,
|
||
seen=seen,
|
||
max_pages_per_bucket=100,
|
||
secondary_only=True,
|
||
on_bucket=None,
|
||
)
|
||
|
||
# 3 страницы с лотами + 4-я (None) → 4 фетча, затем стоп.
|
||
assert counter["fetched"] == 4
|
||
assert set(seen.keys()) == {"p1", "p2", "p3"}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_all_none_dates_grace_stop(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Страницы all-None дат: page1 не стопит сразу, но 2 подряд undated → grace-стоп на page2."""
|
||
scraper = _make_scraper()
|
||
pages: list[list[ScrapedLot] | None] = [
|
||
[_lot("p1", None)], # undated → не стоп, streak=1
|
||
[_lot("p2", None)], # undated → streak=2 → grace-стоп ПОСЛЕ page2
|
||
[_lot("p3", None)], # НЕ должна фетчиться
|
||
]
|
||
counter = _install_pages(scraper, monkeypatch, pages)
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
|
||
await scraper._paginate_incremental_bracket(
|
||
room_slug="rooms-slug",
|
||
room_label="2к",
|
||
lo=0,
|
||
hi=None,
|
||
since=SINCE,
|
||
seen=seen,
|
||
max_pages_per_bucket=100,
|
||
secondary_only=True,
|
||
on_bucket=None,
|
||
)
|
||
|
||
assert counter["fetched"] == 2 # page3 не фетчилась (grace bound)
|
||
assert set(seen.keys()) == {"p1", "p2"}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_none_date_does_not_stop_then_fresh_continues(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Одна undated-страница между свежими не триггерит стоп (streak сбрасывается)."""
|
||
scraper = _make_scraper()
|
||
pages: list[list[ScrapedLot] | None] = [
|
||
[_lot("p1", date(2026, 6, 15))], # fresh
|
||
[_lot("p2", None)], # undated → streak=1, продолжаем
|
||
[_lot("p3", date(2026, 6, 12))], # fresh → streak сброшен
|
||
[_lot("p4", date(2026, 6, 2))], # all older → стоп после page4
|
||
[_lot("p5", date(2026, 5, 1))], # НЕ фетчится
|
||
]
|
||
counter = _install_pages(scraper, monkeypatch, pages)
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
|
||
await scraper._paginate_incremental_bracket(
|
||
room_slug="rooms-slug",
|
||
room_label="2к",
|
||
lo=0,
|
||
hi=None,
|
||
since=SINCE,
|
||
seen=seen,
|
||
max_pages_per_bucket=100,
|
||
secondary_only=True,
|
||
on_bucket=None,
|
||
)
|
||
|
||
assert counter["fetched"] == 4
|
||
assert set(seen.keys()) == {"p1", "p2", "p3", "p4"}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_only_filters_novostroyki(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""secondary_only=True отбрасывает novostroyki до дедупа в seen (как leaf-bucket)."""
|
||
scraper = _make_scraper()
|
||
pages: list[list[ScrapedLot] | None] = [
|
||
[
|
||
_lot("vt1", date(2026, 6, 15), segment="vtorichka"),
|
||
_lot("nb1", date(2026, 6, 15), segment="novostroyki"),
|
||
],
|
||
None,
|
||
]
|
||
_install_pages(scraper, monkeypatch, pages)
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
|
||
await scraper._paginate_incremental_bracket(
|
||
room_slug="rooms-slug",
|
||
room_label="2к",
|
||
lo=0,
|
||
hi=None,
|
||
since=SINCE,
|
||
seen=seen,
|
||
max_pages_per_bucket=100,
|
||
secondary_only=True,
|
||
on_bucket=None,
|
||
)
|
||
|
||
assert set(seen.keys()) == {"vt1"} # novostroyki отфильтрована
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_blocked_error_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""AvitoBlockedError из page-фетча пробрасывается наверх (не глушится)."""
|
||
scraper = _make_scraper()
|
||
|
||
async def fake_fetch(room_slug: str, page: int, lo: int | None, hi: int | None) -> str | None:
|
||
raise AvitoBlockedError("hard block")
|
||
|
||
monkeypatch.setattr(scraper, "_fetch_rooms_page_html", fake_fetch)
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
with patch.object(avito_mod.asyncio, "sleep", AsyncMock()):
|
||
with pytest.raises(AvitoBlockedError):
|
||
await scraper._paginate_incremental_bracket(
|
||
room_slug="rooms-slug",
|
||
room_label="2к",
|
||
lo=0,
|
||
hi=None,
|
||
since=SINCE,
|
||
seen=seen,
|
||
max_pages_per_bucket=100,
|
||
secondary_only=True,
|
||
on_bucket=None,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_since_none_uses_exhaustive_path(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""since=None → НЕ зовёт _paginate_incremental_bracket; идёт бисекция (_walk_price_range)."""
|
||
scraper = _make_scraper()
|
||
|
||
walk_calls = {"n": 0}
|
||
incr_calls = {"n": 0}
|
||
|
||
async def fake_walk(**kwargs: object) -> None:
|
||
walk_calls["n"] += 1
|
||
|
||
async def fake_incr(**kwargs: object) -> None:
|
||
incr_calls["n"] += 1
|
||
|
||
monkeypatch.setattr(scraper, "_walk_price_range", fake_walk)
|
||
monkeypatch.setattr(scraper, "_paginate_incremental_bracket", fake_incr)
|
||
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("2-комнатные", "rooms-slug")],
|
||
since=None,
|
||
)
|
||
|
||
# Один room × N seed-брекетов → _walk_price_range зовётся, incremental — нет.
|
||
assert walk_calls["n"] == len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
|
||
assert incr_calls["n"] == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_since_date_uses_incremental_path(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""since=date → зовёт _paginate_incremental_bracket вместо _walk_price_range."""
|
||
scraper = _make_scraper()
|
||
|
||
walk_calls = {"n": 0}
|
||
incr_calls = {"n": 0}
|
||
|
||
async def fake_walk(**kwargs: object) -> None:
|
||
walk_calls["n"] += 1
|
||
|
||
async def fake_incr(**kwargs: object) -> None:
|
||
incr_calls["n"] += 1
|
||
|
||
monkeypatch.setattr(scraper, "_walk_price_range", fake_walk)
|
||
monkeypatch.setattr(scraper, "_paginate_incremental_bracket", fake_incr)
|
||
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("2-комнатные", "rooms-slug")],
|
||
since=SINCE,
|
||
)
|
||
|
||
assert incr_calls["n"] == len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
|
||
assert walk_calls["n"] == 0
|