"""Тесты exhaustive-загрузки Avito: fetch_all_secondary + _walk_price_range.
Зеркало tests/scrapers/test_cian_exhaustive.py. Мокаем сетевые вызовы
(_fetch_rooms_page_html, _extract_total_count, _parse_html) для детерминированного
контроля бисекции без реального HTTP.
Дополнительно: _build_rooms_url с pmin/pmax (backward-compat) и _extract_total_count
на реальном span-фрагменте `data-marker="page-title/count"`.
"""
from __future__ import annotations
import os
from unittest.mock import MagicMock, patch
# Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
import httpx
import pytest
from app.services.scrapers.avito import (
_AVITO_MAX_PRICE,
_AVITO_MIN_BRACKET,
_AVITO_OFFERS_PER_PAGE,
_AVITO_PRICE_SEED_BRACKETS,
AvitoScraper,
)
from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.base import ScrapedLot
def _make_lot(source_id: str, price: int = 3_000_000, segment: str = "vtorichka") -> ScrapedLot:
"""Минимальный ScrapedLot для теста."""
return ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
source_id=source_id,
address="Екатеринбург",
price_rub=price,
listing_segment=segment,
)
@pytest.fixture
def scraper() -> AvitoScraper:
"""AvitoScraper без реального __aenter__ (без HTTP/IP-rotation)."""
s = AvitoScraper.__new__(AvitoScraper)
s.name = "avito"
s.base_url = "https://www.avito.ru"
s.request_delay_sec = 0.0
s._cffi = MagicMock() # не None
s._browser = None
s.parse_failures = 0
return s
# ── _build_rooms_url с pmin/pmax ──────────────────────────────────────────────
def test_build_rooms_url_both_prices(scraper: AvitoScraper) -> None:
"""min_price + max_price → pmin/pmax оба в query."""
url = scraper._build_rooms_url(
"1-komnatnye-X", page=2, min_price=3_000_000, max_price=4_000_000
)
assert "pmin=3000000" in url
assert "pmax=4000000" in url
assert "p=2" in url
assert "s=104" in url
assert "/kvartiry/prodam/1-komnatnye-X?" in url
def test_build_rooms_url_min_only(scraper: AvitoScraper) -> None:
"""Только min_price → pmin в query, pmax отсутствует."""
url = scraper._build_rooms_url("1-komnatnye-X", min_price=3_000_000)
assert "pmin=3000000" in url
assert "pmax" not in url
def test_build_rooms_url_no_prices_backward_compat(scraper: AvitoScraper) -> None:
"""Без цен URL не меняется (backward-compat для fetch_by_rooms)."""
url = scraper._build_rooms_url("1-komnatnye-X", page=1)
assert "pmin" not in url
assert "pmax" not in url
assert url.endswith("/kvartiry/prodam/1-komnatnye-X?s=104&p=1")
# ── _extract_total_count ──────────────────────────────────────────────────────
# Реальный span-фрагмент из живого Avito SERP (NBSP-разделитель разрядов).
_REAL_COUNT_SPAN = '1\xa0178'
def test_extract_total_count_real_span(scraper: AvitoScraper) -> None:
"""Реальный span с NBSP `1\\xa0178` → 1178."""
html = f"
Квартиры
{_REAL_COUNT_SPAN}
"
assert scraper._extract_total_count(html) == 1178
def test_extract_total_count_nbsp_entity(scraper: AvitoScraper) -> None:
"""Вариант с ` `-entity → корректно очищается."""
html = '12 345'
assert scraper._extract_total_count(html) == 12345
def test_extract_total_count_zero(scraper: AvitoScraper) -> None:
"""0 результатов → 0 (не None)."""
html = '0'
assert scraper._extract_total_count(html) == 0
def test_extract_total_count_missing_returns_none(scraper: AvitoScraper) -> None:
"""Captcha/firewall (нет span, нет no-results-маркера) → None."""
html = "Доступ ограничен"
assert scraper._extract_total_count(html) is None
def test_extract_total_count_no_results_marker_returns_zero(scraper: AvitoScraper) -> None:
"""Валидная пустая выборка: нет `page-title/count`, но есть «ничего не найдено» → 0.
Даже если на странице есть fallback-«похожие» объявления (item-id'ы) — это
не подходящие под фильтр листинги, total всё равно 0. Это предотвращает
дробление пустого брекета + ротацию IP (прод-инцидент run 209).
"""
html = (
''
''
"
По вашему запросу ничего не найдено
"
)
assert scraper._extract_total_count(html) == 0
def test_extract_total_count_no_results_marker_case_insensitive(scraper: AvitoScraper) -> None:
"""Маркер регистронезависим: «Ничего не найдено» с заглавной → тоже 0."""
html = "
Ничего не найдено
Попробуйте изменить фильтры
"
assert scraper._extract_total_count(html) == 0
def test_extract_total_count_count_wins_over_no_results(scraper: AvitoScraper) -> None:
"""Непустая выборка с count имеет приоритет, даже если где-то ниже есть маркер."""
html = (
f"
Квартиры
{_REAL_COUNT_SPAN}
"
""
)
assert scraper._extract_total_count(html) == 1178
# ── Бисекция: split / paginate / dedup ────────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_all_secondary_splits_on_cap(scraper: AvitoScraper) -> None:
"""Внутри seed-брекета total > cap → бакет делится пополам; суб-бакеты пагинируются."""
# Первый закрытый seed-брекет (0, 4_000_000) → walk_hi=3_999_999.
br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
assert br_hi_raw is not None
br_hi = br_hi_raw - 1 # эксклюзивный верх
mid = (br_lo + br_hi) // 2
call_log: list[tuple] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
call_log.append((page, min_price, max_price))
return f"page={page} min={min_price} max={max_price}"
def fake_total(html: str) -> int | None:
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0
max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else None
# Внутри первого брекета [0, 3_999_999]: весь брекет > cap → split;
# половинки < cap → пагинируются. Прочие seed-брекеты — пусто (0).
if min_p == 0 and max_p == br_hi:
return 3000 # весь первый брекет — split
if min_p == 0 and max_p == mid:
return 800 # левая половина — пагинировать
if min_p == mid + 1 and max_p == br_hi:
return 700 # правая половина — пагинировать
return 0 # остальные seed-брекеты пусты
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "MAX"
if page == 1:
return [
_make_lot(f"lot_{min_p}_{max_p}_1"),
_make_lot(f"lot_{min_p}_{max_p}_2"),
]
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
)
assert len(lots) > 0
# Должны быть пробы суб-бакетов (после split весь брекет не пагинируется).
probed_max = [mx for (p, _mn, mx) in call_log if p == 1]
assert any(mx == mid for mx in probed_max), f"Ожидался probe левой половины, got {probed_max}"
@pytest.mark.asyncio
async def test_fetch_all_secondary_paginates_when_under_cap(scraper: AvitoScraper) -> None:
"""total <= cap → пагинирует бакет полностью."""
pages_fetched: list[int] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
pages_fetched.append(page)
return f"page={page}"
def fake_total(html: str) -> int | None:
return 100 # ceil(100/50) = 2 страницы
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
if page <= 2:
return [_make_lot(f"lot_p{page}_{i}") for i in range(_AVITO_OFFERS_PER_PAGE)]
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("2-комн.", "2-komnatnye-X")],
price_cap_per_bucket=1400,
)
assert len(lots) == 100, f"Ожидалось 100 уникальных лотов, получено {len(lots)}"
assert 2 in pages_fetched, "Должна быть запрошена страница 2"
@pytest.mark.asyncio
async def test_fetch_all_secondary_min_bracket_guard(scraper: AvitoScraper) -> None:
"""hi - lo < MIN_BRACKET → пагинируем как есть (нет бесконечной рекурсии)."""
call_count = 0
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
nonlocal call_count
call_count += 1
if call_count > 200:
raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал")
return f"page={page}"
def fake_total(html: str) -> int | None:
return 5000 # всегда > cap → без guard делилось бы бесконечно
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper._walk_price_range(
room_slug="1-komnatnye-X",
room_label="room_1_komn",
lo=1_000_000,
hi=1_000_000 + _AVITO_MIN_BRACKET - 1,
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
)
# Guard сработал: рекурсии нет, только один проход пагинации (probe + до cap страниц).
assert call_count <= 101, f"Слишком много вызовов ({call_count}) — guard не сработал"
@pytest.mark.asyncio
async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: AvitoScraper) -> None:
"""Параллельная пагинация не ломает дедуп по source_id."""
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"page={page}"
def fake_total(html: str) -> int | None:
return 150 # ceil(150/50) = 3 страницы
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
if page == 3:
return [_make_lot(f"lot_p1_{i}") for i in range(10)] # дубликаты стр 1
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
concurrency=5,
)
# Страницы 1+2 = 20 уникальных, страница 3 дублирует → 20
assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}"
@pytest.mark.asyncio
async def test_on_bucket_receives_key_and_lots(scraper: AvitoScraper) -> None:
"""on_bucket получает (bucket_key, lots) на каждый seed-брекет.
bucket_key закрытого брекета = "room_label:lo:hi" (hi = br_hi-1, эксклюзивный);
открытого (последнего) = "room_label:lo:open". Одна on_bucket-нотификация
на каждый непустой seed-брекет (здесь все непустые → len(brackets) вызовов).
"""
received: list[tuple[str, int]] = []
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
received.append((bucket_key, len(lots)))
# Уникальные source_id на каждый брекет, чтобы on_bucket получал непустые
# списки во ВСЕХ брекетах (дедуп в seen не должен опустошать поздние брекеты).
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"page={page} min={min_price} max={max_price}"
def fake_total(html: str) -> int | None:
return 5 # 1 страница
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "open"
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_{min_p}_{max_p}_p{page}_{i}") for i in range(5)]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper.fetch_all_secondary(
rooms_buckets=[("2-комн.", "2-komnatnye-X")],
price_cap_per_bucket=1400,
on_bucket=fake_on_bucket,
)
# Один on_bucket на каждый seed-брекет (все непустые).
assert len(received) == len(
_AVITO_PRICE_SEED_BRACKETS
), f"Ожидалось {len(_AVITO_PRICE_SEED_BRACKETS)} вызовов on_bucket, получено {len(received)}"
keys = [k for (k, _c) in received]
# Закрытые брекеты: hi = br_hi - 1. Открытый (последний): ':open'.
first_lo, first_hi = _AVITO_PRICE_SEED_BRACKETS[0]
assert first_hi is not None
assert keys[0] == f"room_2_komn:{first_lo}:{first_hi - 1}", f"Неверный ключ: {keys[0]!r}"
last_lo, last_hi = _AVITO_PRICE_SEED_BRACKETS[-1]
assert last_hi is None
assert keys[-1] == f"room_2_komn:{last_lo}:open", f"Открытый ключ неверен: {keys[-1]!r}"
assert all(count == 5 for (_k, count) in received)
@pytest.mark.asyncio
async def test_skip_buckets_skips_pagination(scraper: AvitoScraper) -> None:
"""skip_buckets: для skip-бакета probe выполняется, пагинация и on_bucket — нет."""
fetch_calls: list[tuple[int, int | None, int | None]] = []
on_bucket_calls: list[str] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
fetch_calls.append((page, min_price, max_price))
return f"page={page} min={min_price} max={max_price}"
def fake_total(html: str) -> int | None:
return 5
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
return [_make_lot(f"lot_{min_m.group(1) if min_m else 0}_{max_m.group(1) if max_m else 0}")]
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
on_bucket_calls.append(bucket_key)
# Skip ВСЕ брекеты → ни одной on_bucket-нотификации, но каждый брекет
# всё равно делает probe (page=1).
first_lo, first_hi = _AVITO_PRICE_SEED_BRACKETS[0]
assert first_hi is not None
first_key = f"room_1_komn:{first_lo}:{first_hi - 1}"
skip_all: set[str] = set()
for br_lo, br_hi in _AVITO_PRICE_SEED_BRACKETS:
hi_repr = "open" if br_hi is None else str(br_hi - 1)
skip_all.add(f"room_1_komn:{br_lo}:{hi_repr}")
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
on_bucket=fake_on_bucket,
skip_buckets=skip_all,
)
# Ровно один probe (page=1) на каждый seed-брекет, без доп. страниц.
assert first_key in skip_all
assert all(
p == 1 for (p, _mn, _mx) in fetch_calls
), f"Ожидались только probe-вызовы: {fetch_calls}"
assert len(fetch_calls) == len(
_AVITO_PRICE_SEED_BRACKETS
), f"Ожидался 1 probe на брекет ({len(_AVITO_PRICE_SEED_BRACKETS)}), got {len(fetch_calls)}"
assert on_bucket_calls == [], "on_bucket не должен вызываться для skip-бакетов"
assert lots == []
@pytest.mark.asyncio
async def test_secondary_only_drops_novostroyki(scraper: AvitoScraper) -> None:
"""secondary_only=True: новостройки (listing_segment=='novostroyki') отброшены."""
mixed = [
_make_lot("nb_1", segment="novostroyki"),
_make_lot("vt_1", segment="vtorichka"),
_make_lot("vt_2", segment="vtorichka"),
]
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"page={page}"
def fake_total(html: str) -> int | None:
return 3
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return mixed if page == 1 else []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
secondary_only=True,
)
ids = {lot.source_id for lot in lots}
assert "nb_1" not in ids
assert ids == {"vt_1", "vt_2"}
@pytest.mark.asyncio
async def test_on_bucket_cancel_stops_run(scraper: AvitoScraper) -> None:
"""on_bucket кидает RuntimeError('cancelled') → прогон прерывается."""
def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
raise RuntimeError("cancelled")
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"page={page}"
def fake_total(html: str) -> int | None:
return 5
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
return [_make_lot("lot_1")]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
with pytest.raises(RuntimeError, match="cancelled"):
await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X"), ("2-комн.", "2-komnatnye-X")],
price_cap_per_bucket=1400,
on_bucket=cancel_on_bucket,
)
# ── Probe-устойчивость к browser-ошибкам (HTTP 500 / timeout) ─────────────────
def _http_500_error() -> httpx.HTTPStatusError:
"""Сборка httpx.HTTPStatusError(500) как из browser_fetcher.raise_for_status()."""
request = httpx.Request("POST", "http://tradein-browser:3000/fetch")
response = httpx.Response(500, request=request)
return httpx.HTTPStatusError("Server error '500'", request=request, response=response)
@pytest.mark.asyncio
async def test_probe_browser_500_on_wide_bucket_splits_not_crash(
scraper: AvitoScraper,
) -> None:
"""Probe бросает HTTP 500 на ШИРОКОМ bucket → НЕ крах, происходит split.
Воспроизводит прод-инцидент run 199: студии pmax=200M таймаутят на goto →
browser HTTP 500. Раньше это роняло весь full-load с 0 uniq. Теперь широкий
bucket принудительно делится пополам; под-бакеты пагинируются нормально.
"""
# Первый закрытый seed-брекет (0, 4_000_000) → walk_hi=3_999_999.
_br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
assert br_hi_raw is not None
br_hi = br_hi_raw - 1
probe_calls: list[tuple[int | None, int | None]] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
# Probe верхней границы ШИРОКОГО первого брекета падает (имитация HTTP 500
# от tradein-browser на тяжёлой выдаче) → должен произойти split.
if page == 1 and (min_price in (None, 0)) and max_price == br_hi:
probe_calls.append((min_price, max_price))
raise _http_500_error()
return f"page={page} min={min_price} max={max_price}"
def fake_total(html: str) -> int | None:
return 50
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "MAX"
if page == 1:
return [_make_lot(f"lot_{min_p}_{max_p}_{i}") for i in range(3)]
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("студии", "studii-X")],
price_cap_per_bucket=1400,
)
assert probe_calls, "Probe первого (широкого) брекета должен был вызваться и упасть"
assert len(lots) > 0, f"Ожидались лоты из под-бакетов после split, got {len(lots)}"
@pytest.mark.asyncio
async def test_probe_timeout_on_wide_bucket_splits_not_crash(
scraper: AvitoScraper,
) -> None:
"""httpx.TimeoutException на широком bucket обрабатывается как HTTP 500 → split."""
_br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
assert br_hi_raw is not None
br_hi = br_hi_raw - 1
probe_failed = False
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
nonlocal probe_failed
if page == 1 and (min_price in (None, 0)) and max_price == br_hi:
probe_failed = True
raise httpx.TimeoutException("Page.goto timeout")
return f"page={page} min={min_price} max={max_price}"
def fake_total(html: str) -> int | None:
return 40
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "MAX"
return [_make_lot(f"lot_{min_p}_{max_p}_{i}") for i in range(2)] if page == 1 else []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("студии", "studii-X")],
price_cap_per_bucket=1400,
)
assert probe_failed, "Probe должен был таймаутнуть"
assert len(lots) > 0, "После split под-бакеты должны собрать лоты"
@pytest.mark.asyncio
async def test_probe_browser_500_on_narrow_bucket_skips_not_crash(
scraper: AvitoScraper,
) -> None:
"""Probe бросает HTTP 500 на УЗКОМ bucket (< MIN_BRACKET) → skip+return, без краха.
Узкий bucket делить бесполезно — хвост теряется осознанно. Прогон не падает,
остальные комнатности продолжаются.
"""
second_room_paginated = False
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
nonlocal second_room_paginated
if room_slug == "narrow-X":
raise _http_500_error()
if room_slug == "ok-X" and page == 1:
second_room_paginated = True
return f"page={page}"
def fake_total(html: str) -> int | None:
return 5
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
return [_make_lot("ok_lot")] if "page=1" in html else []
narrow_lo = 1_000_000
narrow_hi = narrow_lo + _AVITO_MIN_BRACKET - 1
seen: dict[str, ScrapedLot] = {}
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
await scraper._walk_price_range(
room_slug="narrow-X",
room_label="room_narrow",
lo=narrow_lo,
hi=narrow_hi,
seen=seen,
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
)
assert seen == {}, "Узкий bucket с упавшим probe не должен дать лотов"
await scraper._walk_price_range(
room_slug="ok-X",
room_label="room_ok",
lo=0,
hi=_AVITO_MAX_PRICE,
seen=seen,
price_cap_per_bucket=1400,
max_pages_per_bucket=100,
)
assert second_room_paginated, "Вторая комнатность должна пагинироваться после skip"
assert "ok_lot" in seen
@pytest.mark.asyncio
async def test_probe_avito_blocked_propagates_not_swallowed(
scraper: AvitoScraper,
) -> None:
"""AvitoBlockedError из probe ВСПЛЫВАЕТ (hard-block), а не глушится как total=None."""
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
raise AvitoBlockedError("Avito SERP firewall (browser-mode) — IP banned")
def fake_total(html: str) -> int | None:
return 100
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=lambda h, source_url_base: []),
patch.object(scraper, "_rotate_ip", return_value=False),
):
with pytest.raises(AvitoBlockedError):
await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=1400,
)
# ── Seed price-brackets entry point (perf: open top bucket) ───────────────────
@pytest.mark.asyncio
async def test_fetch_all_secondary_walks_seed_brackets_per_room(
scraper: AvitoScraper,
) -> None:
"""fetch_all_secondary зовёт _walk_price_range РОВНО len(seed) раз на комнатность.
Аргументы lo/hi совпадают с seed-брекетами: закрытые передаются с hi=br_hi-1
(эксклюзивный верх, без overlap на стыках), последний (открытый) — hi=None.
"""
walk_calls: list[tuple[int, int | None]] = []
async def spy_walk(*args: object, **kwargs: object) -> None:
walk_calls.append((kwargs["lo"], kwargs["hi"])) # type: ignore[arg-type]
# Не уходим в реальную рекурсию — фиксируем только верхнеуровневые входы.
return None
with patch.object(scraper, "_walk_price_range", side_effect=spy_walk):
await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=2000,
)
expected = [
(br_lo, (br_hi - 1) if br_hi is not None else None)
for (br_lo, br_hi) in _AVITO_PRICE_SEED_BRACKETS
]
assert walk_calls == expected, f"Неверные входы _walk_price_range: {walk_calls}"
# Последний — открытый (hi=None).
assert walk_calls[-1][1] is None, "Последний seed-брекет должен быть открытым (hi=None)"
# Ровно один вызов на seed-брекет (1 комнатность).
assert len(walk_calls) == len(_AVITO_PRICE_SEED_BRACKETS)
@pytest.mark.asyncio
async def test_open_bracket_probes_pmax_none_and_paginates_without_split(
scraper: AvitoScraper,
) -> None:
"""Открытый брекет (hi=None): probe с pmax=None, пагинация БЕЗ попытки split.
Даже если total > cap (мокнуто) — НЕ должно быть рекурсивного деления:
при hi=None mid=(lo+hi)//2 невозможен, поэтому leaf пагинируется напрямую.
Проверяем: (1) probe идёт с max_price=None (pmax не ставится), (2) НЕТ
под-вызовов с пересчитанным mid (никаких max_price != None в fetch'ах).
"""
open_lo = _AVITO_PRICE_SEED_BRACKETS[-1][0]
assert _AVITO_PRICE_SEED_BRACKETS[-1][1] is None
fetch_args: list[tuple[int, int | None, int | None]] = []
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
fetch_args.append((page, min_price, max_price))
return f"page={page} min={min_price} max={max_price}"
# total > cap для ОТКРЫТОГО брекета — провоцируем split-соблазн. Закрытые
# брекеты возвращают 0 (не пагинируются), чтобы изолировать открытый.
def fake_total(html: str) -> int | None:
import re
max_m = re.search(r"max=(\w+)", html)
max_raw = max_m.group(1) if max_m else "None"
if max_raw == "None": # открытый брекет (pmax отсутствует)
return 5000 # > cap=2000 — но split НЕ должен случиться
return 0
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
# Только открытый брекет (max=None) отдаёт лоты.
if "max=None" not in html:
return []
return [_make_lot(f"open_lot_p{page}_{i}") for i in range(3)]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=2000,
max_pages_per_bucket=100,
)
# Probe открытого брекета: min_price=open_lo, max_price=None.
open_probes = [
(mn, mx) for (p, mn, mx) in fetch_args if p == 1 and mn == open_lo and mx is None
]
assert open_probes, f"Ожидался probe открытого брекета с pmax=None: {fetch_args}"
# Все fetch'и открытого брекета (min=open_lo) идут с max_price=None — НЕТ
# под-брекетов с вычисленным mid (что означало бы split).
open_fetches = [(mn, mx) for (p, mn, mx) in fetch_args if mn == open_lo]
assert all(
mx is None for (_mn, mx) in open_fetches
), f"Открытый брекет НЕ должен порождать closed-под-брекеты (split): {open_fetches}"
# Открытый брекет всё же собрал лоты (пагинация прошла, total=5000 → cap pages).
assert len(lots) > 0, "Открытый брекет должен пагинироваться и собрать лоты"
@pytest.mark.asyncio
async def test_closed_bracket_under_cap_single_probe_then_paginate(
scraper: AvitoScraper,
) -> None:
"""Закрытый брекет total str:
fetch_args.append((page, min_price, max_price))
return f"page={page} min={min_price} max={max_price}"
def fake_total(html: str) -> int | None:
import re
min_m = re.search(r"min=(\w+)", html)
max_m = re.search(r"max=(\w+)", html)
min_p = min_m.group(1) if min_m else "0"
max_p = max_m.group(1) if max_m else "None"
if min_p == str(target_lo) and max_p == str(target_hi):
return 60 # ceil(60/50)=2 страницы, < cap → без split
return 0
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
import re
if f"min={target_lo} max={target_hi}" not in html:
return []
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
if page <= 2:
return [_make_lot(f"t_lot_p{page}_{i}") for i in range(30)]
return []
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=2000,
)
# Целевой брекет: probe (page=1) + страница 2, без под-брекетов (split).
target_pages = sorted(p for (p, mn, mx) in fetch_args if mn == target_lo and mx == target_hi)
assert target_pages == [1, 2], f"Ожидались probe+page2 без split: {target_pages}"
assert len(lots) == 60, f"Ожидалось 60 лотов (2 страницы × 30), got {len(lots)}"
@pytest.mark.asyncio
async def test_dedup_source_id_across_brackets(scraper: AvitoScraper) -> None:
"""Дедуп по source_id работает между разными seed-брекетами (общий seen)."""
async def fake_fetch(
room_slug: str, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"page={page} min={min_price} max={max_price}"
def fake_total(html: str) -> int | None:
return 3 # 1 страница на каждый брекет
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
# ВСЕ брекеты отдают ОДИН и тот же source_id → должен остаться 1 уникальный.
return [_make_lot("shared_dup_id")]
with (
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
patch.object(scraper, "_parse_html", side_effect=fake_parse),
patch.object(scraper, "_rotate_ip", return_value=False),
):
lots = await scraper.fetch_all_secondary(
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
price_cap_per_bucket=2000,
)
assert len(lots) == 1, f"Дедуп между брекетами должен дать 1 уникальный лот, got {len(lots)}"
assert lots[0].source_id == "shared_dup_id"