921 lines
40 KiB
Python
921 lines
40 KiB
Python
"""Тесты exhaustive-загрузки Avito: fetch_all_secondary + _walk_price_range.
|
||
|
||
Зеркало tests/scrapers/test_cian_exhaustive.py. Мокаем сетевые вызовы
|
||
(_fetch_rooms_page_html, _extract_total_count, _parse_html) для детерминированного
|
||
контроля бисекции без реального HTTP.
|
||
|
||
Дополнительно: _build_rooms_url с pmin/pmax (backward-compat) и _extract_total_count
|
||
на реальном span-фрагменте `data-marker="page-title/count"`.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
# Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from app.services.scrapers.avito import (
|
||
_AVITO_MAX_PRICE,
|
||
_AVITO_MIN_BRACKET,
|
||
_AVITO_OFFERS_PER_PAGE,
|
||
_AVITO_PRICE_SEED_BRACKETS,
|
||
AvitoScraper,
|
||
)
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
|
||
def _make_lot(source_id: str, price: int = 3_000_000, segment: str = "vtorichka") -> ScrapedLot:
|
||
"""Минимальный ScrapedLot для теста."""
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
|
||
source_id=source_id,
|
||
address="Екатеринбург",
|
||
price_rub=price,
|
||
listing_segment=segment,
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def scraper() -> AvitoScraper:
|
||
"""AvitoScraper без реального __aenter__ (без HTTP/IP-rotation)."""
|
||
s = AvitoScraper.__new__(AvitoScraper)
|
||
s.name = "avito"
|
||
s.base_url = "https://www.avito.ru"
|
||
s.request_delay_sec = 0.0
|
||
s._cffi = MagicMock() # не None
|
||
s._browser = None
|
||
s.parse_failures = 0
|
||
return s
|
||
|
||
|
||
# ── _build_rooms_url с pmin/pmax ──────────────────────────────────────────────
|
||
|
||
|
||
def test_build_rooms_url_both_prices(scraper: AvitoScraper) -> None:
|
||
"""min_price + max_price → pmin/pmax оба в query."""
|
||
url = scraper._build_rooms_url(
|
||
"1-komnatnye-X", page=2, min_price=3_000_000, max_price=4_000_000
|
||
)
|
||
assert "pmin=3000000" in url
|
||
assert "pmax=4000000" in url
|
||
assert "p=2" in url
|
||
assert "s=104" in url
|
||
assert "/kvartiry/prodam/1-komnatnye-X?" in url
|
||
|
||
|
||
def test_build_rooms_url_min_only(scraper: AvitoScraper) -> None:
|
||
"""Только min_price → pmin в query, pmax отсутствует."""
|
||
url = scraper._build_rooms_url("1-komnatnye-X", min_price=3_000_000)
|
||
assert "pmin=3000000" in url
|
||
assert "pmax" not in url
|
||
|
||
|
||
def test_build_rooms_url_no_prices_backward_compat(scraper: AvitoScraper) -> None:
|
||
"""Без цен URL не меняется (backward-compat для fetch_by_rooms)."""
|
||
url = scraper._build_rooms_url("1-komnatnye-X", page=1)
|
||
assert "pmin" not in url
|
||
assert "pmax" not in url
|
||
assert url.endswith("/kvartiry/prodam/1-komnatnye-X?s=104&p=1")
|
||
|
||
|
||
# ── _extract_total_count ──────────────────────────────────────────────────────
|
||
|
||
# Реальный span-фрагмент из живого Avito SERP (NBSP-разделитель разрядов).
|
||
_REAL_COUNT_SPAN = '<span class="some-css" data-marker="page-title/count">1\xa0178</span>'
|
||
|
||
|
||
def test_extract_total_count_real_span(scraper: AvitoScraper) -> None:
|
||
"""Реальный span с NBSP `1\\xa0178` → 1178."""
|
||
html = f"<div><h1>Квартиры</h1>{_REAL_COUNT_SPAN}</div>"
|
||
assert scraper._extract_total_count(html) == 1178
|
||
|
||
|
||
def test_extract_total_count_nbsp_entity(scraper: AvitoScraper) -> None:
|
||
"""Вариант с ` `-entity → корректно очищается."""
|
||
html = '<span data-marker="page-title/count">12 345</span>'
|
||
assert scraper._extract_total_count(html) == 12345
|
||
|
||
|
||
def test_extract_total_count_zero(scraper: AvitoScraper) -> None:
|
||
"""0 результатов → 0 (не None)."""
|
||
html = '<span data-marker="page-title/count">0</span>'
|
||
assert scraper._extract_total_count(html) == 0
|
||
|
||
|
||
def test_extract_total_count_missing_returns_none(scraper: AvitoScraper) -> None:
|
||
"""Captcha/firewall (нет span, нет no-results-маркера) → None."""
|
||
html = "<html><head><title>Доступ ограничен</title></head></html>"
|
||
assert scraper._extract_total_count(html) is None
|
||
|
||
|
||
def test_extract_total_count_no_results_marker_returns_zero(scraper: AvitoScraper) -> None:
|
||
"""Валидная пустая выборка: нет `page-title/count`, но есть «ничего не найдено» → 0.
|
||
|
||
Даже если на странице есть fallback-«похожие» объявления (item-id'ы) — это
|
||
не подходящие под фильтр листинги, total всё равно 0. Это предотвращает
|
||
дробление пустого брекета + ротацию IP (прод-инцидент run 209).
|
||
"""
|
||
html = (
|
||
'<div data-marker="item" data-item-id="1234567"></div>'
|
||
'<div data-marker="item" data-item-id="7654321"></div>'
|
||
"<p>По вашему запросу ничего не найдено</p>"
|
||
)
|
||
assert scraper._extract_total_count(html) == 0
|
||
|
||
|
||
def test_extract_total_count_no_results_marker_case_insensitive(scraper: AvitoScraper) -> None:
|
||
"""Маркер регистронезависим: «Ничего не найдено» с заглавной → тоже 0."""
|
||
html = "<div><h2>Ничего не найдено</h2><p>Попробуйте изменить фильтры</p></div>"
|
||
assert scraper._extract_total_count(html) == 0
|
||
|
||
|
||
def test_extract_total_count_count_wins_over_no_results(scraper: AvitoScraper) -> None:
|
||
"""Непустая выборка с count имеет приоритет, даже если где-то ниже есть маркер."""
|
||
html = (
|
||
f"<div><h1>Квартиры</h1>{_REAL_COUNT_SPAN}</div>"
|
||
"<footer>Если ничего не найдено, измените запрос</footer>"
|
||
)
|
||
assert scraper._extract_total_count(html) == 1178
|
||
|
||
|
||
# ── Бисекция: split / paginate / dedup ────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_splits_on_cap(scraper: AvitoScraper) -> None:
|
||
"""Внутри seed-брекета total > cap → бакет делится пополам; суб-бакеты пагинируются."""
|
||
# Первый закрытый seed-брекет (0, 4_000_000) → walk_hi=3_999_999.
|
||
br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
|
||
assert br_hi_raw is not None
|
||
br_hi = br_hi_raw - 1 # эксклюзивный верх
|
||
mid = (br_lo + br_hi) // 2
|
||
call_log: list[tuple] = []
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
call_log.append((page, min_price, max_price))
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
import re
|
||
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0
|
||
max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else None
|
||
# Внутри первого брекета [0, 3_999_999]: весь брекет > cap → split;
|
||
# половинки < cap → пагинируются. Прочие seed-брекеты — пусто (0).
|
||
if min_p == 0 and max_p == br_hi:
|
||
return 3000 # весь первый брекет — split
|
||
if min_p == 0 and max_p == mid:
|
||
return 800 # левая половина — пагинировать
|
||
if min_p == mid + 1 and max_p == br_hi:
|
||
return 700 # правая половина — пагинировать
|
||
return 0 # остальные seed-брекеты пусты
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
min_p = min_m.group(1) if min_m else "0"
|
||
max_p = max_m.group(1) if max_m else "MAX"
|
||
if page == 1:
|
||
return [
|
||
_make_lot(f"lot_{min_p}_{max_p}_1"),
|
||
_make_lot(f"lot_{min_p}_{max_p}_2"),
|
||
]
|
||
return []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
assert len(lots) > 0
|
||
# Должны быть пробы суб-бакетов (после split весь брекет не пагинируется).
|
||
probed_max = [mx for (p, _mn, mx) in call_log if p == 1]
|
||
assert any(mx == mid for mx in probed_max), f"Ожидался probe левой половины, got {probed_max}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_paginates_when_under_cap(scraper: AvitoScraper) -> None:
|
||
"""total <= cap → пагинирует бакет полностью."""
|
||
pages_fetched: list[int] = []
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
pages_fetched.append(page)
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 100 # ceil(100/50) = 2 страницы
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
if page <= 2:
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(_AVITO_OFFERS_PER_PAGE)]
|
||
return []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("2-комн.", "2-komnatnye-X")],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
assert len(lots) == 100, f"Ожидалось 100 уникальных лотов, получено {len(lots)}"
|
||
assert 2 in pages_fetched, "Должна быть запрошена страница 2"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_min_bracket_guard(scraper: AvitoScraper) -> None:
|
||
"""hi - lo < MIN_BRACKET → пагинируем как есть (нет бесконечной рекурсии)."""
|
||
call_count = 0
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
if call_count > 200:
|
||
raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал")
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 5000 # всегда > cap → без guard делилось бы бесконечно
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
return []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
await scraper._walk_price_range(
|
||
room_slug="1-komnatnye-X",
|
||
room_label="room_1_komn",
|
||
lo=1_000_000,
|
||
hi=1_000_000 + _AVITO_MIN_BRACKET - 1,
|
||
seen={},
|
||
price_cap_per_bucket=1400,
|
||
max_pages_per_bucket=100,
|
||
)
|
||
|
||
# Guard сработал: рекурсии нет, только один проход пагинации (probe + до cap страниц).
|
||
assert call_count <= 101, f"Слишком много вызовов ({call_count}) — guard не сработал"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: AvitoScraper) -> None:
|
||
"""Параллельная пагинация не ломает дедуп по source_id."""
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 150 # ceil(150/50) = 3 страницы
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
if page == 3:
|
||
return [_make_lot(f"lot_p1_{i}") for i in range(10)] # дубликаты стр 1
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=1400,
|
||
concurrency=5,
|
||
)
|
||
|
||
# Страницы 1+2 = 20 уникальных, страница 3 дублирует → 20
|
||
assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_on_bucket_receives_key_and_lots(scraper: AvitoScraper) -> None:
|
||
"""on_bucket получает (bucket_key, lots) на каждый seed-брекет.
|
||
|
||
bucket_key закрытого брекета = "room_label:lo:hi" (hi = br_hi-1, эксклюзивный);
|
||
открытого (последнего) = "room_label:lo:open". Одна on_bucket-нотификация
|
||
на каждый непустой seed-брекет (здесь все непустые → len(brackets) вызовов).
|
||
"""
|
||
received: list[tuple[str, int]] = []
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
received.append((bucket_key, len(lots)))
|
||
|
||
# Уникальные source_id на каждый брекет, чтобы on_bucket получал непустые
|
||
# списки во ВСЕХ брекетах (дедуп в seen не должен опустошать поздние брекеты).
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 5 # 1 страница
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
min_p = min_m.group(1) if min_m else "0"
|
||
max_p = max_m.group(1) if max_m else "open"
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return [_make_lot(f"lot_{min_p}_{max_p}_p{page}_{i}") for i in range(5)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("2-комн.", "2-komnatnye-X")],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=fake_on_bucket,
|
||
)
|
||
|
||
# Один on_bucket на каждый seed-брекет (все непустые).
|
||
assert len(received) == len(
|
||
_AVITO_PRICE_SEED_BRACKETS
|
||
), f"Ожидалось {len(_AVITO_PRICE_SEED_BRACKETS)} вызовов on_bucket, получено {len(received)}"
|
||
keys = [k for (k, _c) in received]
|
||
# Закрытые брекеты: hi = br_hi - 1. Открытый (последний): ':open'.
|
||
first_lo, first_hi = _AVITO_PRICE_SEED_BRACKETS[0]
|
||
assert first_hi is not None
|
||
assert keys[0] == f"room_2_komn:{first_lo}:{first_hi - 1}", f"Неверный ключ: {keys[0]!r}"
|
||
last_lo, last_hi = _AVITO_PRICE_SEED_BRACKETS[-1]
|
||
assert last_hi is None
|
||
assert keys[-1] == f"room_2_komn:{last_lo}:open", f"Открытый ключ неверен: {keys[-1]!r}"
|
||
assert all(count == 5 for (_k, count) in received)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_skip_buckets_skips_pagination(scraper: AvitoScraper) -> None:
|
||
"""skip_buckets: для skip-бакета probe выполняется, пагинация и on_bucket — нет."""
|
||
fetch_calls: list[tuple[int, int | None, int | None]] = []
|
||
on_bucket_calls: list[str] = []
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
fetch_calls.append((page, min_price, max_price))
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 5
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
return [_make_lot(f"lot_{min_m.group(1) if min_m else 0}_{max_m.group(1) if max_m else 0}")]
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
on_bucket_calls.append(bucket_key)
|
||
|
||
# Skip ВСЕ брекеты → ни одной on_bucket-нотификации, но каждый брекет
|
||
# всё равно делает probe (page=1).
|
||
first_lo, first_hi = _AVITO_PRICE_SEED_BRACKETS[0]
|
||
assert first_hi is not None
|
||
first_key = f"room_1_komn:{first_lo}:{first_hi - 1}"
|
||
skip_all: set[str] = set()
|
||
for br_lo, br_hi in _AVITO_PRICE_SEED_BRACKETS:
|
||
hi_repr = "open" if br_hi is None else str(br_hi - 1)
|
||
skip_all.add(f"room_1_komn:{br_lo}:{hi_repr}")
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=fake_on_bucket,
|
||
skip_buckets=skip_all,
|
||
)
|
||
|
||
# Ровно один probe (page=1) на каждый seed-брекет, без доп. страниц.
|
||
assert first_key in skip_all
|
||
assert all(
|
||
p == 1 for (p, _mn, _mx) in fetch_calls
|
||
), f"Ожидались только probe-вызовы: {fetch_calls}"
|
||
assert len(fetch_calls) == len(
|
||
_AVITO_PRICE_SEED_BRACKETS
|
||
), f"Ожидался 1 probe на брекет ({len(_AVITO_PRICE_SEED_BRACKETS)}), got {len(fetch_calls)}"
|
||
assert on_bucket_calls == [], "on_bucket не должен вызываться для skip-бакетов"
|
||
assert lots == []
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_only_drops_novostroyki(scraper: AvitoScraper) -> None:
|
||
"""secondary_only=True: новостройки (listing_segment=='novostroyki') отброшены."""
|
||
mixed = [
|
||
_make_lot("nb_1", segment="novostroyki"),
|
||
_make_lot("vt_1", segment="vtorichka"),
|
||
_make_lot("vt_2", segment="vtorichka"),
|
||
]
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 3
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return mixed if page == 1 else []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=1400,
|
||
secondary_only=True,
|
||
)
|
||
|
||
ids = {lot.source_id for lot in lots}
|
||
assert "nb_1" not in ids
|
||
assert ids == {"vt_1", "vt_2"}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_on_bucket_cancel_stops_run(scraper: AvitoScraper) -> None:
|
||
"""on_bucket кидает RuntimeError('cancelled') → прогон прерывается."""
|
||
|
||
def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
raise RuntimeError("cancelled")
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 5
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
return [_make_lot("lot_1")]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
with pytest.raises(RuntimeError, match="cancelled"):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X"), ("2-комн.", "2-komnatnye-X")],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=cancel_on_bucket,
|
||
)
|
||
|
||
|
||
# ── Probe-устойчивость к browser-ошибкам (HTTP 500 / timeout) ─────────────────
|
||
|
||
|
||
def _http_500_error() -> httpx.HTTPStatusError:
|
||
"""Сборка httpx.HTTPStatusError(500) как из browser_fetcher.raise_for_status()."""
|
||
request = httpx.Request("POST", "http://tradein-browser:3000/fetch")
|
||
response = httpx.Response(500, request=request)
|
||
return httpx.HTTPStatusError("Server error '500'", request=request, response=response)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_probe_browser_500_on_wide_bucket_splits_not_crash(
|
||
scraper: AvitoScraper,
|
||
) -> None:
|
||
"""Probe бросает HTTP 500 на ШИРОКОМ bucket → НЕ крах, происходит split.
|
||
|
||
Воспроизводит прод-инцидент run 199: студии pmax=200M таймаутят на goto →
|
||
browser HTTP 500. Раньше это роняло весь full-load с 0 uniq. Теперь широкий
|
||
bucket принудительно делится пополам; под-бакеты пагинируются нормально.
|
||
"""
|
||
# Первый закрытый seed-брекет (0, 4_000_000) → walk_hi=3_999_999.
|
||
_br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
|
||
assert br_hi_raw is not None
|
||
br_hi = br_hi_raw - 1
|
||
probe_calls: list[tuple[int | None, int | None]] = []
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
# Probe верхней границы ШИРОКОГО первого брекета падает (имитация HTTP 500
|
||
# от tradein-browser на тяжёлой выдаче) → должен произойти split.
|
||
if page == 1 and (min_price in (None, 0)) and max_price == br_hi:
|
||
probe_calls.append((min_price, max_price))
|
||
raise _http_500_error()
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 50
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
min_p = min_m.group(1) if min_m else "0"
|
||
max_p = max_m.group(1) if max_m else "MAX"
|
||
if page == 1:
|
||
return [_make_lot(f"lot_{min_p}_{max_p}_{i}") for i in range(3)]
|
||
return []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("студии", "studii-X")],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
assert probe_calls, "Probe первого (широкого) брекета должен был вызваться и упасть"
|
||
assert len(lots) > 0, f"Ожидались лоты из под-бакетов после split, got {len(lots)}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_probe_timeout_on_wide_bucket_splits_not_crash(
|
||
scraper: AvitoScraper,
|
||
) -> None:
|
||
"""httpx.TimeoutException на широком bucket обрабатывается как HTTP 500 → split."""
|
||
_br_lo, br_hi_raw = _AVITO_PRICE_SEED_BRACKETS[0]
|
||
assert br_hi_raw is not None
|
||
br_hi = br_hi_raw - 1
|
||
probe_failed = False
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
nonlocal probe_failed
|
||
if page == 1 and (min_price in (None, 0)) and max_price == br_hi:
|
||
probe_failed = True
|
||
raise httpx.TimeoutException("Page.goto timeout")
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 40
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
min_p = min_m.group(1) if min_m else "0"
|
||
max_p = max_m.group(1) if max_m else "MAX"
|
||
return [_make_lot(f"lot_{min_p}_{max_p}_{i}") for i in range(2)] if page == 1 else []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("студии", "studii-X")],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
assert probe_failed, "Probe должен был таймаутнуть"
|
||
assert len(lots) > 0, "После split под-бакеты должны собрать лоты"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_probe_browser_500_on_narrow_bucket_skips_not_crash(
|
||
scraper: AvitoScraper,
|
||
) -> None:
|
||
"""Probe бросает HTTP 500 на УЗКОМ bucket (< MIN_BRACKET) → skip+return, без краха.
|
||
|
||
Узкий bucket делить бесполезно — хвост теряется осознанно. Прогон не падает,
|
||
остальные комнатности продолжаются.
|
||
"""
|
||
second_room_paginated = False
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
nonlocal second_room_paginated
|
||
if room_slug == "narrow-X":
|
||
raise _http_500_error()
|
||
if room_slug == "ok-X" and page == 1:
|
||
second_room_paginated = True
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 5
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
return [_make_lot("ok_lot")] if "page=1" in html else []
|
||
|
||
narrow_lo = 1_000_000
|
||
narrow_hi = narrow_lo + _AVITO_MIN_BRACKET - 1
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
await scraper._walk_price_range(
|
||
room_slug="narrow-X",
|
||
room_label="room_narrow",
|
||
lo=narrow_lo,
|
||
hi=narrow_hi,
|
||
seen=seen,
|
||
price_cap_per_bucket=1400,
|
||
max_pages_per_bucket=100,
|
||
)
|
||
assert seen == {}, "Узкий bucket с упавшим probe не должен дать лотов"
|
||
|
||
await scraper._walk_price_range(
|
||
room_slug="ok-X",
|
||
room_label="room_ok",
|
||
lo=0,
|
||
hi=_AVITO_MAX_PRICE,
|
||
seen=seen,
|
||
price_cap_per_bucket=1400,
|
||
max_pages_per_bucket=100,
|
||
)
|
||
|
||
assert second_room_paginated, "Вторая комнатность должна пагинироваться после skip"
|
||
assert "ok_lot" in seen
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_probe_avito_blocked_propagates_not_swallowed(
|
||
scraper: AvitoScraper,
|
||
) -> None:
|
||
"""AvitoBlockedError из probe ВСПЛЫВАЕТ (hard-block), а не глушится как total=None."""
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
raise AvitoBlockedError("Avito SERP firewall (browser-mode) — IP banned")
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 100
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=lambda h, source_url_base: []),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
with pytest.raises(AvitoBlockedError):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
|
||
# ── Seed price-brackets entry point (perf: open top bucket) ───────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_walks_seed_brackets_per_room(
|
||
scraper: AvitoScraper,
|
||
) -> None:
|
||
"""fetch_all_secondary зовёт _walk_price_range РОВНО len(seed) раз на комнатность.
|
||
|
||
Аргументы lo/hi совпадают с seed-брекетами: закрытые передаются с hi=br_hi-1
|
||
(эксклюзивный верх, без overlap на стыках), последний (открытый) — hi=None.
|
||
"""
|
||
walk_calls: list[tuple[int, int | None]] = []
|
||
|
||
async def spy_walk(*args: object, **kwargs: object) -> None:
|
||
walk_calls.append((kwargs["lo"], kwargs["hi"])) # type: ignore[arg-type]
|
||
# Не уходим в реальную рекурсию — фиксируем только верхнеуровневые входы.
|
||
return None
|
||
|
||
with patch.object(scraper, "_walk_price_range", side_effect=spy_walk):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=2000,
|
||
)
|
||
|
||
expected = [
|
||
(br_lo, (br_hi - 1) if br_hi is not None else None)
|
||
for (br_lo, br_hi) in _AVITO_PRICE_SEED_BRACKETS
|
||
]
|
||
assert walk_calls == expected, f"Неверные входы _walk_price_range: {walk_calls}"
|
||
# Последний — открытый (hi=None).
|
||
assert walk_calls[-1][1] is None, "Последний seed-брекет должен быть открытым (hi=None)"
|
||
# Ровно один вызов на seed-брекет (1 комнатность).
|
||
assert len(walk_calls) == len(_AVITO_PRICE_SEED_BRACKETS)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_open_bracket_probes_pmax_none_and_paginates_without_split(
|
||
scraper: AvitoScraper,
|
||
) -> None:
|
||
"""Открытый брекет (hi=None): probe с pmax=None, пагинация БЕЗ попытки split.
|
||
|
||
Даже если total > cap (мокнуто) — НЕ должно быть рекурсивного деления:
|
||
при hi=None mid=(lo+hi)//2 невозможен, поэтому leaf пагинируется напрямую.
|
||
Проверяем: (1) probe идёт с max_price=None (pmax не ставится), (2) НЕТ
|
||
под-вызовов с пересчитанным mid (никаких max_price != None в fetch'ах).
|
||
"""
|
||
open_lo = _AVITO_PRICE_SEED_BRACKETS[-1][0]
|
||
assert _AVITO_PRICE_SEED_BRACKETS[-1][1] is None
|
||
|
||
fetch_args: list[tuple[int, int | None, int | None]] = []
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
fetch_args.append((page, min_price, max_price))
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
# total > cap для ОТКРЫТОГО брекета — провоцируем split-соблазн. Закрытые
|
||
# брекеты возвращают 0 (не пагинируются), чтобы изолировать открытый.
|
||
def fake_total(html: str) -> int | None:
|
||
import re
|
||
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
max_raw = max_m.group(1) if max_m else "None"
|
||
if max_raw == "None": # открытый брекет (pmax отсутствует)
|
||
return 5000 # > cap=2000 — но split НЕ должен случиться
|
||
return 0
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
# Только открытый брекет (max=None) отдаёт лоты.
|
||
if "max=None" not in html:
|
||
return []
|
||
return [_make_lot(f"open_lot_p{page}_{i}") for i in range(3)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=2000,
|
||
max_pages_per_bucket=100,
|
||
)
|
||
|
||
# Probe открытого брекета: min_price=open_lo, max_price=None.
|
||
open_probes = [
|
||
(mn, mx) for (p, mn, mx) in fetch_args if p == 1 and mn == open_lo and mx is None
|
||
]
|
||
assert open_probes, f"Ожидался probe открытого брекета с pmax=None: {fetch_args}"
|
||
# Все fetch'и открытого брекета (min=open_lo) идут с max_price=None — НЕТ
|
||
# под-брекетов с вычисленным mid (что означало бы split).
|
||
open_fetches = [(mn, mx) for (p, mn, mx) in fetch_args if mn == open_lo]
|
||
assert all(
|
||
mx is None for (_mn, mx) in open_fetches
|
||
), f"Открытый брекет НЕ должен порождать closed-под-брекеты (split): {open_fetches}"
|
||
# Открытый брекет всё же собрал лоты (пагинация прошла, total=5000 → cap pages).
|
||
assert len(lots) > 0, "Открытый брекет должен пагинироваться и собрать лоты"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_closed_bracket_under_cap_single_probe_then_paginate(
|
||
scraper: AvitoScraper,
|
||
) -> None:
|
||
"""Закрытый брекет total<cap → 1 probe + прямая пагинация (без split)."""
|
||
# Изолируем второй закрытый брекет (4_000_000, 5_000_000) → walk_hi=4_999_999.
|
||
target_lo, target_hi_raw = _AVITO_PRICE_SEED_BRACKETS[1]
|
||
assert target_hi_raw is not None
|
||
target_hi = target_hi_raw - 1
|
||
|
||
fetch_args: list[tuple[int, int | None, int | None]] = []
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
fetch_args.append((page, min_price, max_price))
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
import re
|
||
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
min_p = min_m.group(1) if min_m else "0"
|
||
max_p = max_m.group(1) if max_m else "None"
|
||
if min_p == str(target_lo) and max_p == str(target_hi):
|
||
return 60 # ceil(60/50)=2 страницы, < cap → без split
|
||
return 0
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
if f"min={target_lo} max={target_hi}" not in html:
|
||
return []
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
if page <= 2:
|
||
return [_make_lot(f"t_lot_p{page}_{i}") for i in range(30)]
|
||
return []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=2000,
|
||
)
|
||
|
||
# Целевой брекет: probe (page=1) + страница 2, без под-брекетов (split).
|
||
target_pages = sorted(p for (p, mn, mx) in fetch_args if mn == target_lo and mx == target_hi)
|
||
assert target_pages == [1, 2], f"Ожидались probe+page2 без split: {target_pages}"
|
||
assert len(lots) == 60, f"Ожидалось 60 лотов (2 страницы × 30), got {len(lots)}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_dedup_source_id_across_brackets(scraper: AvitoScraper) -> None:
|
||
"""Дедуп по source_id работает между разными seed-брекетами (общий seen)."""
|
||
|
||
async def fake_fetch(
|
||
room_slug: str, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_total(html: str) -> int | None:
|
||
return 3 # 1 страница на каждый брекет
|
||
|
||
def fake_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
# ВСЕ брекеты отдают ОДИН и тот же source_id → должен остаться 1 уникальный.
|
||
return [_make_lot("shared_dup_id")]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_rooms_page_html", side_effect=fake_fetch),
|
||
patch.object(scraper, "_extract_total_count", side_effect=fake_total),
|
||
patch.object(scraper, "_parse_html", side_effect=fake_parse),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[("1-комн.", "1-komnatnye-X")],
|
||
price_cap_per_bucket=2000,
|
||
)
|
||
|
||
assert len(lots) == 1, f"Дедуп между брекетами должен дать 1 уникальный лот, got {len(lots)}"
|
||
assert lots[0].source_id == "shared_dup_id"
|