792 lines
35 KiB
Python
792 lines
35 KiB
Python
"""Тесты exhaustive-загрузки Cian: fetch_all_secondary + _walk_price_range.
|
||
|
||
Мокаем сетевые вызовы (_fetch_page_html, _extract_total_offers, sleep_between_requests)
|
||
для детерминированного контроля поведения без реального HTTP.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
# Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
import logging
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.base import ScrapedLot
|
||
from app.services.scrapers.cian import (
|
||
_CIAN_OFFERS_PER_PAGE,
|
||
_MAX_PRICE,
|
||
_MIN_BRACKET,
|
||
CianScraper,
|
||
)
|
||
from app.services.scrapers.price_brackets import get_price_seed_brackets
|
||
|
||
_SEED = get_price_seed_brackets()
|
||
|
||
|
||
def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot:
|
||
"""Минимальный ScrapedLot для теста."""
|
||
return ScrapedLot(
|
||
source="cian",
|
||
source_url=f"https://ekb.cian.ru/sale/flat/{source_id}/",
|
||
source_id=source_id,
|
||
address="Екатеринбург",
|
||
price_rub=price,
|
||
)
|
||
|
||
|
||
def _make_html_stub(total_offers: int, lot_ids: list[str]) -> str:
|
||
"""Заглушка HTML — не парсится напрямую; используется через мокнутые методы."""
|
||
return f"<html>total={total_offers} ids={lot_ids}</html>"
|
||
|
||
|
||
@pytest.fixture
|
||
def scraper() -> CianScraper:
|
||
"""CianScraper без реального __aenter__ (без HTTP warm-up)."""
|
||
s = CianScraper.__new__(CianScraper)
|
||
s.name = "cian"
|
||
s.base_url = "https://ekb.cian.ru"
|
||
s.request_delay_sec = 0.0
|
||
s._browser = MagicMock() # не None → assert self._browser is not None пройдёт
|
||
return s
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_splits_on_cap(scraper: CianScraper) -> None:
|
||
"""totalOffers > cap → бакет разбивается; в итоге оба суб-бакета пагинируются."""
|
||
# Настройка: rooms=(1,), один бакет [0, None]
|
||
# Probe [0, None]: totalOffers=3000 > cap=1400 → split на [0, MAX_PRICE]
|
||
# Probe [0, MAX_PRICE]: totalOffers=3000 > 1400 → split на [0, mid] и [mid+1, MAX_PRICE]
|
||
# Probe [0, mid] (mid = MAX_PRICE//2 = 100_000_000): totalOffers=800 <= cap → пагинировать
|
||
# Probe [mid+1, MAX_PRICE]: totalOffers=700 <= cap → пагинировать
|
||
|
||
mid = _MAX_PRICE // 2
|
||
|
||
call_log: list[tuple] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
call_log.append((rooms, page, min_price, max_price))
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
# Парсим min/max из fake html
|
||
import re
|
||
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0
|
||
max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else _MAX_PRICE
|
||
|
||
# Имитируем разделение: весь диапазон > 1400, левая и правая половины <= 1400
|
||
if max_p is None or max_p >= _MAX_PRICE:
|
||
if min_p == 0:
|
||
return 3000 # весь диапазон — нужно split
|
||
if max_p == mid:
|
||
return 800 # левая половина — пагинировать
|
||
if min_p == mid + 1:
|
||
return 700 # правая половина — пагинировать
|
||
return 3000 # всё остальное — split
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
min_p = min_m.group(1) if min_m else "0"
|
||
max_p = max_m.group(1) if max_m else "MAX"
|
||
# Возвращаем по 2 лота на первой странице каждого пагинируемого бакета
|
||
if page == 1:
|
||
return [
|
||
_make_lot(f"lot_{min_p}_{max_p}_p{page}_1"),
|
||
_make_lot(f"lot_{min_p}_{max_p}_p{page}_2"),
|
||
]
|
||
return [] # страница 2+ — пусто → early stop
|
||
|
||
async def fake_sleep(*_: object, **__: object) -> None:
|
||
pass
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
# Должны получить лоты из ОБОИХ суб-бакетов после split
|
||
assert len(lots) > 0, "Должны получить лоты после разделения бакета"
|
||
|
||
# Проверяем что был probe с большим диапазоном И с sub-бакетами
|
||
probed_ranges = [(min_p, max_p) for (_, page, min_p, max_p) in call_log if page == 1]
|
||
# Должен быть probe [0, None] или [0, MAX] — начальный probe всего диапазона
|
||
initial_probes = [r for r in probed_ranges if r[0] is None or r[0] == 0]
|
||
assert (
|
||
len(initial_probes) >= 1
|
||
), f"Ожидался probe всего диапазона, probed_ranges={probed_ranges}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_paginates_when_under_cap(scraper: CianScraper) -> None:
|
||
"""totalOffers <= cap → пагинирует бакет полностью (страница за страницей)."""
|
||
pages_fetched: list[int] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
pages_fetched.append(page)
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 56 # 56 офферов → ceil(56/28) = 2 страницы
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
if page <= 2:
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(28)]
|
||
return []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(2,)],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
# 28 лотов на стр 1 + 28 на стр 2 = 56, дедуп по source_id
|
||
assert len(lots) == 56, f"Ожидалось 56 уникальных лотов, получено {len(lots)}"
|
||
# Должно быть 2 страницы пагинации (page=1 probe + page=2)
|
||
assert 2 in pages_fetched, "Должна быть запрошена страница 2"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_min_bracket_guard(scraper: CianScraper) -> None:
|
||
"""Если hi - lo < MIN_BRACKET → пагинируем как есть (не делим бесконечно)."""
|
||
call_count = 0
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
if call_count > 100:
|
||
raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал")
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
# Всегда > cap → без guard делилось бы бесконечно
|
||
return 5000
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
return [] # пустой → early stop в пагинации
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
# Запускаем с очень узким диапазоном — guard должен остановить деление
|
||
await scraper._walk_price_range(
|
||
rooms=(1,),
|
||
lo=1_000_000,
|
||
hi=1_000_000 + _MIN_BRACKET - 1, # bracket < MIN_BRACKET
|
||
seen={},
|
||
price_cap_per_bucket=1400,
|
||
max_pages_per_bucket=54,
|
||
)
|
||
|
||
# Guard сработал: нет RuntimeError (бесконечная рекурсия не возникла).
|
||
# Параллельная пагинация запускает до max_pages_per_bucket=54 страниц единовременно
|
||
# (все страницы бакета параллельны), поэтому call_count может быть до 54.
|
||
# Важно что нет рекурсии (bracket < MIN_BRACKET → один проход пагинации, не деление).
|
||
assert call_count <= 55, f"Слишком много вызовов ({call_count}) — guard не сработал"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_on_bucket_called_per_leaf(scraper: CianScraper) -> None:
|
||
"""on_bucket вызывается после каждого leaf-бакета с лотами бакета."""
|
||
# totalOffers=56 ≤ cap → leaf-бакет, пагинируется параллельно 2 страницы
|
||
bucket_calls: list[tuple[str, list[ScrapedLot]]] = []
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
bucket_calls.append((bucket_key, list(lots)))
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 56 # ceil(56/28) = 2 страницы
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
concurrency=3,
|
||
on_bucket=fake_on_bucket,
|
||
)
|
||
|
||
# on_bucket вызван по разу на каждый seed-брекет (все непустые leaf-бакеты).
|
||
assert len(bucket_calls) == len(
|
||
_SEED
|
||
), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(bucket_calls)}"
|
||
key, lots = bucket_calls[0]
|
||
# Ключ бакета содержит room_label:lo:hi
|
||
assert "room1" in key, f"bucket_key должен содержать room1, got {key!r}"
|
||
# Лоты из обеих страниц переданы в on_bucket
|
||
assert len(lots) == 20, f"Ожидалось 20 лотов в on_bucket (2 стр × 10), получено {len(lots)}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_on_bucket_cancel_stops_run(scraper: CianScraper) -> None:
|
||
"""Если on_bucket кидает RuntimeError('cancelled') — прогон прерывается."""
|
||
|
||
def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
raise RuntimeError("cancelled")
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 28 # один лист → 1 страница
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
return [_make_lot("lot_1")]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
with pytest.raises(RuntimeError, match="cancelled"):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,), (2,)],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=cancel_on_bucket,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: CianScraper) -> None:
|
||
"""Параллельная пагинация не ломает дедупликацию по source_id."""
|
||
fetch_pages: list[int] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
fetch_pages.append(page)
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 84 # ceil(84/28) = 3 страницы
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
# Страница 1 и 2 дают уникальные лоты; страница 3 дублирует страницу 1
|
||
if page == 3:
|
||
return [_make_lot(f"lot_p1_{i}") for i in range(10)] # дубликаты
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
concurrency=5,
|
||
)
|
||
|
||
# Страницы 1+2 = 20 уникальных, страница 3 дублирует → итого 20
|
||
assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}"
|
||
# Все 3 страницы запрошены (параллельно или нет — нам важен факт)
|
||
assert 3 in fetch_pages or 3 in [
|
||
p for p in fetch_pages
|
||
], f"Страница 3 не была запрошена, fetch_pages={fetch_pages}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_skip_buckets_skips_pagination_and_on_bucket(scraper: CianScraper) -> None:
|
||
"""skip_buckets: для скипнутого бакета пагинация не вызывается, on_bucket не вызывается."""
|
||
fetch_calls: list[tuple] = []
|
||
on_bucket_calls: list[str] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
fetch_calls.append((rooms, page, min_price, max_price))
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 28 # один leaf-бакет, 1 страница
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
return [_make_lot("lot_1")]
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
on_bucket_calls.append(bucket_key)
|
||
|
||
# Скипаем ВСЕ seed-бакеты комнатности room1. Закрытые → ':<hi-1>', открытый → ':open'.
|
||
skip_set: set[str] = set()
|
||
for br_lo, br_hi in _SEED:
|
||
hi_repr = "open" if br_hi is None else str(br_hi - 1)
|
||
skip_set.add(f"room1:{br_lo}:{hi_repr}")
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=fake_on_bucket,
|
||
skip_buckets=skip_set,
|
||
)
|
||
|
||
# Probe (page=1) ВСЁ РАВНО вызывается на каждый seed-брекет (нужен для split-решения),
|
||
# но НЕ должно быть страниц пагинации (все бакеты в skip_buckets).
|
||
# Probe = page=1; пагинация пропущена ДО fetch остальных страниц → ровно len(_SEED) probe.
|
||
assert len(fetch_calls) == len(
|
||
_SEED
|
||
), f"Ожидалось {len(_SEED)} probe-вызовов, получено {len(fetch_calls)}: {fetch_calls}"
|
||
assert all(c[1] == 1 for c in fetch_calls), "Все вызовы должны быть probe page=1"
|
||
# on_bucket не вызывается для skip-бакетов
|
||
assert (
|
||
len(on_bucket_calls) == 0
|
||
), f"on_bucket не должен вызываться для skip-бакетов, got: {on_bucket_calls}"
|
||
# lots пустой — ничего не собрали (пагинация пропущена)
|
||
assert len(lots) == 0, f"Ожидалось 0 лотов для skip-бакетов, got {len(lots)}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_on_bucket_receives_key_and_lots(scraper: CianScraper) -> None:
|
||
"""on_bucket получает (bucket_key, lots) — bucket_key в формате room_label:lo:hi."""
|
||
received: list[tuple[str, int]] = [] # (key, len(lots))
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
received.append((bucket_key, len(lots)))
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 28 # 1 страница
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(5)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(2,)],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=fake_on_bucket,
|
||
)
|
||
|
||
assert len(received) == len(
|
||
_SEED
|
||
), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(received)}"
|
||
keys = [k for (k, _c) in received]
|
||
# Закрытые брекеты: hi = br_hi - 1. Открытый (последний): ':open'.
|
||
first_lo, first_hi = _SEED[0]
|
||
assert first_hi is not None
|
||
assert keys[0] == f"room2:{first_lo}:{first_hi - 1}", f"Неверный ключ: {keys[0]!r}"
|
||
last_lo, last_hi = _SEED[-1]
|
||
assert last_hi is None
|
||
assert keys[-1] == f"room2:{last_lo}:open", f"Открытый ключ неверен: {keys[-1]!r}"
|
||
assert all(count == 5 for (_k, count) in received)
|
||
|
||
|
||
# ── secondary_only фильтр ─────────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_only_drops_novostroyki(scraper: CianScraper) -> None:
|
||
"""secondary_only=True: новостройки (listing_segment=='novostroyki') не попадают
|
||
в on_bucket и в итоговый список; вторичка и None-сегмент — пропускаются.
|
||
"""
|
||
on_bucket_calls: list[list[ScrapedLot]] = []
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
on_bucket_calls.append(list(lots))
|
||
|
||
# Микс лотов: 2 новостройки + 2 вторички + 1 None-сегмент
|
||
mixed_lots = [
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/nb_1/",
|
||
source_id="nb_1",
|
||
address="ЖК Новый",
|
||
price_rub=5_000_000,
|
||
listing_segment="novostroyki",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/nb_2/",
|
||
source_id="nb_2",
|
||
address="ЖК Новый 2",
|
||
price_rub=6_000_000,
|
||
listing_segment="novostroyki",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/vt_1/",
|
||
source_id="vt_1",
|
||
address="ул. Ленина",
|
||
price_rub=3_500_000,
|
||
listing_segment="vtorichka",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/vt_2/",
|
||
source_id="vt_2",
|
||
address="ул. Мира",
|
||
price_rub=4_000_000,
|
||
listing_segment="vtorichka",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/none_seg/",
|
||
source_id="none_seg",
|
||
address="ул. Пушкина",
|
||
price_rub=3_000_000,
|
||
listing_segment=None,
|
||
),
|
||
]
|
||
|
||
page_call_count = 0
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
nonlocal page_call_count
|
||
page_call_count += 1
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 5 # <= cap → leaf-бакет
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
# Первая страница отдаёт весь микс; вторая — пусто
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return mixed_lots if page == 1 else []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
secondary_only=True,
|
||
on_bucket=fake_on_bucket,
|
||
)
|
||
|
||
# Итоговый список: только vtorichka + None-сегмент (3 лота), без новостроек
|
||
assert len(lots) == 3, f"Ожидалось 3 лота (без novostroyki), получено {len(lots)}"
|
||
lot_ids = {lot.source_id for lot in lots}
|
||
assert "nb_1" not in lot_ids, "nb_1 (novostroyki) не должен присутствовать"
|
||
assert "nb_2" not in lot_ids, "nb_2 (novostroyki) не должен присутствовать"
|
||
assert "vt_1" in lot_ids, "vt_1 (vtorichka) должен присутствовать"
|
||
assert "vt_2" in lot_ids, "vt_2 (vtorichka) должен присутствовать"
|
||
assert "none_seg" in lot_ids, "none_seg (listing_segment=None) должен присутствовать"
|
||
|
||
# on_bucket вызывается на каждый seed-брекет (тот же микс на page=1 каждого),
|
||
# но новостройки отфильтрованы в каждом вызове.
|
||
assert len(on_bucket_calls) == len(
|
||
_SEED
|
||
), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(on_bucket_calls)}"
|
||
for call_lots in on_bucket_calls:
|
||
bucket_ids = {lot.source_id for lot in call_lots}
|
||
assert "nb_1" not in bucket_ids
|
||
assert "nb_2" not in bucket_ids
|
||
assert len(call_lots) == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None:
|
||
"""secondary_only=False: новостройки НЕ отфильтровываются — все лоты проходят."""
|
||
mixed_lots = [
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/nb_1/",
|
||
source_id="nb_1",
|
||
address="ЖК Новый",
|
||
price_rub=5_000_000,
|
||
listing_segment="novostroyki",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/vt_1/",
|
||
source_id="vt_1",
|
||
address="ул. Ленина",
|
||
price_rub=3_500_000,
|
||
listing_segment="vtorichka",
|
||
),
|
||
]
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 2
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return mixed_lots if page == 1 else []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
secondary_only=False,
|
||
)
|
||
|
||
assert len(lots) == 2, f"Ожидалось 2 лота (фильтр выключен), получено {len(lots)}"
|
||
lot_ids = {lot.source_id for lot in lots}
|
||
assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False"
|
||
assert "vt_1" in lot_ids
|
||
|
||
|
||
# ── #1393: tail-loss warning при превышении page cap ─────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_walk_price_range_warns_on_page_cap_tail_loss(
|
||
scraper: CianScraper, caplog: pytest.LogCaptureFixture
|
||
) -> None:
|
||
"""#1393: когда totalOffers > max_pages_per_bucket × offers_per_page,
|
||
_walk_price_range логирует WARNING с числом потерянных офферов.
|
||
|
||
Сценарий: bracket < _MIN_BRACKET (guard срабатывает → force-paginate),
|
||
totalOffers = max_pages * offers_per_page + 50 → tail_loss = 50.
|
||
"""
|
||
max_pages_cap = 5 # нарочно мало, чтобы быстро дойти до cap
|
||
offers_per_page = _CIAN_OFFERS_PER_PAGE # 28
|
||
# total > cap (2000 > любой price_cap_per_bucket=1400) И bracket < _MIN_BRACKET
|
||
# → force-paginate path; pages_needed = ceil(2000/28) = 72 > max_pages_cap=5
|
||
total_offers = max_pages_cap * offers_per_page + 50 # 190; tail_loss=50
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return total_offers
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(offers_per_page)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
|
||
):
|
||
await scraper._walk_price_range(
|
||
rooms=(1,),
|
||
# Узкий диапазон: bracket < _MIN_BRACKET → force-paginate guard
|
||
lo=1_000_000,
|
||
hi=1_000_000 + _MIN_BRACKET - 1,
|
||
seen={},
|
||
price_cap_per_bucket=1400, # total(190) < 1400 → нет split, только leaf
|
||
max_pages_per_bucket=max_pages_cap,
|
||
)
|
||
|
||
# Должно быть предупреждение о tail-loss
|
||
tail_warnings = [
|
||
r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
|
||
]
|
||
assert tail_warnings, (
|
||
f"Ожидалось WARNING о tail loss, но записей нет. "
|
||
f"Все warnings: {[r.message for r in caplog.records if r.levelno == logging.WARNING]}"
|
||
)
|
||
# Сообщение должно содержать число потерянных офферов (50)
|
||
assert any(
|
||
"50" in r.message for r in tail_warnings
|
||
), f"WARNING должен содержать tail_loss=50, got: {[r.message for r in tail_warnings]}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_walk_price_range_no_warning_when_within_page_cap(
|
||
scraper: CianScraper, caplog: pytest.LogCaptureFixture
|
||
) -> None:
|
||
"""#1393: когда totalOffers <= max_pages_per_bucket × offers_per_page,
|
||
tail-loss WARNING не должен эмитироваться (данные собраны полностью).
|
||
"""
|
||
max_pages_cap = 54
|
||
total_offers = 56 # ceil(56/28) = 2 страницы < 54 cap → нет потерь
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return total_offers
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
|
||
):
|
||
await scraper._walk_price_range(
|
||
rooms=(1,),
|
||
lo=0,
|
||
hi=_MAX_PRICE,
|
||
seen={},
|
||
price_cap_per_bucket=1400,
|
||
max_pages_per_bucket=max_pages_cap,
|
||
)
|
||
|
||
tail_warnings = [
|
||
r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
|
||
]
|
||
assert not tail_warnings, (
|
||
f"Не ожидалось WARNING о tail loss при total={total_offers} <= page cap, "
|
||
f"но получено: {[r.message for r in tail_warnings]}"
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_walk_price_range_open_bracket_no_split(scraper: CianScraper) -> None:
|
||
"""Открытый брекет (hi=None) не делится даже при total > cap — leaf без maxprice.
|
||
|
||
bucket_key оканчивается на ':open'; maxprice в запросах всегда None (без потолка).
|
||
"""
|
||
on_bucket_keys: list[str] = []
|
||
seen_max: list[int | None] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
seen_max.append(max_price)
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 5000 # > cap, но открытый брекет делить нельзя → leaf
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return [_make_lot(f"lot_p{page}")] if page == 1 else []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
await scraper._walk_price_range(
|
||
rooms=(1,),
|
||
lo=250_000_000,
|
||
hi=None,
|
||
seen={},
|
||
price_cap_per_bucket=1400,
|
||
max_pages_per_bucket=54,
|
||
on_bucket=lambda k, lots: on_bucket_keys.append(k),
|
||
)
|
||
|
||
# Открытый брекет: ровно один leaf-бакет с ключом ':open', maxprice всегда None.
|
||
assert on_bucket_keys == [
|
||
"room1:250000000:open"
|
||
], f"Открытый брекет должен дать один ':open'-бакет, got {on_bucket_keys}"
|
||
assert all(
|
||
m is None for m in seen_max
|
||
), f"Открытый брекет: maxprice всегда None (без потолка), got {seen_max}"
|