bot-backend
77ed21a479
object_type SERP-фильтр Cian не разделяет сегмент надёжно (object_type=1 ≠ вторичка), поэтому отсекаем новостройки по authoritative listing_segment (newbuilding.id>0) после пагинации, до save. fetch_all_secondary получает secondary_only=True (default). city-sweep не затронут.
599 lines
27 KiB
Python
599 lines
27 KiB
Python
"""Тесты exhaustive-загрузки Cian: fetch_all_secondary + _walk_price_range.
|
||
|
||
Мокаем сетевые вызовы (_fetch_page_html, _extract_total_offers, sleep_between_requests)
|
||
для детерминированного контроля поведения без реального HTTP.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
# Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.base import ScrapedLot
|
||
from app.services.scrapers.cian import _MAX_PRICE, _MIN_BRACKET, CianScraper
|
||
|
||
|
||
def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot:
|
||
"""Минимальный ScrapedLot для теста."""
|
||
return ScrapedLot(
|
||
source="cian",
|
||
source_url=f"https://ekb.cian.ru/sale/flat/{source_id}/",
|
||
source_id=source_id,
|
||
address="Екатеринбург",
|
||
price_rub=price,
|
||
)
|
||
|
||
|
||
def _make_html_stub(total_offers: int, lot_ids: list[str]) -> str:
|
||
"""Заглушка HTML — не парсится напрямую; используется через мокнутые методы."""
|
||
return f"<html>total={total_offers} ids={lot_ids}</html>"
|
||
|
||
|
||
@pytest.fixture
|
||
def scraper() -> CianScraper:
|
||
"""CianScraper без реального __aenter__ (без HTTP warm-up)."""
|
||
s = CianScraper.__new__(CianScraper)
|
||
s.name = "cian"
|
||
s.base_url = "https://ekb.cian.ru"
|
||
s.request_delay_sec = 0.0
|
||
s._cffi = MagicMock() # не None → assert self._cffi is not None пройдёт
|
||
return s
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_splits_on_cap(scraper: CianScraper) -> None:
|
||
"""totalOffers > cap → бакет разбивается; в итоге оба суб-бакета пагинируются."""
|
||
# Настройка: rooms=(1,), один бакет [0, None]
|
||
# Probe [0, None]: totalOffers=3000 > cap=1400 → split на [0, MAX_PRICE]
|
||
# Probe [0, MAX_PRICE]: totalOffers=3000 > 1400 → split на [0, mid] и [mid+1, MAX_PRICE]
|
||
# Probe [0, mid] (mid = MAX_PRICE//2 = 100_000_000): totalOffers=800 <= cap → пагинировать
|
||
# Probe [mid+1, MAX_PRICE]: totalOffers=700 <= cap → пагинировать
|
||
|
||
mid = _MAX_PRICE // 2
|
||
|
||
call_log: list[tuple] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
call_log.append((rooms, page, min_price, max_price))
|
||
return f"<html>page={page} min={min_price} max={max_price}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
# Парсим min/max из fake html
|
||
import re
|
||
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0
|
||
max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else _MAX_PRICE
|
||
|
||
# Имитируем разделение: весь диапазон > 1400, левая и правая половины <= 1400
|
||
if max_p is None or max_p >= _MAX_PRICE:
|
||
if min_p == 0:
|
||
return 3000 # весь диапазон — нужно split
|
||
if max_p == mid:
|
||
return 800 # левая половина — пагинировать
|
||
if min_p == mid + 1:
|
||
return 700 # правая половина — пагинировать
|
||
return 3000 # всё остальное — split
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
min_m = re.search(r"min=(\w+)", html)
|
||
max_m = re.search(r"max=(\w+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
min_p = min_m.group(1) if min_m else "0"
|
||
max_p = max_m.group(1) if max_m else "MAX"
|
||
# Возвращаем по 2 лота на первой странице каждого пагинируемого бакета
|
||
if page == 1:
|
||
return [
|
||
_make_lot(f"lot_{min_p}_{max_p}_p{page}_1"),
|
||
_make_lot(f"lot_{min_p}_{max_p}_p{page}_2"),
|
||
]
|
||
return [] # страница 2+ — пусто → early stop
|
||
|
||
async def fake_sleep(*_: object, **__: object) -> None:
|
||
pass
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
# Должны получить лоты из ОБОИХ суб-бакетов после split
|
||
assert len(lots) > 0, "Должны получить лоты после разделения бакета"
|
||
|
||
# Проверяем что был probe с большим диапазоном И с sub-бакетами
|
||
probed_ranges = [(min_p, max_p) for (_, page, min_p, max_p) in call_log if page == 1]
|
||
# Должен быть probe [0, None] или [0, MAX] — начальный probe всего диапазона
|
||
initial_probes = [r for r in probed_ranges if r[0] is None or r[0] == 0]
|
||
assert (
|
||
len(initial_probes) >= 1
|
||
), f"Ожидался probe всего диапазона, probed_ranges={probed_ranges}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_paginates_when_under_cap(scraper: CianScraper) -> None:
|
||
"""totalOffers <= cap → пагинирует бакет полностью (страница за страницей)."""
|
||
pages_fetched: list[int] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
pages_fetched.append(page)
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 56 # 56 офферов → ceil(56/28) = 2 страницы
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
if page <= 2:
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(28)]
|
||
return []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(2,)],
|
||
price_cap_per_bucket=1400,
|
||
)
|
||
|
||
# 28 лотов на стр 1 + 28 на стр 2 = 56, дедуп по source_id
|
||
assert len(lots) == 56, f"Ожидалось 56 уникальных лотов, получено {len(lots)}"
|
||
# Должно быть 2 страницы пагинации (page=1 probe + page=2)
|
||
assert 2 in pages_fetched, "Должна быть запрошена страница 2"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_min_bracket_guard(scraper: CianScraper) -> None:
|
||
"""Если hi - lo < MIN_BRACKET → пагинируем как есть (не делим бесконечно)."""
|
||
call_count = 0
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
if call_count > 100:
|
||
raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал")
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
# Всегда > cap → без guard делилось бы бесконечно
|
||
return 5000
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
return [] # пустой → early stop в пагинации
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
# Запускаем с очень узким диапазоном — guard должен остановить деление
|
||
await scraper._walk_price_range(
|
||
rooms=(1,),
|
||
lo=1_000_000,
|
||
hi=1_000_000 + _MIN_BRACKET - 1, # bracket < MIN_BRACKET
|
||
seen={},
|
||
price_cap_per_bucket=1400,
|
||
max_pages_per_bucket=54,
|
||
)
|
||
|
||
# Guard сработал: нет RuntimeError (бесконечная рекурсия не возникла).
|
||
# Параллельная пагинация запускает до max_pages_per_bucket=54 страниц единовременно
|
||
# (все страницы бакета параллельны), поэтому call_count может быть до 54.
|
||
# Важно что нет рекурсии (bracket < MIN_BRACKET → один проход пагинации, не деление).
|
||
assert call_count <= 55, f"Слишком много вызовов ({call_count}) — guard не сработал"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_on_bucket_called_per_leaf(scraper: CianScraper) -> None:
|
||
"""on_bucket вызывается после каждого leaf-бакета с лотами бакета."""
|
||
# totalOffers=56 ≤ cap → leaf-бакет, пагинируется параллельно 2 страницы
|
||
bucket_calls: list[tuple[str, list[ScrapedLot]]] = []
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
bucket_calls.append((bucket_key, list(lots)))
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 56 # ceil(56/28) = 2 страницы
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
concurrency=3,
|
||
on_bucket=fake_on_bucket,
|
||
)
|
||
|
||
# on_bucket вызван ровно 1 раз (один leaf-бакет для одной комнатности)
|
||
assert len(bucket_calls) == 1, f"Ожидался 1 вызов on_bucket, получено {len(bucket_calls)}"
|
||
key, lots = bucket_calls[0]
|
||
# Ключ бакета содержит room_label:lo:hi
|
||
assert "room1" in key, f"bucket_key должен содержать room1, got {key!r}"
|
||
# Лоты из обеих страниц переданы в on_bucket
|
||
assert len(lots) == 20, f"Ожидалось 20 лотов в on_bucket (2 стр × 10), получено {len(lots)}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_on_bucket_cancel_stops_run(scraper: CianScraper) -> None:
|
||
"""Если on_bucket кидает RuntimeError('cancelled') — прогон прерывается."""
|
||
|
||
def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
raise RuntimeError("cancelled")
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 28 # один лист → 1 страница
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
return [_make_lot("lot_1")]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
with pytest.raises(RuntimeError, match="cancelled"):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,), (2,)],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=cancel_on_bucket,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: CianScraper) -> None:
|
||
"""Параллельная пагинация не ломает дедупликацию по source_id."""
|
||
fetch_pages: list[int] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
fetch_pages.append(page)
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 84 # ceil(84/28) = 3 страницы
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
# Страница 1 и 2 дают уникальные лоты; страница 3 дублирует страницу 1
|
||
if page == 3:
|
||
return [_make_lot(f"lot_p1_{i}") for i in range(10)] # дубликаты
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
concurrency=5,
|
||
)
|
||
|
||
# Страницы 1+2 = 20 уникальных, страница 3 дублирует → итого 20
|
||
assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}"
|
||
# Все 3 страницы запрошены (параллельно или нет — нам важен факт)
|
||
assert 3 in fetch_pages or 3 in [
|
||
p for p in fetch_pages
|
||
], f"Страница 3 не была запрошена, fetch_pages={fetch_pages}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_skip_buckets_skips_pagination_and_on_bucket(scraper: CianScraper) -> None:
|
||
"""skip_buckets: для скипнутого бакета пагинация не вызывается, on_bucket не вызывается."""
|
||
fetch_calls: list[tuple] = []
|
||
on_bucket_calls: list[str] = []
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
fetch_calls.append((rooms, page, min_price, max_price))
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 28 # один leaf-бакет, 1 страница
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
return [_make_lot("lot_1")]
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
on_bucket_calls.append(bucket_key)
|
||
|
||
# Сформируем ключ того же бакета, который получится при walk (room1:0:200000000)
|
||
bucket_key_to_skip = f"room1:0:{_MAX_PRICE}"
|
||
skip_set = {bucket_key_to_skip}
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=fake_on_bucket,
|
||
skip_buckets=skip_set,
|
||
)
|
||
|
||
# Probe (page=1) ВСЁ РАВНО вызывается (нужен для split-решения),
|
||
# но НЕ должно быть страниц пагинации (т.к. бакет в skip_buckets).
|
||
# Поскольку probe = page=1 и мы пропускаем ДО пагинации — fetch_page_html не должен вызываться.
|
||
assert (
|
||
len(fetch_calls) == 1
|
||
), f"Ожидался ровно 1 вызов (probe), получено {len(fetch_calls)}: {fetch_calls}"
|
||
assert fetch_calls[0][1] == 1, "Probe должен быть page=1"
|
||
# on_bucket не вызывается для skip-бакета
|
||
assert (
|
||
len(on_bucket_calls) == 0
|
||
), f"on_bucket не должен вызываться для skip-бакета, got: {on_bucket_calls}"
|
||
# lots пустой — ничего не собрали (пагинация пропущена)
|
||
assert len(lots) == 0, f"Ожидалось 0 лотов для skip-бакета, got {len(lots)}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_on_bucket_receives_key_and_lots(scraper: CianScraper) -> None:
|
||
"""on_bucket получает (bucket_key, lots) — bucket_key в формате room_label:lo:hi."""
|
||
received: list[tuple[str, int]] = [] # (key, len(lots))
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
received.append((bucket_key, len(lots)))
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 28 # 1 страница
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return [_make_lot(f"lot_p{page}_{i}") for i in range(5)]
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(2,)],
|
||
price_cap_per_bucket=1400,
|
||
on_bucket=fake_on_bucket,
|
||
)
|
||
|
||
assert len(received) == 1, f"Ожидался 1 вызов on_bucket, получено {len(received)}"
|
||
key, count = received[0]
|
||
# Ключ должен быть «room2:0:<MAX_PRICE>»
|
||
assert key == f"room2:0:{_MAX_PRICE}", f"Неверный bucket_key: {key!r}"
|
||
assert count == 5, f"Ожидалось 5 лотов, получено {count}"
|
||
|
||
|
||
# ── secondary_only фильтр ─────────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_only_drops_novostroyki(scraper: CianScraper) -> None:
|
||
"""secondary_only=True: новостройки (listing_segment=='novostroyki') не попадают
|
||
в on_bucket и в итоговый список; вторичка и None-сегмент — пропускаются.
|
||
"""
|
||
on_bucket_calls: list[list[ScrapedLot]] = []
|
||
|
||
def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
|
||
on_bucket_calls.append(list(lots))
|
||
|
||
# Микс лотов: 2 новостройки + 2 вторички + 1 None-сегмент
|
||
mixed_lots = [
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/nb_1/",
|
||
source_id="nb_1",
|
||
address="ЖК Новый",
|
||
price_rub=5_000_000,
|
||
listing_segment="novostroyki",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/nb_2/",
|
||
source_id="nb_2",
|
||
address="ЖК Новый 2",
|
||
price_rub=6_000_000,
|
||
listing_segment="novostroyki",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/vt_1/",
|
||
source_id="vt_1",
|
||
address="ул. Ленина",
|
||
price_rub=3_500_000,
|
||
listing_segment="vtorichka",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/vt_2/",
|
||
source_id="vt_2",
|
||
address="ул. Мира",
|
||
price_rub=4_000_000,
|
||
listing_segment="vtorichka",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/none_seg/",
|
||
source_id="none_seg",
|
||
address="ул. Пушкина",
|
||
price_rub=3_000_000,
|
||
listing_segment=None,
|
||
),
|
||
]
|
||
|
||
page_call_count = 0
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
nonlocal page_call_count
|
||
page_call_count += 1
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 5 # <= cap → leaf-бакет
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
# Первая страница отдаёт весь микс; вторая — пусто
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return mixed_lots if page == 1 else []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
secondary_only=True,
|
||
on_bucket=fake_on_bucket,
|
||
)
|
||
|
||
# Итоговый список: только vtorichka + None-сегмент (3 лота), без новостроек
|
||
assert len(lots) == 3, f"Ожидалось 3 лота (без novostroyki), получено {len(lots)}"
|
||
lot_ids = {lot.source_id for lot in lots}
|
||
assert "nb_1" not in lot_ids, "nb_1 (novostroyki) не должен присутствовать"
|
||
assert "nb_2" not in lot_ids, "nb_2 (novostroyki) не должен присутствовать"
|
||
assert "vt_1" in lot_ids, "vt_1 (vtorichka) должен присутствовать"
|
||
assert "vt_2" in lot_ids, "vt_2 (vtorichka) должен присутствовать"
|
||
assert "none_seg" in lot_ids, "none_seg (listing_segment=None) должен присутствовать"
|
||
|
||
# on_bucket тоже не должен получить новостройки
|
||
assert len(on_bucket_calls) == 1, f"Ожидался 1 вызов on_bucket, получено {len(on_bucket_calls)}"
|
||
bucket_ids = {lot.source_id for lot in on_bucket_calls[0]}
|
||
assert "nb_1" not in bucket_ids
|
||
assert "nb_2" not in bucket_ids
|
||
assert len(on_bucket_calls[0]) == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None:
|
||
"""secondary_only=False: новостройки НЕ отфильтровываются — все лоты проходят."""
|
||
mixed_lots = [
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/nb_1/",
|
||
source_id="nb_1",
|
||
address="ЖК Новый",
|
||
price_rub=5_000_000,
|
||
listing_segment="novostroyki",
|
||
),
|
||
ScrapedLot(
|
||
source="cian",
|
||
source_url="https://ekb.cian.ru/sale/flat/vt_1/",
|
||
source_id="vt_1",
|
||
address="ул. Ленина",
|
||
price_rub=3_500_000,
|
||
listing_segment="vtorichka",
|
||
),
|
||
]
|
||
|
||
async def fake_fetch_page_html(
|
||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||
) -> str:
|
||
return f"<html>page={page}</html>"
|
||
|
||
def fake_extract_total_offers(html: str) -> int | None:
|
||
return 2
|
||
|
||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||
import re
|
||
|
||
page_m = re.search(r"page=(\d+)", html)
|
||
page = int(page_m.group(1)) if page_m else 1
|
||
return mixed_lots if page == 1 else []
|
||
|
||
with (
|
||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||
):
|
||
lots = await scraper.fetch_all_secondary(
|
||
rooms_buckets=[(1,)],
|
||
price_cap_per_bucket=1400,
|
||
secondary_only=False,
|
||
)
|
||
|
||
assert len(lots) == 2, f"Ожидалось 2 лота (фильтр выключен), получено {len(lots)}"
|
||
lot_ids = {lot.source_id for lot in lots}
|
||
assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False"
|
||
assert "vt_1" in lot_ids
|