"""Тесты exhaustive-загрузки Cian: fetch_all_secondary + _walk_price_range. Мокаем сетевые вызовы (_fetch_page_html, _extract_total_offers, sleep_between_requests) для детерминированного контроля поведения без реального HTTP. """ from __future__ import annotations import os from unittest.mock import AsyncMock, MagicMock, patch # Settings requires DATABASE_URL at import time — set dummy DSN before any app import. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") import logging import pytest from app.services.scrapers.base import ScrapedLot from app.services.scrapers.cian import ( _CIAN_OFFERS_PER_PAGE, _MAX_PRICE, _MIN_BRACKET, CianScraper, ) def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot: """Минимальный ScrapedLot для теста.""" return ScrapedLot( source="cian", source_url=f"https://ekb.cian.ru/sale/flat/{source_id}/", source_id=source_id, address="Екатеринбург", price_rub=price, ) def _make_html_stub(total_offers: int, lot_ids: list[str]) -> str: """Заглушка HTML — не парсится напрямую; используется через мокнутые методы.""" return f"total={total_offers} ids={lot_ids}" @pytest.fixture def scraper() -> CianScraper: """CianScraper без реального __aenter__ (без HTTP warm-up).""" s = CianScraper.__new__(CianScraper) s.name = "cian" s.base_url = "https://ekb.cian.ru" s.request_delay_sec = 0.0 s._cffi = MagicMock() # не None → assert self._cffi is not None пройдёт return s @pytest.mark.asyncio async def test_fetch_all_secondary_splits_on_cap(scraper: CianScraper) -> None: """totalOffers > cap → бакет разбивается; в итоге оба суб-бакета пагинируются.""" # Настройка: rooms=(1,), один бакет [0, None] # Probe [0, None]: totalOffers=3000 > cap=1400 → split на [0, MAX_PRICE] # Probe [0, MAX_PRICE]: totalOffers=3000 > 1400 → split на [0, mid] и [mid+1, MAX_PRICE] # Probe [0, mid] (mid = MAX_PRICE//2 = 100_000_000): totalOffers=800 <= cap → пагинировать # Probe [mid+1, MAX_PRICE]: totalOffers=700 <= cap → пагинировать mid = _MAX_PRICE // 2 call_log: list[tuple] = [] async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: call_log.append((rooms, page, min_price, max_price)) return f"page={page} min={min_price} max={max_price}" def fake_extract_total_offers(html: str) -> int | None: # Парсим min/max из fake html import re min_m = re.search(r"min=(\w+)", html) max_m = re.search(r"max=(\w+)", html) min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0 max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else _MAX_PRICE # Имитируем разделение: весь диапазон > 1400, левая и правая половины <= 1400 if max_p is None or max_p >= _MAX_PRICE: if min_p == 0: return 3000 # весь диапазон — нужно split if max_p == mid: return 800 # левая половина — пагинировать if min_p == mid + 1: return 700 # правая половина — пагинировать return 3000 # всё остальное — split def fake_parse_serp_html(html: str) -> list[ScrapedLot]: import re page_m = re.search(r"page=(\d+)", html) min_m = re.search(r"min=(\w+)", html) max_m = re.search(r"max=(\w+)", html) page = int(page_m.group(1)) if page_m else 1 min_p = min_m.group(1) if min_m else "0" max_p = max_m.group(1) if max_m else "MAX" # Возвращаем по 2 лота на первой странице каждого пагинируемого бакета if page == 1: return [ _make_lot(f"lot_{min_p}_{max_p}_p{page}_1"), _make_lot(f"lot_{min_p}_{max_p}_p{page}_2"), ] return [] # страница 2+ — пусто → early stop async def fake_sleep(*_: object, **__: object) -> None: pass with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): lots = await scraper.fetch_all_secondary( rooms_buckets=[(1,)], price_cap_per_bucket=1400, ) # Должны получить лоты из ОБОИХ суб-бакетов после split assert len(lots) > 0, "Должны получить лоты после разделения бакета" # Проверяем что был probe с большим диапазоном И с sub-бакетами probed_ranges = [(min_p, max_p) for (_, page, min_p, max_p) in call_log if page == 1] # Должен быть probe [0, None] или [0, MAX] — начальный probe всего диапазона initial_probes = [r for r in probed_ranges if r[0] is None or r[0] == 0] assert ( len(initial_probes) >= 1 ), f"Ожидался probe всего диапазона, probed_ranges={probed_ranges}" @pytest.mark.asyncio async def test_fetch_all_secondary_paginates_when_under_cap(scraper: CianScraper) -> None: """totalOffers <= cap → пагинирует бакет полностью (страница за страницей).""" pages_fetched: list[int] = [] async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: pages_fetched.append(page) return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return 56 # 56 офферов → ceil(56/28) = 2 страницы def fake_parse_serp_html(html: str) -> list[ScrapedLot]: import re page_m = re.search(r"page=(\d+)", html) page = int(page_m.group(1)) if page_m else 1 if page <= 2: return [_make_lot(f"lot_p{page}_{i}") for i in range(28)] return [] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): lots = await scraper.fetch_all_secondary( rooms_buckets=[(2,)], price_cap_per_bucket=1400, ) # 28 лотов на стр 1 + 28 на стр 2 = 56, дедуп по source_id assert len(lots) == 56, f"Ожидалось 56 уникальных лотов, получено {len(lots)}" # Должно быть 2 страницы пагинации (page=1 probe + page=2) assert 2 in pages_fetched, "Должна быть запрошена страница 2" @pytest.mark.asyncio async def test_fetch_all_secondary_min_bracket_guard(scraper: CianScraper) -> None: """Если hi - lo < MIN_BRACKET → пагинируем как есть (не делим бесконечно).""" call_count = 0 async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: nonlocal call_count call_count += 1 if call_count > 100: raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал") return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: # Всегда > cap → без guard делилось бы бесконечно return 5000 def fake_parse_serp_html(html: str) -> list[ScrapedLot]: return [] # пустой → early stop в пагинации with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): # Запускаем с очень узким диапазоном — guard должен остановить деление await scraper._walk_price_range( rooms=(1,), lo=1_000_000, hi=1_000_000 + _MIN_BRACKET - 1, # bracket < MIN_BRACKET seen={}, price_cap_per_bucket=1400, max_pages_per_bucket=54, ) # Guard сработал: нет RuntimeError (бесконечная рекурсия не возникла). # Параллельная пагинация запускает до max_pages_per_bucket=54 страниц единовременно # (все страницы бакета параллельны), поэтому call_count может быть до 54. # Важно что нет рекурсии (bracket < MIN_BRACKET → один проход пагинации, не деление). assert call_count <= 55, f"Слишком много вызовов ({call_count}) — guard не сработал" @pytest.mark.asyncio async def test_fetch_all_secondary_on_bucket_called_per_leaf(scraper: CianScraper) -> None: """on_bucket вызывается после каждого leaf-бакета с лотами бакета.""" # totalOffers=56 ≤ cap → leaf-бакет, пагинируется параллельно 2 страницы bucket_calls: list[tuple[str, list[ScrapedLot]]] = [] def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: bucket_calls.append((bucket_key, list(lots))) async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return 56 # ceil(56/28) = 2 страницы def fake_parse_serp_html(html: str) -> list[ScrapedLot]: import re page_m = re.search(r"page=(\d+)", html) page = int(page_m.group(1)) if page_m else 1 return [_make_lot(f"lot_p{page}_{i}") for i in range(10)] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): await scraper.fetch_all_secondary( rooms_buckets=[(1,)], price_cap_per_bucket=1400, concurrency=3, on_bucket=fake_on_bucket, ) # on_bucket вызван ровно 1 раз (один leaf-бакет для одной комнатности) assert len(bucket_calls) == 1, f"Ожидался 1 вызов on_bucket, получено {len(bucket_calls)}" key, lots = bucket_calls[0] # Ключ бакета содержит room_label:lo:hi assert "room1" in key, f"bucket_key должен содержать room1, got {key!r}" # Лоты из обеих страниц переданы в on_bucket assert len(lots) == 20, f"Ожидалось 20 лотов в on_bucket (2 стр × 10), получено {len(lots)}" @pytest.mark.asyncio async def test_fetch_all_secondary_on_bucket_cancel_stops_run(scraper: CianScraper) -> None: """Если on_bucket кидает RuntimeError('cancelled') — прогон прерывается.""" def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: raise RuntimeError("cancelled") async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return 28 # один лист → 1 страница def fake_parse_serp_html(html: str) -> list[ScrapedLot]: return [_make_lot("lot_1")] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): with pytest.raises(RuntimeError, match="cancelled"): await scraper.fetch_all_secondary( rooms_buckets=[(1,), (2,)], price_cap_per_bucket=1400, on_bucket=cancel_on_bucket, ) @pytest.mark.asyncio async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: CianScraper) -> None: """Параллельная пагинация не ломает дедупликацию по source_id.""" fetch_pages: list[int] = [] async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: fetch_pages.append(page) return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return 84 # ceil(84/28) = 3 страницы def fake_parse_serp_html(html: str) -> list[ScrapedLot]: import re page_m = re.search(r"page=(\d+)", html) page = int(page_m.group(1)) if page_m else 1 # Страница 1 и 2 дают уникальные лоты; страница 3 дублирует страницу 1 if page == 3: return [_make_lot(f"lot_p1_{i}") for i in range(10)] # дубликаты return [_make_lot(f"lot_p{page}_{i}") for i in range(10)] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): lots = await scraper.fetch_all_secondary( rooms_buckets=[(1,)], price_cap_per_bucket=1400, concurrency=5, ) # Страницы 1+2 = 20 уникальных, страница 3 дублирует → итого 20 assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}" # Все 3 страницы запрошены (параллельно или нет — нам важен факт) assert 3 in fetch_pages or 3 in [ p for p in fetch_pages ], f"Страница 3 не была запрошена, fetch_pages={fetch_pages}" @pytest.mark.asyncio async def test_skip_buckets_skips_pagination_and_on_bucket(scraper: CianScraper) -> None: """skip_buckets: для скипнутого бакета пагинация не вызывается, on_bucket не вызывается.""" fetch_calls: list[tuple] = [] on_bucket_calls: list[str] = [] async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: fetch_calls.append((rooms, page, min_price, max_price)) return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return 28 # один leaf-бакет, 1 страница def fake_parse_serp_html(html: str) -> list[ScrapedLot]: return [_make_lot("lot_1")] def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: on_bucket_calls.append(bucket_key) # Сформируем ключ того же бакета, который получится при walk (room1:0:200000000) bucket_key_to_skip = f"room1:0:{_MAX_PRICE}" skip_set = {bucket_key_to_skip} with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): lots = await scraper.fetch_all_secondary( rooms_buckets=[(1,)], price_cap_per_bucket=1400, on_bucket=fake_on_bucket, skip_buckets=skip_set, ) # Probe (page=1) ВСЁ РАВНО вызывается (нужен для split-решения), # но НЕ должно быть страниц пагинации (т.к. бакет в skip_buckets). # Поскольку probe = page=1 и мы пропускаем ДО пагинации — fetch_page_html не должен вызываться. assert ( len(fetch_calls) == 1 ), f"Ожидался ровно 1 вызов (probe), получено {len(fetch_calls)}: {fetch_calls}" assert fetch_calls[0][1] == 1, "Probe должен быть page=1" # on_bucket не вызывается для skip-бакета assert ( len(on_bucket_calls) == 0 ), f"on_bucket не должен вызываться для skip-бакета, got: {on_bucket_calls}" # lots пустой — ничего не собрали (пагинация пропущена) assert len(lots) == 0, f"Ожидалось 0 лотов для skip-бакета, got {len(lots)}" @pytest.mark.asyncio async def test_on_bucket_receives_key_and_lots(scraper: CianScraper) -> None: """on_bucket получает (bucket_key, lots) — bucket_key в формате room_label:lo:hi.""" received: list[tuple[str, int]] = [] # (key, len(lots)) def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: received.append((bucket_key, len(lots))) async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return 28 # 1 страница def fake_parse_serp_html(html: str) -> list[ScrapedLot]: import re page_m = re.search(r"page=(\d+)", html) page = int(page_m.group(1)) if page_m else 1 return [_make_lot(f"lot_p{page}_{i}") for i in range(5)] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): await scraper.fetch_all_secondary( rooms_buckets=[(2,)], price_cap_per_bucket=1400, on_bucket=fake_on_bucket, ) assert len(received) == 1, f"Ожидался 1 вызов on_bucket, получено {len(received)}" key, count = received[0] # Ключ должен быть «room2:0:» assert key == f"room2:0:{_MAX_PRICE}", f"Неверный bucket_key: {key!r}" assert count == 5, f"Ожидалось 5 лотов, получено {count}" # ── secondary_only фильтр ───────────────────────────────────────────────────── @pytest.mark.asyncio async def test_secondary_only_drops_novostroyki(scraper: CianScraper) -> None: """secondary_only=True: новостройки (listing_segment=='novostroyki') не попадают в on_bucket и в итоговый список; вторичка и None-сегмент — пропускаются. """ on_bucket_calls: list[list[ScrapedLot]] = [] def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None: on_bucket_calls.append(list(lots)) # Микс лотов: 2 новостройки + 2 вторички + 1 None-сегмент mixed_lots = [ ScrapedLot( source="cian", source_url="https://ekb.cian.ru/sale/flat/nb_1/", source_id="nb_1", address="ЖК Новый", price_rub=5_000_000, listing_segment="novostroyki", ), ScrapedLot( source="cian", source_url="https://ekb.cian.ru/sale/flat/nb_2/", source_id="nb_2", address="ЖК Новый 2", price_rub=6_000_000, listing_segment="novostroyki", ), ScrapedLot( source="cian", source_url="https://ekb.cian.ru/sale/flat/vt_1/", source_id="vt_1", address="ул. Ленина", price_rub=3_500_000, listing_segment="vtorichka", ), ScrapedLot( source="cian", source_url="https://ekb.cian.ru/sale/flat/vt_2/", source_id="vt_2", address="ул. Мира", price_rub=4_000_000, listing_segment="vtorichka", ), ScrapedLot( source="cian", source_url="https://ekb.cian.ru/sale/flat/none_seg/", source_id="none_seg", address="ул. Пушкина", price_rub=3_000_000, listing_segment=None, ), ] page_call_count = 0 async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: nonlocal page_call_count page_call_count += 1 return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return 5 # <= cap → leaf-бакет def fake_parse_serp_html(html: str) -> list[ScrapedLot]: # Первая страница отдаёт весь микс; вторая — пусто import re page_m = re.search(r"page=(\d+)", html) page = int(page_m.group(1)) if page_m else 1 return mixed_lots if page == 1 else [] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): lots = await scraper.fetch_all_secondary( rooms_buckets=[(1,)], price_cap_per_bucket=1400, secondary_only=True, on_bucket=fake_on_bucket, ) # Итоговый список: только vtorichka + None-сегмент (3 лота), без новостроек assert len(lots) == 3, f"Ожидалось 3 лота (без novostroyki), получено {len(lots)}" lot_ids = {lot.source_id for lot in lots} assert "nb_1" not in lot_ids, "nb_1 (novostroyki) не должен присутствовать" assert "nb_2" not in lot_ids, "nb_2 (novostroyki) не должен присутствовать" assert "vt_1" in lot_ids, "vt_1 (vtorichka) должен присутствовать" assert "vt_2" in lot_ids, "vt_2 (vtorichka) должен присутствовать" assert "none_seg" in lot_ids, "none_seg (listing_segment=None) должен присутствовать" # on_bucket тоже не должен получить новостройки assert len(on_bucket_calls) == 1, f"Ожидался 1 вызов on_bucket, получено {len(on_bucket_calls)}" bucket_ids = {lot.source_id for lot in on_bucket_calls[0]} assert "nb_1" not in bucket_ids assert "nb_2" not in bucket_ids assert len(on_bucket_calls[0]) == 3 @pytest.mark.asyncio async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None: """secondary_only=False: новостройки НЕ отфильтровываются — все лоты проходят.""" mixed_lots = [ ScrapedLot( source="cian", source_url="https://ekb.cian.ru/sale/flat/nb_1/", source_id="nb_1", address="ЖК Новый", price_rub=5_000_000, listing_segment="novostroyki", ), ScrapedLot( source="cian", source_url="https://ekb.cian.ru/sale/flat/vt_1/", source_id="vt_1", address="ул. Ленина", price_rub=3_500_000, listing_segment="vtorichka", ), ] async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return 2 def fake_parse_serp_html(html: str) -> list[ScrapedLot]: import re page_m = re.search(r"page=(\d+)", html) page = int(page_m.group(1)) if page_m else 1 return mixed_lots if page == 1 else [] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), ): lots = await scraper.fetch_all_secondary( rooms_buckets=[(1,)], price_cap_per_bucket=1400, secondary_only=False, ) assert len(lots) == 2, f"Ожидалось 2 лота (фильтр выключен), получено {len(lots)}" lot_ids = {lot.source_id for lot in lots} assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False" assert "vt_1" in lot_ids # ── #1393: tail-loss warning при превышении page cap ───────────────────────── @pytest.mark.asyncio async def test_walk_price_range_warns_on_page_cap_tail_loss( scraper: CianScraper, caplog: pytest.LogCaptureFixture ) -> None: """#1393: когда totalOffers > max_pages_per_bucket × offers_per_page, _walk_price_range логирует WARNING с числом потерянных офферов. Сценарий: bracket < _MIN_BRACKET (guard срабатывает → force-paginate), totalOffers = max_pages * offers_per_page + 50 → tail_loss = 50. """ max_pages_cap = 5 # нарочно мало, чтобы быстро дойти до cap offers_per_page = _CIAN_OFFERS_PER_PAGE # 28 # total > cap (2000 > любой price_cap_per_bucket=1400) И bracket < _MIN_BRACKET # → force-paginate path; pages_needed = ceil(2000/28) = 72 > max_pages_cap=5 total_offers = max_pages_cap * offers_per_page + 50 # 190; tail_loss=50 async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return total_offers def fake_parse_serp_html(html: str) -> list[ScrapedLot]: import re page_m = re.search(r"page=(\d+)", html) page = int(page_m.group(1)) if page_m else 1 return [_make_lot(f"lot_p{page}_{i}") for i in range(offers_per_page)] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"), ): await scraper._walk_price_range( rooms=(1,), # Узкий диапазон: bracket < _MIN_BRACKET → force-paginate guard lo=1_000_000, hi=1_000_000 + _MIN_BRACKET - 1, seen={}, price_cap_per_bucket=1400, # total(190) < 1400 → нет split, только leaf max_pages_per_bucket=max_pages_cap, ) # Должно быть предупреждение о tail-loss tail_warnings = [ r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING ] assert tail_warnings, ( f"Ожидалось WARNING о tail loss, но записей нет. " f"Все warnings: {[r.message for r in caplog.records if r.levelno == logging.WARNING]}" ) # Сообщение должно содержать число потерянных офферов (50) assert any( "50" in r.message for r in tail_warnings ), f"WARNING должен содержать tail_loss=50, got: {[r.message for r in tail_warnings]}" @pytest.mark.asyncio async def test_walk_price_range_no_warning_when_within_page_cap( scraper: CianScraper, caplog: pytest.LogCaptureFixture ) -> None: """#1393: когда totalOffers <= max_pages_per_bucket × offers_per_page, tail-loss WARNING не должен эмитироваться (данные собраны полностью). """ max_pages_cap = 54 total_offers = 56 # ceil(56/28) = 2 страницы < 54 cap → нет потерь async def fake_fetch_page_html( rooms: tuple, page: int, min_price: int | None, max_price: int | None ) -> str: return f"page={page}" def fake_extract_total_offers(html: str) -> int | None: return total_offers def fake_parse_serp_html(html: str) -> list[ScrapedLot]: import re page_m = re.search(r"page=(\d+)", html) page = int(page_m.group(1)) if page_m else 1 return [_make_lot(f"lot_p{page}_{i}") for i in range(10)] with ( patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), patch.object(scraper, "_rotate_ip", return_value=False), caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"), ): await scraper._walk_price_range( rooms=(1,), lo=0, hi=_MAX_PRICE, seen={}, price_cap_per_bucket=1400, max_pages_per_bucket=max_pages_cap, ) tail_warnings = [ r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING ] assert not tail_warnings, ( f"Не ожидалось WARNING о tail loss при total={total_offers} <= page cap, " f"но получено: {[r.message for r in tail_warnings]}" )