From 17c494a063d3dbe86fbdc4737171fa1e36d8f536 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Wed, 17 Jun 2026 20:39:16 +0300 Subject: [PATCH] fix(scrapers): cian _walk_price_range stop silently truncating price-range tail (#1393) --- .../backend/app/services/scrapers/cian.py | 24 +++- .../tests/scrapers/test_cian_exhaustive.py | 126 +++++++++++++++++- 2 files changed, 145 insertions(+), 5 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 71f82133..5b1f0bdb 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -531,10 +531,26 @@ class CianScraper(BaseScraper): return # ── Параллельная пагинация leaf-бакета ──────────────────────────────── - max_pages = min( - math.ceil(total / _CIAN_OFFERS_PER_PAGE), - max_pages_per_bucket, - ) + pages_needed = math.ceil(total / _CIAN_OFFERS_PER_PAGE) + max_pages = min(pages_needed, max_pages_per_bucket) + + if pages_needed > max_pages_per_bucket: + # Cian hard-cap срабатывает: часть офферов недостижима через пагинацию. + # Это происходит когда bracket < _MIN_BRACKET но totalOffers всё равно > cap. + tail_loss = total - max_pages_per_bucket * _CIAN_OFFERS_PER_PAGE + logger.warning( + "cian: %s [%d, %d] totalOffers=%d exceeds page cap " + "(max_pages=%d × %d=%d offers) — tail loss ~%d offers, " + "consider narrowing price_cap_per_bucket or _MIN_BRACKET", + room_label, + lo, + hi, + total, + max_pages_per_bucket, + _CIAN_OFFERS_PER_PAGE, + max_pages_per_bucket * _CIAN_OFFERS_PER_PAGE, + tail_loss, + ) # Страница 1 уже есть (html из probe выше); остальные — параллельно. sem = asyncio.Semaphore(concurrency) diff --git a/tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py b/tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py index 3c2521db..56f12916 100644 --- a/tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py +++ b/tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py @@ -12,10 +12,17 @@ from unittest.mock import AsyncMock, MagicMock, patch # Settings requires DATABASE_URL at import time — set dummy DSN before any app import. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") +import logging + import pytest from app.services.scrapers.base import ScrapedLot -from app.services.scrapers.cian import _MAX_PRICE, _MIN_BRACKET, CianScraper +from app.services.scrapers.cian import ( + _CIAN_OFFERS_PER_PAGE, + _MAX_PRICE, + _MIN_BRACKET, + CianScraper, +) def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot: @@ -597,3 +604,120 @@ async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None: lot_ids = {lot.source_id for lot in lots} assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False" assert "vt_1" in lot_ids + + +# ── #1393: tail-loss warning при превышении page cap ───────────────────────── + + +@pytest.mark.asyncio +async def test_walk_price_range_warns_on_page_cap_tail_loss( + scraper: CianScraper, caplog: pytest.LogCaptureFixture +) -> None: + """#1393: когда totalOffers > max_pages_per_bucket × offers_per_page, + _walk_price_range логирует WARNING с числом потерянных офферов. + + Сценарий: bracket < _MIN_BRACKET (guard срабатывает → force-paginate), + totalOffers = max_pages * offers_per_page + 50 → tail_loss = 50. + """ + max_pages_cap = 5 # нарочно мало, чтобы быстро дойти до cap + offers_per_page = _CIAN_OFFERS_PER_PAGE # 28 + # total > cap (2000 > любой price_cap_per_bucket=1400) И bracket < _MIN_BRACKET + # → force-paginate path; pages_needed = ceil(2000/28) = 72 > max_pages_cap=5 + total_offers = max_pages_cap * offers_per_page + 50 # 190; tail_loss=50 + + async def fake_fetch_page_html( + rooms: tuple, page: int, min_price: int | None, max_price: int | None + ) -> str: + return f"page={page}" + + def fake_extract_total_offers(html: str) -> int | None: + return total_offers + + def fake_parse_serp_html(html: str) -> list[ScrapedLot]: + import re + + page_m = re.search(r"page=(\d+)", html) + page = int(page_m.group(1)) if page_m else 1 + return [_make_lot(f"lot_p{page}_{i}") for i in range(offers_per_page)] + + with ( + patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), + patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), + patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), + patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), + patch.object(scraper, "_rotate_ip", return_value=False), + caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"), + ): + await scraper._walk_price_range( + rooms=(1,), + # Узкий диапазон: bracket < _MIN_BRACKET → force-paginate guard + lo=1_000_000, + hi=1_000_000 + _MIN_BRACKET - 1, + seen={}, + price_cap_per_bucket=1400, # total(190) < 1400 → нет split, только leaf + max_pages_per_bucket=max_pages_cap, + ) + + # Должно быть предупреждение о tail-loss + tail_warnings = [ + r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING + ] + assert tail_warnings, ( + f"Ожидалось WARNING о tail loss, но записей нет. " + f"Все warnings: {[r.message for r in caplog.records if r.levelno == logging.WARNING]}" + ) + # Сообщение должно содержать число потерянных офферов (50) + assert any( + "50" in r.message for r in tail_warnings + ), f"WARNING должен содержать tail_loss=50, got: {[r.message for r in tail_warnings]}" + + +@pytest.mark.asyncio +async def test_walk_price_range_no_warning_when_within_page_cap( + scraper: CianScraper, caplog: pytest.LogCaptureFixture +) -> None: + """#1393: когда totalOffers <= max_pages_per_bucket × offers_per_page, + tail-loss WARNING не должен эмитироваться (данные собраны полностью). + """ + max_pages_cap = 54 + total_offers = 56 # ceil(56/28) = 2 страницы < 54 cap → нет потерь + + async def fake_fetch_page_html( + rooms: tuple, page: int, min_price: int | None, max_price: int | None + ) -> str: + return f"page={page}" + + def fake_extract_total_offers(html: str) -> int | None: + return total_offers + + def fake_parse_serp_html(html: str) -> list[ScrapedLot]: + import re + + page_m = re.search(r"page=(\d+)", html) + page = int(page_m.group(1)) if page_m else 1 + return [_make_lot(f"lot_p{page}_{i}") for i in range(10)] + + with ( + patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html), + patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers), + patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html), + patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock), + patch.object(scraper, "_rotate_ip", return_value=False), + caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"), + ): + await scraper._walk_price_range( + rooms=(1,), + lo=0, + hi=_MAX_PRICE, + seen={}, + price_cap_per_bucket=1400, + max_pages_per_bucket=max_pages_cap, + ) + + tail_warnings = [ + r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING + ] + assert not tail_warnings, ( + f"Не ожидалось WARNING о tail loss при total={total_offers} <= page cap, " + f"но получено: {[r.message for r in tail_warnings]}" + ) -- 2.45.3