fix(scrapers): cian _walk_price_range stop silently truncating tail (#1393) #1689
2 changed files with 145 additions and 5 deletions
|
|
@ -531,10 +531,26 @@ class CianScraper(BaseScraper):
|
|||
return
|
||||
|
||||
# ── Параллельная пагинация leaf-бакета ────────────────────────────────
|
||||
max_pages = min(
|
||||
math.ceil(total / _CIAN_OFFERS_PER_PAGE),
|
||||
max_pages_per_bucket,
|
||||
)
|
||||
pages_needed = math.ceil(total / _CIAN_OFFERS_PER_PAGE)
|
||||
max_pages = min(pages_needed, max_pages_per_bucket)
|
||||
|
||||
if pages_needed > max_pages_per_bucket:
|
||||
# Cian hard-cap срабатывает: часть офферов недостижима через пагинацию.
|
||||
# Это происходит когда bracket < _MIN_BRACKET но totalOffers всё равно > cap.
|
||||
tail_loss = total - max_pages_per_bucket * _CIAN_OFFERS_PER_PAGE
|
||||
logger.warning(
|
||||
"cian: %s [%d, %d] totalOffers=%d exceeds page cap "
|
||||
"(max_pages=%d × %d=%d offers) — tail loss ~%d offers, "
|
||||
"consider narrowing price_cap_per_bucket or _MIN_BRACKET",
|
||||
room_label,
|
||||
lo,
|
||||
hi,
|
||||
total,
|
||||
max_pages_per_bucket,
|
||||
_CIAN_OFFERS_PER_PAGE,
|
||||
max_pages_per_bucket * _CIAN_OFFERS_PER_PAGE,
|
||||
tail_loss,
|
||||
)
|
||||
|
||||
# Страница 1 уже есть (html из probe выше); остальные — параллельно.
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
|
|
|
|||
|
|
@ -12,10 +12,17 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
# Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.base import ScrapedLot
|
||||
from app.services.scrapers.cian import _MAX_PRICE, _MIN_BRACKET, CianScraper
|
||||
from app.services.scrapers.cian import (
|
||||
_CIAN_OFFERS_PER_PAGE,
|
||||
_MAX_PRICE,
|
||||
_MIN_BRACKET,
|
||||
CianScraper,
|
||||
)
|
||||
|
||||
|
||||
def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot:
|
||||
|
|
@ -597,3 +604,120 @@ async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None:
|
|||
lot_ids = {lot.source_id for lot in lots}
|
||||
assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False"
|
||||
assert "vt_1" in lot_ids
|
||||
|
||||
|
||||
# ── #1393: tail-loss warning при превышении page cap ─────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_walk_price_range_warns_on_page_cap_tail_loss(
|
||||
scraper: CianScraper, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""#1393: когда totalOffers > max_pages_per_bucket × offers_per_page,
|
||||
_walk_price_range логирует WARNING с числом потерянных офферов.
|
||||
|
||||
Сценарий: bracket < _MIN_BRACKET (guard срабатывает → force-paginate),
|
||||
totalOffers = max_pages * offers_per_page + 50 → tail_loss = 50.
|
||||
"""
|
||||
max_pages_cap = 5 # нарочно мало, чтобы быстро дойти до cap
|
||||
offers_per_page = _CIAN_OFFERS_PER_PAGE # 28
|
||||
# total > cap (2000 > любой price_cap_per_bucket=1400) И bracket < _MIN_BRACKET
|
||||
# → force-paginate path; pages_needed = ceil(2000/28) = 72 > max_pages_cap=5
|
||||
total_offers = max_pages_cap * offers_per_page + 50 # 190; tail_loss=50
|
||||
|
||||
async def fake_fetch_page_html(
|
||||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||||
) -> str:
|
||||
return f"<html>page={page}</html>"
|
||||
|
||||
def fake_extract_total_offers(html: str) -> int | None:
|
||||
return total_offers
|
||||
|
||||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||||
import re
|
||||
|
||||
page_m = re.search(r"page=(\d+)", html)
|
||||
page = int(page_m.group(1)) if page_m else 1
|
||||
return [_make_lot(f"lot_p{page}_{i}") for i in range(offers_per_page)]
|
||||
|
||||
with (
|
||||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||||
caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
|
||||
):
|
||||
await scraper._walk_price_range(
|
||||
rooms=(1,),
|
||||
# Узкий диапазон: bracket < _MIN_BRACKET → force-paginate guard
|
||||
lo=1_000_000,
|
||||
hi=1_000_000 + _MIN_BRACKET - 1,
|
||||
seen={},
|
||||
price_cap_per_bucket=1400, # total(190) < 1400 → нет split, только leaf
|
||||
max_pages_per_bucket=max_pages_cap,
|
||||
)
|
||||
|
||||
# Должно быть предупреждение о tail-loss
|
||||
tail_warnings = [
|
||||
r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
|
||||
]
|
||||
assert tail_warnings, (
|
||||
f"Ожидалось WARNING о tail loss, но записей нет. "
|
||||
f"Все warnings: {[r.message for r in caplog.records if r.levelno == logging.WARNING]}"
|
||||
)
|
||||
# Сообщение должно содержать число потерянных офферов (50)
|
||||
assert any(
|
||||
"50" in r.message for r in tail_warnings
|
||||
), f"WARNING должен содержать tail_loss=50, got: {[r.message for r in tail_warnings]}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_walk_price_range_no_warning_when_within_page_cap(
|
||||
scraper: CianScraper, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""#1393: когда totalOffers <= max_pages_per_bucket × offers_per_page,
|
||||
tail-loss WARNING не должен эмитироваться (данные собраны полностью).
|
||||
"""
|
||||
max_pages_cap = 54
|
||||
total_offers = 56 # ceil(56/28) = 2 страницы < 54 cap → нет потерь
|
||||
|
||||
async def fake_fetch_page_html(
|
||||
rooms: tuple, page: int, min_price: int | None, max_price: int | None
|
||||
) -> str:
|
||||
return f"<html>page={page}</html>"
|
||||
|
||||
def fake_extract_total_offers(html: str) -> int | None:
|
||||
return total_offers
|
||||
|
||||
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
|
||||
import re
|
||||
|
||||
page_m = re.search(r"page=(\d+)", html)
|
||||
page = int(page_m.group(1)) if page_m else 1
|
||||
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
|
||||
|
||||
with (
|
||||
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
|
||||
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
|
||||
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
|
||||
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
|
||||
patch.object(scraper, "_rotate_ip", return_value=False),
|
||||
caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
|
||||
):
|
||||
await scraper._walk_price_range(
|
||||
rooms=(1,),
|
||||
lo=0,
|
||||
hi=_MAX_PRICE,
|
||||
seen={},
|
||||
price_cap_per_bucket=1400,
|
||||
max_pages_per_bucket=max_pages_cap,
|
||||
)
|
||||
|
||||
tail_warnings = [
|
||||
r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
|
||||
]
|
||||
assert not tail_warnings, (
|
||||
f"Не ожидалось WARNING о tail loss при total={total_offers} <= page cap, "
|
||||
f"но получено: {[r.message for r in tail_warnings]}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue