lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request 'fix(scrapers): cian _walk_price_range stop silently truncating tail (#1393)' (#1689) from fix/cian-paginate-tail-1393 into main
Some checks are pending
Deploy Trade-In / test (push) Blocked by required conditions
Details
Deploy Trade-In / build-backend (push) Blocked by required conditions
Details
Deploy Trade-In / build-frontend (push) Blocked by required conditions
Details
Deploy Trade-In / build-browser (push) Blocked by required conditions
Details
Deploy Trade-In / deploy (push) Blocked by required conditions
Details
Deploy Trade-In / changes (push) Successful in 7s
Details

Browse source
This commit is contained in:
lekss361 2026-06-17 17:55:36 +00:00
commit 5c44233ed6
2 changed files with 145 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
tradein-mvp/backend/app/services/scrapers/cian.py
View file

@ -531,10 +531,26 @@ class CianScraper(BaseScraper):
return return
# ── Параллельная пагинация leaf-бакета ──────────────────────────────── # ── Параллельная пагинация leaf-бакета ────────────────────────────────
max_pages = min( pages_needed = math.ceil(total / _CIAN_OFFERS_PER_PAGE)
math.ceil(total / _CIAN_OFFERS_PER_PAGE), max_pages = min(pages_needed, max_pages_per_bucket)
max_pages_per_bucket,
) if pages_needed > max_pages_per_bucket:
# Cian hard-cap срабатывает: часть офферов недостижима через пагинацию.
# Это происходит когда bracket < _MIN_BRACKET но totalOffers всё равно > cap.
tail_loss = total - max_pages_per_bucket * _CIAN_OFFERS_PER_PAGE
logger.warning(
"cian: %s [%d, %d] totalOffers=%d exceeds page cap "
"(max_pages=%d × %d=%d offers) — tail loss ~%d offers, "
"consider narrowing price_cap_per_bucket or _MIN_BRACKET",
room_label,
lo,
hi,
total,
max_pages_per_bucket,
_CIAN_OFFERS_PER_PAGE,
max_pages_per_bucket * _CIAN_OFFERS_PER_PAGE,
tail_loss,
)
# Страница 1 уже есть (html из probe выше); остальные — параллельно. # Страница 1 уже есть (html из probe выше); остальные — параллельно.
sem = asyncio.Semaphore(concurrency) sem = asyncio.Semaphore(concurrency)

126
tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py
View file

@ -12,10 +12,17 @@ from unittest.mock import AsyncMock, MagicMock, patch
# Settings requires DATABASE_URL at import time — set dummy DSN before any app import. # Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
import logging
import pytest import pytest
from app.services.scrapers.base import ScrapedLot from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.cian import _MAX_PRICE, _MIN_BRACKET, CianScraper from app.services.scrapers.cian import (
_CIAN_OFFERS_PER_PAGE,
_MAX_PRICE,
_MIN_BRACKET,
CianScraper,
)
def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot: def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot:
@ -597,3 +604,120 @@ async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None:
lot_ids = {lot.source_id for lot in lots} lot_ids = {lot.source_id for lot in lots}
assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False" assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False"
assert "vt_1" in lot_ids assert "vt_1" in lot_ids
# ── #1393: tail-loss warning при превышении page cap ─────────────────────────
@pytest.mark.asyncio
async def test_walk_price_range_warns_on_page_cap_tail_loss(
scraper: CianScraper, caplog: pytest.LogCaptureFixture
) -> None:
"""#1393: когда totalOffers > max_pages_per_bucket × offers_per_page,
_walk_price_range логирует WARNING с числом потерянных офферов.
Сценарий: bracket < _MIN_BRACKET (guard срабатывает → force-paginate),
totalOffers = max_pages * offers_per_page + 50 → tail_loss = 50.
"""
max_pages_cap = 5 # нарочно мало, чтобы быстро дойти до cap
offers_per_page = _CIAN_OFFERS_PER_PAGE # 28
# total > cap (2000 > любой price_cap_per_bucket=1400) И bracket < _MIN_BRACKET
# → force-paginate path; pages_needed = ceil(2000/28) = 72 > max_pages_cap=5
total_offers = max_pages_cap * offers_per_page + 50 # 190; tail_loss=50
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return total_offers
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_p{page}_{i}") for i in range(offers_per_page)]
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
):
await scraper._walk_price_range(
rooms=(1,),
# Узкий диапазон: bracket < _MIN_BRACKET → force-paginate guard
lo=1_000_000,
hi=1_000_000 + _MIN_BRACKET - 1,
seen={},
price_cap_per_bucket=1400, # total(190) < 1400 → нет split, только leaf
max_pages_per_bucket=max_pages_cap,
)
# Должно быть предупреждение о tail-loss
tail_warnings = [
r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
]
assert tail_warnings, (
f"Ожидалось WARNING о tail loss, но записей нет. "
f"Все warnings: {[r.message for r in caplog.records if r.levelno == logging.WARNING]}"
)
# Сообщение должно содержать число потерянных офферов (50)
assert any(
"50" in r.message for r in tail_warnings
), f"WARNING должен содержать tail_loss=50, got: {[r.message for r in tail_warnings]}"
@pytest.mark.asyncio
async def test_walk_price_range_no_warning_when_within_page_cap(
scraper: CianScraper, caplog: pytest.LogCaptureFixture
) -> None:
"""#1393: когда totalOffers <= max_pages_per_bucket × offers_per_page,
tail-loss WARNING не должен эмитироваться (данные собраны полностью).
"""
max_pages_cap = 54
total_offers = 56 # ceil(56/28) = 2 страницы < 54 cap → нет потерь
async def fake_fetch_page_html(
rooms: tuple, page: int, min_price: int | None, max_price: int | None
) -> str:
return f"<html>page={page}</html>"
def fake_extract_total_offers(html: str) -> int | None:
return total_offers
def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
import re
page_m = re.search(r"page=(\d+)", html)
page = int(page_m.group(1)) if page_m else 1
return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
with (
patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
patch.object(scraper, "_rotate_ip", return_value=False),
caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
):
await scraper._walk_price_range(
rooms=(1,),
lo=0,
hi=_MAX_PRICE,
seen={},
price_cap_per_bucket=1400,
max_pages_per_bucket=max_pages_cap,
)
tail_warnings = [
r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
]
assert not tail_warnings, (
f"Не ожидалось WARNING о tail loss при total={total_offers} <= page cap, "
f"но получено: {[r.message for r in tail_warnings]}"
)