270 lines
11 KiB
Python
270 lines
11 KiB
Python
"""Offline smoke for scrape_pipeline. Mocks scrapers + DB session."""
|
||
|
||
from datetime import date, timedelta
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrape_pipeline import (
|
||
_AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT,
|
||
PipelineCounters,
|
||
PipelineResult,
|
||
run_avito_full_load,
|
||
run_avito_pipeline,
|
||
)
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
|
||
def _make_lot(
|
||
item_id: str,
|
||
house_url: str | None = None,
|
||
price: int = 5_000_000,
|
||
) -> ScrapedLot:
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
|
||
source_id=item_id,
|
||
price_rub=price,
|
||
address="Test address",
|
||
house_source="avito" if house_url else None,
|
||
house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
|
||
house_url=house_url,
|
||
listing_segment="vtorichka",
|
||
)
|
||
|
||
|
||
def test_counters_dataclass_default() -> None:
|
||
c = PipelineCounters()
|
||
assert c.lots_fetched == 0
|
||
assert c.unique_houses == 0
|
||
assert c.errors == []
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_search_failure_graceful() -> None:
|
||
"""Если AvitoScraper.fetch_around raises — pipeline возвращает PipelineResult
|
||
с errors, не падает наружу."""
|
||
mock_db = MagicMock()
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(side_effect=RuntimeError("test net error"))
|
||
|
||
with patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert isinstance(result, PipelineResult)
|
||
assert result.counters.lots_fetched == 0
|
||
assert len(result.counters.errors) == 1
|
||
assert "search:" in result.counters.errors[0]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_group_by_house_unique_set() -> None:
|
||
"""3 lots с house_url'ами (2 одинаковых, 1 уникальный) → unique_houses=2."""
|
||
mock_db = MagicMock()
|
||
lots = [
|
||
_make_lot("1", "https://www.avito.ru/catalog/houses/ekb/test/100"),
|
||
_make_lot("2", "https://www.avito.ru/catalog/houses/ekb/test/100"), # дубль
|
||
_make_lot("3", "https://www.avito.ru/catalog/houses/ekb/other/200"),
|
||
_make_lot("4", None), # без дома
|
||
]
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(return_value=lots)
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(4, 0)),
|
||
):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert result.counters.lots_fetched == 4
|
||
assert result.counters.lots_inserted == 4
|
||
assert result.counters.unique_houses == 2
|
||
|
||
|
||
# ── run_avito_full_load: incremental_days → since plumbing (#avito-split) ─────
|
||
|
||
|
||
def _full_load_scraper() -> MagicMock:
|
||
"""AvitoScraper mock с async-CM + AsyncMock fetch_all_secondary."""
|
||
scraper = MagicMock()
|
||
scraper.__aenter__ = AsyncMock(return_value=scraper)
|
||
scraper.__aexit__ = AsyncMock(return_value=None)
|
||
scraper.fetch_all_secondary = AsyncMock(return_value=[])
|
||
return scraper
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_full_load_incremental_days_passes_since() -> None:
|
||
"""incremental_days=2 → fetch_all_secondary(since=date.today()-2d)."""
|
||
mock_db = MagicMock()
|
||
scraper = _full_load_scraper()
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
|
||
patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
|
||
):
|
||
await run_avito_full_load(mock_db, run_id=1, incremental_days=2)
|
||
|
||
kwargs = scraper.fetch_all_secondary.call_args.kwargs
|
||
assert kwargs["since"] == date.today() - timedelta(days=2)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_full_load_no_incremental_days_since_none() -> None:
|
||
"""incremental_days=None (default) → fetch_all_secondary(since=None) — exhaustive."""
|
||
mock_db = MagicMock()
|
||
scraper = _full_load_scraper()
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
|
||
patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
|
||
):
|
||
await run_avito_full_load(mock_db, run_id=2)
|
||
|
||
kwargs = scraper.fetch_all_secondary.call_args.kwargs
|
||
assert kwargs["since"] is None
|
||
|
||
|
||
# ── #1820: AvitoBlockedError per-item не роняет весь pipeline ─────────────────
|
||
|
||
|
||
def _block_test_scraper(lots: list[ScrapedLot]) -> MagicMock:
|
||
scraper = MagicMock()
|
||
scraper.__aenter__ = AsyncMock(return_value=scraper)
|
||
scraper.__aexit__ = AsyncMock(return_value=None)
|
||
scraper.fetch_around = AsyncMock(return_value=lots)
|
||
scraper._cffi = None
|
||
return scraper
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_single_house_block_does_not_crash() -> None:
|
||
"""#1820: один заблокированный house-фетч НЕ роняет прогон.
|
||
|
||
Listings из SEARCH+SAVE сохранены, блок зафиксирован в counters (houses_failed/errors),
|
||
остальные дома обогащаются дальше (enrichment не прерван < порога).
|
||
"""
|
||
mock_db = MagicMock()
|
||
lots = [
|
||
_make_lot("1", "https://www.avito.ru/catalog/houses/ekb/a/100"),
|
||
_make_lot("2", "https://www.avito.ru/catalog/houses/ekb/b/200"),
|
||
]
|
||
scraper = _block_test_scraper(lots)
|
||
|
||
# 1-й house блокируется, 2-й — успех.
|
||
fetch_house = AsyncMock(side_effect=[AvitoBlockedError("HTTP 403"), MagicMock()])
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(2, 0)),
|
||
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
|
||
patch(
|
||
"app.services.scrape_pipeline.save_house_catalog_enrichment",
|
||
return_value={"house_id": 42},
|
||
),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch(
|
||
"curl_cffi.requests.AsyncSession",
|
||
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
|
||
),
|
||
):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert isinstance(result, PipelineResult)
|
||
# listings сохранены всегда
|
||
assert result.counters.lots_inserted == 2
|
||
# один дом упал по блоку, второй обогащён
|
||
assert result.counters.houses_failed == 1
|
||
assert result.counters.houses_enriched == 1
|
||
assert fetch_house.await_count == 2 # enrichment НЕ прерван — оба дома попробованы
|
||
assert any("BLOCKED_HOUSE" in e for e in result.counters.errors)
|
||
assert 42 in result.touched_house_ids
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_consecutive_house_blocks_abort_but_return_result() -> None:
|
||
"""#1820: N consecutive house-блоков прерывают enrichment, но PipelineResult возвращён."""
|
||
mock_db = MagicMock()
|
||
n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
|
||
# Создаём n+2 дома — после n блоков подряд enrichment прерывается.
|
||
lots = [
|
||
_make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{100 + i}")
|
||
for i in range(n + 2)
|
||
]
|
||
scraper = _block_test_scraper(lots)
|
||
|
||
fetch_house = AsyncMock(side_effect=AvitoRateLimitedError("HTTP 429"))
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)),
|
||
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
|
||
patch("app.services.scrape_pipeline.save_house_catalog_enrichment", return_value={}),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch(
|
||
"curl_cffi.requests.AsyncSession",
|
||
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
|
||
),
|
||
):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert isinstance(result, PipelineResult)
|
||
assert result.counters.lots_inserted == len(lots) # listings сохранены
|
||
# enrichment прерван ровно после порога — лишние дома НЕ фетчились
|
||
assert fetch_house.await_count == n
|
||
assert result.counters.houses_failed == n
|
||
assert any("AVITO_BLOCKED" in e for e in result.counters.errors)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_detail_block_aborts_skips_step5b() -> None:
|
||
"""#1820: consecutive detail-блоки прерывают detail-фазу И Step 5b (houses из detail)."""
|
||
mock_db = MagicMock()
|
||
n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT
|
||
scraper = _block_test_scraper([]) # SERP пуст → сразу detail-фаза по priority_rows
|
||
|
||
# priority_rows: n+1 строк, все блокируются в detail.
|
||
row = MagicMock()
|
||
row.__getitem__ = lambda self, key: (
|
||
"https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
|
||
)
|
||
mappings = MagicMock()
|
||
mappings.all.return_value = [row] * (n + 1)
|
||
execute = MagicMock()
|
||
execute.mappings.return_value = mappings
|
||
mock_db.execute.return_value = execute
|
||
|
||
fetch_detail = AsyncMock(side_effect=AvitoBlockedError("HTTP 403"))
|
||
fetch_house = AsyncMock() # должен НЕ вызываться (Step 5b пропущен)
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
|
||
patch("app.services.scrape_pipeline.fetch_detail", fetch_detail),
|
||
patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch(
|
||
"curl_cffi.requests.AsyncSession",
|
||
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
|
||
),
|
||
):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=n + 1
|
||
)
|
||
|
||
assert isinstance(result, PipelineResult)
|
||
assert fetch_detail.await_count == n # detail прерван на пороге
|
||
assert result.counters.detail_failed == n
|
||
assert fetch_house.await_count == 0 # Step 5b пропущен (enrichment_aborted)
|