86 lines
3 KiB
Python
86 lines
3 KiB
Python
"""Offline smoke for scrape_pipeline. Mocks scrapers + DB session."""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrape_pipeline import (
|
||
PipelineCounters,
|
||
PipelineResult,
|
||
run_avito_pipeline,
|
||
)
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
|
||
def _make_lot(
|
||
item_id: str,
|
||
house_url: str | None = None,
|
||
price: int = 5_000_000,
|
||
) -> ScrapedLot:
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
|
||
source_id=item_id,
|
||
price_rub=price,
|
||
address="Test address",
|
||
house_source="avito" if house_url else None,
|
||
house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
|
||
house_url=house_url,
|
||
listing_segment="vtorichka",
|
||
)
|
||
|
||
|
||
def test_counters_dataclass_default() -> None:
|
||
c = PipelineCounters()
|
||
assert c.lots_fetched == 0
|
||
assert c.unique_houses == 0
|
||
assert c.errors == []
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_search_failure_graceful() -> None:
|
||
"""Если AvitoScraper.fetch_around raises — pipeline возвращает PipelineResult
|
||
с errors, не падает наружу."""
|
||
mock_db = MagicMock()
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(side_effect=RuntimeError("test net error"))
|
||
|
||
with patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert isinstance(result, PipelineResult)
|
||
assert result.counters.lots_fetched == 0
|
||
assert len(result.counters.errors) == 1
|
||
assert "search:" in result.counters.errors[0]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_group_by_house_unique_set() -> None:
|
||
"""3 lots с house_url'ами (2 одинаковых, 1 уникальный) → unique_houses=2."""
|
||
mock_db = MagicMock()
|
||
lots = [
|
||
_make_lot("1", "https://www.avito.ru/catalog/houses/ekb/test/100"),
|
||
_make_lot("2", "https://www.avito.ru/catalog/houses/ekb/test/100"), # дубль
|
||
_make_lot("3", "https://www.avito.ru/catalog/houses/ekb/other/200"),
|
||
_make_lot("4", None), # без дома
|
||
]
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(return_value=lots)
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(4, 0)),
|
||
):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert result.counters.lots_fetched == 4
|
||
assert result.counters.lots_inserted == 4
|
||
assert result.counters.unique_houses == 2
|