bot-backend
bf63683833
run_avito_full_load gains incremental_days param -> passes since= to the (merged) incremental SERP engine. avito_full_load schedule flipped to incremental_days=2 (shallow, date early-stop -> avoids deep-pagination 429 bans). New avito_full_load_exhaustive source runs the full walk weekly to refresh last_seen (10-day delisting TTL) and catch silent price edits.
132 lines
4.7 KiB
Python
132 lines
4.7 KiB
Python
"""Offline smoke for scrape_pipeline. Mocks scrapers + DB session."""
|
||
|
||
from datetime import date, timedelta
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrape_pipeline import (
|
||
PipelineCounters,
|
||
PipelineResult,
|
||
run_avito_full_load,
|
||
run_avito_pipeline,
|
||
)
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
|
||
def _make_lot(
|
||
item_id: str,
|
||
house_url: str | None = None,
|
||
price: int = 5_000_000,
|
||
) -> ScrapedLot:
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
|
||
source_id=item_id,
|
||
price_rub=price,
|
||
address="Test address",
|
||
house_source="avito" if house_url else None,
|
||
house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
|
||
house_url=house_url,
|
||
listing_segment="vtorichka",
|
||
)
|
||
|
||
|
||
def test_counters_dataclass_default() -> None:
|
||
c = PipelineCounters()
|
||
assert c.lots_fetched == 0
|
||
assert c.unique_houses == 0
|
||
assert c.errors == []
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_search_failure_graceful() -> None:
|
||
"""Если AvitoScraper.fetch_around raises — pipeline возвращает PipelineResult
|
||
с errors, не падает наружу."""
|
||
mock_db = MagicMock()
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(side_effect=RuntimeError("test net error"))
|
||
|
||
with patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert isinstance(result, PipelineResult)
|
||
assert result.counters.lots_fetched == 0
|
||
assert len(result.counters.errors) == 1
|
||
assert "search:" in result.counters.errors[0]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_group_by_house_unique_set() -> None:
|
||
"""3 lots с house_url'ами (2 одинаковых, 1 уникальный) → unique_houses=2."""
|
||
mock_db = MagicMock()
|
||
lots = [
|
||
_make_lot("1", "https://www.avito.ru/catalog/houses/ekb/test/100"),
|
||
_make_lot("2", "https://www.avito.ru/catalog/houses/ekb/test/100"), # дубль
|
||
_make_lot("3", "https://www.avito.ru/catalog/houses/ekb/other/200"),
|
||
_make_lot("4", None), # без дома
|
||
]
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(return_value=lots)
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(4, 0)),
|
||
):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert result.counters.lots_fetched == 4
|
||
assert result.counters.lots_inserted == 4
|
||
assert result.counters.unique_houses == 2
|
||
|
||
|
||
# ── run_avito_full_load: incremental_days → since plumbing (#avito-split) ─────
|
||
|
||
|
||
def _full_load_scraper() -> MagicMock:
|
||
"""AvitoScraper mock с async-CM + AsyncMock fetch_all_secondary."""
|
||
scraper = MagicMock()
|
||
scraper.__aenter__ = AsyncMock(return_value=scraper)
|
||
scraper.__aexit__ = AsyncMock(return_value=None)
|
||
scraper.fetch_all_secondary = AsyncMock(return_value=[])
|
||
return scraper
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_full_load_incremental_days_passes_since() -> None:
|
||
"""incremental_days=2 → fetch_all_secondary(since=date.today()-2d)."""
|
||
mock_db = MagicMock()
|
||
scraper = _full_load_scraper()
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
|
||
patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
|
||
):
|
||
await run_avito_full_load(mock_db, run_id=1, incremental_days=2)
|
||
|
||
kwargs = scraper.fetch_all_secondary.call_args.kwargs
|
||
assert kwargs["since"] == date.today() - timedelta(days=2)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_full_load_no_incremental_days_since_none() -> None:
|
||
"""incremental_days=None (default) → fetch_all_secondary(since=None) — exhaustive."""
|
||
mock_db = MagicMock()
|
||
scraper = _full_load_scraper()
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper),
|
||
patch("app.services.scrape_pipeline.scrape_runs", MagicMock()),
|
||
):
|
||
await run_avito_full_load(mock_db, run_id=2)
|
||
|
||
kwargs = scraper.fetch_all_secondary.call_args.kwargs
|
||
assert kwargs["since"] is None
|