0154c2914a
Stage 2e of AvitoScraper_v2.
- run_avito_pipeline(db, lat, lon, radius_m, *, enrich_houses, enrich_detail_top_n)
- 5-step flow:
1. SEARCH: AvitoScraper().fetch_around (uses Stage 2a refactored search)
2. SAVE: save_listings (inserts + tracks house_ext_id from Stage 2a)
3. GROUP: dedupe house path → unique set (path-only для consistency с fetch_house_catalog)
4. ENRICH_HOUSES: fetch_house_catalog + save_house_catalog_enrichment (Stage 2c) per unique house
5. ENRICH_DETAIL: top-N priority listings (detail_enriched_at IS NULL within radius)
→ fetch_detail + save_detail_enrichment (Stage 2b)
- PipelineCounters dataclass: lots_fetched/inserted/updated, unique_houses, houses_enriched/failed,
detail_attempted/enriched/failed, errors[]
- Graceful degradation: per-house and per-detail try/except — single failure не валит pipeline
- IMV ОТСУТСТВУЕТ — он on-demand (Stage 3 estimator integration), не cron
- pytest offline smoke (3 tests: counters default + search failure graceful + group_by_house dedupe)
- Add pytest-asyncio>=0.24.0 to dev deps + asyncio_mode=auto в pyproject.toml
Refs: AvitoScraper_v2_Implementation_Plan Stage 2e.
86 lines
3 KiB
Python
86 lines
3 KiB
Python
"""Offline smoke for scrape_pipeline. Mocks scrapers + DB session."""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrape_pipeline import (
|
||
PipelineCounters,
|
||
PipelineResult,
|
||
run_avito_pipeline,
|
||
)
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
|
||
def _make_lot(
|
||
item_id: str,
|
||
house_url: str | None = None,
|
||
price: int = 5_000_000,
|
||
) -> ScrapedLot:
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/test_{item_id}",
|
||
source_id=item_id,
|
||
price_rub=price,
|
||
address="Test address",
|
||
house_source="avito" if house_url else None,
|
||
house_ext_id=house_url.rstrip("/").split("/")[-1] if house_url else None,
|
||
house_url=house_url,
|
||
listing_segment="vtorichka",
|
||
)
|
||
|
||
|
||
def test_counters_dataclass_default() -> None:
|
||
c = PipelineCounters()
|
||
assert c.lots_fetched == 0
|
||
assert c.unique_houses == 0
|
||
assert c.errors == []
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_search_failure_graceful() -> None:
|
||
"""Если AvitoScraper.fetch_around raises — pipeline возвращает PipelineResult
|
||
с errors, не падает наружу."""
|
||
mock_db = MagicMock()
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(side_effect=RuntimeError("test net error"))
|
||
|
||
with patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert isinstance(result, PipelineResult)
|
||
assert result.counters.lots_fetched == 0
|
||
assert len(result.counters.errors) == 1
|
||
assert "search:" in result.counters.errors[0]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_group_by_house_unique_set() -> None:
|
||
"""3 lots с house_url'ами (2 одинаковых, 1 уникальный) → unique_houses=2."""
|
||
mock_db = MagicMock()
|
||
lots = [
|
||
_make_lot("1", "https://www.avito.ru/catalog/houses/ekb/test/100"),
|
||
_make_lot("2", "https://www.avito.ru/catalog/houses/ekb/test/100"), # дубль
|
||
_make_lot("3", "https://www.avito.ru/catalog/houses/ekb/other/200"),
|
||
_make_lot("4", None), # без дома
|
||
]
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(return_value=lots)
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(4, 0)),
|
||
):
|
||
result = await run_avito_pipeline(
|
||
mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=0
|
||
)
|
||
|
||
assert result.counters.lots_fetched == 4
|
||
assert result.counters.lots_inserted == 4
|
||
assert result.counters.unique_houses == 2
|