"""Stage 1 #915 — route avito pipeline SERP+detail+houses through BrowserFetcher.
Тесты без сети и без БД. BrowserFetcher мокируется объектом с async fetch().
Покрытие:
1. fetch_detail(browser_fetcher=mock) → DetailEnrichment (browser branch taken)
2. fetch_detail(browser_fetcher=mock) → AvitoBlockedError при firewall HTML
3. fetch_house_catalog(browser_fetcher=mock) → browser branch taken
4. run_avito_pipeline в browser-mode → scraper._browser установлен,
fetch_detail/fetch_house_catalog вызваны с browser_fetcher
5. Curl mode (default) — browser_fetcher=None, curl path используется
"""
from __future__ import annotations
import json
import os
import urllib.parse
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.avito_houses import fetch_house_catalog
FIXTURES = Path(__file__).parent / "fixtures"
# Минимальный HTML для fetch_detail — достаточен для parse_detail_html
DETAIL_HTML = """
№ 99887766 · 18 мая в 10:00 · 100 просмотров
5 000 000 ₽
- Количество комнат: 2
- Общая площадь: 50 м²
- Этаж: 5 из 10
"""
# Firewall HTML — содержит маркер который _is_firewall_page ловит
FIREWALL_HTML = """
Доступ ограничен
Доступ ограничен
"""
_MINIMAL_STATE = {
"data": {
"data": {
"page": {
"placeholders": [
{
"type": "housePage",
"props": {
"developmentData": {
"avitoId": 12345,
"id": "abc123",
"title": "ЖК Тест",
"address": "ул. Тестовая, 1",
"fullAddress": "Екатеринбург, ул. Тестовая, 1",
"coords": {"lat": 56.8, "lng": 60.6},
"aboutDevelopment": {"expandParams": {"items": []}},
"developer": {},
"mapPreview": {},
}
},
}
]
}
}
}
}
_encoded_state = urllib.parse.quote(json.dumps(_MINIMAL_STATE, ensure_ascii=False))
HOUSES_HTML = f"""
"""
# ── Helpers ──────────────────────────────────────────────────────────────────
def make_mock_fetcher(html: str) -> MagicMock:
"""Создаёт mock BrowserFetcher с async fetch(), возвращающим заданный HTML."""
mock = MagicMock()
mock.fetch = AsyncMock(return_value=html)
return mock
# ── 1. fetch_detail browser branch → DetailEnrichment ────────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_browser_returns_enrichment() -> None:
"""browser_fetcher.fetch() вызывается, curl AsyncSession не трогается."""
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
item_url = "/ekaterinburg/kvartiry/test_99887766-1234"
result = await fetch_detail(item_url, browser_fetcher=mock_fetcher)
# browser path взят
mock_fetcher.fetch.assert_awaited_once()
called_url = mock_fetcher.fetch.call_args[0][0]
assert called_url.startswith("https://www.avito.ru"), f"unexpected url: {called_url}"
assert called_url.endswith(item_url)
assert result.item_id == "99887766"
assert result.price_rub == 5_000_000
assert result.rooms == 2
@pytest.mark.asyncio
async def test_fetch_detail_browser_full_url_passthrough() -> None:
"""Если item_url уже абсолютный — передаётся без urljoin."""
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
full_url = "https://www.avito.ru/ekaterinburg/kvartiry/test_99887766-1234"
result = await fetch_detail(full_url, browser_fetcher=mock_fetcher)
called_url = mock_fetcher.fetch.call_args[0][0]
assert called_url == full_url
assert result.item_id == "99887766"
@pytest.mark.asyncio
async def test_fetch_detail_browser_no_curl_session_used() -> None:
"""cffi_session не используется когда browser_fetcher передан."""
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
mock_session = MagicMock()
mock_session.get = AsyncMock() # НЕ должен быть вызван
result = await fetch_detail(
"/ekaterinburg/kvartiry/test_99887766-1234",
cffi_session=mock_session,
browser_fetcher=mock_fetcher,
)
mock_session.get.assert_not_awaited()
assert result.item_id == "99887766"
# ── 2. fetch_detail browser firewall → AvitoBlockedError ─────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_browser_firewall_raises() -> None:
"""Firewall HTML через browser path → AvitoBlockedError."""
mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await fetch_detail(
"/ekaterinburg/kvartiry/blocked_item-9999",
browser_fetcher=mock_fetcher,
)
# ── 3. fetch_house_catalog browser branch ────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_house_catalog_browser_branch_taken() -> None:
"""browser_fetcher.fetch() вызывается, curl path не трогается."""
mock_fetcher = make_mock_fetcher(HOUSES_HTML)
house_path = "/catalog/houses/ekaterinburg/ul_test/3171365"
result = await fetch_house_catalog(house_path, browser_fetcher=mock_fetcher)
mock_fetcher.fetch.assert_awaited_once()
called_url = mock_fetcher.fetch.call_args[0][0]
assert "avito.ru" in called_url
assert house_path in called_url
assert result.house.ext_id == 12345
assert result.house.title == "ЖК Тест"
@pytest.mark.asyncio
async def test_fetch_house_catalog_browser_no_curl_used() -> None:
"""cffi_session.get не вызывается в browser mode."""
mock_fetcher = make_mock_fetcher(HOUSES_HTML)
mock_session = MagicMock()
mock_session.get = AsyncMock()
await fetch_house_catalog(
"/catalog/houses/ekaterinburg/ul_test/3171365",
cffi_session=mock_session,
browser_fetcher=mock_fetcher,
)
mock_session.get.assert_not_awaited()
@pytest.mark.asyncio
async def test_fetch_house_catalog_browser_firewall_raises() -> None:
"""Firewall HTML через browser path → AvitoBlockedError."""
mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await fetch_house_catalog(
"/catalog/houses/ekaterinburg/ul_test/9999",
browser_fetcher=mock_fetcher,
)
# ── 4. run_avito_pipeline browser-mode routing ───────────────────────────────
@pytest.mark.asyncio
async def test_run_avito_pipeline_browser_mode_sets_scraper_browser(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""В browser-mode scraper._browser установлен, fetch_detail/fetch_house_catalog
вызываются с browser_fetcher (не с cffi_session).
"""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
mock_bf = make_mock_fetcher(DETAIL_HTML)
captured_scraper: list = []
# Mock AvitoScraper.fetch_around → возвращает пустой список (нет лотов → нет DB)
async def mock_fetch_around(
self: object, lat: float, lon: float, radius_m: int, **kwargs: object
) -> list:
captured_scraper.append(self)
return []
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
with (
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
from app.services.scrape_pipeline import run_avito_pipeline
await run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
radius_m=1500,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
shared_browser=mock_bf,
)
assert len(captured_scraper) == 1
scraper_instance = captured_scraper[0]
assert scraper_instance._browser is mock_bf
# curl session должен быть None — в browser-mode pipeline не создаёт _cffi сессию
assert scraper_instance._cffi is None
@pytest.mark.asyncio
async def test_run_avito_pipeline_browser_mode_own_browser_created(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Если shared_browser не передан в browser-mode, BrowserFetcher создаётся
и __aenter__/__aexit__ вызываются."""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
mock_bf_instance = make_mock_fetcher(DETAIL_HTML)
mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance)
mock_bf_instance.__aexit__ = AsyncMock(return_value=None)
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
return []
with (
patch("app.services.scrape_pipeline.BrowserFetcher", return_value=mock_bf_instance),
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
await scrape_pipeline.run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
)
mock_bf_instance.__aenter__.assert_awaited_once()
mock_bf_instance.__aexit__.assert_awaited_once()
# ── 5. Curl mode (default) — browser_fetcher stays None ──────────────────────
@pytest.mark.asyncio
async def test_run_avito_pipeline_curl_mode_no_browser_fetcher(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""В curl-mode (default) browser_fetcher=None, scraper._cffi установлен."""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "curl_cffi", raising=False)
captured_scraper: list = []
async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
captured_scraper.append(self)
return []
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
with (
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
await scrape_pipeline.run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
)
assert len(captured_scraper) == 1
scraper_instance = captured_scraper[0]
# curl mode: _cffi должен быть установлен
assert scraper_instance._cffi is not None
# browser не установлен в curl mode
assert getattr(scraper_instance, "_browser", None) is None
@pytest.mark.asyncio
async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
"""Если browser_fetcher=None, curl path используется (собственная сессия)."""
from unittest.mock import patch as _patch
# Мокаем создание собственной сессии чтобы не нужен реальный curl_cffi
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.text = DETAIL_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(return_value=mock_resp)
mock_session.close = AsyncMock()
with _patch(
"app.services.scrapers.avito_detail.AsyncSession",
return_value=mock_session,
):
result = await fetch_detail("/ekaterinburg/kvartiry/test_99887766-1234")
# curl path взят — session.get был вызван
mock_session.get.assert_awaited_once()
assert result.item_id == "99887766"
# ── 6. SERP routing: real _fetch_serp_html routes through browser ────────────
@pytest.mark.asyncio
async def test_fetch_serp_html_routes_through_browser() -> None:
"""With _browser set and _cffi=None, the REAL _fetch_serp_html returns the
browser HTML and never dereferences _cffi (no AssertionError). Proves #915
SERP routing on top of #901's browser branch."""
from unittest.mock import AsyncMock
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
assert scraper._cffi is None
scraper._browser = AsyncMock()
scraper._browser.fetch = AsyncMock(return_value="real serp listing")
html = await scraper._fetch_serp_html(
"https://www.avito.ru/ekaterinburg/kvartiry/prodam", page=1
)
assert html == "real serp listing"
scraper._browser.fetch.assert_awaited_once()