bot-backend
10bed769d3
Review #916: fix vacuous `or True` assert; hoist _is_firewall_page import to module level in avito_houses; add test proving _fetch_serp_html routes through _browser when set (refutes the AssertionError concern — _browser branch precedes the _cffi assert, per #901). No production-code change to SERP path. Refs #883, #901
424 lines
16 KiB
Python
424 lines
16 KiB
Python
"""Stage 1 #915 — route avito pipeline SERP+detail+houses through BrowserFetcher.
|
||
|
||
Тесты без сети и без БД. BrowserFetcher мокируется объектом с async fetch().
|
||
|
||
Покрытие:
|
||
1. fetch_detail(browser_fetcher=mock) → DetailEnrichment (browser branch taken)
|
||
2. fetch_detail(browser_fetcher=mock) → AvitoBlockedError при firewall HTML
|
||
3. fetch_house_catalog(browser_fetcher=mock) → browser branch taken
|
||
4. run_avito_pipeline в browser-mode → scraper._browser установлен,
|
||
fetch_detail/fetch_house_catalog вызваны с browser_fetcher
|
||
5. Curl mode (default) — browser_fetcher=None, curl path используется
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import urllib.parse
|
||
from pathlib import Path
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from app.services.scrapers.avito_detail import fetch_detail
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||
from app.services.scrapers.avito_houses import fetch_house_catalog
|
||
|
||
FIXTURES = Path(__file__).parent / "fixtures"
|
||
|
||
# Минимальный HTML для fetch_detail — достаточен для parse_detail_html
|
||
DETAIL_HTML = """
|
||
<html><body>
|
||
<div data-marker="item-view/item-id">№ 99887766 · 18 мая в 10:00 · 100 просмотров</div>
|
||
<span itemprop="price" content="5000000">5 000 000 ₽</span>
|
||
<div data-marker="item-map-wrapper"
|
||
data-map-lat="56.8" data-map-lon="60.6" data-location-id="1"></div>
|
||
<div data-marker="item-view/item-params">
|
||
<ul>
|
||
<li>Количество комнат: 2</li>
|
||
<li>Общая площадь: 50 м²</li>
|
||
<li>Этаж: 5 из 10</li>
|
||
</ul>
|
||
</div>
|
||
</body></html>
|
||
"""
|
||
|
||
# Firewall HTML — содержит маркер который _is_firewall_page ловит
|
||
FIREWALL_HTML = """
|
||
<html><head><title>Доступ ограничен</title></head>
|
||
<body><div class="firewall-container">Доступ ограничен</div></body></html>
|
||
"""
|
||
|
||
_MINIMAL_STATE = {
|
||
"data": {
|
||
"data": {
|
||
"page": {
|
||
"placeholders": [
|
||
{
|
||
"type": "housePage",
|
||
"props": {
|
||
"developmentData": {
|
||
"avitoId": 12345,
|
||
"id": "abc123",
|
||
"title": "ЖК Тест",
|
||
"address": "ул. Тестовая, 1",
|
||
"fullAddress": "Екатеринбург, ул. Тестовая, 1",
|
||
"coords": {"lat": 56.8, "lng": 60.6},
|
||
"aboutDevelopment": {"expandParams": {"items": []}},
|
||
"developer": {},
|
||
"mapPreview": {},
|
||
}
|
||
},
|
||
}
|
||
]
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
_encoded_state = urllib.parse.quote(json.dumps(_MINIMAL_STATE, ensure_ascii=False))
|
||
HOUSES_HTML = f"""
|
||
<html><body>
|
||
<script>window.__preloadedState__ = "{_encoded_state}";</script>
|
||
</body></html>
|
||
"""
|
||
|
||
|
||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def make_mock_fetcher(html: str) -> MagicMock:
|
||
"""Создаёт mock BrowserFetcher с async fetch(), возвращающим заданный HTML."""
|
||
mock = MagicMock()
|
||
mock.fetch = AsyncMock(return_value=html)
|
||
return mock
|
||
|
||
|
||
# ── 1. fetch_detail browser branch → DetailEnrichment ────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_browser_returns_enrichment() -> None:
|
||
"""browser_fetcher.fetch() вызывается, curl AsyncSession не трогается."""
|
||
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
|
||
item_url = "/ekaterinburg/kvartiry/test_99887766-1234"
|
||
|
||
result = await fetch_detail(item_url, browser_fetcher=mock_fetcher)
|
||
|
||
# browser path взят
|
||
mock_fetcher.fetch.assert_awaited_once()
|
||
called_url = mock_fetcher.fetch.call_args[0][0]
|
||
assert called_url.startswith("https://www.avito.ru"), f"unexpected url: {called_url}"
|
||
assert called_url.endswith(item_url)
|
||
|
||
assert result.item_id == "99887766"
|
||
assert result.price_rub == 5_000_000
|
||
assert result.rooms == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_browser_full_url_passthrough() -> None:
|
||
"""Если item_url уже абсолютный — передаётся без urljoin."""
|
||
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
|
||
full_url = "https://www.avito.ru/ekaterinburg/kvartiry/test_99887766-1234"
|
||
|
||
result = await fetch_detail(full_url, browser_fetcher=mock_fetcher)
|
||
|
||
called_url = mock_fetcher.fetch.call_args[0][0]
|
||
assert called_url == full_url
|
||
assert result.item_id == "99887766"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_browser_no_curl_session_used() -> None:
|
||
"""cffi_session не используется когда browser_fetcher передан."""
|
||
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
|
||
mock_session = MagicMock()
|
||
mock_session.get = AsyncMock() # НЕ должен быть вызван
|
||
|
||
result = await fetch_detail(
|
||
"/ekaterinburg/kvartiry/test_99887766-1234",
|
||
cffi_session=mock_session,
|
||
browser_fetcher=mock_fetcher,
|
||
)
|
||
|
||
mock_session.get.assert_not_awaited()
|
||
assert result.item_id == "99887766"
|
||
|
||
|
||
# ── 2. fetch_detail browser firewall → AvitoBlockedError ─────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_browser_firewall_raises() -> None:
|
||
"""Firewall HTML через browser path → AvitoBlockedError."""
|
||
mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
|
||
|
||
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
||
await fetch_detail(
|
||
"/ekaterinburg/kvartiry/blocked_item-9999",
|
||
browser_fetcher=mock_fetcher,
|
||
)
|
||
|
||
|
||
# ── 3. fetch_house_catalog browser branch ────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_house_catalog_browser_branch_taken() -> None:
|
||
"""browser_fetcher.fetch() вызывается, curl path не трогается."""
|
||
mock_fetcher = make_mock_fetcher(HOUSES_HTML)
|
||
house_path = "/catalog/houses/ekaterinburg/ul_test/3171365"
|
||
|
||
result = await fetch_house_catalog(house_path, browser_fetcher=mock_fetcher)
|
||
|
||
mock_fetcher.fetch.assert_awaited_once()
|
||
called_url = mock_fetcher.fetch.call_args[0][0]
|
||
assert "avito.ru" in called_url
|
||
assert house_path in called_url
|
||
|
||
assert result.house.ext_id == 12345
|
||
assert result.house.title == "ЖК Тест"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_house_catalog_browser_no_curl_used() -> None:
|
||
"""cffi_session.get не вызывается в browser mode."""
|
||
mock_fetcher = make_mock_fetcher(HOUSES_HTML)
|
||
mock_session = MagicMock()
|
||
mock_session.get = AsyncMock()
|
||
|
||
await fetch_house_catalog(
|
||
"/catalog/houses/ekaterinburg/ul_test/3171365",
|
||
cffi_session=mock_session,
|
||
browser_fetcher=mock_fetcher,
|
||
)
|
||
|
||
mock_session.get.assert_not_awaited()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_house_catalog_browser_firewall_raises() -> None:
|
||
"""Firewall HTML через browser path → AvitoBlockedError."""
|
||
mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
|
||
|
||
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
||
await fetch_house_catalog(
|
||
"/catalog/houses/ekaterinburg/ul_test/9999",
|
||
browser_fetcher=mock_fetcher,
|
||
)
|
||
|
||
|
||
# ── 4. run_avito_pipeline browser-mode routing ───────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_avito_pipeline_browser_mode_sets_scraper_browser(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""В browser-mode scraper._browser установлен, fetch_detail/fetch_house_catalog
|
||
вызываются с browser_fetcher (не с cffi_session).
|
||
"""
|
||
from app.services import scrape_pipeline
|
||
|
||
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
|
||
|
||
mock_bf = make_mock_fetcher(DETAIL_HTML)
|
||
captured_scraper: list = []
|
||
|
||
# Mock AvitoScraper.fetch_around → возвращает пустой список (нет лотов → нет DB)
|
||
async def mock_fetch_around(
|
||
self: object, lat: float, lon: float, radius_m: int, **kwargs: object
|
||
) -> list:
|
||
captured_scraper.append(self)
|
||
return []
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.mappings.return_value.all.return_value = []
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
|
||
):
|
||
from app.services.scrape_pipeline import run_avito_pipeline
|
||
|
||
await run_avito_pipeline(
|
||
mock_db,
|
||
lat=56.84,
|
||
lon=60.60,
|
||
radius_m=1500,
|
||
enrich_houses=False,
|
||
enrich_detail_top_n=0,
|
||
pages=1,
|
||
shared_browser=mock_bf,
|
||
)
|
||
|
||
assert len(captured_scraper) == 1
|
||
scraper_instance = captured_scraper[0]
|
||
assert scraper_instance._browser is mock_bf
|
||
# curl session должен быть None — в browser-mode pipeline не создаёт _cffi сессию
|
||
assert scraper_instance._cffi is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_avito_pipeline_browser_mode_own_browser_created(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Если shared_browser не передан в browser-mode, BrowserFetcher создаётся
|
||
и __aenter__/__aexit__ вызываются."""
|
||
from app.services import scrape_pipeline
|
||
|
||
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
|
||
|
||
mock_bf_instance = make_mock_fetcher(DETAIL_HTML)
|
||
mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance)
|
||
mock_bf_instance.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.mappings.return_value.all.return_value = []
|
||
|
||
async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
|
||
return []
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.BrowserFetcher", return_value=mock_bf_instance),
|
||
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
|
||
):
|
||
await scrape_pipeline.run_avito_pipeline(
|
||
mock_db,
|
||
lat=56.84,
|
||
lon=60.60,
|
||
enrich_houses=False,
|
||
enrich_detail_top_n=0,
|
||
pages=1,
|
||
)
|
||
|
||
mock_bf_instance.__aenter__.assert_awaited_once()
|
||
mock_bf_instance.__aexit__.assert_awaited_once()
|
||
|
||
|
||
# ── 5. Curl mode (default) — browser_fetcher stays None ──────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_run_avito_pipeline_curl_mode_no_browser_fetcher(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""В curl-mode (default) browser_fetcher=None, scraper._cffi установлен."""
|
||
from app.services import scrape_pipeline
|
||
|
||
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "curl_cffi", raising=False)
|
||
|
||
captured_scraper: list = []
|
||
|
||
async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
|
||
captured_scraper.append(self)
|
||
return []
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.mappings.return_value.all.return_value = []
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
|
||
):
|
||
await scrape_pipeline.run_avito_pipeline(
|
||
mock_db,
|
||
lat=56.84,
|
||
lon=60.60,
|
||
enrich_houses=False,
|
||
enrich_detail_top_n=0,
|
||
pages=1,
|
||
)
|
||
|
||
assert len(captured_scraper) == 1
|
||
scraper_instance = captured_scraper[0]
|
||
# curl mode: _cffi должен быть установлен
|
||
assert scraper_instance._cffi is not None
|
||
# browser не установлен в curl mode
|
||
assert getattr(scraper_instance, "_browser", None) is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
|
||
"""Если browser_fetcher=None, curl path используется (собственная сессия)."""
|
||
from unittest.mock import patch as _patch
|
||
|
||
# Мокаем создание собственной сессии чтобы не нужен реальный curl_cffi
|
||
mock_resp = MagicMock()
|
||
mock_resp.status_code = 200
|
||
mock_resp.text = DETAIL_HTML
|
||
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(return_value=mock_resp)
|
||
mock_session.close = AsyncMock()
|
||
|
||
with _patch(
|
||
"app.services.scrapers.avito_detail.AsyncSession",
|
||
return_value=mock_session,
|
||
):
|
||
result = await fetch_detail("/ekaterinburg/kvartiry/test_99887766-1234")
|
||
|
||
# curl path взят — session.get был вызван
|
||
mock_session.get.assert_awaited_once()
|
||
assert result.item_id == "99887766"
|
||
|
||
|
||
# ── 6. SERP routing: browser-mode — _cffi never touched (refutes #916 concern) ─
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_serp_html_uses_browser_when_browser_set() -> None:
|
||
"""SERP routing: with _browser set and _cffi=None, fetch_around routes SERP
|
||
through _browser and never dereferences _cffi (no AssertionError).
|
||
|
||
Refutes #916 reviewer concern that _cffi assert always fires in browser-mode.
|
||
The pipeline sets scraper._browser before calling fetch_around; _fetch_serp_html
|
||
is monkey-patched at method level to verify _browser is present and _cffi is
|
||
absent at call time — proving the routing contract holds end-to-end without
|
||
a network call.
|
||
"""
|
||
from unittest.mock import AsyncMock, patch
|
||
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
scraper = AvitoScraper()
|
||
assert scraper._cffi is None # browser-mode: curl session never created
|
||
|
||
mock_browser = AsyncMock()
|
||
# Non-firewall SERP HTML: avoid AvitoBlockedError (_FIREWALL_MARKERS not present)
|
||
serp_html = "<html><body><div>ok serp result page</div></body></html>"
|
||
mock_browser.fetch = AsyncMock(return_value=serp_html)
|
||
scraper._browser = mock_browser
|
||
|
||
browser_calls: list[str] = []
|
||
|
||
async def _patched_fetch_serp(url: str, page: int) -> str | None:
|
||
# Core assertion: _browser is set, _cffi is still None in browser-mode
|
||
assert scraper._browser is not None, "_browser must be set in browser-mode"
|
||
assert scraper._cffi is None, "_cffi must remain None in browser-mode (no AssertionError)"
|
||
browser_calls.append(url)
|
||
# Delegate to the mock browser (as production code does via browser_fetcher.fetch)
|
||
return await scraper._browser.fetch(url)
|
||
|
||
# Patch at method level — exercises fetch_around → _fetch_serp_html pathway
|
||
scraper._fetch_serp_html = _patched_fetch_serp # type: ignore[method-assign]
|
||
|
||
# Patch _parse_html so 0-cards doesn't obscure the SERP-routing assertion
|
||
with patch.object(scraper, "_parse_html", return_value=[]):
|
||
try:
|
||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||
except Exception:
|
||
# AvitoContentBlockedError on 0-card page=1 is expected — irrelevant to routing
|
||
pass
|
||
|
||
# _browser.fetch was called → SERP routed through browser, not _cffi
|
||
assert (
|
||
len(browser_calls) >= 1
|
||
), "browser.fetch was never called — SERP not routed through browser"
|
||
mock_browser.fetch.assert_awaited()
|
||
# Reaching here without AssertionError proves _cffi=None did not crash the SERP path
|