lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

feat(tradein): route avito pipeline SERP+detail+houses through BrowserFetcher (#915)

Browse source 
Stage 1 of #915. run_avito_pipeline / run_avito_city_sweep acquire a shared
BrowserFetcher when scraper_fetch_mode==browser (instead of injecting a curl
_cffi session); fetch_detail / fetch_house_catalog gain a browser_fetcher param.
Fixes #901's gap where the pipeline bypassed browser-mode. IMV stays on curl
(Stage 3). Default scraper_fetch_mode unchanged.

Refs #883, #901
This commit is contained in:
bot-backend 2026-05-31 20:30:53 +03:00
parent 91d51e60d6
commit 5f3f55faef
4 changed files with 511 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
tradein-mvp/backend/app/services/scrape_pipeline.py
View file

@ -30,6 +30,7 @@ from __future__ import annotations
import asyncio
import logging
import random
from contextlib import AsyncExitStack
from dataclasses import dataclass, field, fields
from urllib.parse import urlparse
@ -44,6 +45,7 @@ from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichm
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
from app.services.scrapers.avito_houses import fetch_house_catalog, save_house_catalog_enrichment
from app.services.scrapers.base import ScrapedLot, save_listings
from app.services.scrapers.browser_fetcher import BrowserFetcher
from app.services.scrapers.n1 import N1Scraper
from app.services.scrapers.yandex_realty import YandexRealtyScraper
@ -137,6 +139,7 @@ async def run_avito_pipeline(
pages: int = 1,
request_delay_sec: float | None = None,
shared_session: AsyncSession | None = None,
shared_browser: BrowserFetcher | None = None,
) -> PipelineResult:
"""Full Avito search → houses → detail enrichment pipeline.
@ -148,24 +151,39 @@ async def run_avito_pipeline(
5. ENRICH_DETAIL: top-N listings → fetch_detail + save (abort on 3 blocks)
shared_session — если передан, используется без закрытия (lifecycle у вызывающего).
shared_browser — если передан в browser-mode, используется без закрытия.
AvitoBlockedError / AvitoRateLimitedError propagate наружу.
"""
counters = PipelineCounters()
lots: list[ScrapedLot] = []
detail_delay = request_delay_sec if request_delay_sec is not None else 7.0
own_session = shared_session is None
session = shared_session or AsyncSession(
impersonate="chrome120",
timeout=25,
headers=_CHROME_HEADERS,
proxies=_avito_proxies(),
)
browser_mode = settings.scraper_fetch_mode == "browser"
session: AsyncSession | None = None
browser_fetcher: BrowserFetcher | None = None
own_session = False
own_browser = False
scraper = AvitoScraper()
if browser_mode:
browser_fetcher = shared_browser
if browser_fetcher is None:
browser_fetcher = BrowserFetcher()
await browser_fetcher.__aenter__()
own_browser = True
scraper._browser = browser_fetcher
else:
own_session = shared_session is None
session = shared_session or AsyncSession(
impersonate="chrome120",
timeout=25,
headers=_CHROME_HEADERS,
proxies=_avito_proxies(),
)
scraper._cffi = session
try:
# ── Step 1: search ──────────────────────────────────────
scraper = AvitoScraper()
scraper._cffi = session
try:
lots = await scraper.fetch_around(
lat,
@ -218,7 +236,9 @@ async def run_avito_pipeline(
house_paths_list = list(unique_house_paths)
for idx, house_path in enumerate(house_paths_list):
try:
enrichment = await fetch_house_catalog(house_path, cffi_session=session)
enrichment = await fetch_house_catalog(
house_path, cffi_session=session, browser_fetcher=browser_fetcher
)
house_counters = save_house_catalog_enrichment(db, enrichment)
counters.houses_enriched += 1
hid = house_counters.get("house_id")
@ -282,7 +302,9 @@ async def run_avito_pipeline(
item_url = (
urlparse(source_url).path if source_url.startswith("http") else source_url
)
enrichment_detail = await fetch_detail(item_url, cffi_session=session)
enrichment_detail = await fetch_detail(
item_url, cffi_session=session, browser_fetcher=browser_fetcher
)
if save_detail_enrichment(db, enrichment_detail):
counters.detail_enriched += 1
# #871: house-link есть в detail SSR (в SERP-карточке — JS-only).
@ -332,7 +354,9 @@ async def run_avito_pipeline(
nh_list = list(new_house_paths)
for h_idx, house_path in enumerate(nh_list):
try:
enrichment = await fetch_house_catalog(house_path, cffi_session=session)
enrichment = await fetch_house_catalog(
house_path, cffi_session=session, browser_fetcher=browser_fetcher
)
house_counters = save_house_catalog_enrichment(db, enrichment)
counters.houses_enriched += 1
hid = house_counters.get("house_id")
@ -378,8 +402,10 @@ async def run_avito_pipeline(
)
finally:
if own_session:
if own_session and session is not None:
await session.close()
if own_browser and browser_fetcher is not None:
await browser_fetcher.__aexit__(None, None, None)
@dataclass
@ -432,12 +458,21 @@ async def run_avito_city_sweep(
counters = CitySweepCounters(anchors_total=len(_anchors))
all_touched_house_ids: set[int] = set()
async with AsyncSession(
impersonate="chrome120",
timeout=25,
headers=_CHROME_HEADERS,
proxies=_avito_proxies(),
) as session:
browser_mode = settings.scraper_fetch_mode == "browser"
async with AsyncExitStack() as stack:
session: AsyncSession | None = None
shared_bf: BrowserFetcher | None = None
if browser_mode:
shared_bf = await stack.enter_async_context(BrowserFetcher())
else:
session = await stack.enter_async_context(
AsyncSession(
impersonate="chrome120",
timeout=25,
headers=_CHROME_HEADERS,
proxies=_avito_proxies(),
)
)
try:
for idx, (lat, lon, name) in enumerate(_anchors, start=1):
if scrape_runs.is_cancelled(db, run_id):
@ -472,6 +507,7 @@ async def run_avito_city_sweep(
pages=pages_per_anchor,
request_delay_sec=request_delay_sec,
shared_session=session,
shared_browser=shared_bf,
),
timeout=ANCHOR_TIMEOUT_SEC,
)

16
tradein-mvp/backend/app/services/scrapers/avito_detail.py
View file

@ -24,7 +24,7 @@ import logging
import re
from dataclasses import dataclass, field
from datetime import date
from typing import Any
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin, urlparse
from curl_cffi.requests import AsyncSession
@ -32,10 +32,13 @@ from selectolax.parser import HTMLParser, Node
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.services.scrapers.avito import _clean_address
from app.services.scrapers.avito import _clean_address, _is_firewall_page
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text
if TYPE_CHECKING:
from app.services.scrapers.browser_fetcher import BrowserFetcher
logger = logging.getLogger(__name__)
AVITO_BASE = "https://www.avito.ru"
@ -229,14 +232,23 @@ async def fetch_detail(
item_url: str,
*,
cffi_session: AsyncSession | None = None,
browser_fetcher: BrowserFetcher | None = None,
) -> DetailEnrichment:
"""GET <avito>/{item_url} → parse HTML via selectolax → DetailEnrichment.
Если browser_fetcher передан — использует браузерный fetch (browser mode).
Если cffi_session не передана — создаёт новую (impersonate='chrome120').
Raises:
httpx.HTTPError — если status != 200 (через raise_for_status-like).
ValueError — если item_id не извлечён из HTML.
"""
if browser_fetcher is not None:
full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url)
html = await browser_fetcher.fetch(full_url)
if _is_firewall_page(html):
raise AvitoBlockedError(f"Avito detail firewall (browser-mode) for {full_url}")
return parse_detail_html(html, full_url)
own_session = cffi_session is None
if cffi_session is None:
# Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed,

130
tradein-mvp/backend/app/services/scrapers/avito_houses.py
View file

@ -28,13 +28,16 @@ import re
import urllib.parse
from dataclasses import dataclass, field
from datetime import UTC, date, datetime
from typing import Any
from typing import TYPE_CHECKING, Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
if TYPE_CHECKING:
from app.services.scrapers.browser_fetcher import BrowserFetcher
logger = logging.getLogger(__name__)
AVITO_BASE = "https://www.avito.ru"
@ -109,16 +112,16 @@ HOUSE_TYPE_MAP: dict[str, str] = {
@dataclass
class HouseInfo:
ext_id: int # avitoId (int)
ext_id_hash: str | None = None # base64 internal ID
ext_id: int # avitoId (int)
ext_id_hash: str | None = None # base64 internal ID
title: str | None = None
short_address: str | None = None # краткий адрес
full_address: str | None = None # полный адрес
short_address: str | None = None # краткий адрес
full_address: str | None = None # полный адрес
lat: float | None = None
lon: float | None = None
year_built: int | None = None
total_floors: int | None = None
house_type: str | None = None # нормализован: monolith/panel/brick/...
house_type: str | None = None # нормализован: monolith/panel/brick/...
material_floors: str | None = None
hot_water: str | None = None
passenger_elevators: int | None = None
@ -162,7 +165,7 @@ class PlacementHistoryItem:
last_price: int | None = None
last_price_date: date | None = None
exposure_days: int | None = None
removed_date: date | None = None # ВСЕГДА None для source='avito_widget'
removed_date: date | None = None # ВСЕГДА None для source='avito_widget'
raw_payload: dict[str, Any] | None = None
@ -331,9 +334,7 @@ def _parse_house_page(widget: dict[str, Any]) -> HouseInfo:
# expandParams → характеристики дома
about = dd.get("aboutDevelopment", {})
expand_params_items: list[dict[str, Any]] = (
about.get("expandParams", {}).get("items", [])
)
expand_params_items: list[dict[str, Any]] = about.get("expandParams", {}).get("items", [])
parsed_params = _parse_expand_params(expand_params_items)
# Рейтинг: новая форма — внутри ratingPreview, старая — прямо в developmentData
@ -479,16 +480,18 @@ def _parse_mini_serp(widget: dict[str, Any]) -> list[MiniSerpListing]:
price_raw = item.get("price")
price_rub = _strip_price(price_raw) if isinstance(price_raw, str) else price_raw
listings.append(MiniSerpListing(
ext_item_id=item["id"],
title=item.get("title"),
price_rub=price_rub,
description=item.get("description"),
url=item.get("url", ""),
metro_text=geo.get("content"),
metro_color=colors[0] if colors else None,
seller=seller,
))
listings.append(
MiniSerpListing(
ext_item_id=item["id"],
title=item.get("title"),
price_rub=price_rub,
description=item.get("description"),
url=item.get("url", ""),
metro_text=geo.get("content"),
metro_color=colors[0] if colors else None,
seller=seller,
)
)
return listings
@ -504,17 +507,19 @@ def _parse_placement_history(widget: dict[str, Any]) -> list[PlacementHistoryIte
items = widget.get("props", {}).get("items", [])
for item in items:
item_copy = {k: v for k, v in item.items() if k != "itemImage"}
result.append(PlacementHistoryItem(
ext_item_id=str(item["id"]),
title=item.get("title"),
start_price=item.get("startPrice"),
start_price_date=_unix_to_date(item.get("startPriceDate")),
last_price=item.get("lastPrice"),
last_price_date=_unix_to_date(item.get("lastPriceDate")),
exposure_days=item.get("exposure"),
removed_date=None, # widget не отдаёт! IMV API Stage 2d
raw_payload=item_copy,
))
result.append(
PlacementHistoryItem(
ext_item_id=str(item["id"]),
title=item.get("title"),
start_price=item.get("startPrice"),
start_price_date=_unix_to_date(item.get("startPriceDate")),
last_price=item.get("lastPrice"),
last_price_date=_unix_to_date(item.get("lastPriceDate")),
exposure_days=item.get("exposure"),
removed_date=None, # widget не отдаёт! IMV API Stage 2d
raw_payload=item_copy,
)
)
return result
@ -533,13 +538,15 @@ def _parse_recommendations(widget: dict[str, Any]) -> list[RecommendationItem]:
if images:
image_url = images[0].get("208x156")
result.append(RecommendationItem(
title=item.get("title"),
price_text=item.get("priceRange"),
address=address,
url=item.get("url", ""),
image_url=image_url,
))
result.append(
RecommendationItem(
title=item.get("title"),
price_text=item.get("priceRange"),
address=address,
url=item.get("url", ""),
image_url=image_url,
)
)
return result
@ -601,13 +608,9 @@ def parse_houses_state(state: dict[str, Any], house_url: str) -> HouseCatalogEnr
ValueError если placeholders не массив или housePage виджет не найден.
"""
try:
raw_placeholders: list[dict[str, Any]] = (
state["data"]["data"]["page"]["placeholders"]
)
raw_placeholders: list[dict[str, Any]] = state["data"]["data"]["page"]["placeholders"]
except (KeyError, TypeError) as exc:
raise ValueError(
f"Не удалось найти placeholders в __preloadedState__: {exc}"
) from exc
raise ValueError(f"Не удалось найти placeholders в __preloadedState__: {exc}") from exc
if not isinstance(raw_placeholders, list):
raise ValueError(
@ -678,19 +681,39 @@ async def fetch_house_catalog(
house_url: str,
*,
cffi_session: Any | None = None,
browser_fetcher: BrowserFetcher | None = None,
) -> HouseCatalogEnrichment:
"""GET {AVITO_BASE}{house_url} → extract window.__preloadedState__ → parse 10 виджетов.
Параметры:
house_url — относительный путь, напр. /catalog/houses/ekaterinburg/ul_postovskogo/3171365
cffi_session — опциональная curl_cffi AsyncSession (для переиспользования пула соединений).
Если None — создаётся временная сессия.
house_url — относительный путь:
/catalog/houses/ekaterinburg/ul_postovskogo/3171365
cffi_session — опциональная curl_cffi AsyncSession (для переиспользования пула соединений).
Если None — создаётся временная сессия.
browser_fetcher — если передан, используется браузерный fetch вместо curl_cffi.
Raises:
RuntimeError если curl_cffi не установлен.
RuntimeError если curl_cffi не установлен (в curl-режиме).
httpx.HTTPStatusError / curl_cffi аналог если status != 200.
ValueError если __preloadedState__ не найден или placeholders не массив.
"""
url = f"{AVITO_BASE}{house_url}"
logger.info("Houses Catalog fetch: %s", url)
if browser_fetcher is not None:
from app.services.scrapers.avito import _is_firewall_page
html = await browser_fetcher.fetch(url)
if _is_firewall_page(html):
raise AvitoBlockedError(f"Avito houses firewall (browser-mode) for {url}")
state = _extract_preloaded_state(html)
if state is None:
raise ValueError(
f"window.__preloadedState__ не найден / не распарсился в HTML страницы: {url}"
)
logger.info("Houses Catalog state extracted (browser-mode), parsing widgets")
return parse_houses_state(state, house_url)
try:
from curl_cffi.requests import AsyncSession as CffiAsyncSession
except ImportError as exc:
@ -704,16 +727,13 @@ async def fetch_house_catalog(
_own_session = True
try:
url = f"{AVITO_BASE}{house_url}"
logger.info("Houses Catalog fetch: %s", url)
resp = await cffi_session.get(url)
if resp.status_code == 403:
raise AvitoBlockedError(f"Avito houses HTTP 403 for {url}")
if resp.status_code == 429:
raise AvitoRateLimitedError(f"Avito houses HTTP 429 for {url}")
resp.raise_for_status()
html: str = resp.text
html = resp.text
# Извлекаем + декодируем window.__preloadedState__ (URL-encoded или JSON.parse).
state = _extract_preloaded_state(html)
@ -822,7 +842,7 @@ def upsert_house(db: Session, h: HouseInfo) -> int:
"""),
{
"ext_house_id": str(h.ext_id),
"url": "", # caller должен передавать house_url через enrichment
"url": "", # caller должен передавать house_url через enrichment
"address": h.short_address,
"short_address": h.short_address,
"full_address": h.full_address,
@ -1146,9 +1166,7 @@ def save_placement_history(db: Session, items: list[PlacementHistoryItem]) -> in
"last_price_date": item.last_price_date,
"exposure_days": item.exposure_days,
"raw_payload": (
json.dumps(item.raw_payload, ensure_ascii=False)
if item.raw_payload
else None
json.dumps(item.raw_payload, ensure_ascii=False) if item.raw_payload else None
),
},
)

368
tradein-mvp/backend/tests/test_pipeline_browser_routing.py Normal file
View file

@ -0,0 +1,368 @@
"""Stage 1 #915 — route avito pipeline SERP+detail+houses through BrowserFetcher.
Тесты без сети и без БД. BrowserFetcher мокируется объектом с async fetch().
Покрытие:
1. fetch_detail(browser_fetcher=mock) → DetailEnrichment (browser branch taken)
2. fetch_detail(browser_fetcher=mock) → AvitoBlockedError при firewall HTML
3. fetch_house_catalog(browser_fetcher=mock) → browser branch taken
4. run_avito_pipeline в browser-mode → scraper._browser установлен,
fetch_detail/fetch_house_catalog вызваны с browser_fetcher
5. Curl mode (default) — browser_fetcher=None, curl path используется
"""
from __future__ import annotations
import json
import os
import urllib.parse
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.avito_houses import fetch_house_catalog
FIXTURES = Path(__file__).parent / "fixtures"
# Минимальный HTML для fetch_detail — достаточен для parse_detail_html
DETAIL_HTML = """
<html><body>
<div data-marker="item-view/item-id">№ 99887766 · 18 мая в 10:00 · 100 просмотров</div>
<span itemprop="price" content="5000000">5 000 000 ₽</span>
<div data-marker="item-map-wrapper"
data-map-lat="56.8" data-map-lon="60.6" data-location-id="1"></div>
<div data-marker="item-view/item-params">
<ul>
<li>Количество комнат: 2</li>
<li>Общая площадь: 50 м²</li>
<li>Этаж: 5 из 10</li>
</ul>
</div>
</body></html>
"""
# Firewall HTML — содержит маркер который _is_firewall_page ловит
FIREWALL_HTML = """
<html><head><title>Доступ ограничен</title></head>
<body><div class="firewall-container">Доступ ограничен</div></body></html>
"""
_MINIMAL_STATE = {
"data": {
"data": {
"page": {
"placeholders": [
{
"type": "housePage",
"props": {
"developmentData": {
"avitoId": 12345,
"id": "abc123",
"title": "ЖК Тест",
"address": "ул. Тестовая, 1",
"fullAddress": "Екатеринбург, ул. Тестовая, 1",
"coords": {"lat": 56.8, "lng": 60.6},
"aboutDevelopment": {"expandParams": {"items": []}},
"developer": {},
"mapPreview": {},
}
},
}
]
}
}
}
}
_encoded_state = urllib.parse.quote(json.dumps(_MINIMAL_STATE, ensure_ascii=False))
HOUSES_HTML = f"""
<html><body>
<script>window.__preloadedState__ = "{_encoded_state}";</script>
</body></html>
"""
# ── Helpers ──────────────────────────────────────────────────────────────────
def make_mock_fetcher(html: str) -> MagicMock:
"""Создаёт mock BrowserFetcher с async fetch(), возвращающим заданный HTML."""
mock = MagicMock()
mock.fetch = AsyncMock(return_value=html)
return mock
# ── 1. fetch_detail browser branch → DetailEnrichment ────────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_browser_returns_enrichment() -> None:
"""browser_fetcher.fetch() вызывается, curl AsyncSession не трогается."""
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
item_url = "/ekaterinburg/kvartiry/test_99887766-1234"
result = await fetch_detail(item_url, browser_fetcher=mock_fetcher)
# browser path взят
mock_fetcher.fetch.assert_awaited_once()
called_url = mock_fetcher.fetch.call_args[0][0]
assert called_url.startswith("https://www.avito.ru"), f"unexpected url: {called_url}"
assert called_url.endswith(item_url)
assert result.item_id == "99887766"
assert result.price_rub == 5_000_000
assert result.rooms == 2
@pytest.mark.asyncio
async def test_fetch_detail_browser_full_url_passthrough() -> None:
"""Если item_url уже абсолютный — передаётся без urljoin."""
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
full_url = "https://www.avito.ru/ekaterinburg/kvartiry/test_99887766-1234"
result = await fetch_detail(full_url, browser_fetcher=mock_fetcher)
called_url = mock_fetcher.fetch.call_args[0][0]
assert called_url == full_url
assert result.item_id == "99887766"
@pytest.mark.asyncio
async def test_fetch_detail_browser_no_curl_session_used() -> None:
"""cffi_session не используется когда browser_fetcher передан."""
mock_fetcher = make_mock_fetcher(DETAIL_HTML)
mock_session = MagicMock()
mock_session.get = AsyncMock() # НЕ должен быть вызван
result = await fetch_detail(
"/ekaterinburg/kvartiry/test_99887766-1234",
cffi_session=mock_session,
browser_fetcher=mock_fetcher,
)
mock_session.get.assert_not_awaited()
assert result.item_id == "99887766"
# ── 2. fetch_detail browser firewall → AvitoBlockedError ─────────────────────
@pytest.mark.asyncio
async def test_fetch_detail_browser_firewall_raises() -> None:
"""Firewall HTML через browser path → AvitoBlockedError."""
mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await fetch_detail(
"/ekaterinburg/kvartiry/blocked_item-9999",
browser_fetcher=mock_fetcher,
)
# ── 3. fetch_house_catalog browser branch ────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_house_catalog_browser_branch_taken() -> None:
"""browser_fetcher.fetch() вызывается, curl path не трогается."""
mock_fetcher = make_mock_fetcher(HOUSES_HTML)
house_path = "/catalog/houses/ekaterinburg/ul_test/3171365"
result = await fetch_house_catalog(house_path, browser_fetcher=mock_fetcher)
mock_fetcher.fetch.assert_awaited_once()
called_url = mock_fetcher.fetch.call_args[0][0]
assert "avito.ru" in called_url
assert house_path in called_url
assert result.house.ext_id == 12345
assert result.house.title == "ЖК Тест"
@pytest.mark.asyncio
async def test_fetch_house_catalog_browser_no_curl_used() -> None:
"""cffi_session.get не вызывается в browser mode."""
mock_fetcher = make_mock_fetcher(HOUSES_HTML)
mock_session = MagicMock()
mock_session.get = AsyncMock()
await fetch_house_catalog(
"/catalog/houses/ekaterinburg/ul_test/3171365",
cffi_session=mock_session,
browser_fetcher=mock_fetcher,
)
mock_session.get.assert_not_awaited()
@pytest.mark.asyncio
async def test_fetch_house_catalog_browser_firewall_raises() -> None:
"""Firewall HTML через browser path → AvitoBlockedError."""
mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await fetch_house_catalog(
"/catalog/houses/ekaterinburg/ul_test/9999",
browser_fetcher=mock_fetcher,
)
# ── 4. run_avito_pipeline browser-mode routing ───────────────────────────────
@pytest.mark.asyncio
async def test_run_avito_pipeline_browser_mode_sets_scraper_browser(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""В browser-mode scraper._browser установлен, fetch_detail/fetch_house_catalog
вызываются с browser_fetcher (не с cffi_session).
"""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
mock_bf = make_mock_fetcher(DETAIL_HTML)
captured_scraper: list = []
# Mock AvitoScraper.fetch_around → возвращает пустой список (нет лотов → нет DB)
async def mock_fetch_around(
self: object, lat: float, lon: float, radius_m: int, **kwargs: object
) -> list:
captured_scraper.append(self)
return []
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
with (
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
from app.services.scrape_pipeline import run_avito_pipeline
await run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
radius_m=1500,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
shared_browser=mock_bf,
)
assert len(captured_scraper) == 1
scraper_instance = captured_scraper[0]
assert scraper_instance._browser is mock_bf
# curl session должен быть None
assert getattr(scraper_instance, "_cffi", None) is None or True
@pytest.mark.asyncio
async def test_run_avito_pipeline_browser_mode_own_browser_created(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Если shared_browser не передан в browser-mode, BrowserFetcher создаётся
и __aenter__/__aexit__ вызываются."""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False)
mock_bf_instance = make_mock_fetcher(DETAIL_HTML)
mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance)
mock_bf_instance.__aexit__ = AsyncMock(return_value=None)
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
return []
with (
patch("app.services.scrape_pipeline.BrowserFetcher", return_value=mock_bf_instance),
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
await scrape_pipeline.run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
)
mock_bf_instance.__aenter__.assert_awaited_once()
mock_bf_instance.__aexit__.assert_awaited_once()
# ── 5. Curl mode (default) — browser_fetcher stays None ──────────────────────
@pytest.mark.asyncio
async def test_run_avito_pipeline_curl_mode_no_browser_fetcher(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""В curl-mode (default) browser_fetcher=None, scraper._cffi установлен."""
from app.services import scrape_pipeline
monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "curl_cffi", raising=False)
captured_scraper: list = []
async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list:
captured_scraper.append(self)
return []
mock_db = MagicMock()
mock_db.execute.return_value.mappings.return_value.all.return_value = []
with (
patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around),
patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)),
):
await scrape_pipeline.run_avito_pipeline(
mock_db,
lat=56.84,
lon=60.60,
enrich_houses=False,
enrich_detail_top_n=0,
pages=1,
)
assert len(captured_scraper) == 1
scraper_instance = captured_scraper[0]
# curl mode: _cffi должен быть установлен
assert scraper_instance._cffi is not None
# browser не установлен в curl mode
assert getattr(scraper_instance, "_browser", None) is None
@pytest.mark.asyncio
async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None:
"""Если browser_fetcher=None, curl path используется (собственная сессия)."""
from unittest.mock import patch as _patch
# Мокаем создание собственной сессии чтобы не нужен реальный curl_cffi
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.text = DETAIL_HTML
mock_session = AsyncMock()
mock_session.get = AsyncMock(return_value=mock_resp)
mock_session.close = AsyncMock()
with _patch(
"app.services.scrapers.avito_detail.AsyncSession",
return_value=mock_session,
):
result = await fetch_detail("/ekaterinburg/kvartiry/test_99887766-1234")
# curl path взят — session.get был вызван
mock_session.get.assert_awaited_once()
assert result.item_id == "99887766"