diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index 66558735..f968d998 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -30,6 +30,7 @@ from __future__ import annotations import asyncio import logging import random +from contextlib import AsyncExitStack from dataclasses import dataclass, field, fields from urllib.parse import urlparse @@ -44,6 +45,7 @@ from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichm from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError from app.services.scrapers.avito_houses import fetch_house_catalog, save_house_catalog_enrichment from app.services.scrapers.base import ScrapedLot, save_listings +from app.services.scrapers.browser_fetcher import BrowserFetcher from app.services.scrapers.n1 import N1Scraper from app.services.scrapers.yandex_realty import YandexRealtyScraper @@ -137,6 +139,7 @@ async def run_avito_pipeline( pages: int = 1, request_delay_sec: float | None = None, shared_session: AsyncSession | None = None, + shared_browser: BrowserFetcher | None = None, ) -> PipelineResult: """Full Avito search → houses → detail enrichment pipeline. @@ -148,24 +151,39 @@ async def run_avito_pipeline( 5. ENRICH_DETAIL: top-N listings → fetch_detail + save (abort on 3 blocks) shared_session — если передан, используется без закрытия (lifecycle у вызывающего). + shared_browser — если передан в browser-mode, используется без закрытия. AvitoBlockedError / AvitoRateLimitedError propagate наружу. """ counters = PipelineCounters() lots: list[ScrapedLot] = [] detail_delay = request_delay_sec if request_delay_sec is not None else 7.0 - own_session = shared_session is None - session = shared_session or AsyncSession( - impersonate="chrome120", - timeout=25, - headers=_CHROME_HEADERS, - proxies=_avito_proxies(), - ) + browser_mode = settings.scraper_fetch_mode == "browser" + session: AsyncSession | None = None + browser_fetcher: BrowserFetcher | None = None + own_session = False + own_browser = False + + scraper = AvitoScraper() + if browser_mode: + browser_fetcher = shared_browser + if browser_fetcher is None: + browser_fetcher = BrowserFetcher() + await browser_fetcher.__aenter__() + own_browser = True + scraper._browser = browser_fetcher + else: + own_session = shared_session is None + session = shared_session or AsyncSession( + impersonate="chrome120", + timeout=25, + headers=_CHROME_HEADERS, + proxies=_avito_proxies(), + ) + scraper._cffi = session try: # ── Step 1: search ────────────────────────────────────── - scraper = AvitoScraper() - scraper._cffi = session try: lots = await scraper.fetch_around( lat, @@ -218,7 +236,9 @@ async def run_avito_pipeline( house_paths_list = list(unique_house_paths) for idx, house_path in enumerate(house_paths_list): try: - enrichment = await fetch_house_catalog(house_path, cffi_session=session) + enrichment = await fetch_house_catalog( + house_path, cffi_session=session, browser_fetcher=browser_fetcher + ) house_counters = save_house_catalog_enrichment(db, enrichment) counters.houses_enriched += 1 hid = house_counters.get("house_id") @@ -282,7 +302,9 @@ async def run_avito_pipeline( item_url = ( urlparse(source_url).path if source_url.startswith("http") else source_url ) - enrichment_detail = await fetch_detail(item_url, cffi_session=session) + enrichment_detail = await fetch_detail( + item_url, cffi_session=session, browser_fetcher=browser_fetcher + ) if save_detail_enrichment(db, enrichment_detail): counters.detail_enriched += 1 # #871: house-link есть в detail SSR (в SERP-карточке — JS-only). @@ -332,7 +354,9 @@ async def run_avito_pipeline( nh_list = list(new_house_paths) for h_idx, house_path in enumerate(nh_list): try: - enrichment = await fetch_house_catalog(house_path, cffi_session=session) + enrichment = await fetch_house_catalog( + house_path, cffi_session=session, browser_fetcher=browser_fetcher + ) house_counters = save_house_catalog_enrichment(db, enrichment) counters.houses_enriched += 1 hid = house_counters.get("house_id") @@ -378,8 +402,10 @@ async def run_avito_pipeline( ) finally: - if own_session: + if own_session and session is not None: await session.close() + if own_browser and browser_fetcher is not None: + await browser_fetcher.__aexit__(None, None, None) @dataclass @@ -432,12 +458,21 @@ async def run_avito_city_sweep( counters = CitySweepCounters(anchors_total=len(_anchors)) all_touched_house_ids: set[int] = set() - async with AsyncSession( - impersonate="chrome120", - timeout=25, - headers=_CHROME_HEADERS, - proxies=_avito_proxies(), - ) as session: + browser_mode = settings.scraper_fetch_mode == "browser" + async with AsyncExitStack() as stack: + session: AsyncSession | None = None + shared_bf: BrowserFetcher | None = None + if browser_mode: + shared_bf = await stack.enter_async_context(BrowserFetcher()) + else: + session = await stack.enter_async_context( + AsyncSession( + impersonate="chrome120", + timeout=25, + headers=_CHROME_HEADERS, + proxies=_avito_proxies(), + ) + ) try: for idx, (lat, lon, name) in enumerate(_anchors, start=1): if scrape_runs.is_cancelled(db, run_id): @@ -472,6 +507,7 @@ async def run_avito_city_sweep( pages=pages_per_anchor, request_delay_sec=request_delay_sec, shared_session=session, + shared_browser=shared_bf, ), timeout=ANCHOR_TIMEOUT_SEC, ) diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index 55a592c4..35a7f52c 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -24,7 +24,7 @@ import logging import re from dataclasses import dataclass, field from datetime import date -from typing import Any +from typing import TYPE_CHECKING, Any from urllib.parse import urljoin, urlparse from curl_cffi.requests import AsyncSession @@ -32,10 +32,13 @@ from selectolax.parser import HTMLParser, Node from sqlalchemy import text from sqlalchemy.orm import Session -from app.services.scrapers.avito import _clean_address +from app.services.scrapers.avito import _clean_address, _is_firewall_page from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text +if TYPE_CHECKING: + from app.services.scrapers.browser_fetcher import BrowserFetcher + logger = logging.getLogger(__name__) AVITO_BASE = "https://www.avito.ru" @@ -229,14 +232,23 @@ async def fetch_detail( item_url: str, *, cffi_session: AsyncSession | None = None, + browser_fetcher: BrowserFetcher | None = None, ) -> DetailEnrichment: """GET /{item_url} → parse HTML via selectolax → DetailEnrichment. + Если browser_fetcher передан — использует браузерный fetch (browser mode). Если cffi_session не передана — создаёт новую (impersonate='chrome120'). Raises: httpx.HTTPError — если status != 200 (через raise_for_status-like). ValueError — если item_id не извлечён из HTML. """ + if browser_fetcher is not None: + full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url) + html = await browser_fetcher.fetch(full_url) + if _is_firewall_page(html): + raise AvitoBlockedError(f"Avito detail firewall (browser-mode) for {full_url}") + return parse_detail_html(html, full_url) + own_session = cffi_session is None if cffi_session is None: # Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed, diff --git a/tradein-mvp/backend/app/services/scrapers/avito_houses.py b/tradein-mvp/backend/app/services/scrapers/avito_houses.py index 45b8b11d..c443b810 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_houses.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_houses.py @@ -28,13 +28,17 @@ import re import urllib.parse from dataclasses import dataclass, field from datetime import UTC, date, datetime -from typing import Any +from typing import TYPE_CHECKING, Any from sqlalchemy import text from sqlalchemy.orm import Session +from app.services.scrapers.avito import _is_firewall_page from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError +if TYPE_CHECKING: + from app.services.scrapers.browser_fetcher import BrowserFetcher + logger = logging.getLogger(__name__) AVITO_BASE = "https://www.avito.ru" @@ -109,16 +113,16 @@ HOUSE_TYPE_MAP: dict[str, str] = { @dataclass class HouseInfo: - ext_id: int # avitoId (int) - ext_id_hash: str | None = None # base64 internal ID + ext_id: int # avitoId (int) + ext_id_hash: str | None = None # base64 internal ID title: str | None = None - short_address: str | None = None # краткий адрес - full_address: str | None = None # полный адрес + short_address: str | None = None # краткий адрес + full_address: str | None = None # полный адрес lat: float | None = None lon: float | None = None year_built: int | None = None total_floors: int | None = None - house_type: str | None = None # нормализован: monolith/panel/brick/... + house_type: str | None = None # нормализован: monolith/panel/brick/... material_floors: str | None = None hot_water: str | None = None passenger_elevators: int | None = None @@ -162,7 +166,7 @@ class PlacementHistoryItem: last_price: int | None = None last_price_date: date | None = None exposure_days: int | None = None - removed_date: date | None = None # ВСЕГДА None для source='avito_widget' + removed_date: date | None = None # ВСЕГДА None для source='avito_widget' raw_payload: dict[str, Any] | None = None @@ -331,9 +335,7 @@ def _parse_house_page(widget: dict[str, Any]) -> HouseInfo: # expandParams → характеристики дома about = dd.get("aboutDevelopment", {}) - expand_params_items: list[dict[str, Any]] = ( - about.get("expandParams", {}).get("items", []) - ) + expand_params_items: list[dict[str, Any]] = about.get("expandParams", {}).get("items", []) parsed_params = _parse_expand_params(expand_params_items) # Рейтинг: новая форма — внутри ratingPreview, старая — прямо в developmentData @@ -479,16 +481,18 @@ def _parse_mini_serp(widget: dict[str, Any]) -> list[MiniSerpListing]: price_raw = item.get("price") price_rub = _strip_price(price_raw) if isinstance(price_raw, str) else price_raw - listings.append(MiniSerpListing( - ext_item_id=item["id"], - title=item.get("title"), - price_rub=price_rub, - description=item.get("description"), - url=item.get("url", ""), - metro_text=geo.get("content"), - metro_color=colors[0] if colors else None, - seller=seller, - )) + listings.append( + MiniSerpListing( + ext_item_id=item["id"], + title=item.get("title"), + price_rub=price_rub, + description=item.get("description"), + url=item.get("url", ""), + metro_text=geo.get("content"), + metro_color=colors[0] if colors else None, + seller=seller, + ) + ) return listings @@ -504,17 +508,19 @@ def _parse_placement_history(widget: dict[str, Any]) -> list[PlacementHistoryIte items = widget.get("props", {}).get("items", []) for item in items: item_copy = {k: v for k, v in item.items() if k != "itemImage"} - result.append(PlacementHistoryItem( - ext_item_id=str(item["id"]), - title=item.get("title"), - start_price=item.get("startPrice"), - start_price_date=_unix_to_date(item.get("startPriceDate")), - last_price=item.get("lastPrice"), - last_price_date=_unix_to_date(item.get("lastPriceDate")), - exposure_days=item.get("exposure"), - removed_date=None, # widget не отдаёт! IMV API Stage 2d - raw_payload=item_copy, - )) + result.append( + PlacementHistoryItem( + ext_item_id=str(item["id"]), + title=item.get("title"), + start_price=item.get("startPrice"), + start_price_date=_unix_to_date(item.get("startPriceDate")), + last_price=item.get("lastPrice"), + last_price_date=_unix_to_date(item.get("lastPriceDate")), + exposure_days=item.get("exposure"), + removed_date=None, # widget не отдаёт! IMV API Stage 2d + raw_payload=item_copy, + ) + ) return result @@ -533,13 +539,15 @@ def _parse_recommendations(widget: dict[str, Any]) -> list[RecommendationItem]: if images: image_url = images[0].get("208x156") - result.append(RecommendationItem( - title=item.get("title"), - price_text=item.get("priceRange"), - address=address, - url=item.get("url", ""), - image_url=image_url, - )) + result.append( + RecommendationItem( + title=item.get("title"), + price_text=item.get("priceRange"), + address=address, + url=item.get("url", ""), + image_url=image_url, + ) + ) return result @@ -601,13 +609,9 @@ def parse_houses_state(state: dict[str, Any], house_url: str) -> HouseCatalogEnr ValueError если placeholders не массив или housePage виджет не найден. """ try: - raw_placeholders: list[dict[str, Any]] = ( - state["data"]["data"]["page"]["placeholders"] - ) + raw_placeholders: list[dict[str, Any]] = state["data"]["data"]["page"]["placeholders"] except (KeyError, TypeError) as exc: - raise ValueError( - f"Не удалось найти placeholders в __preloadedState__: {exc}" - ) from exc + raise ValueError(f"Не удалось найти placeholders в __preloadedState__: {exc}") from exc if not isinstance(raw_placeholders, list): raise ValueError( @@ -678,19 +682,37 @@ async def fetch_house_catalog( house_url: str, *, cffi_session: Any | None = None, + browser_fetcher: BrowserFetcher | None = None, ) -> HouseCatalogEnrichment: """GET {AVITO_BASE}{house_url} → extract window.__preloadedState__ → parse 10 виджетов. Параметры: - house_url — относительный путь, напр. /catalog/houses/ekaterinburg/ul_postovskogo/3171365 - cffi_session — опциональная curl_cffi AsyncSession (для переиспользования пула соединений). - Если None — создаётся временная сессия. + house_url — относительный путь: + /catalog/houses/ekaterinburg/ul_postovskogo/3171365 + cffi_session — опциональная curl_cffi AsyncSession (для переиспользования пула соединений). + Если None — создаётся временная сессия. + browser_fetcher — если передан, используется браузерный fetch вместо curl_cffi. Raises: - RuntimeError если curl_cffi не установлен. + RuntimeError если curl_cffi не установлен (в curl-режиме). httpx.HTTPStatusError / curl_cffi аналог если status != 200. ValueError если __preloadedState__ не найден или placeholders не массив. """ + url = f"{AVITO_BASE}{house_url}" + logger.info("Houses Catalog fetch: %s", url) + + if browser_fetcher is not None: + html = await browser_fetcher.fetch(url) + if _is_firewall_page(html): + raise AvitoBlockedError(f"Avito houses firewall (browser-mode) for {url}") + state = _extract_preloaded_state(html) + if state is None: + raise ValueError( + f"window.__preloadedState__ не найден / не распарсился в HTML страницы: {url}" + ) + logger.info("Houses Catalog state extracted (browser-mode), parsing widgets") + return parse_houses_state(state, house_url) + try: from curl_cffi.requests import AsyncSession as CffiAsyncSession except ImportError as exc: @@ -704,16 +726,13 @@ async def fetch_house_catalog( _own_session = True try: - url = f"{AVITO_BASE}{house_url}" - logger.info("Houses Catalog fetch: %s", url) - resp = await cffi_session.get(url) if resp.status_code == 403: raise AvitoBlockedError(f"Avito houses HTTP 403 for {url}") if resp.status_code == 429: raise AvitoRateLimitedError(f"Avito houses HTTP 429 for {url}") resp.raise_for_status() - html: str = resp.text + html = resp.text # Извлекаем + декодируем window.__preloadedState__ (URL-encoded или JSON.parse). state = _extract_preloaded_state(html) @@ -822,7 +841,7 @@ def upsert_house(db: Session, h: HouseInfo) -> int: """), { "ext_house_id": str(h.ext_id), - "url": "", # caller должен передавать house_url через enrichment + "url": "", # caller должен передавать house_url через enrichment "address": h.short_address, "short_address": h.short_address, "full_address": h.full_address, @@ -1146,9 +1165,7 @@ def save_placement_history(db: Session, items: list[PlacementHistoryItem]) -> in "last_price_date": item.last_price_date, "exposure_days": item.exposure_days, "raw_payload": ( - json.dumps(item.raw_payload, ensure_ascii=False) - if item.raw_payload - else None + json.dumps(item.raw_payload, ensure_ascii=False) if item.raw_payload else None ), }, ) diff --git a/tradein-mvp/backend/tests/test_pipeline_browser_routing.py b/tradein-mvp/backend/tests/test_pipeline_browser_routing.py new file mode 100644 index 00000000..3777a290 --- /dev/null +++ b/tradein-mvp/backend/tests/test_pipeline_browser_routing.py @@ -0,0 +1,393 @@ +"""Stage 1 #915 — route avito pipeline SERP+detail+houses through BrowserFetcher. + +Тесты без сети и без БД. BrowserFetcher мокируется объектом с async fetch(). + +Покрытие: + 1. fetch_detail(browser_fetcher=mock) → DetailEnrichment (browser branch taken) + 2. fetch_detail(browser_fetcher=mock) → AvitoBlockedError при firewall HTML + 3. fetch_house_catalog(browser_fetcher=mock) → browser branch taken + 4. run_avito_pipeline в browser-mode → scraper._browser установлен, + fetch_detail/fetch_house_catalog вызваны с browser_fetcher + 5. Curl mode (default) — browser_fetcher=None, curl path используется +""" + +from __future__ import annotations + +import json +import os +import urllib.parse +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from app.services.scrapers.avito_detail import fetch_detail +from app.services.scrapers.avito_exceptions import AvitoBlockedError +from app.services.scrapers.avito_houses import fetch_house_catalog + +FIXTURES = Path(__file__).parent / "fixtures" + +# Минимальный HTML для fetch_detail — достаточен для parse_detail_html +DETAIL_HTML = """ + +
№ 99887766 · 18 мая в 10:00 · 100 просмотров
+5 000 000 ₽ +
+
+ +
+ +""" + +# Firewall HTML — содержит маркер который _is_firewall_page ловит +FIREWALL_HTML = """ +Доступ ограничен +
Доступ ограничен
+""" + +_MINIMAL_STATE = { + "data": { + "data": { + "page": { + "placeholders": [ + { + "type": "housePage", + "props": { + "developmentData": { + "avitoId": 12345, + "id": "abc123", + "title": "ЖК Тест", + "address": "ул. Тестовая, 1", + "fullAddress": "Екатеринбург, ул. Тестовая, 1", + "coords": {"lat": 56.8, "lng": 60.6}, + "aboutDevelopment": {"expandParams": {"items": []}}, + "developer": {}, + "mapPreview": {}, + } + }, + } + ] + } + } + } +} + +_encoded_state = urllib.parse.quote(json.dumps(_MINIMAL_STATE, ensure_ascii=False)) +HOUSES_HTML = f""" + + + +""" + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def make_mock_fetcher(html: str) -> MagicMock: + """Создаёт mock BrowserFetcher с async fetch(), возвращающим заданный HTML.""" + mock = MagicMock() + mock.fetch = AsyncMock(return_value=html) + return mock + + +# ── 1. fetch_detail browser branch → DetailEnrichment ──────────────────────── + + +@pytest.mark.asyncio +async def test_fetch_detail_browser_returns_enrichment() -> None: + """browser_fetcher.fetch() вызывается, curl AsyncSession не трогается.""" + mock_fetcher = make_mock_fetcher(DETAIL_HTML) + item_url = "/ekaterinburg/kvartiry/test_99887766-1234" + + result = await fetch_detail(item_url, browser_fetcher=mock_fetcher) + + # browser path взят + mock_fetcher.fetch.assert_awaited_once() + called_url = mock_fetcher.fetch.call_args[0][0] + assert called_url.startswith("https://www.avito.ru"), f"unexpected url: {called_url}" + assert called_url.endswith(item_url) + + assert result.item_id == "99887766" + assert result.price_rub == 5_000_000 + assert result.rooms == 2 + + +@pytest.mark.asyncio +async def test_fetch_detail_browser_full_url_passthrough() -> None: + """Если item_url уже абсолютный — передаётся без urljoin.""" + mock_fetcher = make_mock_fetcher(DETAIL_HTML) + full_url = "https://www.avito.ru/ekaterinburg/kvartiry/test_99887766-1234" + + result = await fetch_detail(full_url, browser_fetcher=mock_fetcher) + + called_url = mock_fetcher.fetch.call_args[0][0] + assert called_url == full_url + assert result.item_id == "99887766" + + +@pytest.mark.asyncio +async def test_fetch_detail_browser_no_curl_session_used() -> None: + """cffi_session не используется когда browser_fetcher передан.""" + mock_fetcher = make_mock_fetcher(DETAIL_HTML) + mock_session = MagicMock() + mock_session.get = AsyncMock() # НЕ должен быть вызван + + result = await fetch_detail( + "/ekaterinburg/kvartiry/test_99887766-1234", + cffi_session=mock_session, + browser_fetcher=mock_fetcher, + ) + + mock_session.get.assert_not_awaited() + assert result.item_id == "99887766" + + +# ── 2. fetch_detail browser firewall → AvitoBlockedError ───────────────────── + + +@pytest.mark.asyncio +async def test_fetch_detail_browser_firewall_raises() -> None: + """Firewall HTML через browser path → AvitoBlockedError.""" + mock_fetcher = make_mock_fetcher(FIREWALL_HTML) + + with pytest.raises(AvitoBlockedError, match="browser-mode"): + await fetch_detail( + "/ekaterinburg/kvartiry/blocked_item-9999", + browser_fetcher=mock_fetcher, + ) + + +# ── 3. fetch_house_catalog browser branch ──────────────────────────────────── + + +@pytest.mark.asyncio +async def test_fetch_house_catalog_browser_branch_taken() -> None: + """browser_fetcher.fetch() вызывается, curl path не трогается.""" + mock_fetcher = make_mock_fetcher(HOUSES_HTML) + house_path = "/catalog/houses/ekaterinburg/ul_test/3171365" + + result = await fetch_house_catalog(house_path, browser_fetcher=mock_fetcher) + + mock_fetcher.fetch.assert_awaited_once() + called_url = mock_fetcher.fetch.call_args[0][0] + assert "avito.ru" in called_url + assert house_path in called_url + + assert result.house.ext_id == 12345 + assert result.house.title == "ЖК Тест" + + +@pytest.mark.asyncio +async def test_fetch_house_catalog_browser_no_curl_used() -> None: + """cffi_session.get не вызывается в browser mode.""" + mock_fetcher = make_mock_fetcher(HOUSES_HTML) + mock_session = MagicMock() + mock_session.get = AsyncMock() + + await fetch_house_catalog( + "/catalog/houses/ekaterinburg/ul_test/3171365", + cffi_session=mock_session, + browser_fetcher=mock_fetcher, + ) + + mock_session.get.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_fetch_house_catalog_browser_firewall_raises() -> None: + """Firewall HTML через browser path → AvitoBlockedError.""" + mock_fetcher = make_mock_fetcher(FIREWALL_HTML) + + with pytest.raises(AvitoBlockedError, match="browser-mode"): + await fetch_house_catalog( + "/catalog/houses/ekaterinburg/ul_test/9999", + browser_fetcher=mock_fetcher, + ) + + +# ── 4. run_avito_pipeline browser-mode routing ─────────────────────────────── + + +@pytest.mark.asyncio +async def test_run_avito_pipeline_browser_mode_sets_scraper_browser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """В browser-mode scraper._browser установлен, fetch_detail/fetch_house_catalog + вызываются с browser_fetcher (не с cffi_session). + """ + from app.services import scrape_pipeline + + monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False) + + mock_bf = make_mock_fetcher(DETAIL_HTML) + captured_scraper: list = [] + + # Mock AvitoScraper.fetch_around → возвращает пустой список (нет лотов → нет DB) + async def mock_fetch_around( + self: object, lat: float, lon: float, radius_m: int, **kwargs: object + ) -> list: + captured_scraper.append(self) + return [] + + mock_db = MagicMock() + mock_db.execute.return_value.mappings.return_value.all.return_value = [] + + with ( + patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around), + patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)), + ): + from app.services.scrape_pipeline import run_avito_pipeline + + await run_avito_pipeline( + mock_db, + lat=56.84, + lon=60.60, + radius_m=1500, + enrich_houses=False, + enrich_detail_top_n=0, + pages=1, + shared_browser=mock_bf, + ) + + assert len(captured_scraper) == 1 + scraper_instance = captured_scraper[0] + assert scraper_instance._browser is mock_bf + # curl session должен быть None — в browser-mode pipeline не создаёт _cffi сессию + assert scraper_instance._cffi is None + + +@pytest.mark.asyncio +async def test_run_avito_pipeline_browser_mode_own_browser_created( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Если shared_browser не передан в browser-mode, BrowserFetcher создаётся + и __aenter__/__aexit__ вызываются.""" + from app.services import scrape_pipeline + + monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "browser", raising=False) + + mock_bf_instance = make_mock_fetcher(DETAIL_HTML) + mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance) + mock_bf_instance.__aexit__ = AsyncMock(return_value=None) + + mock_db = MagicMock() + mock_db.execute.return_value.mappings.return_value.all.return_value = [] + + async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list: + return [] + + with ( + patch("app.services.scrape_pipeline.BrowserFetcher", return_value=mock_bf_instance), + patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around), + patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)), + ): + await scrape_pipeline.run_avito_pipeline( + mock_db, + lat=56.84, + lon=60.60, + enrich_houses=False, + enrich_detail_top_n=0, + pages=1, + ) + + mock_bf_instance.__aenter__.assert_awaited_once() + mock_bf_instance.__aexit__.assert_awaited_once() + + +# ── 5. Curl mode (default) — browser_fetcher stays None ────────────────────── + + +@pytest.mark.asyncio +async def test_run_avito_pipeline_curl_mode_no_browser_fetcher( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """В curl-mode (default) browser_fetcher=None, scraper._cffi установлен.""" + from app.services import scrape_pipeline + + monkeypatch.setattr(scrape_pipeline.settings, "scraper_fetch_mode", "curl_cffi", raising=False) + + captured_scraper: list = [] + + async def mock_fetch_around(self: object, *args: object, **kwargs: object) -> list: + captured_scraper.append(self) + return [] + + mock_db = MagicMock() + mock_db.execute.return_value.mappings.return_value.all.return_value = [] + + with ( + patch("app.services.scrape_pipeline.AvitoScraper.fetch_around", mock_fetch_around), + patch("app.services.scrape_pipeline.save_listings", return_value=(0, 0)), + ): + await scrape_pipeline.run_avito_pipeline( + mock_db, + lat=56.84, + lon=60.60, + enrich_houses=False, + enrich_detail_top_n=0, + pages=1, + ) + + assert len(captured_scraper) == 1 + scraper_instance = captured_scraper[0] + # curl mode: _cffi должен быть установлен + assert scraper_instance._cffi is not None + # browser не установлен в curl mode + assert getattr(scraper_instance, "_browser", None) is None + + +@pytest.mark.asyncio +async def test_fetch_detail_curl_mode_no_browser_fetcher() -> None: + """Если browser_fetcher=None, curl path используется (собственная сессия).""" + from unittest.mock import patch as _patch + + # Мокаем создание собственной сессии чтобы не нужен реальный curl_cffi + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.text = DETAIL_HTML + + mock_session = AsyncMock() + mock_session.get = AsyncMock(return_value=mock_resp) + mock_session.close = AsyncMock() + + with _patch( + "app.services.scrapers.avito_detail.AsyncSession", + return_value=mock_session, + ): + result = await fetch_detail("/ekaterinburg/kvartiry/test_99887766-1234") + + # curl path взят — session.get был вызван + mock_session.get.assert_awaited_once() + assert result.item_id == "99887766" + + +# ── 6. SERP routing: real _fetch_serp_html routes through browser ──────────── + + +@pytest.mark.asyncio +async def test_fetch_serp_html_routes_through_browser() -> None: + """With _browser set and _cffi=None, the REAL _fetch_serp_html returns the + browser HTML and never dereferences _cffi (no AssertionError). Proves #915 + SERP routing on top of #901's browser branch.""" + from unittest.mock import AsyncMock + + from app.services.scrapers.avito import AvitoScraper + + scraper = AvitoScraper() + assert scraper._cffi is None + scraper._browser = AsyncMock() + scraper._browser.fetch = AsyncMock(return_value="real serp listing") + + html = await scraper._fetch_serp_html( + "https://www.avito.ru/ekaterinburg/kvartiry/prodam", page=1 + ) + + assert html == "real serp listing" + scraper._browser.fetch.assert_awaited_once()