"""Живой прогрев сессии + батч-на-сессию (#1551). warm_up_session: GET Yandex SERP (best-effort) -> GET Avito EKB search-страница с Referer=yandex сеет антибот-куки. Прогретая сессия потом держит батч detail-GET'ов без 403 (доказано пробами на проде: 22/22 карточек подряд, 0 блоков). fetch_detail(referer=..., reconnect_on_block=False): шлёт Referer detail-GET'а и при 403/firewall поднимает AvitoBlockedError СРАЗУ — без холодных per-card reconnect'ов (их делает outer-loop через rebuild+rewarm прогретой сессии в avito_detail_backfill). Стиль фикстур/моков зеркалит test_avito_detail_403_reconnect.py. """ from unittest.mock import AsyncMock, MagicMock, patch import pytest from app.core.config import settings from app.services.scrapers import avito_detail as detail_mod from app.services.scrapers.avito_detail import ( fetch_detail, research_in_session, warm_up_session, ) from app.services.scrapers.avito_exceptions import AvitoBlockedError def _resp(status_code: int, text: str = "") -> MagicMock: r = MagicMock() r.status_code = status_code r.text = text return r def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619") # ── warm_up_session ─────────────────────────────────────────────────────────── @pytest.mark.asyncio async def test_warm_up_session_search_with_yandex_referer_true_on_200() -> None: """warm_up_session шлёт GET на _AVITO_WARM_SEARCH_URL с Referer=yandex и возвращает True на 200/non-firewall.""" session = AsyncMock() session.get = AsyncMock(return_value=_resp(200, "ok")) with ( patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()), patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False), ): ok = await warm_up_session(session) assert ok is True # search-GET (на avito-search URL) ушёл ровно один раз и с Referer=yandex. search_calls = [ c for c in session.get.await_args_list if c.args and c.args[0] == detail_mod._AVITO_WARM_SEARCH_URL ] assert len(search_calls) == 1 assert search_calls[0].kwargs["headers"]["Referer"] == detail_mod._AVITO_WARM_YANDEX_REFERER # Yandex SERP — органический referer-источник — тоже фетчился (best-effort, до search). assert any( c.args and c.args[0] == detail_mod._AVITO_WARM_YANDEX_URL for c in session.get.await_args_list ) @pytest.mark.asyncio async def test_warm_up_session_false_on_403() -> None: """search-страница вернула 403 → warm_up_session возвращает False.""" session = AsyncMock() session.get = AsyncMock(return_value=_resp(403)) with ( patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()), patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False), ): ok = await warm_up_session(session) assert ok is False # ── fetch_detail referer + reconnect_on_block=False ─────────────────────────── @pytest.mark.asyncio async def test_fetch_detail_referer_forwarded_and_block_raises_immediately( monkeypatch: pytest.MonkeyPatch, ) -> None: """fetch_detail(referer="X", reconnect_on_block=False): Referer="X" уходит в detail-GET, а 403 поднимает AvitoBlockedError СРАЗУ — без холодного reconnect (даже когда backconnect-proxy включён → гейт = reconnect_on_block, не наличие proxy). """ _enable_backconnect(monkeypatch) caller = AsyncMock() caller.get = AsyncMock(return_value=_resp(403)) build = MagicMock() # _build_detail_session НЕ должна дёргаться (нет reconnect'а) with patch.object(detail_mod, "_build_detail_session", build): with pytest.raises(AvitoBlockedError, match="HTTP 403"): await fetch_detail( "/ekaterinburg/kvartiry/test-123", cffi_session=caller, referer="X", reconnect_on_block=False, ) build.assert_not_called() # никаких холодных reconnect'ов assert caller.get.await_count == 1 # один detail-GET → сразу raise assert caller.get.await_args.kwargs["headers"] == {"Referer": "X"} # ── research_in_session (in-session перепоиск, #1551 follow-up) ──────────────── @pytest.mark.asyncio async def test_research_in_session_re_get_search_with_yandex_referer_true_on_200() -> None: """research_in_session re-GET'ит avito-search на ТОЙ ЖЕ сессии с Referer=yandex и возвращает True на 200/non-firewall (свежесть куки без смены exit-IP).""" session = AsyncMock() session.get = AsyncMock(return_value=_resp(200, "ok")) with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False): ok = await research_in_session(session) assert ok is True assert session.get.await_count == 1 # ровно один re-GET, без yandex-фазы (in-session) assert session.get.await_args.args[0] == detail_mod._AVITO_WARM_SEARCH_URL sent_referer = session.get.await_args.kwargs["headers"]["Referer"] assert sent_referer == detail_mod._AVITO_WARM_YANDEX_REFERER @pytest.mark.asyncio async def test_research_in_session_false_on_403() -> None: """search вернул 403 → research_in_session возвращает False (best-effort, не fatal).""" session = AsyncMock() session.get = AsyncMock(return_value=_resp(403)) with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False): ok = await research_in_session(session) assert ok is False