lekss361
69845ddcee
All checks were successful
Deploy Trade-In / changes (push) Successful in 12s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m27s
Deploy Trade-In / build-backend (push) Successful in 51s
Deploy Trade-In / deploy (push) Successful in 1m37s
140 lines
6.2 KiB
Python
140 lines
6.2 KiB
Python
"""Живой прогрев сессии + батч-на-сессию (#1551).
|
||
|
||
warm_up_session: GET Yandex SERP (best-effort) -> GET Avito EKB search-страница с
|
||
Referer=yandex сеет антибот-куки. Прогретая сессия потом держит батч detail-GET'ов
|
||
без 403 (доказано пробами на проде: 22/22 карточек подряд, 0 блоков).
|
||
|
||
fetch_detail(referer=..., reconnect_on_block=False): шлёт Referer detail-GET'а и при
|
||
403/firewall поднимает AvitoBlockedError СРАЗУ — без холодных per-card reconnect'ов
|
||
(их делает outer-loop через rebuild+rewarm прогретой сессии в avito_detail_backfill).
|
||
|
||
Стиль фикстур/моков зеркалит test_avito_detail_403_reconnect.py.
|
||
"""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.core.config import settings
|
||
from app.services.scrapers import avito_detail as detail_mod
|
||
from app.services.scrapers.avito_detail import (
|
||
fetch_detail,
|
||
research_in_session,
|
||
warm_up_session,
|
||
)
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||
|
||
|
||
def _resp(status_code: int, text: str = "") -> MagicMock:
|
||
r = MagicMock()
|
||
r.status_code = status_code
|
||
r.text = text
|
||
return r
|
||
|
||
|
||
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
|
||
|
||
|
||
# ── warm_up_session ───────────────────────────────────────────────────────────
|
||
@pytest.mark.asyncio
|
||
async def test_warm_up_session_search_with_yandex_referer_true_on_200() -> None:
|
||
"""warm_up_session шлёт GET на _AVITO_WARM_SEARCH_URL с Referer=yandex и
|
||
возвращает True на 200/non-firewall."""
|
||
session = AsyncMock()
|
||
session.get = AsyncMock(return_value=_resp(200, "<html>ok</html>"))
|
||
|
||
with (
|
||
patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False),
|
||
):
|
||
ok = await warm_up_session(session)
|
||
|
||
assert ok is True
|
||
# search-GET (на avito-search URL) ушёл ровно один раз и с Referer=yandex.
|
||
search_calls = [
|
||
c
|
||
for c in session.get.await_args_list
|
||
if c.args and c.args[0] == detail_mod._AVITO_WARM_SEARCH_URL
|
||
]
|
||
assert len(search_calls) == 1
|
||
assert search_calls[0].kwargs["headers"]["Referer"] == detail_mod._AVITO_WARM_YANDEX_REFERER
|
||
# Yandex SERP — органический referer-источник — тоже фетчился (best-effort, до search).
|
||
assert any(
|
||
c.args and c.args[0] == detail_mod._AVITO_WARM_YANDEX_URL
|
||
for c in session.get.await_args_list
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_warm_up_session_false_on_403() -> None:
|
||
"""search-страница вернула 403 → warm_up_session возвращает False."""
|
||
session = AsyncMock()
|
||
session.get = AsyncMock(return_value=_resp(403))
|
||
|
||
with (
|
||
patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False),
|
||
):
|
||
ok = await warm_up_session(session)
|
||
|
||
assert ok is False
|
||
|
||
|
||
# ── fetch_detail referer + reconnect_on_block=False ───────────────────────────
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_referer_forwarded_and_block_raises_immediately(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""fetch_detail(referer="X", reconnect_on_block=False): Referer="X" уходит в
|
||
detail-GET, а 403 поднимает AvitoBlockedError СРАЗУ — без холодного reconnect
|
||
(даже когда backconnect-proxy включён → гейт = reconnect_on_block, не наличие proxy).
|
||
"""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
caller = AsyncMock()
|
||
caller.get = AsyncMock(return_value=_resp(403))
|
||
build = MagicMock() # _build_detail_session НЕ должна дёргаться (нет reconnect'а)
|
||
|
||
with patch.object(detail_mod, "_build_detail_session", build):
|
||
with pytest.raises(AvitoBlockedError, match="HTTP 403"):
|
||
await fetch_detail(
|
||
"/ekaterinburg/kvartiry/test-123",
|
||
cffi_session=caller,
|
||
referer="X",
|
||
reconnect_on_block=False,
|
||
)
|
||
|
||
build.assert_not_called() # никаких холодных reconnect'ов
|
||
assert caller.get.await_count == 1 # один detail-GET → сразу raise
|
||
assert caller.get.await_args.kwargs["headers"] == {"Referer": "X"}
|
||
|
||
|
||
# ── research_in_session (in-session перепоиск, #1551 follow-up) ────────────────
|
||
@pytest.mark.asyncio
|
||
async def test_research_in_session_re_get_search_with_yandex_referer_true_on_200() -> None:
|
||
"""research_in_session re-GET'ит avito-search на ТОЙ ЖЕ сессии с Referer=yandex
|
||
и возвращает True на 200/non-firewall (свежесть куки без смены exit-IP)."""
|
||
session = AsyncMock()
|
||
session.get = AsyncMock(return_value=_resp(200, "<html>ok</html>"))
|
||
|
||
with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False):
|
||
ok = await research_in_session(session)
|
||
|
||
assert ok is True
|
||
assert session.get.await_count == 1 # ровно один re-GET, без yandex-фазы (in-session)
|
||
assert session.get.await_args.args[0] == detail_mod._AVITO_WARM_SEARCH_URL
|
||
sent_referer = session.get.await_args.kwargs["headers"]["Referer"]
|
||
assert sent_referer == detail_mod._AVITO_WARM_YANDEX_REFERER
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_research_in_session_false_on_403() -> None:
|
||
"""search вернул 403 → research_in_session возвращает False (best-effort, не fatal)."""
|
||
session = AsyncMock()
|
||
session.get = AsyncMock(return_value=_resp(403))
|
||
|
||
with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False):
|
||
ok = await research_in_session(session)
|
||
|
||
assert ok is False
|