bot-backend
757f6486f5
Два связанных фикса из диагностики 2026-06-27: ФИКС A (_rotate_proxy_ip): вместо одношотного GET с timeout=30s — retry-loop (proxy_rotate_attempts=3 попыток по proxy_rotate_attempt_timeout_s=8s каждая). Зависший changeip-сервис больше не блокирует весь sweep на 30s. Первый успешный attempt → True; все провалились → False (логируется как error). ФИКС B (run_avito_city_sweep): когда SERP-фаза уже сохранила лоты (lots_inserted + lots_updated > 0) и упала только detail/houses-фаза (AvitoBlockedError / RateLimitedError), помечаем run как done (не banned) при avito_serp_ok_not_banned=True (default). Метрика banned зарезервирована для случаев где SERP сам заблокирован (0 лотов). Новые настройки (ENV): PROXY_ROTATE_ATTEMPT_TIMEOUT_S=8.0 — timeout одной changeip-попытки PROXY_ROTATE_ATTEMPTS=3 — число попыток перед отказом AVITO_SERP_OK_NOT_BANNED=true — флаг нового поведения (default on)
336 lines
13 KiB
Python
336 lines
13 KiB
Python
"""Тесты устойчивости скрейперов: avito changeip-retry (#1950) + cian anti-zombie (#1949).
|
||
|
||
Fix A (#1950): _rotate_proxy_ip теперь делает до proxy_rotate_attempts попыток
|
||
по proxy_rotate_attempt_timeout_s каждая.
|
||
|
||
Fix B (#1950): run_avito_city_sweep при AvitoBlockedError/RateLimited с
|
||
lots_inserted+lots_updated>0 вызывает mark_done (не mark_banned)
|
||
когда avito_serp_ok_not_banned=True.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
# ──────────────────────────────────────────────────────────────────────────────
|
||
# Fix A: _rotate_proxy_ip retry loop (#1950)
|
||
# ──────────────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _make_session_mock(get_side_effect: Any) -> AsyncMock:
|
||
"""Вспомогательный: AsyncMock-сессия с заданным поведением .get()."""
|
||
mock_session = AsyncMock()
|
||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||
mock_session.get = AsyncMock(side_effect=get_side_effect)
|
||
return mock_session
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_changeip_retry_success_on_second_attempt() -> None:
|
||
"""#1950-A: первая попытка таймаут, вторая успех → возвращает True; ровно 2 GET-вызова."""
|
||
from app.services.scrape_pipeline import _rotate_proxy_ip
|
||
|
||
get_call_count = 0
|
||
|
||
async def fake_get(url: str) -> MagicMock:
|
||
nonlocal get_call_count
|
||
get_call_count += 1
|
||
if get_call_count == 1:
|
||
raise TimeoutError("connect timeout")
|
||
resp = MagicMock()
|
||
resp.json.return_value = {"new_ip": "1.2.3.4"}
|
||
return resp
|
||
|
||
mock_session = _make_session_mock(fake_get)
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrape_pipeline.settings") as s,
|
||
):
|
||
s.proxy_rotate_attempts = 3
|
||
s.proxy_rotate_attempt_timeout_s = 8.0
|
||
s.avito_proxy_rotate_settle_s = 0.0
|
||
s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
|
||
s.avito_proxy_max_rotations = 3
|
||
|
||
result = await _rotate_proxy_ip(
|
||
source="avito",
|
||
reason="test-detail-block",
|
||
rotations_done=0,
|
||
)
|
||
|
||
assert result is True
|
||
assert get_call_count == 2, f"expected 2 GET calls, got {get_call_count}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_changeip_retry_all_fail_returns_false() -> None:
|
||
"""#1950-A: все 3 попытки падают с TimeoutError → False; ровно 3 GET-вызова."""
|
||
from app.services.scrape_pipeline import _rotate_proxy_ip
|
||
|
||
get_call_count = 0
|
||
|
||
async def failing_get(url: str) -> None:
|
||
nonlocal get_call_count
|
||
get_call_count += 1
|
||
raise TimeoutError("connect timeout")
|
||
|
||
mock_session = _make_session_mock(failing_get)
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrape_pipeline.settings") as s,
|
||
):
|
||
s.proxy_rotate_attempts = 3
|
||
s.proxy_rotate_attempt_timeout_s = 8.0
|
||
s.avito_proxy_rotate_settle_s = 0.0
|
||
s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
|
||
s.avito_proxy_max_rotations = 3
|
||
|
||
result = await _rotate_proxy_ip(
|
||
source="avito",
|
||
reason="test-detail-block",
|
||
rotations_done=0,
|
||
)
|
||
|
||
assert result is False
|
||
assert (
|
||
get_call_count == 3
|
||
), f"expected exactly 3 attempts (proxy_rotate_attempts=3), got {get_call_count}"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_changeip_retry_bounded_by_setting() -> None:
|
||
"""#1950-A: proxy_rotate_attempts=2 → ровно 2 попытки, не больше."""
|
||
from app.services.scrape_pipeline import _rotate_proxy_ip
|
||
|
||
call_count = 0
|
||
|
||
async def failing_get(url: str) -> None:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
raise TimeoutError("timeout")
|
||
|
||
mock_session = _make_session_mock(failing_get)
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrape_pipeline.settings") as s,
|
||
):
|
||
s.proxy_rotate_attempts = 2
|
||
s.proxy_rotate_attempt_timeout_s = 8.0
|
||
s.avito_proxy_rotate_settle_s = 0.0
|
||
s.cian_proxy_rotate_url = "http://changeip.example.com/rotate"
|
||
s.avito_proxy_rotate_url = None
|
||
s.cian_proxy_max_rotations = 3
|
||
|
||
result = await _rotate_proxy_ip(
|
||
source="cian",
|
||
reason="serp-block",
|
||
rotations_done=1,
|
||
)
|
||
|
||
assert result is False
|
||
assert call_count == 2, f"proxy_rotate_attempts=2 should make exactly 2 calls, got {call_count}"
|
||
|
||
|
||
# ──────────────────────────────────────────────────────────────────────────────
|
||
# Fix B: честный статус run_avito_city_sweep (#1950)
|
||
# ──────────────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _make_city_sweep_mocks() -> tuple[MagicMock, MagicMock, MagicMock]:
|
||
"""Возвращает (mock_db, mock_runs, mock_scraper) для тестов run_avito_city_sweep."""
|
||
mock_db = MagicMock()
|
||
|
||
# priority_rows для detail-фазы: 3 строки с source_url
|
||
class _DetailRow:
|
||
def __getitem__(self, key: str) -> str:
|
||
return "https://www.avito.ru/ekaterinburg/kvartiry/test-1"
|
||
|
||
mock_db.execute.return_value.mappings.return_value.all.return_value = [
|
||
_DetailRow(),
|
||
_DetailRow(),
|
||
_DetailRow(),
|
||
]
|
||
|
||
mock_runs = MagicMock()
|
||
mock_runs.is_cancelled.return_value = False
|
||
mock_runs.mark_done = MagicMock()
|
||
mock_runs.mark_banned = MagicMock()
|
||
mock_runs.mark_failed = MagicMock()
|
||
mock_runs.update_heartbeat = MagicMock()
|
||
|
||
mock_scraper = MagicMock()
|
||
# AvitoScraper НЕ используется как async CM в _avito_anchor_phases (просто instantiate)
|
||
mock_scraper.fetch_around = AsyncMock(
|
||
return_value=[
|
||
ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/e/k/{i}",
|
||
source_id=str(i),
|
||
price_rub=5_000_000,
|
||
address="ЕКБ test",
|
||
listing_segment="vtorichka",
|
||
)
|
||
for i in range(3)
|
||
]
|
||
)
|
||
|
||
return mock_db, mock_runs, mock_scraper
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_avito_city_sweep_serp_ok_marks_done_not_banned() -> None:
|
||
"""#1950-B: SERP собрал лоты (ins=3), detail заблокировало → mark_done не mark_banned.
|
||
|
||
Условие: avito_serp_ok_not_banned=True (default), lots_inserted+lots_updated > 0.
|
||
"""
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
|
||
|
||
# Shared AsyncSession mock (browser_mode=False)
|
||
mock_session = AsyncMock()
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
||
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
|
||
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
|
||
patch(
|
||
"app.services.scrape_pipeline.fetch_detail",
|
||
AsyncMock(side_effect=AvitoBlockedError("HTTP 403 banned")),
|
||
),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrape_pipeline.settings") as s,
|
||
):
|
||
s.scraper_fetch_mode = "curl_cffi"
|
||
s.scraper_proxy_url = None
|
||
s.avito_proxy_max_rotations = 0 # no rotation budget
|
||
s.avito_serp_ok_not_banned = True # ← наш флаг
|
||
s.avito_proxy_rotate_settle_s = 0.0
|
||
s.proxy_rotate_attempts = 3
|
||
s.proxy_rotate_attempt_timeout_s = 8.0
|
||
|
||
await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=1,
|
||
anchors=[(56.84, 60.61, "test-anchor")],
|
||
detail_top_n=3,
|
||
enrich_houses=False,
|
||
enrich_imv=False,
|
||
)
|
||
|
||
assert (
|
||
mock_runs.mark_done.called
|
||
), "mark_done должен быть вызван: SERP OK (ins=3), detail blocked"
|
||
assert (
|
||
not mock_runs.mark_banned.called
|
||
), "mark_banned НЕ должен быть вызван когда SERP уже сохранил лоты"
|
||
# enrichment_abort_note передан в counters
|
||
call_args = mock_runs.mark_done.call_args
|
||
# mark_done(db, run_id, counters_dict)
|
||
passed_counters: dict = call_args[0][2] if call_args[0] else {}
|
||
assert (
|
||
"enrichment_abort_note" in passed_counters
|
||
), "enrichment_abort_note должен присутствовать в counters переданных mark_done"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_avito_city_sweep_serp_zero_lots_marks_banned() -> None:
|
||
"""#1950-B: SERP вернул 0 лотов, SERP-блок → mark_banned (не mark_done).
|
||
|
||
Условие: lots_inserted+lots_updated == 0.
|
||
"""
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db, mock_runs, _ = _make_city_sweep_mocks()
|
||
|
||
# SERP сам поднимает AvitoBlockedError → anchor_lots не сохраняется
|
||
mock_scraper_serp_block = MagicMock()
|
||
mock_scraper_serp_block.fetch_around = AsyncMock(side_effect=AvitoBlockedError("SERP HTTP 403"))
|
||
|
||
mock_session = AsyncMock()
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper_serp_block),
|
||
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
|
||
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrape_pipeline.settings") as s,
|
||
):
|
||
s.scraper_fetch_mode = "curl_cffi"
|
||
s.scraper_proxy_url = None
|
||
s.avito_proxy_max_rotations = 0
|
||
s.avito_serp_ok_not_banned = True # включён флаг, но лотов нет
|
||
s.avito_proxy_rotate_settle_s = 0.0
|
||
s.proxy_rotate_attempts = 3
|
||
s.proxy_rotate_attempt_timeout_s = 8.0
|
||
|
||
await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=2,
|
||
anchors=[(56.84, 60.61, "test-anchor")],
|
||
detail_top_n=3,
|
||
enrich_houses=False,
|
||
enrich_imv=False,
|
||
)
|
||
|
||
assert (
|
||
mock_runs.mark_banned.called
|
||
), "mark_banned должен быть вызван: SERP сам заблокирован, lots=0"
|
||
assert not mock_runs.mark_done.called, "mark_done НЕ должен быть вызван при lots_inserted == 0"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_avito_city_sweep_flag_off_still_marks_banned() -> None:
|
||
"""#1950-B: avito_serp_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
|
||
mock_session = AsyncMock()
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
||
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
|
||
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
|
||
patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
|
||
patch(
|
||
"app.services.scrape_pipeline.fetch_detail",
|
||
AsyncMock(side_effect=AvitoBlockedError("HTTP 403")),
|
||
),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch("app.services.scrape_pipeline.settings") as s,
|
||
):
|
||
s.scraper_fetch_mode = "curl_cffi"
|
||
s.scraper_proxy_url = None
|
||
s.avito_proxy_max_rotations = 0
|
||
s.avito_serp_ok_not_banned = False # ← флаг выключен
|
||
s.avito_proxy_rotate_settle_s = 0.0
|
||
s.proxy_rotate_attempts = 3
|
||
s.proxy_rotate_attempt_timeout_s = 8.0
|
||
|
||
await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=3,
|
||
anchors=[(56.84, 60.61, "test-anchor")],
|
||
detail_top_n=3,
|
||
enrich_houses=False,
|
||
enrich_imv=False,
|
||
)
|
||
|
||
assert (
|
||
mock_runs.mark_banned.called
|
||
), "avito_serp_ok_not_banned=False → mark_banned (backward-compat)"
|
||
assert not mock_runs.mark_done.called
|