lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fix(scrapers): avito changeip-resilience (retry короткими попытками) + честный статус когда SERP отработал (#1950)

Browse source 
Два связанных фикса из диагностики 2026-06-27:

ФИКС A (_rotate_proxy_ip): вместо одношотного GET с timeout=30s — retry-loop
(proxy_rotate_attempts=3 попыток по proxy_rotate_attempt_timeout_s=8s каждая).
Зависший changeip-сервис больше не блокирует весь sweep на 30s.
Первый успешный attempt → True; все провалились → False (логируется как error).

ФИКС B (run_avito_city_sweep): когда SERP-фаза уже сохранила лоты
(lots_inserted + lots_updated > 0) и упала только detail/houses-фаза
(AvitoBlockedError / RateLimitedError), помечаем run как done (не banned)
при avito_serp_ok_not_banned=True (default). Метрика banned зарезервирована
для случаев где SERP сам заблокирован (0 лотов).

Новые настройки (ENV):
  PROXY_ROTATE_ATTEMPT_TIMEOUT_S=8.0   — timeout одной changeip-попытки
  PROXY_ROTATE_ATTEMPTS=3              — число попыток перед отказом
  AVITO_SERP_OK_NOT_BANNED=true        — флаг нового поведения (default on)
This commit is contained in:
bot-backend 2026-06-27 11:24:54 +03:00
parent 9c9b50ceca
commit 757f6486f5
4 changed files with 421 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
tradein-mvp/backend/app/core/config.py
View file

@ -359,6 +359,17 @@ class Settings(BaseSettings):
# Settle-sleep после changeip-вызова: мобильный модем поднимает новый IP.
# ~9с по умолчанию (эмпирика mobileproxy.space). ENV: AVITO_PROXY_ROTATE_SETTLE_S.
avito_proxy_rotate_settle_s: float = 9.0
# #1950: retry-параметры changeip-GET (_rotate_proxy_ip). Вместо одношотного 30s-timeout
# делаем proxy_rotate_attempts попыток по proxy_rotate_attempt_timeout_s каждая.
# Короткий timeout (8s) означает, что зависший changeip не блокирует весь run на 30s.
# ENV: PROXY_ROTATE_ATTEMPT_TIMEOUT_S / PROXY_ROTATE_ATTEMPTS.
proxy_rotate_attempt_timeout_s: float = 8.0
proxy_rotate_attempts: int = 3
# #1950: если SERP уже сохранил лоты (ins+upd > 0) и упали только detail/houses,
# ставим 'done' а не 'banned' — partial intake сохранён, 'banned' лишний.
# False = старое поведение. ENV: AVITO_SERP_OK_NOT_BANNED.
avito_serp_ok_not_banned: bool = True
# ── Cian dedicated mobile proxy (separate egress from Avito) ──────────────
# Cian и Avito делят один мобильный IP при общем scraper_proxy_url → конкуренция

49
tradein-mvp/backend/app/services/scrape_pipeline.py
View file

@ -81,8 +81,15 @@ async def _rotate_proxy_ip(*, reason: str, rotations_done: int, source: str = "a
if not rotate_url:
return False
sep = "&" if "?" in rotate_url else "?"
# #1950: retry changeip-GET короткими попытками вместо одношотного 30s-timeout.
# Если changeip-сервер завис на 30s → весь прогон ждёт зря + ротация считается
# успешной (False возвращался). Теперь: proxy_rotate_attempts попыток по
# proxy_rotate_attempt_timeout_s каждая; возвращаем True при первом успехе.
_attempts = settings.proxy_rotate_attempts
_attempt_timeout = settings.proxy_rotate_attempt_timeout_s
for attempt in range(_attempts):
try:
async with AsyncSession(timeout=30) as rot:
async with AsyncSession(timeout=_attempt_timeout) as rot:
resp = await rot.get(f"{rotate_url}{sep}format=json")
new_ip: str = ""
try:
@ -93,22 +100,35 @@ async def _rotate_proxy_ip(*, reason: str, rotations_done: int, source: str = "a
await asyncio.sleep(settings.avito_proxy_rotate_settle_s)
logger.info(
"pipeline: IP rotated via changeip — source=%s reason=%s "
"rotation=#%d remaining=%d new_ip=%s",
"rotation=#%d attempt=%d/%d remaining=%d new_ip=%s",
source,
reason,
rotations_done + 1,
attempt + 1,
_attempts,
max_rot - rotations_done - 1,
new_ip or "unknown",
)
return True
except Exception:
logger.warning(
"pipeline: IP rotation failed (source=%s reason=%s rotation=#%d)",
"pipeline: IP rotation attempt %d/%d failed " "(source=%s reason=%s rotation=#%d)",
attempt + 1,
_attempts,
source,
reason,
rotations_done + 1,
exc_info=True,
)
if attempt < _attempts - 1:
await asyncio.sleep(1.0)
logger.error(
"pipeline: all %d changeip attempts failed " "(source=%s reason=%s rotation=#%d)",
_attempts,
source,
reason,
rotations_done + 1,
)
return False
@ -1187,6 +1207,29 @@ async def run_avito_city_sweep(
counters.errors_count += 1
counters.anchors_done = idx
scrape_runs.update_heartbeat(db, run_id, counters.to_dict())
# #1950: если SERP уже собрал лоты и заблокировало только detail/houses,
# ставим 'done' (не 'banned') — partial intake сохранён.
# За флагом avito_serp_ok_not_banned (default True).
_serp_ok = (counters.lots_inserted + counters.lots_updated) > 0
if settings.avito_serp_ok_not_banned and _serp_ok:
_note = (
f"detail enrichment aborted ({e}) но SERP intake завершён: "
f"{counters.lots_inserted} ins {counters.lots_updated} upd"
)
logger.warning(
"city-sweep run_id=%d: SERP OK (ins=%d upd=%d), "
"detail blocked — DONE not banned. %s",
run_id,
counters.lots_inserted,
counters.lots_updated,
_note,
)
scrape_runs.mark_done(
db,
run_id,
{**counters.to_dict(), "enrichment_abort_note": _note}, # type: ignore[arg-type]
)
else:
scrape_runs.mark_banned(db, run_id, str(e), counters.to_dict())
return counters
except Exception:

3
tradein-mvp/backend/tests/test_provider_rotation_1848.py
View file

@ -200,6 +200,7 @@ async def test_rotate_proxy_ip_returns_false_on_network_error() -> None:
with (
patch("app.services.scrape_pipeline.settings") as s,
patch("app.services.scrape_pipeline.AsyncSession", mock_session_cls),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
):
s.cian_proxy_rotate_url = "http://cian-rotate.test/"
s.avito_proxy_rotate_url = None
@ -208,6 +209,8 @@ async def test_rotate_proxy_ip_returns_false_on_network_error() -> None:
s.avito_proxy_max_rotations = 4
s.yandex_proxy_max_rotations = 4
s.avito_proxy_rotate_settle_s = 0.0
s.proxy_rotate_attempts = 1 # одна попытка, без retry-sleep (#1950)
s.proxy_rotate_attempt_timeout_s = 8.0
result = await _rotate_proxy_ip(reason="ban", rotations_done=0, source="cian")

336
tradein-mvp/backend/tests/test_scraper_resilience_1949_1950.py Normal file
View file

@ -0,0 +1,336 @@
"""Тесты устойчивости скрейперов: avito changeip-retry (#1950) + cian anti-zombie (#1949).
Fix A (#1950): _rotate_proxy_ip теперь делает до proxy_rotate_attempts попыток
по proxy_rotate_attempt_timeout_s каждая.
Fix B (#1950): run_avito_city_sweep при AvitoBlockedError/RateLimited с
lots_inserted+lots_updated>0 вызывает mark_done (не mark_banned)
когда avito_serp_ok_not_banned=True.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrapers.avito_exceptions import AvitoBlockedError
from app.services.scrapers.base import ScrapedLot
# ──────────────────────────────────────────────────────────────────────────────
# Fix A: _rotate_proxy_ip retry loop (#1950)
# ──────────────────────────────────────────────────────────────────────────────
def _make_session_mock(get_side_effect: Any) -> AsyncMock:
"""Вспомогательный: AsyncMock-сессия с заданным поведением .get()."""
mock_session = AsyncMock()
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
mock_session.__aexit__ = AsyncMock(return_value=None)
mock_session.get = AsyncMock(side_effect=get_side_effect)
return mock_session
@pytest.mark.asyncio
async def test_changeip_retry_success_on_second_attempt() -> None:
"""#1950-A: первая попытка таймаут, вторая успех → возвращает True; ровно 2 GET-вызова."""
from app.services.scrape_pipeline import _rotate_proxy_ip
get_call_count = 0
async def fake_get(url: str) -> MagicMock:
nonlocal get_call_count
get_call_count += 1
if get_call_count == 1:
raise TimeoutError("connect timeout")
resp = MagicMock()
resp.json.return_value = {"new_ip": "1.2.3.4"}
return resp
mock_session = _make_session_mock(fake_get)
with (
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
s.avito_proxy_rotate_settle_s = 0.0
s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
s.avito_proxy_max_rotations = 3
result = await _rotate_proxy_ip(
source="avito",
reason="test-detail-block",
rotations_done=0,
)
assert result is True
assert get_call_count == 2, f"expected 2 GET calls, got {get_call_count}"
@pytest.mark.asyncio
async def test_changeip_retry_all_fail_returns_false() -> None:
"""#1950-A: все 3 попытки падают с TimeoutError → False; ровно 3 GET-вызова."""
from app.services.scrape_pipeline import _rotate_proxy_ip
get_call_count = 0
async def failing_get(url: str) -> None:
nonlocal get_call_count
get_call_count += 1
raise TimeoutError("connect timeout")
mock_session = _make_session_mock(failing_get)
with (
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
s.avito_proxy_rotate_settle_s = 0.0
s.avito_proxy_rotate_url = "http://changeip.example.com/rotate?token=X"
s.avito_proxy_max_rotations = 3
result = await _rotate_proxy_ip(
source="avito",
reason="test-detail-block",
rotations_done=0,
)
assert result is False
assert (
get_call_count == 3
), f"expected exactly 3 attempts (proxy_rotate_attempts=3), got {get_call_count}"
@pytest.mark.asyncio
async def test_changeip_retry_bounded_by_setting() -> None:
"""#1950-A: proxy_rotate_attempts=2 → ровно 2 попытки, не больше."""
from app.services.scrape_pipeline import _rotate_proxy_ip
call_count = 0
async def failing_get(url: str) -> None:
nonlocal call_count
call_count += 1
raise TimeoutError("timeout")
mock_session = _make_session_mock(failing_get)
with (
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.proxy_rotate_attempts = 2
s.proxy_rotate_attempt_timeout_s = 8.0
s.avito_proxy_rotate_settle_s = 0.0
s.cian_proxy_rotate_url = "http://changeip.example.com/rotate"
s.avito_proxy_rotate_url = None
s.cian_proxy_max_rotations = 3
result = await _rotate_proxy_ip(
source="cian",
reason="serp-block",
rotations_done=1,
)
assert result is False
assert call_count == 2, f"proxy_rotate_attempts=2 should make exactly 2 calls, got {call_count}"
# ──────────────────────────────────────────────────────────────────────────────
# Fix B: честный статус run_avito_city_sweep (#1950)
# ──────────────────────────────────────────────────────────────────────────────
def _make_city_sweep_mocks() -> tuple[MagicMock, MagicMock, MagicMock]:
"""Возвращает (mock_db, mock_runs, mock_scraper) для тестов run_avito_city_sweep."""
mock_db = MagicMock()
# priority_rows для detail-фазы: 3 строки с source_url
class _DetailRow:
def __getitem__(self, key: str) -> str:
return "https://www.avito.ru/ekaterinburg/kvartiry/test-1"
mock_db.execute.return_value.mappings.return_value.all.return_value = [
_DetailRow(),
_DetailRow(),
_DetailRow(),
]
mock_runs = MagicMock()
mock_runs.is_cancelled.return_value = False
mock_runs.mark_done = MagicMock()
mock_runs.mark_banned = MagicMock()
mock_runs.mark_failed = MagicMock()
mock_runs.update_heartbeat = MagicMock()
mock_scraper = MagicMock()
# AvitoScraper НЕ используется как async CM в _avito_anchor_phases (просто instantiate)
mock_scraper.fetch_around = AsyncMock(
return_value=[
ScrapedLot(
source="avito",
source_url=f"https://www.avito.ru/e/k/{i}",
source_id=str(i),
price_rub=5_000_000,
address="ЕКБ test",
listing_segment="vtorichka",
)
for i in range(3)
]
)
return mock_db, mock_runs, mock_scraper
@pytest.mark.asyncio
async def test_avito_city_sweep_serp_ok_marks_done_not_banned() -> None:
"""#1950-B: SERP собрал лоты (ins=3), detail заблокировало → mark_done не mark_banned.
Условие: avito_serp_ok_not_banned=True (default), lots_inserted+lots_updated > 0.
"""
from app.services.scrape_pipeline import run_avito_city_sweep
mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
# Shared AsyncSession mock (browser_mode=False)
mock_session = AsyncMock()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
patch(
"app.services.scrape_pipeline.fetch_detail",
AsyncMock(side_effect=AvitoBlockedError("HTTP 403 banned")),
),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.scraper_fetch_mode = "curl_cffi"
s.scraper_proxy_url = None
s.avito_proxy_max_rotations = 0 # no rotation budget
s.avito_serp_ok_not_banned = True # ← наш флаг
s.avito_proxy_rotate_settle_s = 0.0
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
await run_avito_city_sweep(
mock_db,
run_id=1,
anchors=[(56.84, 60.61, "test-anchor")],
detail_top_n=3,
enrich_houses=False,
enrich_imv=False,
)
assert (
mock_runs.mark_done.called
), "mark_done должен быть вызван: SERP OK (ins=3), detail blocked"
assert (
not mock_runs.mark_banned.called
), "mark_banned НЕ должен быть вызван когда SERP уже сохранил лоты"
# enrichment_abort_note передан в counters
call_args = mock_runs.mark_done.call_args
# mark_done(db, run_id, counters_dict)
passed_counters: dict = call_args[0][2] if call_args[0] else {}
assert (
"enrichment_abort_note" in passed_counters
), "enrichment_abort_note должен присутствовать в counters переданных mark_done"
@pytest.mark.asyncio
async def test_avito_city_sweep_serp_zero_lots_marks_banned() -> None:
"""#1950-B: SERP вернул 0 лотов, SERP-блок → mark_banned (не mark_done).
Условие: lots_inserted+lots_updated == 0.
"""
from app.services.scrape_pipeline import run_avito_city_sweep
mock_db, mock_runs, _ = _make_city_sweep_mocks()
# SERP сам поднимает AvitoBlockedError → anchor_lots не сохраняется
mock_scraper_serp_block = MagicMock()
mock_scraper_serp_block.fetch_around = AsyncMock(side_effect=AvitoBlockedError("SERP HTTP 403"))
mock_session = AsyncMock()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper_serp_block),
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.scraper_fetch_mode = "curl_cffi"
s.scraper_proxy_url = None
s.avito_proxy_max_rotations = 0
s.avito_serp_ok_not_banned = True # включён флаг, но лотов нет
s.avito_proxy_rotate_settle_s = 0.0
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
await run_avito_city_sweep(
mock_db,
run_id=2,
anchors=[(56.84, 60.61, "test-anchor")],
detail_top_n=3,
enrich_houses=False,
enrich_imv=False,
)
assert (
mock_runs.mark_banned.called
), "mark_banned должен быть вызван: SERP сам заблокирован, lots=0"
assert not mock_runs.mark_done.called, "mark_done НЕ должен быть вызван при lots_inserted == 0"
@pytest.mark.asyncio
async def test_avito_city_sweep_flag_off_still_marks_banned() -> None:
"""#1950-B: avito_serp_ok_not_banned=False → mark_banned даже при lots>0 (backward-compat)."""
from app.services.scrape_pipeline import run_avito_city_sweep
mock_db, mock_runs, mock_scraper = _make_city_sweep_mocks()
mock_session = AsyncMock()
with (
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
patch("app.services.scrape_pipeline.AsyncSession", return_value=mock_session),
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
patch("app.services.scrape_pipeline.save_listings", return_value=(3, 0)),
patch(
"app.services.scrape_pipeline.fetch_detail",
AsyncMock(side_effect=AvitoBlockedError("HTTP 403")),
),
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
patch("app.services.scrape_pipeline.settings") as s,
):
s.scraper_fetch_mode = "curl_cffi"
s.scraper_proxy_url = None
s.avito_proxy_max_rotations = 0
s.avito_serp_ok_not_banned = False # ← флаг выключен
s.avito_proxy_rotate_settle_s = 0.0
s.proxy_rotate_attempts = 3
s.proxy_rotate_attempt_timeout_s = 8.0
await run_avito_city_sweep(
mock_db,
run_id=3,
anchors=[(56.84, 60.61, "test-anchor")],
detail_top_n=3,
enrich_houses=False,
enrich_imv=False,
)
assert (
mock_runs.mark_banned.called
), "avito_serp_ok_not_banned=False → mark_banned (backward-compat)"
assert not mock_runs.mark_done.called