5bf650c843
Корень банов city-sweep: pipeline создаёт собственную shared AsyncSession и присваивает её scraper._cffi, ПЕРЕЗАТИРАЯ проксированную сессию из AvitoScraper.__aenter__ (#788). Через эту же сессию идёт и SERP, и detail-энричмент (fetch_detail), и houses. Сессия создавалась без proxies → detail-страницы летели с datacenter-IP → HTTP 429 → весь run mark_banned (scrape_runs id=9: «Avito detail HTTP 429 …»). Единичный /admin/scrape работал, т.к. зовёт fetch_around напрямую (проксированный _cffi из __aenter__), а не pipeline. - scrape_pipeline.py: helper _avito_proxies() (из settings.avito_proxy_url) → proxies= в обе shared AsyncSession (run_avito_pipeline + run_avito_city_sweep). Пусто/нет env → прямое подключение (dev no-op). n1/yandex не трогаем (им прокси не нужен, не жжём мобильный трафик). - test: _avito_proxies None без env / {http,https} при заданном URL. Проверено: 18 passed (anti-bot + serp_date). ruff clean. Требует deploy (cron city-sweep каждые 2ч на проде продолжит банить detail-429 до выката). Refs #623, #788, #562.
432 lines
16 KiB
Python
432 lines
16 KiB
Python
"""Test anti-bot hardening: blocked exceptions + early abort."""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.avito_exceptions import (
|
||
AvitoBlockedError,
|
||
AvitoContentBlockedError,
|
||
AvitoRateLimitedError,
|
||
)
|
||
|
||
# ── fetch_detail tests ──────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_raises_blocked_on_403() -> None:
|
||
"""fetch_detail должен raise AvitoBlockedError на HTTP 403."""
|
||
from app.services.scrapers.avito_detail import fetch_detail
|
||
|
||
mock_session = AsyncMock()
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 403
|
||
mock_session.get = AsyncMock(return_value=mock_response)
|
||
|
||
with pytest.raises(AvitoBlockedError, match="403"):
|
||
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_raises_rate_limited_on_429() -> None:
|
||
"""fetch_detail должен raise AvitoRateLimitedError на HTTP 429."""
|
||
from app.services.scrapers.avito_detail import fetch_detail
|
||
|
||
mock_session = AsyncMock()
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 429
|
||
mock_session.get = AsyncMock(return_value=mock_response)
|
||
|
||
with pytest.raises(AvitoRateLimitedError, match="429"):
|
||
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_detail_raises_value_error_on_other_non_200() -> None:
|
||
"""fetch_detail должен raise ValueError на прочие non-200 (не 403/429)."""
|
||
from app.services.scrapers.avito_detail import fetch_detail
|
||
|
||
mock_session = AsyncMock()
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 503
|
||
mock_session.get = AsyncMock(return_value=mock_response)
|
||
|
||
with pytest.raises(ValueError, match="503"):
|
||
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
|
||
|
||
|
||
# ── SERP (avito.py) tests ───────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_content_blocked_on_page1_empty_html() -> None:
|
||
"""fetch_around должен raise AvitoContentBlockedError на HTTP 200 + 0 карточек на странице 1."""
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
scraper = AvitoScraper()
|
||
mock_session = AsyncMock()
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 200
|
||
# Garbage HTML: нет data-marker="item" → _parse_html вернёт []
|
||
mock_response.text = "<html><body><p>captcha challenge</p></body></html>"
|
||
mock_session.get = AsyncMock(return_value=mock_response)
|
||
scraper._cffi = mock_session
|
||
|
||
with pytest.raises(AvitoContentBlockedError, match="content-block"):
|
||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_content_blocked_is_subtype_of_blocked() -> None:
|
||
"""AvitoContentBlockedError должен быть поймать как AvitoBlockedError (наследование)."""
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
scraper = AvitoScraper()
|
||
mock_session = AsyncMock()
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 200
|
||
mock_response.text = "<html><body></body></html>"
|
||
mock_session.get = AsyncMock(return_value=mock_response)
|
||
scraper._cffi = mock_session
|
||
|
||
with pytest.raises(AvitoBlockedError):
|
||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_page_gt1_empty_returns_gracefully() -> None:
|
||
"""fetch_around на page>1 с 0 карточек должен тихо завершить пагинацию (не raise)."""
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
scraper = AvitoScraper()
|
||
mock_session = AsyncMock()
|
||
|
||
# Страница 1: нормальный HTML с одной карточкой
|
||
valid_html = (
|
||
"<html><body>"
|
||
'<div data-marker="item" data-item-id="123456789">'
|
||
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
|
||
"2-к. квартира, 54,3 м², 5/9 эт."
|
||
"</a>"
|
||
'<meta itemprop="price" content="4500000"/>'
|
||
"</div>"
|
||
"</body></html>"
|
||
)
|
||
# Страница 2: пустой HTML → end of pagination
|
||
empty_html = "<html><body><p>no listings</p></body></html>"
|
||
|
||
page1_resp = MagicMock()
|
||
page1_resp.status_code = 200
|
||
page1_resp.text = valid_html
|
||
|
||
page2_resp = MagicMock()
|
||
page2_resp.status_code = 200
|
||
page2_resp.text = empty_html
|
||
|
||
mock_session.get = AsyncMock(side_effect=[page1_resp, page2_resp])
|
||
scraper._cffi = mock_session
|
||
|
||
with patch("app.services.scrapers.avito.AvitoScraper.sleep_between_requests", AsyncMock()):
|
||
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=2)
|
||
|
||
# Должен вернуть лоты со страницы 1, не raise на странице 2
|
||
assert len(result) >= 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_blocked_on_403() -> None:
|
||
"""AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403."""
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
scraper = AvitoScraper()
|
||
mock_session = AsyncMock()
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 403
|
||
mock_session.get = AsyncMock(return_value=mock_response)
|
||
scraper._cffi = mock_session
|
||
|
||
with pytest.raises(AvitoBlockedError):
|
||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_rate_limited_on_429() -> None:
|
||
"""AvitoScraper.fetch_around должен raise AvitoRateLimitedError на HTTP 429."""
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
scraper = AvitoScraper()
|
||
mock_session = AsyncMock()
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 429
|
||
mock_session.get = AsyncMock(return_value=mock_response)
|
||
scraper._cffi = mock_session
|
||
|
||
with pytest.raises(AvitoRateLimitedError):
|
||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||
|
||
|
||
# ── Pipeline abort tests ────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_aborts_after_3_consecutive_blocks() -> None:
|
||
"""Pipeline должен прервать sweep после 3 consecutive 403 (re-raise)."""
|
||
from app.services.scrape_pipeline import run_avito_pipeline
|
||
|
||
mock_db = MagicMock()
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(return_value=[])
|
||
mock_scraper._cffi = None
|
||
|
||
mock_row = MagicMock()
|
||
mock_row.__getitem__ = lambda self, key: (
|
||
"https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
|
||
)
|
||
mock_mappings = MagicMock()
|
||
mock_mappings.all.return_value = [mock_row, mock_row, mock_row, mock_row]
|
||
mock_execute = MagicMock()
|
||
mock_execute.mappings.return_value = mock_mappings
|
||
mock_db.execute.return_value = mock_execute
|
||
|
||
block_error = AvitoBlockedError("HTTP 403")
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
||
patch(
|
||
"app.services.scrape_pipeline.fetch_detail",
|
||
AsyncMock(side_effect=block_error),
|
||
),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
||
patch(
|
||
"curl_cffi.requests.AsyncSession",
|
||
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
|
||
),
|
||
):
|
||
with pytest.raises(AvitoBlockedError):
|
||
await run_avito_pipeline(
|
||
mock_db,
|
||
56.8,
|
||
60.6,
|
||
1000,
|
||
enrich_houses=False,
|
||
enrich_detail_top_n=4,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pipeline_sleep_between_detail_requests() -> None:
|
||
"""Pipeline должен вызывать asyncio.sleep между каждым detail request."""
|
||
from app.services.scrape_pipeline import run_avito_pipeline
|
||
|
||
mock_db = MagicMock()
|
||
mock_scraper = MagicMock()
|
||
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
||
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
||
mock_scraper.fetch_around = AsyncMock(return_value=[])
|
||
mock_scraper._cffi = None
|
||
|
||
mock_row = MagicMock()
|
||
mock_row.__getitem__ = lambda self, key: (
|
||
"https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
|
||
)
|
||
mock_mappings = MagicMock()
|
||
mock_mappings.all.return_value = [mock_row, mock_row, mock_row]
|
||
mock_execute = MagicMock()
|
||
mock_execute.mappings.return_value = mock_mappings
|
||
mock_db.execute.return_value = mock_execute
|
||
|
||
mock_enrichment = MagicMock()
|
||
sleep_mock = AsyncMock()
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
||
patch(
|
||
"app.services.scrape_pipeline.fetch_detail",
|
||
AsyncMock(return_value=mock_enrichment),
|
||
),
|
||
patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
|
||
patch("app.services.scrape_pipeline.asyncio.sleep", sleep_mock),
|
||
patch(
|
||
"curl_cffi.requests.AsyncSession",
|
||
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
|
||
),
|
||
patch("app.services.scrape_pipeline.random.uniform", return_value=1.0),
|
||
):
|
||
await run_avito_pipeline(
|
||
mock_db,
|
||
56.8,
|
||
60.6,
|
||
1000,
|
||
enrich_houses=False,
|
||
enrich_detail_top_n=3,
|
||
request_delay_sec=7.0,
|
||
)
|
||
|
||
assert sleep_mock.call_count == 2
|
||
for c in sleep_mock.call_args_list:
|
||
assert c == call(7.0)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_city_sweep_marks_banned_on_block() -> None:
|
||
"""City sweep mark'ит run как banned при AvitoBlockedError.
|
||
|
||
Migration 015 задокументировал 'banned' как 'Avito вернул 403/captcha' — именно этот сценарий.
|
||
"""
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
mock_db = MagicMock()
|
||
mock_runs = MagicMock()
|
||
mock_runs.is_cancelled.return_value = False
|
||
mock_runs.update_heartbeat = MagicMock()
|
||
mock_runs.mark_banned = MagicMock()
|
||
mock_runs.mark_done = MagicMock()
|
||
|
||
block_error = AvitoBlockedError("IP blocked at page=1")
|
||
|
||
with (
|
||
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
|
||
patch(
|
||
"app.services.scrape_pipeline.run_avito_pipeline",
|
||
AsyncMock(side_effect=block_error),
|
||
),
|
||
patch(
|
||
"curl_cffi.requests.AsyncSession",
|
||
return_value=AsyncMock(
|
||
__aenter__=AsyncMock(return_value=AsyncMock()),
|
||
__aexit__=AsyncMock(return_value=None),
|
||
),
|
||
),
|
||
):
|
||
result = await run_avito_city_sweep(
|
||
mock_db,
|
||
run_id=42,
|
||
anchors=[(56.84, 60.60, "Центр"), (56.79, 60.53, "ЮЗ")],
|
||
)
|
||
|
||
assert mock_runs.mark_banned.called
|
||
assert not mock_runs.mark_done.called
|
||
assert result.anchors_done == 1
|
||
|
||
|
||
# ── #623 firewall-детект + ротация IP ────────────────────────────
|
||
|
||
_CARD_HTML = (
|
||
"<html><body>"
|
||
'<div data-marker="item" data-item-id="123456789">'
|
||
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
|
||
"2-к. квартира, 54,3 м², 5/9 эт."
|
||
"</a>"
|
||
'<meta itemprop="price" content="4500000"/>'
|
||
"</div>"
|
||
"</body></html>"
|
||
)
|
||
|
||
# Firewall-интерстициал Avito (HTTP 200, не 403) — IP-блок datacenter.
|
||
_FIREWALL_HTML = (
|
||
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
|
||
'<body><div class="firewall-container">...</div></body></html>'
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_blocked_on_firewall_html_200() -> None:
|
||
"""HTTP 200 + firewall-страница «Доступ ограничен» → AvitoBlockedError (#623).
|
||
|
||
Avito банит datacenter-IP отдачей firewall-интерстициала с кодом 200 (не 403) —
|
||
раньше это молча проходило как «0 cards / content-block», теперь явный IP-блок.
|
||
"""
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
scraper = AvitoScraper()
|
||
mock_session = AsyncMock()
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 200
|
||
mock_response.text = _FIREWALL_HTML
|
||
mock_session.get = AsyncMock(return_value=mock_response)
|
||
scraper._cffi = mock_session
|
||
|
||
with pytest.raises(AvitoBlockedError, match="firewall"):
|
||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_rotates_ip_then_succeeds(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""При firewall + заданной changeip-ссылке: ротация IP и повтор страницы (#623)."""
|
||
from app.services.scrapers import avito as avito_mod
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
monkeypatch.setattr(
|
||
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
|
||
)
|
||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
|
||
|
||
scraper = AvitoScraper()
|
||
fw = MagicMock()
|
||
fw.status_code = 200
|
||
fw.text = _FIREWALL_HTML
|
||
ok = MagicMock()
|
||
ok.status_code = 200
|
||
ok.text = _CARD_HTML
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(side_effect=[fw, ok])
|
||
scraper._cffi = mock_session
|
||
|
||
with (
|
||
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
|
||
patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()),
|
||
):
|
||
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||
|
||
rot.assert_awaited_once()
|
||
assert len(result) == 1
|
||
assert mock_session.get.await_count == 2 # firewall → rotate → retry OK
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_blocked_when_rotation_exhausted(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Если ротация не сняла бан (max_rotations исчерпан) → AvitoBlockedError (#623)."""
|
||
from app.services.scrapers import avito as avito_mod
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
|
||
monkeypatch.setattr(
|
||
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
|
||
)
|
||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
|
||
|
||
scraper = AvitoScraper()
|
||
fw = MagicMock()
|
||
fw.status_code = 200
|
||
fw.text = _FIREWALL_HTML
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(return_value=fw) # всегда firewall
|
||
scraper._cffi = mock_session
|
||
|
||
with patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot:
|
||
with pytest.raises(AvitoBlockedError, match="firewall"):
|
||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||
|
||
assert rot.await_count == 1 # одна попытка ротации, потом сдаёмся
|
||
|
||
|
||
# ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ───
|
||
|
||
|
||
def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""_avito_proxies: None без env, dict {http,https} при заданном AVITO_PROXY_URL.
|
||
|
||
Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail —
|
||
без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup).
|
||
"""
|
||
from app.services import scrape_pipeline as sp
|
||
|
||
monkeypatch.setattr(sp.settings, "avito_proxy_url", None)
|
||
assert sp._avito_proxies() is None
|
||
|
||
url = "http://u:p@mproxy.site:16928"
|
||
monkeypatch.setattr(sp.settings, "avito_proxy_url", url)
|
||
assert sp._avito_proxies() == {"http": url, "https": url}
|