229 lines
8.3 KiB
Python
229 lines
8.3 KiB
Python
"""Test anti-bot hardening: blocked exceptions + early abort."""
|
|
|
|
from unittest.mock import AsyncMock, MagicMock, call, patch
|
|
|
|
import pytest
|
|
|
|
from app.services.scrapers.avito_exceptions import (
|
|
AvitoBlockedError,
|
|
AvitoRateLimitedError,
|
|
)
|
|
|
|
# ── fetch_detail tests ──────────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_detail_raises_blocked_on_403() -> None:
|
|
"""fetch_detail должен raise AvitoBlockedError на HTTP 403."""
|
|
from app.services.scrapers.avito_detail import fetch_detail
|
|
|
|
mock_session = AsyncMock()
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 403
|
|
mock_session.get = AsyncMock(return_value=mock_response)
|
|
|
|
with pytest.raises(AvitoBlockedError, match="403"):
|
|
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_detail_raises_rate_limited_on_429() -> None:
|
|
"""fetch_detail должен raise AvitoRateLimitedError на HTTP 429."""
|
|
from app.services.scrapers.avito_detail import fetch_detail
|
|
|
|
mock_session = AsyncMock()
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 429
|
|
mock_session.get = AsyncMock(return_value=mock_response)
|
|
|
|
with pytest.raises(AvitoRateLimitedError, match="429"):
|
|
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_detail_raises_value_error_on_other_non_200() -> None:
|
|
"""fetch_detail должен raise ValueError на прочие non-200 (не 403/429)."""
|
|
from app.services.scrapers.avito_detail import fetch_detail
|
|
|
|
mock_session = AsyncMock()
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 503
|
|
mock_session.get = AsyncMock(return_value=mock_response)
|
|
|
|
with pytest.raises(ValueError, match="503"):
|
|
await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session)
|
|
|
|
|
|
# ── SERP (avito.py) tests ───────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_raises_blocked_on_403() -> None:
|
|
"""AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403."""
|
|
from app.services.scrapers.avito import AvitoScraper
|
|
|
|
scraper = AvitoScraper()
|
|
mock_session = AsyncMock()
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 403
|
|
mock_session.get = AsyncMock(return_value=mock_response)
|
|
scraper._cffi = mock_session
|
|
|
|
with pytest.raises(AvitoBlockedError):
|
|
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_raises_rate_limited_on_429() -> None:
|
|
"""AvitoScraper.fetch_around должен raise AvitoRateLimitedError на HTTP 429."""
|
|
from app.services.scrapers.avito import AvitoScraper
|
|
|
|
scraper = AvitoScraper()
|
|
mock_session = AsyncMock()
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 429
|
|
mock_session.get = AsyncMock(return_value=mock_response)
|
|
scraper._cffi = mock_session
|
|
|
|
with pytest.raises(AvitoRateLimitedError):
|
|
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
|
|
|
|
|
# ── Pipeline abort tests ────────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_aborts_after_3_consecutive_blocks() -> None:
|
|
"""Pipeline должен прервать sweep после 3 consecutive 403 (re-raise)."""
|
|
from app.services.scrape_pipeline import run_avito_pipeline
|
|
|
|
mock_db = MagicMock()
|
|
mock_scraper = MagicMock()
|
|
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
|
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
|
mock_scraper.fetch_around = AsyncMock(return_value=[])
|
|
mock_scraper._cffi = None
|
|
|
|
mock_row = MagicMock()
|
|
mock_row.__getitem__ = lambda self, key: (
|
|
"https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
|
|
)
|
|
mock_mappings = MagicMock()
|
|
mock_mappings.all.return_value = [mock_row, mock_row, mock_row, mock_row]
|
|
mock_execute = MagicMock()
|
|
mock_execute.mappings.return_value = mock_mappings
|
|
mock_db.execute.return_value = mock_execute
|
|
|
|
block_error = AvitoBlockedError("HTTP 403")
|
|
|
|
with (
|
|
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
|
patch(
|
|
"app.services.scrape_pipeline.fetch_detail",
|
|
AsyncMock(side_effect=block_error),
|
|
),
|
|
patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()),
|
|
patch(
|
|
"curl_cffi.requests.AsyncSession",
|
|
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
|
|
),
|
|
):
|
|
with pytest.raises(AvitoBlockedError):
|
|
await run_avito_pipeline(
|
|
mock_db, 56.8, 60.6, 1000,
|
|
enrich_houses=False,
|
|
enrich_detail_top_n=4,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_sleep_between_detail_requests() -> None:
|
|
"""Pipeline должен вызывать asyncio.sleep между каждым detail request."""
|
|
from app.services.scrape_pipeline import run_avito_pipeline
|
|
|
|
mock_db = MagicMock()
|
|
mock_scraper = MagicMock()
|
|
mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper)
|
|
mock_scraper.__aexit__ = AsyncMock(return_value=None)
|
|
mock_scraper.fetch_around = AsyncMock(return_value=[])
|
|
mock_scraper._cffi = None
|
|
|
|
mock_row = MagicMock()
|
|
mock_row.__getitem__ = lambda self, key: (
|
|
"https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None
|
|
)
|
|
mock_mappings = MagicMock()
|
|
mock_mappings.all.return_value = [mock_row, mock_row, mock_row]
|
|
mock_execute = MagicMock()
|
|
mock_execute.mappings.return_value = mock_mappings
|
|
mock_db.execute.return_value = mock_execute
|
|
|
|
mock_enrichment = MagicMock()
|
|
sleep_mock = AsyncMock()
|
|
|
|
with (
|
|
patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper),
|
|
patch(
|
|
"app.services.scrape_pipeline.fetch_detail",
|
|
AsyncMock(return_value=mock_enrichment),
|
|
),
|
|
patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True),
|
|
patch("app.services.scrape_pipeline.asyncio.sleep", sleep_mock),
|
|
patch(
|
|
"curl_cffi.requests.AsyncSession",
|
|
return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()),
|
|
),
|
|
patch("app.services.scrape_pipeline.random.uniform", return_value=1.0),
|
|
):
|
|
await run_avito_pipeline(
|
|
mock_db, 56.8, 60.6, 1000,
|
|
enrich_houses=False,
|
|
enrich_detail_top_n=3,
|
|
request_delay_sec=7.0,
|
|
)
|
|
|
|
assert sleep_mock.call_count == 2
|
|
for c in sleep_mock.call_args_list:
|
|
assert c == call(7.0)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_city_sweep_marks_banned_on_block() -> None:
|
|
"""City sweep mark'ит run как banned при AvitoBlockedError.
|
|
|
|
Migration 015 задокументировал 'banned' как 'Avito вернул 403/captcha' — именно этот сценарий.
|
|
"""
|
|
from app.services.scrape_pipeline import run_avito_city_sweep
|
|
|
|
mock_db = MagicMock()
|
|
mock_runs = MagicMock()
|
|
mock_runs.is_cancelled.return_value = False
|
|
mock_runs.update_heartbeat = MagicMock()
|
|
mock_runs.mark_banned = MagicMock()
|
|
mock_runs.mark_done = MagicMock()
|
|
|
|
block_error = AvitoBlockedError("IP blocked at page=1")
|
|
|
|
with (
|
|
patch("app.services.scrape_pipeline.scrape_runs", mock_runs),
|
|
patch(
|
|
"app.services.scrape_pipeline.run_avito_pipeline",
|
|
AsyncMock(side_effect=block_error),
|
|
),
|
|
patch(
|
|
"curl_cffi.requests.AsyncSession",
|
|
return_value=AsyncMock(
|
|
__aenter__=AsyncMock(return_value=AsyncMock()),
|
|
__aexit__=AsyncMock(return_value=None),
|
|
),
|
|
),
|
|
):
|
|
result = await run_avito_city_sweep(
|
|
mock_db,
|
|
run_id=42,
|
|
anchors=[(56.84, 60.60, "Центр"), (56.79, 60.53, "ЮЗ")],
|
|
)
|
|
|
|
assert mock_runs.mark_banned.called
|
|
assert not mock_runs.mark_done.called
|
|
assert result.anchors_done == 1
|