fix(scrapers): avito_detail_backfill — снапшот ЕКБ+active, detail browser-only без curl-fallback на бан (#1814) #1815
4 changed files with 163 additions and 3 deletions
|
|
@ -294,8 +294,24 @@ async def fetch_detail(
|
|||
ValueError — если item_id не извлечён из HTML.
|
||||
"""
|
||||
if browser_fetcher is not None:
|
||||
# Browser-only режим (#1814): когда передан browser_fetcher, fetch идёт
|
||||
# СТРОГО через camoufox-сервис. Никакой curl_cffi-сессии и никакого
|
||||
# curl-fallback — avito-антибот банит curl-фингерпринт под нагрузкой
|
||||
# (429 CONNECT tunnel failed), а browser проходит. Любой firewall/ошибка
|
||||
# браузера → AvitoBlockedError; caller (backfill) пометит failed и пойдёт
|
||||
# дальше, НЕ трогая curl.
|
||||
full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url)
|
||||
html = await browser_fetcher.fetch(full_url)
|
||||
try:
|
||||
html = await browser_fetcher.fetch(full_url)
|
||||
except (AvitoBlockedError, AvitoRateLimitedError):
|
||||
raise
|
||||
except Exception as exc:
|
||||
# browser-сервис вернул HTTPError / ConnectError / 502 (firewall, dead
|
||||
# listing, redirect). НЕ откатываемся в curl — поднимаем block, чтобы
|
||||
# caller пометил listing и продолжил без curl-фингерпринта.
|
||||
raise AvitoBlockedError(
|
||||
f"Avito detail browser fetch failed for {full_url}: {exc}"
|
||||
) from exc
|
||||
if _is_firewall_page(html):
|
||||
raise AvitoBlockedError(f"Avito detail firewall (browser-mode) for {full_url}")
|
||||
return parse_detail_html(html, full_url)
|
||||
|
|
|
|||
|
|
@ -118,7 +118,12 @@ async def run_avito_detail_backfill(
|
|||
runs_mod.update_heartbeat(db, run_id, current_counters)
|
||||
|
||||
# SNAPSHOT: single SELECT at start -- NOT re-selected in loop.
|
||||
# Priority: is_active DESC (active first), scraped_at DESC (newest first).
|
||||
# Scope (#1814): только активные ЕКБ-листинги. region_code на insert
|
||||
# хардкодится в 66 (base.py) → НЕ дискриминирует legacy не-ЕКБ; реальный
|
||||
# признак региона у Avito — путь URL (/ekaterinburg/ для ЕКБ; legacy
|
||||
# Москва/СПб/Тюмень — /moskva//sankt-peterburg//tyumen/). browser-fetch
|
||||
# на legacy не-ЕКБ спотыкается → curl-fallback → 429-бан curl-фингерпринта.
|
||||
# Не тратим фетчи на мёртвые (is_active) и не-ЕКБ.
|
||||
snapshot = (
|
||||
db.execute(
|
||||
text(
|
||||
|
|
@ -128,7 +133,9 @@ async def run_avito_detail_backfill(
|
|||
WHERE source = 'avito'
|
||||
AND detail_enriched_at IS NULL
|
||||
AND source_url IS NOT NULL
|
||||
ORDER BY is_active DESC NULLS LAST, scraped_at DESC NULLS LAST
|
||||
AND is_active = TRUE
|
||||
AND source_url LIKE '%/ekaterinburg/%'
|
||||
ORDER BY scraped_at DESC NULLS LAST
|
||||
LIMIT CAST(:batch_size AS int)
|
||||
"""
|
||||
),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,109 @@
|
|||
"""Browser-only режим fetch_detail (#1814).
|
||||
|
||||
Когда передан browser_fetcher, detail-fetch идёт СТРОГО через camoufox-сервис.
|
||||
Никакой curl_cffi-сессии (AsyncSession) и никакого curl-fallback — avito-антибот
|
||||
банит curl-фингерпринт под нагрузкой (429 CONNECT tunnel failed). Любой
|
||||
firewall/ошибка браузера → AvitoBlockedError; caller (backfill) метит failed и идёт
|
||||
дальше, НЕ создавая curl-сессию.
|
||||
|
||||
Тесты проверяют именно отсутствие инстанцирования AsyncSession и поднятие
|
||||
AvitoBlockedError на browser-firewall/ошибке.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.avito_detail import fetch_detail
|
||||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||||
|
||||
_SENTINEL = object()
|
||||
_FIREWALL_HTML = (
|
||||
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
|
||||
'<body><div class="firewall-container">...</div></body></html>'
|
||||
)
|
||||
_OK_HTML = "<html><body>ok</body></html>"
|
||||
|
||||
# AsyncSession (curl_cffi) в модуле avito_detail — НЕ должна инстанцироваться в browser-mode.
|
||||
_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession"
|
||||
_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session"
|
||||
_PARSE = "app.services.scrapers.avito_detail.parse_detail_html"
|
||||
|
||||
|
||||
def _browser(*, fetch_side: object) -> MagicMock:
|
||||
bf = MagicMock()
|
||||
if isinstance(fetch_side, BaseException):
|
||||
bf.fetch = AsyncMock(side_effect=fetch_side)
|
||||
else:
|
||||
bf.fetch = AsyncMock(return_value=fetch_side)
|
||||
return bf
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_mode_success_no_curl_session() -> None:
|
||||
"""browser_fetcher OK → parse, AsyncSession НЕ создаётся."""
|
||||
bf = _browser(fetch_side=_OK_HTML)
|
||||
with (
|
||||
patch(_ASYNC_SESSION) as async_session,
|
||||
patch(_BUILD_SESSION) as build_session,
|
||||
patch(_PARSE, return_value=_SENTINEL) as parse,
|
||||
):
|
||||
result = await fetch_detail("/items/123", browser_fetcher=bf)
|
||||
|
||||
assert result is _SENTINEL
|
||||
bf.fetch.assert_awaited_once()
|
||||
parse.assert_called_once()
|
||||
async_session.assert_not_called()
|
||||
build_session.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_firewall_raises_blocked_no_curl_session() -> None:
|
||||
"""browser вернул firewall-HTML (status 200) → AvitoBlockedError, без curl."""
|
||||
bf = _browser(fetch_side=_FIREWALL_HTML)
|
||||
with (
|
||||
patch(_ASYNC_SESSION) as async_session,
|
||||
patch(_BUILD_SESSION) as build_session,
|
||||
patch(_PARSE, return_value=_SENTINEL),
|
||||
):
|
||||
with pytest.raises(AvitoBlockedError):
|
||||
await fetch_detail("/items/123", browser_fetcher=bf)
|
||||
|
||||
async_session.assert_not_called()
|
||||
build_session.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_http_error_raises_blocked_no_curl_session() -> None:
|
||||
"""browser_fetcher.fetch кидает HTTPError (502/firewall сервиса) → AvitoBlockedError,
|
||||
AsyncSession НЕ инстанцируется (НЕ откатываемся в curl-фингерпринт → 429-бан)."""
|
||||
bf = _browser(fetch_side=httpx.HTTPError("browser service 502"))
|
||||
with (
|
||||
patch(_ASYNC_SESSION) as async_session,
|
||||
patch(_BUILD_SESSION) as build_session,
|
||||
patch(_PARSE, return_value=_SENTINEL),
|
||||
):
|
||||
with pytest.raises(AvitoBlockedError):
|
||||
await fetch_detail("/items/123", browser_fetcher=bf)
|
||||
|
||||
async_session.assert_not_called()
|
||||
build_session.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_blocked_error_propagates_unchanged() -> None:
|
||||
"""Если browser_fetcher сам кинул AvitoBlockedError — пробрасываем как есть, без curl."""
|
||||
original = AvitoBlockedError("already blocked")
|
||||
bf = _browser(fetch_side=original)
|
||||
with (
|
||||
patch(_ASYNC_SESSION) as async_session,
|
||||
patch(_BUILD_SESSION) as build_session,
|
||||
patch(_PARSE, return_value=_SENTINEL),
|
||||
):
|
||||
with pytest.raises(AvitoBlockedError) as exc_info:
|
||||
await fetch_detail("/items/123", browser_fetcher=bf)
|
||||
|
||||
assert exc_info.value is original
|
||||
async_session.assert_not_called()
|
||||
build_session.assert_not_called()
|
||||
|
|
@ -215,6 +215,34 @@ async def test_backfill_rotate_ip_called_on_each_block() -> None:
|
|||
runs.mark_done.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_snapshot_filters_ekb_active_only() -> None:
|
||||
"""Снапшот-SELECT (#1814) фильтрует только активные ЕКБ-листинги.
|
||||
|
||||
Проверяем, что текст запроса содержит `is_active = TRUE` и
|
||||
`LIKE '%/ekaterinburg/%'` — legacy не-ЕКБ (moskva/spb/tyumen) и мёртвые
|
||||
листинги не попадают в фетч, иначе browser спотыкается → curl-бан 429.
|
||||
"""
|
||||
db = _mock_db([])
|
||||
runs = MagicMock()
|
||||
fake_settings = MagicMock(scraper_fetch_mode="cffi")
|
||||
with (
|
||||
patch(_SETTINGS, fake_settings),
|
||||
patch(_SESSION, return_value=AsyncMock()),
|
||||
patch(_SCRAPER),
|
||||
patch(_RUNS, runs),
|
||||
patch(_FETCH),
|
||||
):
|
||||
await run_avito_detail_backfill(db, run_id=8, params={"batch_size": 10, "budget_sec": 60})
|
||||
|
||||
# Первый (и единственный при пустом снапшоте) execute — это SELECT-снапшот.
|
||||
snapshot_call = db.execute.call_args_list[0]
|
||||
sql_text = str(snapshot_call.args[0])
|
||||
assert "is_active = TRUE" in sql_text
|
||||
assert "/ekaterinburg/" in sql_text
|
||||
assert "detail_enriched_at IS NULL" in sql_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_fetch_exception_continues() -> None:
|
||||
"""RuntimeError on one listing -> failed++, db.rollback(), loop continues for next."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue