lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fix(scrapers): avito_detail_backfill — снапшот ЕКБ+active, detail browser-only без curl-fallback на бан (#1814)
All checks were successful
CI / changes (pull_request) Successful in 6s
Details
CI / backend-tests (pull_request) Has been skipped
Details
CI / frontend-tests (pull_request) Has been skipped
Details
CI / openapi-codegen-check (pull_request) Has been skipped
Details

Browse source
This commit is contained in:
bot-backend 2026-06-19 21:58:35 +03:00
parent 2402287d64
commit 37fc50ad08
4 changed files with 163 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
tradein-mvp/backend/app/services/scrapers/avito_detail.py
View file

@ -294,8 +294,24 @@ async def fetch_detail(
ValueError — если item_id не извлечён из HTML. ValueError — если item_id не извлечён из HTML.
""" """
if browser_fetcher is not None: if browser_fetcher is not None:
# Browser-only режим (#1814): когда передан browser_fetcher, fetch идёт
# СТРОГО через camoufox-сервис. Никакой curl_cffi-сессии и никакого
# curl-fallback — avito-антибот банит curl-фингерпринт под нагрузкой
# (429 CONNECT tunnel failed), а browser проходит. Любой firewall/ошибка
# браузера → AvitoBlockedError; caller (backfill) пометит failed и пойдёт
# дальше, НЕ трогая curl.
full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url) full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url)
html = await browser_fetcher.fetch(full_url) try:
html = await browser_fetcher.fetch(full_url)
except (AvitoBlockedError, AvitoRateLimitedError):
raise
except Exception as exc:
# browser-сервис вернул HTTPError / ConnectError / 502 (firewall, dead
# listing, redirect). НЕ откатываемся в curl — поднимаем block, чтобы
# caller пометил listing и продолжил без curl-фингерпринта.
raise AvitoBlockedError(
f"Avito detail browser fetch failed for {full_url}: {exc}"
) from exc
if _is_firewall_page(html): if _is_firewall_page(html):
raise AvitoBlockedError(f"Avito detail firewall (browser-mode) for {full_url}") raise AvitoBlockedError(f"Avito detail firewall (browser-mode) for {full_url}")
return parse_detail_html(html, full_url) return parse_detail_html(html, full_url)

11
tradein-mvp/backend/app/tasks/avito_detail_backfill.py
View file

@ -118,7 +118,12 @@ async def run_avito_detail_backfill(
runs_mod.update_heartbeat(db, run_id, current_counters) runs_mod.update_heartbeat(db, run_id, current_counters)
# SNAPSHOT: single SELECT at start -- NOT re-selected in loop. # SNAPSHOT: single SELECT at start -- NOT re-selected in loop.
# Priority: is_active DESC (active first), scraped_at DESC (newest first). # Scope (#1814): только активные ЕКБ-листинги. region_code на insert
# хардкодится в 66 (base.py) → НЕ дискриминирует legacy не-ЕКБ; реальный
# признак региона у Avito — путь URL (/ekaterinburg/ для ЕКБ; legacy
# Москва/СПб/Тюмень — /moskva//sankt-peterburg//tyumen/). browser-fetch
# на legacy не-ЕКБ спотыкается → curl-fallback → 429-бан curl-фингерпринта.
# Не тратим фетчи на мёртвые (is_active) и не-ЕКБ.
snapshot = ( snapshot = (
db.execute( db.execute(
text( text(
@ -128,7 +133,9 @@ async def run_avito_detail_backfill(
WHERE source = 'avito' WHERE source = 'avito'
AND detail_enriched_at IS NULL AND detail_enriched_at IS NULL
AND source_url IS NOT NULL AND source_url IS NOT NULL
ORDER BY is_active DESC NULLS LAST, scraped_at DESC NULLS LAST AND is_active = TRUE
AND source_url LIKE '%/ekaterinburg/%'
ORDER BY scraped_at DESC NULLS LAST
LIMIT CAST(:batch_size AS int) LIMIT CAST(:batch_size AS int)
""" """
), ),

109
tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py Normal file
View file

@ -0,0 +1,109 @@
"""Browser-only режим fetch_detail (#1814).
Когда передан browser_fetcher, detail-fetch идёт СТРОГО через camoufox-сервис.
Никакой curl_cffi-сессии (AsyncSession) и никакого curl-fallback — avito-антибот
банит curl-фингерпринт под нагрузкой (429 CONNECT tunnel failed). Любой
firewall/ошибка браузера → AvitoBlockedError; caller (backfill) метит failed и идёт
дальше, НЕ создавая curl-сессию.
Тесты проверяют именно отсутствие инстанцирования AsyncSession и поднятие
AvitoBlockedError на browser-firewall/ошибке.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from app.services.scrapers.avito_detail import fetch_detail
from app.services.scrapers.avito_exceptions import AvitoBlockedError
_SENTINEL = object()
_FIREWALL_HTML = (
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
'<body><div class="firewall-container">...</div></body></html>'
)
_OK_HTML = "<html><body>ok</body></html>"
# AsyncSession (curl_cffi) в модуле avito_detail — НЕ должна инстанцироваться в browser-mode.
_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession"
_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session"
_PARSE = "app.services.scrapers.avito_detail.parse_detail_html"
def _browser(*, fetch_side: object) -> MagicMock:
bf = MagicMock()
if isinstance(fetch_side, BaseException):
bf.fetch = AsyncMock(side_effect=fetch_side)
else:
bf.fetch = AsyncMock(return_value=fetch_side)
return bf
@pytest.mark.asyncio
async def test_browser_mode_success_no_curl_session() -> None:
"""browser_fetcher OK → parse, AsyncSession НЕ создаётся."""
bf = _browser(fetch_side=_OK_HTML)
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL) as parse,
):
result = await fetch_detail("/items/123", browser_fetcher=bf)
assert result is _SENTINEL
bf.fetch.assert_awaited_once()
parse.assert_called_once()
async_session.assert_not_called()
build_session.assert_not_called()
@pytest.mark.asyncio
async def test_browser_firewall_raises_blocked_no_curl_session() -> None:
"""browser вернул firewall-HTML (status 200) → AvitoBlockedError, без curl."""
bf = _browser(fetch_side=_FIREWALL_HTML)
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL),
):
with pytest.raises(AvitoBlockedError):
await fetch_detail("/items/123", browser_fetcher=bf)
async_session.assert_not_called()
build_session.assert_not_called()
@pytest.mark.asyncio
async def test_browser_http_error_raises_blocked_no_curl_session() -> None:
"""browser_fetcher.fetch кидает HTTPError (502/firewall сервиса) → AvitoBlockedError,
AsyncSession НЕ инстанцируется (НЕ откатываемся в curl-фингерпринт → 429-бан)."""
bf = _browser(fetch_side=httpx.HTTPError("browser service 502"))
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL),
):
with pytest.raises(AvitoBlockedError):
await fetch_detail("/items/123", browser_fetcher=bf)
async_session.assert_not_called()
build_session.assert_not_called()
@pytest.mark.asyncio
async def test_browser_blocked_error_propagates_unchanged() -> None:
"""Если browser_fetcher сам кинул AvitoBlockedError — пробрасываем как есть, без curl."""
original = AvitoBlockedError("already blocked")
bf = _browser(fetch_side=original)
with (
patch(_ASYNC_SESSION) as async_session,
patch(_BUILD_SESSION) as build_session,
patch(_PARSE, return_value=_SENTINEL),
):
with pytest.raises(AvitoBlockedError) as exc_info:
await fetch_detail("/items/123", browser_fetcher=bf)
assert exc_info.value is original
async_session.assert_not_called()
build_session.assert_not_called()

28
tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py
View file

@ -215,6 +215,34 @@ async def test_backfill_rotate_ip_called_on_each_block() -> None:
runs.mark_done.assert_called_once() runs.mark_done.assert_called_once()
@pytest.mark.asyncio
async def test_backfill_snapshot_filters_ekb_active_only() -> None:
"""Снапшот-SELECT (#1814) фильтрует только активные ЕКБ-листинги.
Проверяем, что текст запроса содержит `is_active = TRUE` и
`LIKE '%/ekaterinburg/%'` — legacy не-ЕКБ (moskva/spb/tyumen) и мёртвые
листинги не попадают в фетч, иначе browser спотыкается → curl-бан 429.
"""
db = _mock_db([])
runs = MagicMock()
fake_settings = MagicMock(scraper_fetch_mode="cffi")
with (
patch(_SETTINGS, fake_settings),
patch(_SESSION, return_value=AsyncMock()),
patch(_SCRAPER),
patch(_RUNS, runs),
patch(_FETCH),
):
await run_avito_detail_backfill(db, run_id=8, params={"batch_size": 10, "budget_sec": 60})
# Первый (и единственный при пустом снапшоте) execute — это SELECT-снапшот.
snapshot_call = db.execute.call_args_list[0]
sql_text = str(snapshot_call.args[0])
assert "is_active = TRUE" in sql_text
assert "/ekaterinburg/" in sql_text
assert "detail_enriched_at IS NULL" in sql_text
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_backfill_fetch_exception_continues() -> None: async def test_backfill_fetch_exception_continues() -> None:
"""RuntimeError on one listing -> failed++, db.rollback(), loop continues for next.""" """RuntimeError on one listing -> failed++, db.rollback(), loop continues for next."""