From 37fc50ad08edf8ee9d1adbcdca57bf87737cffe9 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Fri, 19 Jun 2026 21:58:35 +0300 Subject: [PATCH] =?UTF-8?q?fix(scrapers):=20avito=5Fdetail=5Fbackfill=20?= =?UTF-8?q?=E2=80=94=20=D1=81=D0=BD=D0=B0=D0=BF=D1=88=D0=BE=D1=82=20=D0=95?= =?UTF-8?q?=D0=9A=D0=91+active,=20detail=20browser-only=20=D0=B1=D0=B5?= =?UTF-8?q?=D0=B7=20curl-fallback=20=D0=BD=D0=B0=20=D0=B1=D0=B0=D0=BD=20(#?= =?UTF-8?q?1814)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/services/scrapers/avito_detail.py | 18 ++- .../app/tasks/avito_detail_backfill.py | 11 +- .../test_avito_detail_browser_only.py | 109 ++++++++++++++++++ .../tests/tasks/test_avito_detail_backfill.py | 28 +++++ 4 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index 5c02432f..f502c50a 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -294,8 +294,24 @@ async def fetch_detail( ValueError — если item_id не извлечён из HTML. """ if browser_fetcher is not None: + # Browser-only режим (#1814): когда передан browser_fetcher, fetch идёт + # СТРОГО через camoufox-сервис. Никакой curl_cffi-сессии и никакого + # curl-fallback — avito-антибот банит curl-фингерпринт под нагрузкой + # (429 CONNECT tunnel failed), а browser проходит. Любой firewall/ошибка + # браузера → AvitoBlockedError; caller (backfill) пометит failed и пойдёт + # дальше, НЕ трогая curl. full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url) - html = await browser_fetcher.fetch(full_url) + try: + html = await browser_fetcher.fetch(full_url) + except (AvitoBlockedError, AvitoRateLimitedError): + raise + except Exception as exc: + # browser-сервис вернул HTTPError / ConnectError / 502 (firewall, dead + # listing, redirect). НЕ откатываемся в curl — поднимаем block, чтобы + # caller пометил listing и продолжил без curl-фингерпринта. + raise AvitoBlockedError( + f"Avito detail browser fetch failed for {full_url}: {exc}" + ) from exc if _is_firewall_page(html): raise AvitoBlockedError(f"Avito detail firewall (browser-mode) for {full_url}") return parse_detail_html(html, full_url) diff --git a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py index 601e7f3f..2ed2648f 100644 --- a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py +++ b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py @@ -118,7 +118,12 @@ async def run_avito_detail_backfill( runs_mod.update_heartbeat(db, run_id, current_counters) # SNAPSHOT: single SELECT at start -- NOT re-selected in loop. - # Priority: is_active DESC (active first), scraped_at DESC (newest first). + # Scope (#1814): только активные ЕКБ-листинги. region_code на insert + # хардкодится в 66 (base.py) → НЕ дискриминирует legacy не-ЕКБ; реальный + # признак региона у Avito — путь URL (/ekaterinburg/ для ЕКБ; legacy + # Москва/СПб/Тюмень — /moskva//sankt-peterburg//tyumen/). browser-fetch + # на legacy не-ЕКБ спотыкается → curl-fallback → 429-бан curl-фингерпринта. + # Не тратим фетчи на мёртвые (is_active) и не-ЕКБ. snapshot = ( db.execute( text( @@ -128,7 +133,9 @@ async def run_avito_detail_backfill( WHERE source = 'avito' AND detail_enriched_at IS NULL AND source_url IS NOT NULL - ORDER BY is_active DESC NULLS LAST, scraped_at DESC NULLS LAST + AND is_active = TRUE + AND source_url LIKE '%/ekaterinburg/%' + ORDER BY scraped_at DESC NULLS LAST LIMIT CAST(:batch_size AS int) """ ), diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py new file mode 100644 index 00000000..6d78306b --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_avito_detail_browser_only.py @@ -0,0 +1,109 @@ +"""Browser-only режим fetch_detail (#1814). + +Когда передан browser_fetcher, detail-fetch идёт СТРОГО через camoufox-сервис. +Никакой curl_cffi-сессии (AsyncSession) и никакого curl-fallback — avito-антибот +банит curl-фингерпринт под нагрузкой (429 CONNECT tunnel failed). Любой +firewall/ошибка браузера → AvitoBlockedError; caller (backfill) метит failed и идёт +дальше, НЕ создавая curl-сессию. + +Тесты проверяют именно отсутствие инстанцирования AsyncSession и поднятие +AvitoBlockedError на browser-firewall/ошибке. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from app.services.scrapers.avito_detail import fetch_detail +from app.services.scrapers.avito_exceptions import AvitoBlockedError + +_SENTINEL = object() +_FIREWALL_HTML = ( + "Доступ ограничен: проблема с IP" + '
...
' +) +_OK_HTML = "ok" + +# AsyncSession (curl_cffi) в модуле avito_detail — НЕ должна инстанцироваться в browser-mode. +_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession" +_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session" +_PARSE = "app.services.scrapers.avito_detail.parse_detail_html" + + +def _browser(*, fetch_side: object) -> MagicMock: + bf = MagicMock() + if isinstance(fetch_side, BaseException): + bf.fetch = AsyncMock(side_effect=fetch_side) + else: + bf.fetch = AsyncMock(return_value=fetch_side) + return bf + + +@pytest.mark.asyncio +async def test_browser_mode_success_no_curl_session() -> None: + """browser_fetcher OK → parse, AsyncSession НЕ создаётся.""" + bf = _browser(fetch_side=_OK_HTML) + with ( + patch(_ASYNC_SESSION) as async_session, + patch(_BUILD_SESSION) as build_session, + patch(_PARSE, return_value=_SENTINEL) as parse, + ): + result = await fetch_detail("/items/123", browser_fetcher=bf) + + assert result is _SENTINEL + bf.fetch.assert_awaited_once() + parse.assert_called_once() + async_session.assert_not_called() + build_session.assert_not_called() + + +@pytest.mark.asyncio +async def test_browser_firewall_raises_blocked_no_curl_session() -> None: + """browser вернул firewall-HTML (status 200) → AvitoBlockedError, без curl.""" + bf = _browser(fetch_side=_FIREWALL_HTML) + with ( + patch(_ASYNC_SESSION) as async_session, + patch(_BUILD_SESSION) as build_session, + patch(_PARSE, return_value=_SENTINEL), + ): + with pytest.raises(AvitoBlockedError): + await fetch_detail("/items/123", browser_fetcher=bf) + + async_session.assert_not_called() + build_session.assert_not_called() + + +@pytest.mark.asyncio +async def test_browser_http_error_raises_blocked_no_curl_session() -> None: + """browser_fetcher.fetch кидает HTTPError (502/firewall сервиса) → AvitoBlockedError, + AsyncSession НЕ инстанцируется (НЕ откатываемся в curl-фингерпринт → 429-бан).""" + bf = _browser(fetch_side=httpx.HTTPError("browser service 502")) + with ( + patch(_ASYNC_SESSION) as async_session, + patch(_BUILD_SESSION) as build_session, + patch(_PARSE, return_value=_SENTINEL), + ): + with pytest.raises(AvitoBlockedError): + await fetch_detail("/items/123", browser_fetcher=bf) + + async_session.assert_not_called() + build_session.assert_not_called() + + +@pytest.mark.asyncio +async def test_browser_blocked_error_propagates_unchanged() -> None: + """Если browser_fetcher сам кинул AvitoBlockedError — пробрасываем как есть, без curl.""" + original = AvitoBlockedError("already blocked") + bf = _browser(fetch_side=original) + with ( + patch(_ASYNC_SESSION) as async_session, + patch(_BUILD_SESSION) as build_session, + patch(_PARSE, return_value=_SENTINEL), + ): + with pytest.raises(AvitoBlockedError) as exc_info: + await fetch_detail("/items/123", browser_fetcher=bf) + + assert exc_info.value is original + async_session.assert_not_called() + build_session.assert_not_called() diff --git a/tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py b/tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py index 82d113cc..93edf1ca 100644 --- a/tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py +++ b/tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py @@ -215,6 +215,34 @@ async def test_backfill_rotate_ip_called_on_each_block() -> None: runs.mark_done.assert_called_once() +@pytest.mark.asyncio +async def test_backfill_snapshot_filters_ekb_active_only() -> None: + """Снапшот-SELECT (#1814) фильтрует только активные ЕКБ-листинги. + + Проверяем, что текст запроса содержит `is_active = TRUE` и + `LIKE '%/ekaterinburg/%'` — legacy не-ЕКБ (moskva/spb/tyumen) и мёртвые + листинги не попадают в фетч, иначе browser спотыкается → curl-бан 429. + """ + db = _mock_db([]) + runs = MagicMock() + fake_settings = MagicMock(scraper_fetch_mode="cffi") + with ( + patch(_SETTINGS, fake_settings), + patch(_SESSION, return_value=AsyncMock()), + patch(_SCRAPER), + patch(_RUNS, runs), + patch(_FETCH), + ): + await run_avito_detail_backfill(db, run_id=8, params={"batch_size": 10, "budget_sec": 60}) + + # Первый (и единственный при пустом снапшоте) execute — это SELECT-снапшот. + snapshot_call = db.execute.call_args_list[0] + sql_text = str(snapshot_call.args[0]) + assert "is_active = TRUE" in sql_text + assert "/ekaterinburg/" in sql_text + assert "detail_enriched_at IS NULL" in sql_text + + @pytest.mark.asyncio async def test_backfill_fetch_exception_continues() -> None: """RuntimeError on one listing -> failed++, db.rollback(), loop continues for next.""" -- 2.45.3