From d311efb431b15de6569d0638428e605d20b1089d Mon Sep 17 00:00:00 2001 From: bot-backend Date: Mon, 29 Jun 2026 01:07:44 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein/avito-detail):=20in-session=20?= =?UTF-8?q?=D0=BF=D0=B5=D1=80=D0=B5=D0=BF=D0=BE=D0=B8=D1=81=D0=BA=20(resea?= =?UTF-8?q?rch=5Fevery)=20+=20warm=5Fbatch=3D500=20=E2=80=94=20=D0=B8?= =?UTF-8?q?=D0=BD=D1=82=D0=B5=D1=80=D0=B2=D0=B0=D0=BB=D1=8B=20=D0=BF=D0=BE?= =?UTF-8?q?=20=D1=80=D0=B0=D0=B7=D0=BC=D0=B5=D1=80=D1=83=20SERP-=D1=81?= =?UTF-8?q?=D1=82=D1=80=D0=B0=D0=BD=D0=B8=D1=86=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/services/scrapers/avito_detail.py | 18 ++++++++++ .../app/tasks/avito_detail_backfill.py | 34 ++++++++++++------ .../scrapers/test_avito_detail_warmup.py | 36 ++++++++++++++++++- 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index d0d7fe1d..65055665 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -319,6 +319,24 @@ async def warm_up_session(session: AsyncSession) -> bool: return ok +async def research_in_session(session: AsyncSession) -> bool: + """Лёгкий in-session перепоиск: re-GET avito search-страницы (Referer=yandex) на + ТЕКУЩЕЙ сессии — освежает антибот-куки БЕЗ смены exit-IP. Best-effort (search-страница + иногда 429, это не ломает detail-фетчи). Возвращает True если страница не firewall. + """ + try: + r = await session.get( + _AVITO_WARM_SEARCH_URL, headers={"Referer": _AVITO_WARM_YANDEX_REFERER} + ) + except Exception as exc: + logger.warning("avito re-search: fetch error: %s", exc) + return False + ok = r.status_code == 200 and not _is_firewall_page(r.text) + if not ok: + logger.info("avito re-search: search page sc=%s (non-fatal)", r.status_code) + return ok + + async def build_warmed_session() -> AsyncSession: """Построить detail-сессию (_build_detail_session) и прогреть её. Возвращает прогретую сессию (даже если warm-up soft-failed — caller может всё равно пробовать). diff --git a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py index 4241be98..d56a63ff 100644 --- a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py +++ b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py @@ -35,6 +35,7 @@ from app.services.scrapers.avito_detail import ( _AVITO_WARM_SEARCH_URL, build_warmed_session, fetch_detail, + research_in_session, save_detail_enrichment, ) from app.services.scrapers.avito_exceptions import ( @@ -95,7 +96,8 @@ async def run_avito_detail_backfill( budget_sec = float(params.get("budget_sec", 3600)) request_delay_sec = float(params.get("request_delay_sec", 6.0)) max_consecutive_blocks = int(params.get("max_consecutive_blocks", 5)) - warm_batch = int(params.get("warm_batch", 20)) + warm_batch = int(params.get("warm_batch", 500)) + research_every = int(params.get("research_every", 50)) # #1950: hard-timeout'ы на блокирующие await'ы внутри loop'а. Без них один # зависший fetch_detail/rotate ронял весь run в zombie (run 423 завис 7.7ч). @@ -252,16 +254,26 @@ async def run_avito_detail_backfill( # Normalise URL -> path (mirrors scrape_pipeline.py line 311) item_url = urlparse(source_url).path if source_url.startswith("http") else source_url - # use_curl warm-batch (#1551): ротация прогретой сессии каждые warm_batch - # карточек — свежий exit-IP + свежий органический прогрев, щадит IP-пул - # (1 exit-IP на батч вместо 1 на карточку). - if use_curl and session is not None and items_since_warm >= warm_batch: - try: - await session.close() - except Exception: - pass - session = await build_warmed_session() - items_since_warm = 0 + # use_curl warm-batch (#1551): интервалы по размеру SERP-страницы (~50). + # Полная ротация (новый exit-IP + полный прогрев) каждые warm_batch (~500); + # между ротациями — лёгкий in-session перепоиск каждые research_every (~50) + # на ТОМ ЖЕ exit-IP (освежить куки, без смены IP). On-block rebuild = backstop. + if use_curl and session is not None: + if warm_batch and items_since_warm >= warm_batch: + # полная ротация: новый exit-IP + полный прогрев (yandex->search) + try: + await session.close() + except Exception: + pass + session = await build_warmed_session() + items_since_warm = 0 + elif ( + research_every + and items_since_warm > 0 + and items_since_warm % research_every == 0 + ): + # лёгкий in-session перепоиск: тот же exit-IP, освежить куки + await research_in_session(session) try: # #1950: hard-timeout — зависший fetch (browser hang / curl-stall) не diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py index 4b667358..b741e0f4 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py @@ -17,7 +17,11 @@ import pytest from app.core.config import settings from app.services.scrapers import avito_detail as detail_mod -from app.services.scrapers.avito_detail import fetch_detail, warm_up_session +from app.services.scrapers.avito_detail import ( + fetch_detail, + research_in_session, + warm_up_session, +) from app.services.scrapers.avito_exceptions import AvitoBlockedError @@ -104,3 +108,33 @@ async def test_fetch_detail_referer_forwarded_and_block_raises_immediately( build.assert_not_called() # никаких холодных reconnect'ов assert caller.get.await_count == 1 # один detail-GET → сразу raise assert caller.get.await_args.kwargs["headers"] == {"Referer": "X"} + + +# ── research_in_session (in-session перепоиск, #1551 follow-up) ──────────────── +@pytest.mark.asyncio +async def test_research_in_session_re_get_search_with_yandex_referer_true_on_200() -> None: + """research_in_session re-GET'ит avito-search на ТОЙ ЖЕ сессии с Referer=yandex + и возвращает True на 200/non-firewall (свежесть куки без смены exit-IP).""" + session = AsyncMock() + session.get = AsyncMock(return_value=_resp(200, "ok")) + + with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False): + ok = await research_in_session(session) + + assert ok is True + assert session.get.await_count == 1 # ровно один re-GET, без yandex-фазы (in-session) + assert session.get.await_args.args[0] == detail_mod._AVITO_WARM_SEARCH_URL + sent_referer = session.get.await_args.kwargs["headers"]["Referer"] + assert sent_referer == detail_mod._AVITO_WARM_YANDEX_REFERER + + +@pytest.mark.asyncio +async def test_research_in_session_false_on_403() -> None: + """search вернул 403 → research_in_session возвращает False (best-effort, не fatal).""" + session = AsyncMock() + session.get = AsyncMock(return_value=_resp(403)) + + with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False): + ok = await research_in_session(session) + + assert ok is False