diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index d0d7fe1d..65055665 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -319,6 +319,24 @@ async def warm_up_session(session: AsyncSession) -> bool: return ok +async def research_in_session(session: AsyncSession) -> bool: + """Лёгкий in-session перепоиск: re-GET avito search-страницы (Referer=yandex) на + ТЕКУЩЕЙ сессии — освежает антибот-куки БЕЗ смены exit-IP. Best-effort (search-страница + иногда 429, это не ломает detail-фетчи). Возвращает True если страница не firewall. + """ + try: + r = await session.get( + _AVITO_WARM_SEARCH_URL, headers={"Referer": _AVITO_WARM_YANDEX_REFERER} + ) + except Exception as exc: + logger.warning("avito re-search: fetch error: %s", exc) + return False + ok = r.status_code == 200 and not _is_firewall_page(r.text) + if not ok: + logger.info("avito re-search: search page sc=%s (non-fatal)", r.status_code) + return ok + + async def build_warmed_session() -> AsyncSession: """Построить detail-сессию (_build_detail_session) и прогреть её. Возвращает прогретую сессию (даже если warm-up soft-failed — caller может всё равно пробовать). diff --git a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py index 4241be98..d56a63ff 100644 --- a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py +++ b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py @@ -35,6 +35,7 @@ from app.services.scrapers.avito_detail import ( _AVITO_WARM_SEARCH_URL, build_warmed_session, fetch_detail, + research_in_session, save_detail_enrichment, ) from app.services.scrapers.avito_exceptions import ( @@ -95,7 +96,8 @@ async def run_avito_detail_backfill( budget_sec = float(params.get("budget_sec", 3600)) request_delay_sec = float(params.get("request_delay_sec", 6.0)) max_consecutive_blocks = int(params.get("max_consecutive_blocks", 5)) - warm_batch = int(params.get("warm_batch", 20)) + warm_batch = int(params.get("warm_batch", 500)) + research_every = int(params.get("research_every", 50)) # #1950: hard-timeout'ы на блокирующие await'ы внутри loop'а. Без них один # зависший fetch_detail/rotate ронял весь run в zombie (run 423 завис 7.7ч). @@ -252,16 +254,26 @@ async def run_avito_detail_backfill( # Normalise URL -> path (mirrors scrape_pipeline.py line 311) item_url = urlparse(source_url).path if source_url.startswith("http") else source_url - # use_curl warm-batch (#1551): ротация прогретой сессии каждые warm_batch - # карточек — свежий exit-IP + свежий органический прогрев, щадит IP-пул - # (1 exit-IP на батч вместо 1 на карточку). - if use_curl and session is not None and items_since_warm >= warm_batch: - try: - await session.close() - except Exception: - pass - session = await build_warmed_session() - items_since_warm = 0 + # use_curl warm-batch (#1551): интервалы по размеру SERP-страницы (~50). + # Полная ротация (новый exit-IP + полный прогрев) каждые warm_batch (~500); + # между ротациями — лёгкий in-session перепоиск каждые research_every (~50) + # на ТОМ ЖЕ exit-IP (освежить куки, без смены IP). On-block rebuild = backstop. + if use_curl and session is not None: + if warm_batch and items_since_warm >= warm_batch: + # полная ротация: новый exit-IP + полный прогрев (yandex->search) + try: + await session.close() + except Exception: + pass + session = await build_warmed_session() + items_since_warm = 0 + elif ( + research_every + and items_since_warm > 0 + and items_since_warm % research_every == 0 + ): + # лёгкий in-session перепоиск: тот же exit-IP, освежить куки + await research_in_session(session) try: # #1950: hard-timeout — зависший fetch (browser hang / curl-stall) не diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py index 4b667358..b741e0f4 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_detail_warmup.py @@ -17,7 +17,11 @@ import pytest from app.core.config import settings from app.services.scrapers import avito_detail as detail_mod -from app.services.scrapers.avito_detail import fetch_detail, warm_up_session +from app.services.scrapers.avito_detail import ( + fetch_detail, + research_in_session, + warm_up_session, +) from app.services.scrapers.avito_exceptions import AvitoBlockedError @@ -104,3 +108,33 @@ async def test_fetch_detail_referer_forwarded_and_block_raises_immediately( build.assert_not_called() # никаких холодных reconnect'ов assert caller.get.await_count == 1 # один detail-GET → сразу raise assert caller.get.await_args.kwargs["headers"] == {"Referer": "X"} + + +# ── research_in_session (in-session перепоиск, #1551 follow-up) ──────────────── +@pytest.mark.asyncio +async def test_research_in_session_re_get_search_with_yandex_referer_true_on_200() -> None: + """research_in_session re-GET'ит avito-search на ТОЙ ЖЕ сессии с Referer=yandex + и возвращает True на 200/non-firewall (свежесть куки без смены exit-IP).""" + session = AsyncMock() + session.get = AsyncMock(return_value=_resp(200, "ok")) + + with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False): + ok = await research_in_session(session) + + assert ok is True + assert session.get.await_count == 1 # ровно один re-GET, без yandex-фазы (in-session) + assert session.get.await_args.args[0] == detail_mod._AVITO_WARM_SEARCH_URL + sent_referer = session.get.await_args.kwargs["headers"]["Referer"] + assert sent_referer == detail_mod._AVITO_WARM_YANDEX_REFERER + + +@pytest.mark.asyncio +async def test_research_in_session_false_on_403() -> None: + """search вернул 403 → research_in_session возвращает False (best-effort, не fatal).""" + session = AsyncMock() + session.get = AsyncMock(return_value=_resp(403)) + + with patch("app.services.scrapers.avito_detail._is_firewall_page", return_value=False): + ok = await research_in_session(session) + + assert ok is False