diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index 731e0647..e4df5de6 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -126,6 +126,12 @@ _TOTAL_COUNT_PATHS: list[list[str]] = [ # Regex для fallback-поиска количества офферов в тексте DOM. _RE_TOTAL_COUNT = re.compile(r"(\d[\d ]{0,8})\s+(?:объявлен|квартир|предложен)", re.IGNORECASE) +# Максимальное число повторов fetch_around при статусе 0 (tarpit/timeout) или captcha. +# Каждый retry — rotate IP + asyncio.sleep(2) + новый fetch. +# При worst-case 30 combos × 3 fetches (1 + 2 retry) общий объём умещается +# в watchdog-таймаут anchor (~2190s, #1659); большинство свежих IP не тарпитятся. +_YANDEX_TARPIT_MAX_RETRIES: int = 2 + # ── Subprocess curl transport (SOCKS5 proxy) ────────────────────────────────── # curl_cffi collapses to <1 KB/s over the SOCKS5 mobile proxy; system curl with # --compressed fetches the full SERP in ~17s (proven on prod 2026-06). @@ -848,25 +854,67 @@ class YandexRealtyScraper(BaseScraper): kept for BaseScraper compat + logging. rooms: room-bucket slug (one of ROOM_PATH keys) or None = all rooms. price_min/price_max: price filters in rubles (None = open-ended). + + Tarpit resilience: when Yandex bandwidth-tarpits the current mobile-proxy IP + (status_code==0 from curl timeout) or returns a captcha, the method rotates + the proxy IP and retries up to _YANDEX_TARPIT_MAX_RETRIES times. + Non-tarpit failures (4xx/5xx, i.e. status != 0 and status != 200) are not + retried — rotation would not help. """ url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max) - try: - response = await self._http_get(url) - except Exception: - logger.exception("yandex serp fetch failed: %s", url) - return [] - if response.status_code != 200: - logger.warning("yandex serp returned %d for %s", response.status_code, url) - return [] - # NBSP → space: parse_rub() buggy on \xa0 in capture groups → price=None (T4) - html = response.text.replace("\xa0", " ") + html: str | None = None + for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES): + try: + response = await self._http_get(url) + except Exception: + logger.exception("yandex serp fetch failed: %s", url) + return [] - if _is_captcha(html): + status = response.status_code + + if status == 0: + # Tarpit / curl timeout — rotate IP and retry + logger.warning( + "yandex: status=0 (tarpit?) rooms=%s page=%d — rotating IP + retry attempt %d", + rooms, + page, + attempt + 1, + ) + await self._rotate_ip() + await asyncio.sleep(2) + continue + + if status != 200: + logger.warning("yandex serp returned %d for %s", status, url) + return [] + + # NBSP → space: parse_rub() buggy on   in capture groups → price=None (T4) + raw_html = response.text.replace(" ", " ") + + if _is_captcha(raw_html): + logger.warning( + "yandex serp: captcha detected rooms=%s page=%d url=%s" + " — rotating IP + retry attempt %d", + rooms, + page, + url, + attempt + 1, + ) + await self._rotate_ip() + await asyncio.sleep(2) + continue + + html = raw_html + break + + if html is None: logger.warning( - "yandex serp: captcha detected on page=%d url=%s — returning empty", + "yandex serp: tarpit/captcha persisted after %d retries rooms=%s page=%d" + " — returning empty", + _YANDEX_TARPIT_MAX_RETRIES, + rooms, page, - url, ) return [] diff --git a/tradein-mvp/backend/tests/test_yandex_realty_serp.py b/tradein-mvp/backend/tests/test_yandex_realty_serp.py index 0834e4f2..a5242d57 100644 --- a/tradein-mvp/backend/tests/test_yandex_realty_serp.py +++ b/tradein-mvp/backend/tests/test_yandex_realty_serp.py @@ -9,6 +9,7 @@ import pytest from app.services.scrapers.base import ScrapedLot from app.services.scrapers.yandex_realty import ( _CURL_STATUS_MARKER, + _YANDEX_TARPIT_MAX_RETRIES, DEFAULT_CITY, DEFAULT_PRICE_RANGES, MAX_PAGES, @@ -1014,3 +1015,111 @@ async def test_fetch_all_secondary_deduplicates_across_rooms(): assert "unique_001" in ids assert "unique_002" in ids assert len(ids) == 3 # shared deduplicated — only 3 unique + + +# ── PART H: tarpit resilience — fetch_around rotate-on-status-0 / captcha ──── + + +@pytest.mark.asyncio +async def test_fetch_around_rotates_and_retries_on_status_0(): + """fetch_around rotates IP and retries when status_code==0 (tarpit/curl-timeout). + + Sequence: first call → status 0 (tarpit), second call → status 200 + valid HTML. + Expect: _rotate_ip called once, parsed lots returned from the second attempt. + """ + s = YandexRealtyScraper() + + # Valid SERP page that parses to >=1 lot + valid_html = SINGLE_CARD_HTML.replace(" ", " ") + + call_count = 0 + + async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: + nonlocal call_count + call_count += 1 + if call_count == 1: + # First attempt: tarpit — status 0 + return _CurlResponse(status_code=0, text="") + # Second attempt: success + return _CurlResponse(status_code=200, text=valid_html) + + mock_settings = MagicMock() + mock_settings.yandex_proxy_url = "socks5h://x:y@h:1" + mock_settings.yandex_proxy_rotate_url = "http://rotate.example.com/changeip" + mock_settings.avito_proxy_rotate_url = None + + with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}): + with patch.object(s, "_http_get", side_effect=fake_http_get): + with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate: + with patch("asyncio.sleep", new_callable=AsyncMock): + with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): + lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1") + + assert ( + mock_rotate.call_count == 1 + ), f"_rotate_ip must be called once, got {mock_rotate.call_count}" + assert ( + call_count == 2 + ), f"_http_get must be called twice (tarpit then success), got {call_count}" + assert len(lots) >= 1, f"Expected >=1 parsed lots after retry, got {len(lots)}" + + +@pytest.mark.asyncio +async def test_fetch_around_rotates_on_captcha(): + """fetch_around rotates IP and retries when HTML is a captcha page.""" + s = YandexRealtyScraper() + + captcha_html = "captcha" + valid_html = SINGLE_CARD_HTML.replace(" ", " ") + + call_count = 0 + + async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: + nonlocal call_count + call_count += 1 + if call_count == 1: + return _CurlResponse(status_code=200, text=captcha_html) + return _CurlResponse(status_code=200, text=valid_html) + + with patch.object(s, "_http_get", side_effect=fake_http_get): + with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate: + with patch("asyncio.sleep", new_callable=AsyncMock): + with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): + lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="2") + + assert mock_rotate.call_count == 1 + assert len(lots) >= 1 + + +@pytest.mark.asyncio +async def test_fetch_around_exhausts_retries_returns_empty(): + """fetch_around returns [] when all retries are exhausted (tarpit persists).""" + s = YandexRealtyScraper() + + async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: + return _CurlResponse(status_code=0, text="") + + with patch.object(s, "_http_get", side_effect=fake_http_get): + with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate: + with patch("asyncio.sleep", new_callable=AsyncMock): + lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1") + + # Rotated on every attempt: 1 initial + _YANDEX_TARPIT_MAX_RETRIES retries + assert mock_rotate.call_count == 1 + _YANDEX_TARPIT_MAX_RETRIES + assert lots == [] + + +@pytest.mark.asyncio +async def test_fetch_around_no_rotate_on_404(): + """fetch_around does NOT rotate on non-zero non-200 status (e.g. 404).""" + s = YandexRealtyScraper() + + async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: + return _CurlResponse(status_code=404, text="Not Found") + + with patch.object(s, "_http_get", side_effect=fake_http_get): + with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate: + lots = await s.fetch_around(lat=56.84, lon=60.60) + + assert mock_rotate.call_count == 0, "Must not rotate on 404 — only on status==0 / captcha" + assert lots == []