tradein-mvp/backend/app/services/scrapers/yandex_realty.py

@@ -126,6 +126,12 @@ _TOTAL_COUNT_PATHS: list[list[str]] = [
 # Regex для fallback-поиска количества офферов в тексте DOM.
 _RE_TOTAL_COUNT = re.compile(r"(\d[\d ]{0,8})\s+(?:объявлен|квартир|предложен)", re.IGNORECASE)
 
+# Максимальное число повторов fetch_around при статусе 0 (tarpit/timeout) или captcha.
+# Каждый retry — rotate IP + asyncio.sleep(2) + новый fetch.
+# При worst-case 30 combos × 3 fetches (1 + 2 retry) общий объём умещается
+# в watchdog-таймаут anchor (~2190s, #1659); большинство свежих IP не тарпитятся.
+_YANDEX_TARPIT_MAX_RETRIES: int = 2
+
 # ── Subprocess curl transport (SOCKS5 proxy) ──────────────────────────────────
 # curl_cffi collapses to <1 KB/s over the SOCKS5 mobile proxy; system curl with
 # --compressed fetches the full SERP in ~17s (proven on prod 2026-06).
@@ -848,25 +854,67 @@ class YandexRealtyScraper(BaseScraper):
         kept for BaseScraper compat + logging.
         rooms: room-bucket slug (one of ROOM_PATH keys) or None = all rooms.
         price_min/price_max: price filters in rubles (None = open-ended).
+
+        Tarpit resilience: when Yandex bandwidth-tarpits the current mobile-proxy IP
+        (status_code==0 from curl timeout) or returns a captcha, the method rotates
+        the proxy IP and retries up to _YANDEX_TARPIT_MAX_RETRIES times.
+        Non-tarpit failures (4xx/5xx, i.e. status != 0 and status != 200) are not
+        retried — rotation would not help.
         """
         url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
+
+        html: str | None = None
+        for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
             try:
                 response = await self._http_get(url)
             except Exception:
                 logger.exception("yandex serp fetch failed: %s", url)
                 return []
-        if response.status_code != 200:
-            logger.warning("yandex serp returned %d for %s", response.status_code, url)
+
+            status = response.status_code
+
+            if status == 0:
+                # Tarpit / curl timeout — rotate IP and retry
+                logger.warning(
+                    "yandex: status=0 (tarpit?) rooms=%s page=%d — rotating IP + retry attempt %d",
+                    rooms,
+                    page,
+                    attempt + 1,
+                )
+                await self._rotate_ip()
+                await asyncio.sleep(2)
+                continue
+
+            if status != 200:
+                logger.warning("yandex serp returned %d for %s", status, url)
                 return []
 
-        # NBSP → space: parse_rub() buggy on \xa0 in capture groups → price=None (T4)
-        html = response.text.replace("\xa0", " ")
+            # NBSP → space: parse_rub() buggy on   in capture groups → price=None (T4)
+            raw_html = response.text.replace(" ", " ")
 
-        if _is_captcha(html):
+            if _is_captcha(raw_html):
                 logger.warning(
-                "yandex serp: captcha detected on page=%d url=%s — returning empty",
+                    "yandex serp: captcha detected rooms=%s page=%d url=%s"
+                    " — rotating IP + retry attempt %d",
+                    rooms,
                     page,
                     url,
+                    attempt + 1,
+                )
+                await self._rotate_ip()
+                await asyncio.sleep(2)
+                continue
+
+            html = raw_html
+            break
+
+        if html is None:
+            logger.warning(
+                "yandex serp: tarpit/captcha persisted after %d retries rooms=%s page=%d"
+                " — returning empty",
+                _YANDEX_TARPIT_MAX_RETRIES,
+                rooms,
+                page,
             )
             return []
 

tradein-mvp/backend/tests/test_yandex_realty_serp.py

@@ -9,6 +9,7 @@ import pytest
 from app.services.scrapers.base import ScrapedLot
 from app.services.scrapers.yandex_realty import (
     _CURL_STATUS_MARKER,
+    _YANDEX_TARPIT_MAX_RETRIES,
     DEFAULT_CITY,
     DEFAULT_PRICE_RANGES,
     MAX_PAGES,
@@ -1014,3 +1015,111 @@ async def test_fetch_all_secondary_deduplicates_across_rooms():
     assert "unique_001" in ids
     assert "unique_002" in ids
     assert len(ids) == 3  # shared deduplicated — only 3 unique
+
+
+# ── PART H: tarpit resilience — fetch_around rotate-on-status-0 / captcha ────
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_rotates_and_retries_on_status_0():
+    """fetch_around rotates IP and retries when status_code==0 (tarpit/curl-timeout).
+
+    Sequence: first call → status 0 (tarpit), second call → status 200 + valid HTML.
+    Expect: _rotate_ip called once, parsed lots returned from the second attempt.
+    """
+    s = YandexRealtyScraper()
+
+    # Valid SERP page that parses to >=1 lot
+    valid_html = SINGLE_CARD_HTML.replace(" ", " ")
+
+    call_count = 0
+
+    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
+        nonlocal call_count
+        call_count += 1
+        if call_count == 1:
+            # First attempt: tarpit — status 0
+            return _CurlResponse(status_code=0, text="")
+        # Second attempt: success
+        return _CurlResponse(status_code=200, text=valid_html)
+
+    mock_settings = MagicMock()
+    mock_settings.yandex_proxy_url = "socks5h://x:y@h:1"
+    mock_settings.yandex_proxy_rotate_url = "http://rotate.example.com/changeip"
+    mock_settings.avito_proxy_rotate_url = None
+
+    with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}):
+        with patch.object(s, "_http_get", side_effect=fake_http_get):
+            with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
+                with patch("asyncio.sleep", new_callable=AsyncMock):
+                    with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
+                        lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1")
+
+    assert (
+        mock_rotate.call_count == 1
+    ), f"_rotate_ip must be called once, got {mock_rotate.call_count}"
+    assert (
+        call_count == 2
+    ), f"_http_get must be called twice (tarpit then success), got {call_count}"
+    assert len(lots) >= 1, f"Expected >=1 parsed lots after retry, got {len(lots)}"
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_rotates_on_captcha():
+    """fetch_around rotates IP and retries when HTML is a captcha page."""
+    s = YandexRealtyScraper()
+
+    captcha_html = "<html><body><a href='/showcaptcha'>captcha</a></body></html>"
+    valid_html = SINGLE_CARD_HTML.replace(" ", " ")
+
+    call_count = 0
+
+    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
+        nonlocal call_count
+        call_count += 1
+        if call_count == 1:
+            return _CurlResponse(status_code=200, text=captcha_html)
+        return _CurlResponse(status_code=200, text=valid_html)
+
+    with patch.object(s, "_http_get", side_effect=fake_http_get):
+        with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
+            with patch("asyncio.sleep", new_callable=AsyncMock):
+                with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
+                    lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="2")
+
+    assert mock_rotate.call_count == 1
+    assert len(lots) >= 1
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_exhausts_retries_returns_empty():
+    """fetch_around returns [] when all retries are exhausted (tarpit persists)."""
+    s = YandexRealtyScraper()
+
+    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
+        return _CurlResponse(status_code=0, text="")
+
+    with patch.object(s, "_http_get", side_effect=fake_http_get):
+        with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
+            with patch("asyncio.sleep", new_callable=AsyncMock):
+                lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1")
+
+    # Rotated on every attempt: 1 initial + _YANDEX_TARPIT_MAX_RETRIES retries
+    assert mock_rotate.call_count == 1 + _YANDEX_TARPIT_MAX_RETRIES
+    assert lots == []
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_no_rotate_on_404():
+    """fetch_around does NOT rotate on non-zero non-200 status (e.g. 404)."""
+    s = YandexRealtyScraper()
+
+    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
+        return _CurlResponse(status_code=404, text="Not Found")
+
+    with patch.object(s, "_http_get", side_effect=fake_http_get):
+        with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
+            lots = await s.fetch_around(lat=56.84, lon=60.60)
+
+    assert mock_rotate.call_count == 0, "Must not rotate on 404 — only on status==0 / captcha"
+    assert lots == []