feat(tradein): dedicated Cian mobile proxy — separate egress from Avito (#923)

Co-authored-by: bot-backend <bot-backend@gendsgn.local>
Co-committed-by: bot-backend <bot-backend@gendsgn.local>

tradein-mvp/backend/app/core/config.py

@@ -198,6 +198,21 @@ class Settings(BaseSettings):
     # Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу).
     avito_proxy_max_rotations: int = 2
 
+    # ── Cian dedicated mobile proxy (separate egress from Avito) ──────────────
+    # Cian и Avito делят один мобильный IP при общем scraper_proxy_url → конкуренция
+    # за единственный egress → взаимные таймауты/баны при параллельных прогонах.
+    # Отдельный прокси для Cian устраняет contention. Если не задан — fallback на
+    # общий scraper_proxy_url (backward-compat). ENV: CIAN_PROXY_URL.
+    cian_proxy_url_env: str | None = Field(default=None, validation_alias="CIAN_PROXY_URL")
+    # changeip-ссылка для Cian-прокси (ротация IP при бане/таймауте). Если не задан —
+    # fallback на avito_proxy_rotate_url. ENV: CIAN_PROXY_ROTATE_URL.
+    cian_proxy_rotate_url: str | None = None
+
+    @property
+    def cian_proxy_url(self) -> str | None:
+        """Прокси для Cian-скраперов. CIAN_PROXY_URL > scraper_proxy_url (fallback)."""
+        return self.cian_proxy_url_env or self.scraper_proxy_url
+
     # ── #759: deactivate stale avito listings ───────────────────────────────
     # Avito-объявления, не виденные scraper'ом более avito_stale_ttl_days дней,
     # помечаются is_active=false ночной задачей deactivate_stale_avito_listings.

tradein-mvp/backend/app/services/cian_session.py

@@ -112,7 +112,7 @@ async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
         # proxies: mobile-proxy egress (#806) — Cian блокирует datacenter-IP даже
         # при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403.
         # Пусто (env не задан) → прямое подключение (dev/no-op).
-        _proxy_url = settings.scraper_proxy_url
+        _proxy_url = settings.cian_proxy_url
         _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
         async with AsyncSession(
             impersonate="chrome120",

tradein-mvp/backend/app/services/scrapers/cian.py

@@ -67,7 +67,7 @@ class CianScraper(BaseScraper):
         # proxies: mobile-proxy egress (#806) — аналогично Avito (#623). Без прокси
         # datacenter-IP блокируется Cian (403/captcha) несмотря на валидные cookies.
         # Пусто (SCRAPER_PROXY_URL/AVITO_PROXY_URL не заданы) → прямое подключение (dev).
-        _proxy_url = settings.scraper_proxy_url
+        _proxy_url = settings.cian_proxy_url
         _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
         self._cffi = AsyncSession(
             impersonate="chrome120",

tradein-mvp/backend/app/services/scrapers/cian_detail.py

@@ -81,7 +81,7 @@ async def fetch_detail(
     if session is None:
         # proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian.
         # Пусто (env не задан) → прямое подключение (dev/no-op).
-        _proxy_url = settings.scraper_proxy_url
+        _proxy_url = settings.cian_proxy_url
         _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
         session = AsyncSession(
             impersonate="chrome120",

tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py

@@ -96,7 +96,7 @@ async def fetch_newbuilding(
     if session is None:
         # Mobile proxy wiring (#806 follow-up): Cian ЖК страница — datacenter-IP бан.
         # proxy=None → прямое подключение (dev без прокси).
-        _proxy_url = settings.scraper_proxy_url
+        _proxy_url = settings.cian_proxy_url
         _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
         session = AsyncSession(
             impersonate="chrome120",
@@ -594,7 +594,7 @@ async def resolve_cian_zhk_url(
     close_session = False
     if session is None:
         # Mobile proxy wiring (#806 follow-up): resolve ЖК URL через мобильный прокси.
-        _proxy_url = settings.scraper_proxy_url
+        _proxy_url = settings.cian_proxy_url
         _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
         session = AsyncSession(
             impersonate="chrome120",

tradein-mvp/backend/app/services/scrapers/cian_valuation.py

@@ -176,8 +176,8 @@ async def estimate_via_cian_valuation(
 
     # 4. Fetch с TLS fingerprint (curl_cffi, impersonate chrome120)
     # Mobile proxy wiring (#806 follow-up): Cian валюация — такой же datacenter-бан риск
-    # как SERP. Используем scraper_proxy_url. proxy=None → прямое подключение (dev).
-    _proxy_url = settings.scraper_proxy_url
+    # как SERP. Используем cian_proxy_url. proxy=None → прямое подключение (dev).
+    _proxy_url = settings.cian_proxy_url
     _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
     try:
         async with AsyncSession(

tradein-mvp/backend/tests/test_scraper_proxy.py

@@ -427,3 +427,63 @@ async def test_avito_imv_own_session_receives_proxies():
                 pass  # network/parse failures expected in unit test
 
     assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
+
+
+# ── Settings.cian_proxy_url property ─────────────────────────────────────────
+# cian_proxy_url: CIAN_PROXY_URL > scraper_proxy_url (fallback) > None.
+
+
+def _resolve_cian_proxy_url(
+    cian_proxy_url_env: str | None, scraper_proxy_url: str | None
+) -> str | None:
+    """Inline replica of Settings.cian_proxy_url property logic."""
+    return cian_proxy_url_env or scraper_proxy_url
+
+
+def test_cian_proxy_url_uses_cian_env_when_set():
+    """CIAN_PROXY_URL takes precedence over shared scraper_proxy_url."""
+    result = _resolve_cian_proxy_url(
+        cian_proxy_url_env="http://cian-proxy:2222",
+        scraper_proxy_url="http://shared-proxy:1111",
+    )
+    assert result == "http://cian-proxy:2222"
+
+
+def test_cian_proxy_url_falls_back_to_scraper_proxy_url():
+    """Falls back to shared scraper_proxy_url when CIAN_PROXY_URL is absent."""
+    result = _resolve_cian_proxy_url(
+        cian_proxy_url_env=None,
+        scraper_proxy_url="http://shared-proxy:1111",
+    )
+    assert result == "http://shared-proxy:1111"
+
+
+def test_cian_proxy_url_none_when_both_absent():
+    result = _resolve_cian_proxy_url(cian_proxy_url_env=None, scraper_proxy_url=None)
+    assert result is None
+
+
+def test_settings_binds_cian_proxy_url_env(monkeypatch):
+    """CIAN_PROXY_URL env binds to cian_proxy_url_env via validation_alias."""
+    monkeypatch.setenv("CIAN_PROXY_URL", "http://cian-proxy:2222")
+    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
+    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
+    assert _fresh_settings().cian_proxy_url == "http://cian-proxy:2222"
+
+
+def test_settings_cian_env_overrides_scraper_proxy(monkeypatch):
+    """CIAN_PROXY_URL overrides shared SCRAPER_PROXY_URL for Cian scrapers."""
+    monkeypatch.setenv("CIAN_PROXY_URL", "http://cian-proxy:2222")
+    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://shared-proxy:1111")
+    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
+    s = _fresh_settings()
+    assert s.cian_proxy_url == "http://cian-proxy:2222"
+    assert s.scraper_proxy_url == "http://shared-proxy:1111"
+
+
+def test_settings_cian_proxy_falls_back_to_scraper(monkeypatch):
+    """Without CIAN_PROXY_URL, cian_proxy_url falls back to scraper_proxy_url."""
+    monkeypatch.delenv("CIAN_PROXY_URL", raising=False)
+    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://shared-proxy:1111")
+    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
+    assert _fresh_settings().cian_proxy_url == "http://shared-proxy:1111"