diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index 1d47f7cf..57b78340 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -198,6 +198,21 @@ class Settings(BaseSettings): # Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу). avito_proxy_max_rotations: int = 2 + # ── Cian dedicated mobile proxy (separate egress from Avito) ────────────── + # Cian и Avito делят один мобильный IP при общем scraper_proxy_url → конкуренция + # за единственный egress → взаимные таймауты/баны при параллельных прогонах. + # Отдельный прокси для Cian устраняет contention. Если не задан — fallback на + # общий scraper_proxy_url (backward-compat). ENV: CIAN_PROXY_URL. + cian_proxy_url_env: str | None = Field(default=None, validation_alias="CIAN_PROXY_URL") + # changeip-ссылка для Cian-прокси (ротация IP при бане/таймауте). Если не задан — + # fallback на avito_proxy_rotate_url. ENV: CIAN_PROXY_ROTATE_URL. + cian_proxy_rotate_url: str | None = None + + @property + def cian_proxy_url(self) -> str | None: + """Прокси для Cian-скраперов. CIAN_PROXY_URL > scraper_proxy_url (fallback).""" + return self.cian_proxy_url_env or self.scraper_proxy_url + # ── #759: deactivate stale avito listings ─────────────────────────────── # Avito-объявления, не виденные scraper'ом более avito_stale_ttl_days дней, # помечаются is_active=false ночной задачей deactivate_stale_avito_listings. diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py index 09ad69e8..89324edc 100644 --- a/tradein-mvp/backend/app/services/cian_session.py +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -112,7 +112,7 @@ async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None: # proxies: mobile-proxy egress (#806) — Cian блокирует datacenter-IP даже # при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403. # Пусто (env не задан) → прямое подключение (dev/no-op). - _proxy_url = settings.scraper_proxy_url + _proxy_url = settings.cian_proxy_url _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None async with AsyncSession( impersonate="chrome120", diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index cbace45b..13e0af0b 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -67,7 +67,7 @@ class CianScraper(BaseScraper): # proxies: mobile-proxy egress (#806) — аналогично Avito (#623). Без прокси # datacenter-IP блокируется Cian (403/captcha) несмотря на валидные cookies. # Пусто (SCRAPER_PROXY_URL/AVITO_PROXY_URL не заданы) → прямое подключение (dev). - _proxy_url = settings.scraper_proxy_url + _proxy_url = settings.cian_proxy_url _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None self._cffi = AsyncSession( impersonate="chrome120", diff --git a/tradein-mvp/backend/app/services/scrapers/cian_detail.py b/tradein-mvp/backend/app/services/scrapers/cian_detail.py index 56014d1f..d50495d6 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_detail.py @@ -81,7 +81,7 @@ async def fetch_detail( if session is None: # proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian. # Пусто (env не задан) → прямое подключение (dev/no-op). - _proxy_url = settings.scraper_proxy_url + _proxy_url = settings.cian_proxy_url _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None session = AsyncSession( impersonate="chrome120", diff --git a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py index 2ae7127e..639f9938 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py @@ -96,7 +96,7 @@ async def fetch_newbuilding( if session is None: # Mobile proxy wiring (#806 follow-up): Cian ЖК страница — datacenter-IP бан. # proxy=None → прямое подключение (dev без прокси). - _proxy_url = settings.scraper_proxy_url + _proxy_url = settings.cian_proxy_url _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None session = AsyncSession( impersonate="chrome120", @@ -594,7 +594,7 @@ async def resolve_cian_zhk_url( close_session = False if session is None: # Mobile proxy wiring (#806 follow-up): resolve ЖК URL через мобильный прокси. - _proxy_url = settings.scraper_proxy_url + _proxy_url = settings.cian_proxy_url _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None session = AsyncSession( impersonate="chrome120", diff --git a/tradein-mvp/backend/app/services/scrapers/cian_valuation.py b/tradein-mvp/backend/app/services/scrapers/cian_valuation.py index e5a4dd36..db0afe37 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_valuation.py @@ -176,8 +176,8 @@ async def estimate_via_cian_valuation( # 4. Fetch с TLS fingerprint (curl_cffi, impersonate chrome120) # Mobile proxy wiring (#806 follow-up): Cian валюация — такой же datacenter-бан риск - # как SERP. Используем scraper_proxy_url. proxy=None → прямое подключение (dev). - _proxy_url = settings.scraper_proxy_url + # как SERP. Используем cian_proxy_url. proxy=None → прямое подключение (dev). + _proxy_url = settings.cian_proxy_url _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None try: async with AsyncSession( diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py index 5e1aca70..6e0d7422 100644 --- a/tradein-mvp/backend/tests/test_scraper_proxy.py +++ b/tradein-mvp/backend/tests/test_scraper_proxy.py @@ -427,3 +427,63 @@ async def test_avito_imv_own_session_receives_proxies(): pass # network/parse failures expected in unit test assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES + + +# ── Settings.cian_proxy_url property ───────────────────────────────────────── +# cian_proxy_url: CIAN_PROXY_URL > scraper_proxy_url (fallback) > None. + + +def _resolve_cian_proxy_url( + cian_proxy_url_env: str | None, scraper_proxy_url: str | None +) -> str | None: + """Inline replica of Settings.cian_proxy_url property logic.""" + return cian_proxy_url_env or scraper_proxy_url + + +def test_cian_proxy_url_uses_cian_env_when_set(): + """CIAN_PROXY_URL takes precedence over shared scraper_proxy_url.""" + result = _resolve_cian_proxy_url( + cian_proxy_url_env="http://cian-proxy:2222", + scraper_proxy_url="http://shared-proxy:1111", + ) + assert result == "http://cian-proxy:2222" + + +def test_cian_proxy_url_falls_back_to_scraper_proxy_url(): + """Falls back to shared scraper_proxy_url when CIAN_PROXY_URL is absent.""" + result = _resolve_cian_proxy_url( + cian_proxy_url_env=None, + scraper_proxy_url="http://shared-proxy:1111", + ) + assert result == "http://shared-proxy:1111" + + +def test_cian_proxy_url_none_when_both_absent(): + result = _resolve_cian_proxy_url(cian_proxy_url_env=None, scraper_proxy_url=None) + assert result is None + + +def test_settings_binds_cian_proxy_url_env(monkeypatch): + """CIAN_PROXY_URL env binds to cian_proxy_url_env via validation_alias.""" + monkeypatch.setenv("CIAN_PROXY_URL", "http://cian-proxy:2222") + monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False) + monkeypatch.delenv("AVITO_PROXY_URL", raising=False) + assert _fresh_settings().cian_proxy_url == "http://cian-proxy:2222" + + +def test_settings_cian_env_overrides_scraper_proxy(monkeypatch): + """CIAN_PROXY_URL overrides shared SCRAPER_PROXY_URL for Cian scrapers.""" + monkeypatch.setenv("CIAN_PROXY_URL", "http://cian-proxy:2222") + monkeypatch.setenv("SCRAPER_PROXY_URL", "http://shared-proxy:1111") + monkeypatch.delenv("AVITO_PROXY_URL", raising=False) + s = _fresh_settings() + assert s.cian_proxy_url == "http://cian-proxy:2222" + assert s.scraper_proxy_url == "http://shared-proxy:1111" + + +def test_settings_cian_proxy_falls_back_to_scraper(monkeypatch): + """Without CIAN_PROXY_URL, cian_proxy_url falls back to scraper_proxy_url.""" + monkeypatch.delenv("CIAN_PROXY_URL", raising=False) + monkeypatch.setenv("SCRAPER_PROXY_URL", "http://shared-proxy:1111") + monkeypatch.delenv("AVITO_PROXY_URL", raising=False) + assert _fresh_settings().cian_proxy_url == "http://shared-proxy:1111"