tradein-mvp/backend/app/core/config.py

@@ -1,5 +1,6 @@
 """Минимальный settings для standalone trade-in MVP."""
 
+from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -132,13 +133,33 @@ class Settings(BaseSettings):
         "n1.ru",
     }
 
-    # ── Avito mobile proxy (#623) ─────────────────────────────────────────────
-    # Avito банит datacenter-IP сервера — на HTTP 200 отдаёт firewall-страницу
-    # «Доступ ограничен: проблема с IP». Мобильный прокси (mobileproxy.space)
-    # снимает бан. Пусто = прямое подключение (dev). ENV: AVITO_PROXY_URL.
-    avito_proxy_url: str | None = None
+    # ── Scraper mobile proxy (#806) ──────────────────────────────────────────
+    # Мобильный прокси (RU, mobileproxy.space) используется ВСЕМИ scraper-сессиями:
+    # Avito (#623) + Cian (#806). Datacenter-IP блокируется обоими сайтами.
+    # Пусто = прямое подключение (dev/staging без прокси).
+    #
+    # Приоритет ENV-переменных (precedence):
+    #   1. SCRAPER_PROXY_URL — новый общий ENV; когда задан — используется первым.
+    #   2. AVITO_PROXY_URL   — legacy ENV; fallback, чтобы prod-серверы с уже
+    #      настроенным AVITO_PROXY_URL работали без изменений .env.runtime (#806).
+    # property `scraper_proxy_url` реализует эту логику; используй его везде.
+    # validation_alias привязывает поле к env SCRAPER_PROXY_URL (без него
+    # pydantic-settings читал бы SCRAPER_PROXY_URL_ENV по имени поля — #806 fixup).
+    scraper_proxy_url_env: str | None = Field(default=None, validation_alias="SCRAPER_PROXY_URL")
+    avito_proxy_url: str | None = None  # ENV: AVITO_PROXY_URL (legacy fallback)
+
+    @property
+    def scraper_proxy_url(self) -> str | None:
+        """Единый прокси URL для всех scraper-сессий (Avito + Cian).
+
+        Приоритет: SCRAPER_PROXY_URL > AVITO_PROXY_URL > None (прямое подключение).
+        Prod-серверы с существующим AVITO_PROXY_URL работают без изменений env.
+        """
+        return self.scraper_proxy_url_env or self.avito_proxy_url
+
     # changeip-ссылка mobileproxy: GET меняет мобильный IP за ~9с. Дёргается при
-    # детекте бана перед повтором. Пусто = ротация выключена (raise сразу).
+    # детекте бана Avito перед повтором. Пусто = ротация выключена (raise сразу).
+    # ENV: AVITO_PROXY_ROTATE_URL.
     avito_proxy_rotate_url: str | None = None
     # Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу).
     avito_proxy_max_rotations: int = 2

tradein-mvp/backend/app/services/cian_session.py

@@ -105,10 +105,16 @@ async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
     Никогда не логирует сырые значения cookies.
     """
     try:
+        # proxies: mobile-proxy egress (#806) — Cian блокирует datacenter-IP даже
+        # при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403.
+        # Пусто (env не задан) → прямое подключение (dev/no-op).
+        _proxy_url = settings.scraper_proxy_url
+        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
         async with AsyncSession(
             impersonate="chrome120",
             cookies=cookies,
             timeout=20.0,
+            proxies=_proxies,
         ) as session:
             resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True)
             status = resp.status_code

tradein-mvp/backend/app/services/scrape_pipeline.py

@@ -68,15 +68,17 @@ _CHROME_HEADERS = {
 
 
 def _avito_proxies() -> dict[str, str] | None:
-    """#623: mobile-proxy egress для shared-сессии sweep'а.
+    """#623/#806: mobile-proxy egress для shared-сессии sweep'а.
 
     КРИТИЧНО: sweep создаёт собственную shared AsyncSession и присваивает её
     `scraper._cffi`, перезатирая проксированную сессию из `AvitoScraper.__aenter__`.
     Через эту же сессию идёт и SERP, и detail-энричмент (fetch_detail). Без прокси
     detail-страницы летят с datacenter-IP → HTTP 429 → весь run mark_banned.
     Пусто (env не задан) → прямое подключение (dev/no-op).
+
+    Читает settings.scraper_proxy_url (SCRAPER_PROXY_URL > AVITO_PROXY_URL fallback).
     """
-    url = settings.avito_proxy_url
+    url = settings.scraper_proxy_url
     return {"http": url, "https": url} if url else None
 
 

tradein-mvp/backend/app/services/scrapers/avito.py

@@ -211,8 +211,8 @@ class AvitoScraper(BaseScraper):
     async def __aenter__(self) -> AvitoScraper:
         await super().__aenter__()
         proxies = None
-        if settings.avito_proxy_url:
-            proxies = {"http": settings.avito_proxy_url, "https": settings.avito_proxy_url}
+        if settings.scraper_proxy_url:
+            proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url}
             logger.info("avito: routing through mobile proxy egress (#623)")
         self._cffi = AsyncSession(
             impersonate="chrome120",

tradein-mvp/backend/app/services/scrapers/cian.py

@@ -23,6 +23,7 @@ from urllib.parse import urlencode
 
 from curl_cffi.requests import AsyncSession
 
+from app.core.config import settings
 from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper, ScrapedLot
 from app.services.scrapers.cian_state_parser import extract_state
@@ -61,10 +62,16 @@ class CianScraper(BaseScraper):
 
     async def __aenter__(self) -> CianScraper:
         await super().__aenter__()
-        # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian
+        # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian.
+        # proxies: mobile-proxy egress (#806) — аналогично Avito (#623). Без прокси
+        # datacenter-IP блокируется Cian (403/captcha) несмотря на валидные cookies.
+        # Пусто (SCRAPER_PROXY_URL/AVITO_PROXY_URL не заданы) → прямое подключение (dev).
+        _proxy_url = settings.scraper_proxy_url
+        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
         self._cffi = AsyncSession(
             impersonate="chrome120",
             timeout=30,
+            proxies=_proxies,
             headers={
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                 "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
@@ -114,7 +121,10 @@ class CianScraper(BaseScraper):
         lots = self._parse_serp_html(response.text)
         logger.info(
             "cian: %d lots fetched rooms=%s page=%d url=%s",
-            len(lots), rooms, page, url,
+            len(lots),
+            rooms,
+            page,
+            url,
         )
         await self.sleep_between_requests()
         return lots
@@ -136,9 +146,7 @@ class CianScraper(BaseScraper):
                 try:
                     lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page)
                 except Exception:
-                    logger.exception(
-                        "cian multi-room fetch failed rooms=%s page=%d", rooms, page
-                    )
+                    logger.exception("cian multi-room fetch failed rooms=%s page=%d", rooms, page)
                     continue
                 if not lots:
                     break  # пустая страница → дальше смысла нет
@@ -148,7 +156,10 @@ class CianScraper(BaseScraper):
                         seen[key] = lot
         logger.info(
             "cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)",
-            len(seen), lat, lon, radius_m,
+            len(seen),
+            lat,
+            lon,
+            radius_m,
         )
         return list(seen.values())
 
@@ -187,13 +198,12 @@ class CianScraper(BaseScraper):
             logger.warning(
                 "cian SERP state extraction failed (mfe=%s key=%s) — "
                 "возможно Cian изменил структуру или вернул captcha",
-                _MFE_SERP, _STATE_KEY,
+                _MFE_SERP,
+                _STATE_KEY,
             )
             return []
 
-        offers_data: list[dict[str, Any]] = (
-            state.get("results", {}).get("offers", [])
-        )
+        offers_data: list[dict[str, Any]] = state.get("results", {}).get("offers", [])
         if not offers_data:
             logger.warning(
                 "cian state found but results.offers пуст (totalOffers=%s)",
@@ -299,7 +309,8 @@ class CianScraper(BaseScraper):
             if lat is None or lon is None:
                 logger.debug(
                     "cian offer %s: нет точных координат в state (geo=%s)",
-                    offer_id, coords,
+                    offer_id,
+                    coords,
                 )
                 lat = lon = None
 

tradein-mvp/backend/app/services/scrapers/cian_detail.py

@@ -9,6 +9,7 @@ Different from SERP (cian.py):
 Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
 Stage 5 of CianScraper v1.
 """
+
 from __future__ import annotations
 
 import json
@@ -20,6 +21,7 @@ from curl_cffi.requests import AsyncSession
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
+from app.core.config import settings
 from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
 from app.services.scrapers.repair_state_normalizer import (
     infer_repair_state_from_text,
@@ -36,17 +38,17 @@ class DetailEnrichment:
 
     # Core listing enrichment (extend existing listings row)
     cian_id: int | None = None
-    windows_view_type: str | None = None        # 'yard' / 'street' / 'yardAndStreet'
+    windows_view_type: str | None = None  # 'yard' / 'street' / 'yardAndStreet'
     separate_wcs_count: int | None = None
     combined_wcs_count: int | None = None
-    ceiling_height: float | None = None         # meters
-    repair_type: str | None = None              # 'cosmetic' / 'design' / 'no' — raw Cian value
-    repair_state: str | None = None             # enum: needs_repair/standard/good/excellent
-    views_total: int | None = None              # Cian stats.totalViewsFormattedString → int
+    ceiling_height: float | None = None  # meters
+    repair_type: str | None = None  # 'cosmetic' / 'design' / 'no' — raw Cian value
+    repair_state: str | None = None  # enum: needs_repair/standard/good/excellent
+    views_total: int | None = None  # Cian stats.totalViewsFormattedString → int
     views_today: int | None = None
 
     # BTI (вторичка only — primary buildings don't have БТИ)
-    bti_data: dict[str, Any] | None = None      # raw bti.houseData snapshot
+    bti_data: dict[str, Any] | None = None  # raw bti.houseData snapshot
 
     # Price history (offer.priceChanges)
     price_changes: list[dict[str, Any]] = field(default_factory=list)
@@ -77,9 +79,14 @@ async def fetch_detail(
     """
     close_session = False
     if session is None:
+        # proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian.
+        # Пусто (env не задан) → прямое подключение (dev/no-op).
+        _proxy_url = settings.scraper_proxy_url
+        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
         session = AsyncSession(
             impersonate="chrome120",
             timeout=25.0,
+            proxies=_proxies,
             headers={
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                 "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",

tradein-mvp/backend/tests/test_scraper_proxy.py

@@ -0,0 +1,142 @@
+"""Tests for scraper proxy wiring (#806).
+
+Covers:
+  - scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL
+  - AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
+  - None/empty → direct connection (no proxies dict)
+  - _avito_proxies() helper returns correct dict shape
+"""
+
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
+
+from types import SimpleNamespace
+from unittest.mock import patch
+
+
+def _mock_settings(scraper_proxy_url: str | None) -> SimpleNamespace:
+    """Minimal settings stand-in with only the fields the proxy helpers read."""
+    return SimpleNamespace(scraper_proxy_url=scraper_proxy_url)
+
+
+# ── Settings.scraper_proxy_url property ──────────────────────────────────────
+# Test the property logic directly via inline replica — avoids instantiating
+# pydantic-settings (requires DATABASE_URL) for pure unit tests of the rule.
+
+
+def _resolve_scraper_proxy_url(
+    scraper_proxy_url_env: str | None, avito_proxy_url: str | None
+) -> str | None:
+    """Inline replica of Settings.scraper_proxy_url property logic."""
+    return scraper_proxy_url_env or avito_proxy_url
+
+
+def test_scraper_proxy_url_uses_scraper_env_when_set():
+    result = _resolve_scraper_proxy_url(
+        scraper_proxy_url_env="http://scraper-proxy:1234",
+        avito_proxy_url="http://avito-proxy:5678",
+    )
+    assert result == "http://scraper-proxy:1234"
+
+
+def test_scraper_proxy_url_falls_back_to_avito_proxy_url():
+    """AVITO_PROXY_URL used when SCRAPER_PROXY_URL absent — zero env change on prod."""
+    result = _resolve_scraper_proxy_url(
+        scraper_proxy_url_env=None,
+        avito_proxy_url="http://avito-proxy:5678",
+    )
+    assert result == "http://avito-proxy:5678"
+
+
+def test_scraper_proxy_url_none_when_both_absent():
+    result = _resolve_scraper_proxy_url(scraper_proxy_url_env=None, avito_proxy_url=None)
+    assert result is None
+
+
+def test_scraper_proxy_url_empty_string_falls_through_to_avito():
+    """Empty string is falsy — falls through to avito_proxy_url."""
+    result = _resolve_scraper_proxy_url(
+        scraper_proxy_url_env="",
+        avito_proxy_url="http://avito-proxy:5678",
+    )
+    assert result == "http://avito-proxy:5678"
+
+
+# ── _avito_proxies() dict shape ───────────────────────────────────────────────
+
+
+def test_avito_proxies_returns_correct_dict_shape():
+    from app.services.scrape_pipeline import _avito_proxies
+
+    proxy_url = "http://user:pass@proxy.example.com:8080"
+    with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)):
+        result = _avito_proxies()
+
+    assert result == {"http": proxy_url, "https": proxy_url}
+
+
+def test_avito_proxies_returns_none_when_no_proxy():
+    from app.services.scrape_pipeline import _avito_proxies
+
+    with patch("app.services.scrape_pipeline.settings", _mock_settings(None)):
+        result = _avito_proxies()
+
+    assert result is None
+
+
+def test_avito_proxies_fallback_via_avito_proxy_url():
+    """_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback.
+
+    When settings.scraper_proxy_url resolves to the avito URL (fallback path),
+    the dict shape must be {"http": url, "https": url}.
+    """
+    from app.services.scrape_pipeline import _avito_proxies
+
+    avito_url = "http://mobile-proxy.space:9090"
+    with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)):
+        result = _avito_proxies()
+
+    assert result == {"http": avito_url, "https": avito_url}
+
+
+# ── Real pydantic env-binding (regression for #806 fixup) ─────────────────────
+# The inline-replica tests above prove the OR-logic but NOT that the field
+# actually binds to env SCRAPER_PROXY_URL. `validation_alias` is what makes that
+# work — without it pydantic-settings reads SCRAPER_PROXY_URL_ENV (the field
+# name), so setting SCRAPER_PROXY_URL would be a silent no-op. These tests
+# construct a real Settings and assert end-to-end env binding.
+
+
+def _fresh_settings():
+    from app.core.config import Settings
+
+    return Settings(_env_file=None)
+
+
+def test_settings_binds_scraper_proxy_url_env(monkeypatch):
+    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234")
+    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
+    assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234"
+
+
+def test_settings_scraper_env_overrides_avito(monkeypatch):
+    monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234")
+    monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678")
+    assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234"
+
+
+def test_settings_falls_back_to_avito_env(monkeypatch):
+    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
+    monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678")
+    assert _fresh_settings().scraper_proxy_url == "http://avito-proxy:5678"
+
+
+def test_settings_ignores_misnamed_scraper_proxy_url_env(monkeypatch):
+    """Regression: the old wrong env name SCRAPER_PROXY_URL_ENV must NOT bind."""
+    monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
+    monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
+    monkeypatch.setenv("SCRAPER_PROXY_URL_ENV", "http://wrong-name:1111")
+    assert _fresh_settings().scraper_proxy_url is None