diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index d5faf559..2c92f3f4 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -132,13 +132,31 @@ class Settings(BaseSettings): "n1.ru", } - # ── Avito mobile proxy (#623) ───────────────────────────────────────────── - # Avito банит datacenter-IP сервера — на HTTP 200 отдаёт firewall-страницу - # «Доступ ограничен: проблема с IP». Мобильный прокси (mobileproxy.space) - # снимает бан. Пусто = прямое подключение (dev). ENV: AVITO_PROXY_URL. - avito_proxy_url: str | None = None + # ── Scraper mobile proxy (#806) ────────────────────────────────────────── + # Мобильный прокси (RU, mobileproxy.space) используется ВСЕМИ scraper-сессиями: + # Avito (#623) + Cian (#806). Datacenter-IP блокируется обоими сайтами. + # Пусто = прямое подключение (dev/staging без прокси). + # + # Приоритет ENV-переменных (precedence): + # 1. SCRAPER_PROXY_URL — новый общий ENV; когда задан — используется первым. + # 2. AVITO_PROXY_URL — legacy ENV; fallback, чтобы prod-серверы с уже + # настроенным AVITO_PROXY_URL работали без изменений .env.runtime (#806). + # property `scraper_proxy_url` реализует эту логику; используй его везде. + scraper_proxy_url_env: str | None = None # ENV: SCRAPER_PROXY_URL (новый общий) + avito_proxy_url: str | None = None # ENV: AVITO_PROXY_URL (legacy fallback) + + @property + def scraper_proxy_url(self) -> str | None: + """Единый прокси URL для всех scraper-сессий (Avito + Cian). + + Приоритет: SCRAPER_PROXY_URL > AVITO_PROXY_URL > None (прямое подключение). + Prod-серверы с существующим AVITO_PROXY_URL работают без изменений env. + """ + return self.scraper_proxy_url_env or self.avito_proxy_url + # changeip-ссылка mobileproxy: GET меняет мобильный IP за ~9с. Дёргается при - # детекте бана перед повтором. Пусто = ротация выключена (raise сразу). + # детекте бана Avito перед повтором. Пусто = ротация выключена (raise сразу). + # ENV: AVITO_PROXY_ROTATE_URL. avito_proxy_rotate_url: str | None = None # Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу). avito_proxy_max_rotations: int = 2 diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py index ee45331d..3717355a 100644 --- a/tradein-mvp/backend/app/services/cian_session.py +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -105,10 +105,16 @@ async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None: Никогда не логирует сырые значения cookies. """ try: + # proxies: mobile-proxy egress (#806) — Cian блокирует datacenter-IP даже + # при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403. + # Пусто (env не задан) → прямое подключение (dev/no-op). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None async with AsyncSession( impersonate="chrome120", cookies=cookies, timeout=20.0, + proxies=_proxies, ) as session: resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True) status = resp.status_code diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index fbc4b2ee..382bb53c 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -68,15 +68,17 @@ _CHROME_HEADERS = { def _avito_proxies() -> dict[str, str] | None: - """#623: mobile-proxy egress для shared-сессии sweep'а. + """#623/#806: mobile-proxy egress для shared-сессии sweep'а. КРИТИЧНО: sweep создаёт собственную shared AsyncSession и присваивает её `scraper._cffi`, перезатирая проксированную сессию из `AvitoScraper.__aenter__`. Через эту же сессию идёт и SERP, и detail-энричмент (fetch_detail). Без прокси detail-страницы летят с datacenter-IP → HTTP 429 → весь run mark_banned. Пусто (env не задан) → прямое подключение (dev/no-op). + + Читает settings.scraper_proxy_url (SCRAPER_PROXY_URL > AVITO_PROXY_URL fallback). """ - url = settings.avito_proxy_url + url = settings.scraper_proxy_url return {"http": url, "https": url} if url else None diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 0e55533f..00380ed9 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -211,8 +211,8 @@ class AvitoScraper(BaseScraper): async def __aenter__(self) -> AvitoScraper: await super().__aenter__() proxies = None - if settings.avito_proxy_url: - proxies = {"http": settings.avito_proxy_url, "https": settings.avito_proxy_url} + if settings.scraper_proxy_url: + proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url} logger.info("avito: routing through mobile proxy egress (#623)") self._cffi = AsyncSession( impersonate="chrome120", diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 817fbeb1..42d0adf0 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -23,6 +23,7 @@ from urllib.parse import urlencode from curl_cffi.requests import AsyncSession +from app.core.config import settings from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.cian_state_parser import extract_state @@ -61,10 +62,16 @@ class CianScraper(BaseScraper): async def __aenter__(self) -> CianScraper: await super().__aenter__() - # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian + # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian. + # proxies: mobile-proxy egress (#806) — аналогично Avito (#623). Без прокси + # datacenter-IP блокируется Cian (403/captcha) несмотря на валидные cookies. + # Пусто (SCRAPER_PROXY_URL/AVITO_PROXY_URL не заданы) → прямое подключение (dev). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None self._cffi = AsyncSession( impersonate="chrome120", timeout=30, + proxies=_proxies, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", @@ -114,7 +121,10 @@ class CianScraper(BaseScraper): lots = self._parse_serp_html(response.text) logger.info( "cian: %d lots fetched rooms=%s page=%d url=%s", - len(lots), rooms, page, url, + len(lots), + rooms, + page, + url, ) await self.sleep_between_requests() return lots @@ -136,9 +146,7 @@ class CianScraper(BaseScraper): try: lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page) except Exception: - logger.exception( - "cian multi-room fetch failed rooms=%s page=%d", rooms, page - ) + logger.exception("cian multi-room fetch failed rooms=%s page=%d", rooms, page) continue if not lots: break # пустая страница → дальше смысла нет @@ -148,7 +156,10 @@ class CianScraper(BaseScraper): seen[key] = lot logger.info( "cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)", - len(seen), lat, lon, radius_m, + len(seen), + lat, + lon, + radius_m, ) return list(seen.values()) @@ -187,13 +198,12 @@ class CianScraper(BaseScraper): logger.warning( "cian SERP state extraction failed (mfe=%s key=%s) — " "возможно Cian изменил структуру или вернул captcha", - _MFE_SERP, _STATE_KEY, + _MFE_SERP, + _STATE_KEY, ) return [] - offers_data: list[dict[str, Any]] = ( - state.get("results", {}).get("offers", []) - ) + offers_data: list[dict[str, Any]] = state.get("results", {}).get("offers", []) if not offers_data: logger.warning( "cian state found but results.offers пуст (totalOffers=%s)", @@ -299,7 +309,8 @@ class CianScraper(BaseScraper): if lat is None or lon is None: logger.debug( "cian offer %s: нет точных координат в state (geo=%s)", - offer_id, coords, + offer_id, + coords, ) lat = lon = None diff --git a/tradein-mvp/backend/app/services/scrapers/cian_detail.py b/tradein-mvp/backend/app/services/scrapers/cian_detail.py index 9054193f..56014d1f 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_detail.py @@ -9,6 +9,7 @@ Different from SERP (cian.py): Parses ~88 offer fields + sister data → DetailEnrichment dataclass. Stage 5 of CianScraper v1. """ + from __future__ import annotations import json @@ -20,6 +21,7 @@ from curl_cffi.requests import AsyncSession from sqlalchemy import text from sqlalchemy.orm import Session +from app.core.config import settings from app.services.scrapers.cian_state_parser import extract_all_states, extract_state from app.services.scrapers.repair_state_normalizer import ( infer_repair_state_from_text, @@ -36,17 +38,17 @@ class DetailEnrichment: # Core listing enrichment (extend existing listings row) cian_id: int | None = None - windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet' + windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet' separate_wcs_count: int | None = None combined_wcs_count: int | None = None - ceiling_height: float | None = None # meters - repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value - repair_state: str | None = None # enum: needs_repair/standard/good/excellent - views_total: int | None = None # Cian stats.totalViewsFormattedString → int + ceiling_height: float | None = None # meters + repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value + repair_state: str | None = None # enum: needs_repair/standard/good/excellent + views_total: int | None = None # Cian stats.totalViewsFormattedString → int views_today: int | None = None # BTI (вторичка only — primary buildings don't have БТИ) - bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot + bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot # Price history (offer.priceChanges) price_changes: list[dict[str, Any]] = field(default_factory=list) @@ -77,9 +79,14 @@ async def fetch_detail( """ close_session = False if session is None: + # proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian. + # Пусто (env не задан) → прямое подключение (dev/no-op). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None session = AsyncSession( impersonate="chrome120", timeout=25.0, + proxies=_proxies, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py new file mode 100644 index 00000000..8feee9e6 --- /dev/null +++ b/tradein-mvp/backend/tests/test_scraper_proxy.py @@ -0,0 +1,102 @@ +"""Tests for scraper proxy wiring (#806). + +Covers: + - scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL + - AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent + - None/empty → direct connection (no proxies dict) + - _avito_proxies() helper returns correct dict shape +""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from types import SimpleNamespace +from unittest.mock import patch + + +def _mock_settings(scraper_proxy_url: str | None) -> SimpleNamespace: + """Minimal settings stand-in with only the fields the proxy helpers read.""" + return SimpleNamespace(scraper_proxy_url=scraper_proxy_url) + + +# ── Settings.scraper_proxy_url property ────────────────────────────────────── +# Test the property logic directly via inline replica — avoids instantiating +# pydantic-settings (requires DATABASE_URL) for pure unit tests of the rule. + + +def _resolve_scraper_proxy_url( + scraper_proxy_url_env: str | None, avito_proxy_url: str | None +) -> str | None: + """Inline replica of Settings.scraper_proxy_url property logic.""" + return scraper_proxy_url_env or avito_proxy_url + + +def test_scraper_proxy_url_uses_scraper_env_when_set(): + result = _resolve_scraper_proxy_url( + scraper_proxy_url_env="http://scraper-proxy:1234", + avito_proxy_url="http://avito-proxy:5678", + ) + assert result == "http://scraper-proxy:1234" + + +def test_scraper_proxy_url_falls_back_to_avito_proxy_url(): + """AVITO_PROXY_URL used when SCRAPER_PROXY_URL absent — zero env change on prod.""" + result = _resolve_scraper_proxy_url( + scraper_proxy_url_env=None, + avito_proxy_url="http://avito-proxy:5678", + ) + assert result == "http://avito-proxy:5678" + + +def test_scraper_proxy_url_none_when_both_absent(): + result = _resolve_scraper_proxy_url(scraper_proxy_url_env=None, avito_proxy_url=None) + assert result is None + + +def test_scraper_proxy_url_empty_string_falls_through_to_avito(): + """Empty string is falsy — falls through to avito_proxy_url.""" + result = _resolve_scraper_proxy_url( + scraper_proxy_url_env="", + avito_proxy_url="http://avito-proxy:5678", + ) + assert result == "http://avito-proxy:5678" + + +# ── _avito_proxies() dict shape ─────────────────────────────────────────────── + + +def test_avito_proxies_returns_correct_dict_shape(): + from app.services.scrape_pipeline import _avito_proxies + + proxy_url = "http://user:pass@proxy.example.com:8080" + with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)): + result = _avito_proxies() + + assert result == {"http": proxy_url, "https": proxy_url} + + +def test_avito_proxies_returns_none_when_no_proxy(): + from app.services.scrape_pipeline import _avito_proxies + + with patch("app.services.scrape_pipeline.settings", _mock_settings(None)): + result = _avito_proxies() + + assert result is None + + +def test_avito_proxies_fallback_via_avito_proxy_url(): + """_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback. + + When settings.scraper_proxy_url resolves to the avito URL (fallback path), + the dict shape must be {"http": url, "https": url}. + """ + from app.services.scrape_pipeline import _avito_proxies + + avito_url = "http://mobile-proxy.space:9090" + with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)): + result = _avito_proxies() + + assert result == {"http": avito_url, "https": avito_url}