From e5f4e37415a0d2f571d0b062601b2c2511cf1b1d Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 30 May 2026 22:01:21 +0300 Subject: [PATCH 1/2] feat(scrapers): route cian sessions through mobile proxy (Refs #806) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Снимает egress-часть #639: cian-скрейперы ходили с datacenter-IP (без proxies) → даже валидные куки не фетчили с прод-бокса. Прокидываем тот же mobile-proxy, что у avito (#623/#805). - config: generic scraper_proxy_url (env SCRAPER_PROXY_URL → fallback AVITO_PROXY_URL → None/direct). Прод .env.runtime с AVITO_PROXY_URL работает без изменений env. - avito.py + scrape_pipeline._avito_proxies(): на settings.scraper_proxy_url (поведение идентично). - cian.py / cian_detail.fetch_detail / cian_session.verify_session: proxies= {http,https} из settings.scraper_proxy_url; пусто → direct (dev no-op). Cookie-refresh DMIR_AUTH (#639) — НЕ тронут (needs-human). 7 тестов (precedence + dict-shape + fallback), ruff clean. --- tradein-mvp/backend/app/core/config.py | 30 ++++-- .../backend/app/services/cian_session.py | 6 ++ .../backend/app/services/scrape_pipeline.py | 6 +- .../backend/app/services/scrapers/avito.py | 4 +- .../backend/app/services/scrapers/cian.py | 33 ++++-- .../app/services/scrapers/cian_detail.py | 19 ++-- .../backend/tests/test_scraper_proxy.py | 102 ++++++++++++++++++ 7 files changed, 173 insertions(+), 27 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_scraper_proxy.py diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index d5faf559..2c92f3f4 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -132,13 +132,31 @@ class Settings(BaseSettings): "n1.ru", } - # ── Avito mobile proxy (#623) ───────────────────────────────────────────── - # Avito банит datacenter-IP сервера — на HTTP 200 отдаёт firewall-страницу - # «Доступ ограничен: проблема с IP». Мобильный прокси (mobileproxy.space) - # снимает бан. Пусто = прямое подключение (dev). ENV: AVITO_PROXY_URL. - avito_proxy_url: str | None = None + # ── Scraper mobile proxy (#806) ────────────────────────────────────────── + # Мобильный прокси (RU, mobileproxy.space) используется ВСЕМИ scraper-сессиями: + # Avito (#623) + Cian (#806). Datacenter-IP блокируется обоими сайтами. + # Пусто = прямое подключение (dev/staging без прокси). + # + # Приоритет ENV-переменных (precedence): + # 1. SCRAPER_PROXY_URL — новый общий ENV; когда задан — используется первым. + # 2. AVITO_PROXY_URL — legacy ENV; fallback, чтобы prod-серверы с уже + # настроенным AVITO_PROXY_URL работали без изменений .env.runtime (#806). + # property `scraper_proxy_url` реализует эту логику; используй его везде. + scraper_proxy_url_env: str | None = None # ENV: SCRAPER_PROXY_URL (новый общий) + avito_proxy_url: str | None = None # ENV: AVITO_PROXY_URL (legacy fallback) + + @property + def scraper_proxy_url(self) -> str | None: + """Единый прокси URL для всех scraper-сессий (Avito + Cian). + + Приоритет: SCRAPER_PROXY_URL > AVITO_PROXY_URL > None (прямое подключение). + Prod-серверы с существующим AVITO_PROXY_URL работают без изменений env. + """ + return self.scraper_proxy_url_env or self.avito_proxy_url + # changeip-ссылка mobileproxy: GET меняет мобильный IP за ~9с. Дёргается при - # детекте бана перед повтором. Пусто = ротация выключена (raise сразу). + # детекте бана Avito перед повтором. Пусто = ротация выключена (raise сразу). + # ENV: AVITO_PROXY_ROTATE_URL. avito_proxy_rotate_url: str | None = None # Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу). avito_proxy_max_rotations: int = 2 diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py index ee45331d..3717355a 100644 --- a/tradein-mvp/backend/app/services/cian_session.py +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -105,10 +105,16 @@ async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None: Никогда не логирует сырые значения cookies. """ try: + # proxies: mobile-proxy egress (#806) — Cian блокирует datacenter-IP даже + # при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403. + # Пусто (env не задан) → прямое подключение (dev/no-op). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None async with AsyncSession( impersonate="chrome120", cookies=cookies, timeout=20.0, + proxies=_proxies, ) as session: resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True) status = resp.status_code diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index fbc4b2ee..382bb53c 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -68,15 +68,17 @@ _CHROME_HEADERS = { def _avito_proxies() -> dict[str, str] | None: - """#623: mobile-proxy egress для shared-сессии sweep'а. + """#623/#806: mobile-proxy egress для shared-сессии sweep'а. КРИТИЧНО: sweep создаёт собственную shared AsyncSession и присваивает её `scraper._cffi`, перезатирая проксированную сессию из `AvitoScraper.__aenter__`. Через эту же сессию идёт и SERP, и detail-энричмент (fetch_detail). Без прокси detail-страницы летят с datacenter-IP → HTTP 429 → весь run mark_banned. Пусто (env не задан) → прямое подключение (dev/no-op). + + Читает settings.scraper_proxy_url (SCRAPER_PROXY_URL > AVITO_PROXY_URL fallback). """ - url = settings.avito_proxy_url + url = settings.scraper_proxy_url return {"http": url, "https": url} if url else None diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 0e55533f..00380ed9 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -211,8 +211,8 @@ class AvitoScraper(BaseScraper): async def __aenter__(self) -> AvitoScraper: await super().__aenter__() proxies = None - if settings.avito_proxy_url: - proxies = {"http": settings.avito_proxy_url, "https": settings.avito_proxy_url} + if settings.scraper_proxy_url: + proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url} logger.info("avito: routing through mobile proxy egress (#623)") self._cffi = AsyncSession( impersonate="chrome120", diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 817fbeb1..42d0adf0 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -23,6 +23,7 @@ from urllib.parse import urlencode from curl_cffi.requests import AsyncSession +from app.core.config import settings from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.cian_state_parser import extract_state @@ -61,10 +62,16 @@ class CianScraper(BaseScraper): async def __aenter__(self) -> CianScraper: await super().__aenter__() - # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian + # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian. + # proxies: mobile-proxy egress (#806) — аналогично Avito (#623). Без прокси + # datacenter-IP блокируется Cian (403/captcha) несмотря на валидные cookies. + # Пусто (SCRAPER_PROXY_URL/AVITO_PROXY_URL не заданы) → прямое подключение (dev). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None self._cffi = AsyncSession( impersonate="chrome120", timeout=30, + proxies=_proxies, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", @@ -114,7 +121,10 @@ class CianScraper(BaseScraper): lots = self._parse_serp_html(response.text) logger.info( "cian: %d lots fetched rooms=%s page=%d url=%s", - len(lots), rooms, page, url, + len(lots), + rooms, + page, + url, ) await self.sleep_between_requests() return lots @@ -136,9 +146,7 @@ class CianScraper(BaseScraper): try: lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page) except Exception: - logger.exception( - "cian multi-room fetch failed rooms=%s page=%d", rooms, page - ) + logger.exception("cian multi-room fetch failed rooms=%s page=%d", rooms, page) continue if not lots: break # пустая страница → дальше смысла нет @@ -148,7 +156,10 @@ class CianScraper(BaseScraper): seen[key] = lot logger.info( "cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)", - len(seen), lat, lon, radius_m, + len(seen), + lat, + lon, + radius_m, ) return list(seen.values()) @@ -187,13 +198,12 @@ class CianScraper(BaseScraper): logger.warning( "cian SERP state extraction failed (mfe=%s key=%s) — " "возможно Cian изменил структуру или вернул captcha", - _MFE_SERP, _STATE_KEY, + _MFE_SERP, + _STATE_KEY, ) return [] - offers_data: list[dict[str, Any]] = ( - state.get("results", {}).get("offers", []) - ) + offers_data: list[dict[str, Any]] = state.get("results", {}).get("offers", []) if not offers_data: logger.warning( "cian state found but results.offers пуст (totalOffers=%s)", @@ -299,7 +309,8 @@ class CianScraper(BaseScraper): if lat is None or lon is None: logger.debug( "cian offer %s: нет точных координат в state (geo=%s)", - offer_id, coords, + offer_id, + coords, ) lat = lon = None diff --git a/tradein-mvp/backend/app/services/scrapers/cian_detail.py b/tradein-mvp/backend/app/services/scrapers/cian_detail.py index 9054193f..56014d1f 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_detail.py @@ -9,6 +9,7 @@ Different from SERP (cian.py): Parses ~88 offer fields + sister data → DetailEnrichment dataclass. Stage 5 of CianScraper v1. """ + from __future__ import annotations import json @@ -20,6 +21,7 @@ from curl_cffi.requests import AsyncSession from sqlalchemy import text from sqlalchemy.orm import Session +from app.core.config import settings from app.services.scrapers.cian_state_parser import extract_all_states, extract_state from app.services.scrapers.repair_state_normalizer import ( infer_repair_state_from_text, @@ -36,17 +38,17 @@ class DetailEnrichment: # Core listing enrichment (extend existing listings row) cian_id: int | None = None - windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet' + windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet' separate_wcs_count: int | None = None combined_wcs_count: int | None = None - ceiling_height: float | None = None # meters - repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value - repair_state: str | None = None # enum: needs_repair/standard/good/excellent - views_total: int | None = None # Cian stats.totalViewsFormattedString → int + ceiling_height: float | None = None # meters + repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value + repair_state: str | None = None # enum: needs_repair/standard/good/excellent + views_total: int | None = None # Cian stats.totalViewsFormattedString → int views_today: int | None = None # BTI (вторичка only — primary buildings don't have БТИ) - bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot + bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot # Price history (offer.priceChanges) price_changes: list[dict[str, Any]] = field(default_factory=list) @@ -77,9 +79,14 @@ async def fetch_detail( """ close_session = False if session is None: + # proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian. + # Пусто (env не задан) → прямое подключение (dev/no-op). + _proxy_url = settings.scraper_proxy_url + _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None session = AsyncSession( impersonate="chrome120", timeout=25.0, + proxies=_proxies, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py new file mode 100644 index 00000000..8feee9e6 --- /dev/null +++ b/tradein-mvp/backend/tests/test_scraper_proxy.py @@ -0,0 +1,102 @@ +"""Tests for scraper proxy wiring (#806). + +Covers: + - scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL + - AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent + - None/empty → direct connection (no proxies dict) + - _avito_proxies() helper returns correct dict shape +""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from types import SimpleNamespace +from unittest.mock import patch + + +def _mock_settings(scraper_proxy_url: str | None) -> SimpleNamespace: + """Minimal settings stand-in with only the fields the proxy helpers read.""" + return SimpleNamespace(scraper_proxy_url=scraper_proxy_url) + + +# ── Settings.scraper_proxy_url property ────────────────────────────────────── +# Test the property logic directly via inline replica — avoids instantiating +# pydantic-settings (requires DATABASE_URL) for pure unit tests of the rule. + + +def _resolve_scraper_proxy_url( + scraper_proxy_url_env: str | None, avito_proxy_url: str | None +) -> str | None: + """Inline replica of Settings.scraper_proxy_url property logic.""" + return scraper_proxy_url_env or avito_proxy_url + + +def test_scraper_proxy_url_uses_scraper_env_when_set(): + result = _resolve_scraper_proxy_url( + scraper_proxy_url_env="http://scraper-proxy:1234", + avito_proxy_url="http://avito-proxy:5678", + ) + assert result == "http://scraper-proxy:1234" + + +def test_scraper_proxy_url_falls_back_to_avito_proxy_url(): + """AVITO_PROXY_URL used when SCRAPER_PROXY_URL absent — zero env change on prod.""" + result = _resolve_scraper_proxy_url( + scraper_proxy_url_env=None, + avito_proxy_url="http://avito-proxy:5678", + ) + assert result == "http://avito-proxy:5678" + + +def test_scraper_proxy_url_none_when_both_absent(): + result = _resolve_scraper_proxy_url(scraper_proxy_url_env=None, avito_proxy_url=None) + assert result is None + + +def test_scraper_proxy_url_empty_string_falls_through_to_avito(): + """Empty string is falsy — falls through to avito_proxy_url.""" + result = _resolve_scraper_proxy_url( + scraper_proxy_url_env="", + avito_proxy_url="http://avito-proxy:5678", + ) + assert result == "http://avito-proxy:5678" + + +# ── _avito_proxies() dict shape ─────────────────────────────────────────────── + + +def test_avito_proxies_returns_correct_dict_shape(): + from app.services.scrape_pipeline import _avito_proxies + + proxy_url = "http://user:pass@proxy.example.com:8080" + with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)): + result = _avito_proxies() + + assert result == {"http": proxy_url, "https": proxy_url} + + +def test_avito_proxies_returns_none_when_no_proxy(): + from app.services.scrape_pipeline import _avito_proxies + + with patch("app.services.scrape_pipeline.settings", _mock_settings(None)): + result = _avito_proxies() + + assert result is None + + +def test_avito_proxies_fallback_via_avito_proxy_url(): + """_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback. + + When settings.scraper_proxy_url resolves to the avito URL (fallback path), + the dict shape must be {"http": url, "https": url}. + """ + from app.services.scrape_pipeline import _avito_proxies + + avito_url = "http://mobile-proxy.space:9090" + with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)): + result = _avito_proxies() + + assert result == {"http": avito_url, "https": avito_url} -- 2.45.3 From d815d7ba831f3ffe7751f1a933766df6df82db3a Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 30 May 2026 22:07:53 +0300 Subject: [PATCH 2/2] fixup(scrapers): bind SCRAPER_PROXY_URL via validation_alias + env-binding test (#806) --- tradein-mvp/backend/app/core/config.py | 5 ++- .../backend/tests/test_scraper_proxy.py | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index 2c92f3f4..08681852 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -1,5 +1,6 @@ """Минимальный settings для standalone trade-in MVP.""" +from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -142,7 +143,9 @@ class Settings(BaseSettings): # 2. AVITO_PROXY_URL — legacy ENV; fallback, чтобы prod-серверы с уже # настроенным AVITO_PROXY_URL работали без изменений .env.runtime (#806). # property `scraper_proxy_url` реализует эту логику; используй его везде. - scraper_proxy_url_env: str | None = None # ENV: SCRAPER_PROXY_URL (новый общий) + # validation_alias привязывает поле к env SCRAPER_PROXY_URL (без него + # pydantic-settings читал бы SCRAPER_PROXY_URL_ENV по имени поля — #806 fixup). + scraper_proxy_url_env: str | None = Field(default=None, validation_alias="SCRAPER_PROXY_URL") avito_proxy_url: str | None = None # ENV: AVITO_PROXY_URL (legacy fallback) @property diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py index 8feee9e6..108e2b01 100644 --- a/tradein-mvp/backend/tests/test_scraper_proxy.py +++ b/tradein-mvp/backend/tests/test_scraper_proxy.py @@ -100,3 +100,43 @@ def test_avito_proxies_fallback_via_avito_proxy_url(): result = _avito_proxies() assert result == {"http": avito_url, "https": avito_url} + + +# ── Real pydantic env-binding (regression for #806 fixup) ───────────────────── +# The inline-replica tests above prove the OR-logic but NOT that the field +# actually binds to env SCRAPER_PROXY_URL. `validation_alias` is what makes that +# work — without it pydantic-settings reads SCRAPER_PROXY_URL_ENV (the field +# name), so setting SCRAPER_PROXY_URL would be a silent no-op. These tests +# construct a real Settings and assert end-to-end env binding. + + +def _fresh_settings(): + from app.core.config import Settings + + return Settings(_env_file=None) + + +def test_settings_binds_scraper_proxy_url_env(monkeypatch): + monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234") + monkeypatch.delenv("AVITO_PROXY_URL", raising=False) + assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234" + + +def test_settings_scraper_env_overrides_avito(monkeypatch): + monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234") + monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678") + assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234" + + +def test_settings_falls_back_to_avito_env(monkeypatch): + monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False) + monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678") + assert _fresh_settings().scraper_proxy_url == "http://avito-proxy:5678" + + +def test_settings_ignores_misnamed_scraper_proxy_url_env(monkeypatch): + """Regression: the old wrong env name SCRAPER_PROXY_URL_ENV must NOT bind.""" + monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False) + monkeypatch.delenv("AVITO_PROXY_URL", raising=False) + monkeypatch.setenv("SCRAPER_PROXY_URL_ENV", "http://wrong-name:1111") + assert _fresh_settings().scraper_proxy_url is None -- 2.45.3