feat(scrapers): route cian sessions through mobile proxy (Refs #806)
Снимает egress-часть #639: cian-скрейперы ходили с datacenter-IP (без proxies) → даже валидные куки не фетчили с прод-бокса. Прокидываем тот же mobile-proxy, что у avito (#623/#805). - config: generic scraper_proxy_url (env SCRAPER_PROXY_URL → fallback AVITO_PROXY_URL → None/direct). Прод .env.runtime с AVITO_PROXY_URL работает без изменений env. - avito.py + scrape_pipeline._avito_proxies(): на settings.scraper_proxy_url (поведение идентично). - cian.py / cian_detail.fetch_detail / cian_session.verify_session: proxies= {http,https} из settings.scraper_proxy_url; пусто → direct (dev no-op). Cookie-refresh DMIR_AUTH (#639) — НЕ тронут (needs-human). 7 тестов (precedence + dict-shape + fallback), ruff clean.
This commit is contained in:
parent
a48205391c
commit
e5f4e37415
7 changed files with 173 additions and 27 deletions
|
|
@ -132,13 +132,31 @@ class Settings(BaseSettings):
|
|||
"n1.ru",
|
||||
}
|
||||
|
||||
# ── Avito mobile proxy (#623) ─────────────────────────────────────────────
|
||||
# Avito банит datacenter-IP сервера — на HTTP 200 отдаёт firewall-страницу
|
||||
# «Доступ ограничен: проблема с IP». Мобильный прокси (mobileproxy.space)
|
||||
# снимает бан. Пусто = прямое подключение (dev). ENV: AVITO_PROXY_URL.
|
||||
avito_proxy_url: str | None = None
|
||||
# ── Scraper mobile proxy (#806) ──────────────────────────────────────────
|
||||
# Мобильный прокси (RU, mobileproxy.space) используется ВСЕМИ scraper-сессиями:
|
||||
# Avito (#623) + Cian (#806). Datacenter-IP блокируется обоими сайтами.
|
||||
# Пусто = прямое подключение (dev/staging без прокси).
|
||||
#
|
||||
# Приоритет ENV-переменных (precedence):
|
||||
# 1. SCRAPER_PROXY_URL — новый общий ENV; когда задан — используется первым.
|
||||
# 2. AVITO_PROXY_URL — legacy ENV; fallback, чтобы prod-серверы с уже
|
||||
# настроенным AVITO_PROXY_URL работали без изменений .env.runtime (#806).
|
||||
# property `scraper_proxy_url` реализует эту логику; используй его везде.
|
||||
scraper_proxy_url_env: str | None = None # ENV: SCRAPER_PROXY_URL (новый общий)
|
||||
avito_proxy_url: str | None = None # ENV: AVITO_PROXY_URL (legacy fallback)
|
||||
|
||||
@property
|
||||
def scraper_proxy_url(self) -> str | None:
|
||||
"""Единый прокси URL для всех scraper-сессий (Avito + Cian).
|
||||
|
||||
Приоритет: SCRAPER_PROXY_URL > AVITO_PROXY_URL > None (прямое подключение).
|
||||
Prod-серверы с существующим AVITO_PROXY_URL работают без изменений env.
|
||||
"""
|
||||
return self.scraper_proxy_url_env or self.avito_proxy_url
|
||||
|
||||
# changeip-ссылка mobileproxy: GET меняет мобильный IP за ~9с. Дёргается при
|
||||
# детекте бана перед повтором. Пусто = ротация выключена (raise сразу).
|
||||
# детекте бана Avito перед повтором. Пусто = ротация выключена (raise сразу).
|
||||
# ENV: AVITO_PROXY_ROTATE_URL.
|
||||
avito_proxy_rotate_url: str | None = None
|
||||
# Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу).
|
||||
avito_proxy_max_rotations: int = 2
|
||||
|
|
|
|||
|
|
@ -105,10 +105,16 @@ async def verify_session(cookies: dict[str, str]) -> dict[str, Any] | None:
|
|||
Никогда не логирует сырые значения cookies.
|
||||
"""
|
||||
try:
|
||||
# proxies: mobile-proxy egress (#806) — Cian блокирует datacenter-IP даже
|
||||
# при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403.
|
||||
# Пусто (env не задан) → прямое подключение (dev/no-op).
|
||||
_proxy_url = settings.scraper_proxy_url
|
||||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||
async with AsyncSession(
|
||||
impersonate="chrome120",
|
||||
cookies=cookies,
|
||||
timeout=20.0,
|
||||
proxies=_proxies,
|
||||
) as session:
|
||||
resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True)
|
||||
status = resp.status_code
|
||||
|
|
|
|||
|
|
@ -68,15 +68,17 @@ _CHROME_HEADERS = {
|
|||
|
||||
|
||||
def _avito_proxies() -> dict[str, str] | None:
|
||||
"""#623: mobile-proxy egress для shared-сессии sweep'а.
|
||||
"""#623/#806: mobile-proxy egress для shared-сессии sweep'а.
|
||||
|
||||
КРИТИЧНО: sweep создаёт собственную shared AsyncSession и присваивает её
|
||||
`scraper._cffi`, перезатирая проксированную сессию из `AvitoScraper.__aenter__`.
|
||||
Через эту же сессию идёт и SERP, и detail-энричмент (fetch_detail). Без прокси
|
||||
detail-страницы летят с datacenter-IP → HTTP 429 → весь run mark_banned.
|
||||
Пусто (env не задан) → прямое подключение (dev/no-op).
|
||||
|
||||
Читает settings.scraper_proxy_url (SCRAPER_PROXY_URL > AVITO_PROXY_URL fallback).
|
||||
"""
|
||||
url = settings.avito_proxy_url
|
||||
url = settings.scraper_proxy_url
|
||||
return {"http": url, "https": url} if url else None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -211,8 +211,8 @@ class AvitoScraper(BaseScraper):
|
|||
async def __aenter__(self) -> AvitoScraper:
|
||||
await super().__aenter__()
|
||||
proxies = None
|
||||
if settings.avito_proxy_url:
|
||||
proxies = {"http": settings.avito_proxy_url, "https": settings.avito_proxy_url}
|
||||
if settings.scraper_proxy_url:
|
||||
proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url}
|
||||
logger.info("avito: routing through mobile proxy egress (#623)")
|
||||
self._cffi = AsyncSession(
|
||||
impersonate="chrome120",
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from urllib.parse import urlencode
|
|||
|
||||
from curl_cffi.requests import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
from app.services.scrapers.cian_state_parser import extract_state
|
||||
|
|
@ -61,10 +62,16 @@ class CianScraper(BaseScraper):
|
|||
|
||||
async def __aenter__(self) -> CianScraper:
|
||||
await super().__aenter__()
|
||||
# curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian
|
||||
# curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian.
|
||||
# proxies: mobile-proxy egress (#806) — аналогично Avito (#623). Без прокси
|
||||
# datacenter-IP блокируется Cian (403/captcha) несмотря на валидные cookies.
|
||||
# Пусто (SCRAPER_PROXY_URL/AVITO_PROXY_URL не заданы) → прямое подключение (dev).
|
||||
_proxy_url = settings.scraper_proxy_url
|
||||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||
self._cffi = AsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=30,
|
||||
proxies=_proxies,
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
|
|
@ -114,7 +121,10 @@ class CianScraper(BaseScraper):
|
|||
lots = self._parse_serp_html(response.text)
|
||||
logger.info(
|
||||
"cian: %d lots fetched rooms=%s page=%d url=%s",
|
||||
len(lots), rooms, page, url,
|
||||
len(lots),
|
||||
rooms,
|
||||
page,
|
||||
url,
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
return lots
|
||||
|
|
@ -136,9 +146,7 @@ class CianScraper(BaseScraper):
|
|||
try:
|
||||
lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"cian multi-room fetch failed rooms=%s page=%d", rooms, page
|
||||
)
|
||||
logger.exception("cian multi-room fetch failed rooms=%s page=%d", rooms, page)
|
||||
continue
|
||||
if not lots:
|
||||
break # пустая страница → дальше смысла нет
|
||||
|
|
@ -148,7 +156,10 @@ class CianScraper(BaseScraper):
|
|||
seen[key] = lot
|
||||
logger.info(
|
||||
"cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)",
|
||||
len(seen), lat, lon, radius_m,
|
||||
len(seen),
|
||||
lat,
|
||||
lon,
|
||||
radius_m,
|
||||
)
|
||||
return list(seen.values())
|
||||
|
||||
|
|
@ -187,13 +198,12 @@ class CianScraper(BaseScraper):
|
|||
logger.warning(
|
||||
"cian SERP state extraction failed (mfe=%s key=%s) — "
|
||||
"возможно Cian изменил структуру или вернул captcha",
|
||||
_MFE_SERP, _STATE_KEY,
|
||||
_MFE_SERP,
|
||||
_STATE_KEY,
|
||||
)
|
||||
return []
|
||||
|
||||
offers_data: list[dict[str, Any]] = (
|
||||
state.get("results", {}).get("offers", [])
|
||||
)
|
||||
offers_data: list[dict[str, Any]] = state.get("results", {}).get("offers", [])
|
||||
if not offers_data:
|
||||
logger.warning(
|
||||
"cian state found but results.offers пуст (totalOffers=%s)",
|
||||
|
|
@ -299,7 +309,8 @@ class CianScraper(BaseScraper):
|
|||
if lat is None or lon is None:
|
||||
logger.debug(
|
||||
"cian offer %s: нет точных координат в state (geo=%s)",
|
||||
offer_id, coords,
|
||||
offer_id,
|
||||
coords,
|
||||
)
|
||||
lat = lon = None
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ Different from SERP (cian.py):
|
|||
Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
|
||||
Stage 5 of CianScraper v1.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
|
@ -20,6 +21,7 @@ from curl_cffi.requests import AsyncSession
|
|||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
|
||||
from app.services.scrapers.repair_state_normalizer import (
|
||||
infer_repair_state_from_text,
|
||||
|
|
@ -36,17 +38,17 @@ class DetailEnrichment:
|
|||
|
||||
# Core listing enrichment (extend existing listings row)
|
||||
cian_id: int | None = None
|
||||
windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet'
|
||||
windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet'
|
||||
separate_wcs_count: int | None = None
|
||||
combined_wcs_count: int | None = None
|
||||
ceiling_height: float | None = None # meters
|
||||
repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value
|
||||
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
||||
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
||||
ceiling_height: float | None = None # meters
|
||||
repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value
|
||||
repair_state: str | None = None # enum: needs_repair/standard/good/excellent
|
||||
views_total: int | None = None # Cian stats.totalViewsFormattedString → int
|
||||
views_today: int | None = None
|
||||
|
||||
# BTI (вторичка only — primary buildings don't have БТИ)
|
||||
bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot
|
||||
bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot
|
||||
|
||||
# Price history (offer.priceChanges)
|
||||
price_changes: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
|
@ -77,9 +79,14 @@ async def fetch_detail(
|
|||
"""
|
||||
close_session = False
|
||||
if session is None:
|
||||
# proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian.
|
||||
# Пусто (env не задан) → прямое подключение (dev/no-op).
|
||||
_proxy_url = settings.scraper_proxy_url
|
||||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||||
session = AsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=25.0,
|
||||
proxies=_proxies,
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
|
|
|
|||
102
tradein-mvp/backend/tests/test_scraper_proxy.py
Normal file
102
tradein-mvp/backend/tests/test_scraper_proxy.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
"""Tests for scraper proxy wiring (#806).
|
||||
|
||||
Covers:
|
||||
- scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL
|
||||
- AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
|
||||
- None/empty → direct connection (no proxies dict)
|
||||
- _avito_proxies() helper returns correct dict shape
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
def _mock_settings(scraper_proxy_url: str | None) -> SimpleNamespace:
|
||||
"""Minimal settings stand-in with only the fields the proxy helpers read."""
|
||||
return SimpleNamespace(scraper_proxy_url=scraper_proxy_url)
|
||||
|
||||
|
||||
# ── Settings.scraper_proxy_url property ──────────────────────────────────────
|
||||
# Test the property logic directly via inline replica — avoids instantiating
|
||||
# pydantic-settings (requires DATABASE_URL) for pure unit tests of the rule.
|
||||
|
||||
|
||||
def _resolve_scraper_proxy_url(
|
||||
scraper_proxy_url_env: str | None, avito_proxy_url: str | None
|
||||
) -> str | None:
|
||||
"""Inline replica of Settings.scraper_proxy_url property logic."""
|
||||
return scraper_proxy_url_env or avito_proxy_url
|
||||
|
||||
|
||||
def test_scraper_proxy_url_uses_scraper_env_when_set():
|
||||
result = _resolve_scraper_proxy_url(
|
||||
scraper_proxy_url_env="http://scraper-proxy:1234",
|
||||
avito_proxy_url="http://avito-proxy:5678",
|
||||
)
|
||||
assert result == "http://scraper-proxy:1234"
|
||||
|
||||
|
||||
def test_scraper_proxy_url_falls_back_to_avito_proxy_url():
|
||||
"""AVITO_PROXY_URL used when SCRAPER_PROXY_URL absent — zero env change on prod."""
|
||||
result = _resolve_scraper_proxy_url(
|
||||
scraper_proxy_url_env=None,
|
||||
avito_proxy_url="http://avito-proxy:5678",
|
||||
)
|
||||
assert result == "http://avito-proxy:5678"
|
||||
|
||||
|
||||
def test_scraper_proxy_url_none_when_both_absent():
|
||||
result = _resolve_scraper_proxy_url(scraper_proxy_url_env=None, avito_proxy_url=None)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_scraper_proxy_url_empty_string_falls_through_to_avito():
|
||||
"""Empty string is falsy — falls through to avito_proxy_url."""
|
||||
result = _resolve_scraper_proxy_url(
|
||||
scraper_proxy_url_env="",
|
||||
avito_proxy_url="http://avito-proxy:5678",
|
||||
)
|
||||
assert result == "http://avito-proxy:5678"
|
||||
|
||||
|
||||
# ── _avito_proxies() dict shape ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_avito_proxies_returns_correct_dict_shape():
|
||||
from app.services.scrape_pipeline import _avito_proxies
|
||||
|
||||
proxy_url = "http://user:pass@proxy.example.com:8080"
|
||||
with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)):
|
||||
result = _avito_proxies()
|
||||
|
||||
assert result == {"http": proxy_url, "https": proxy_url}
|
||||
|
||||
|
||||
def test_avito_proxies_returns_none_when_no_proxy():
|
||||
from app.services.scrape_pipeline import _avito_proxies
|
||||
|
||||
with patch("app.services.scrape_pipeline.settings", _mock_settings(None)):
|
||||
result = _avito_proxies()
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_avito_proxies_fallback_via_avito_proxy_url():
|
||||
"""_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback.
|
||||
|
||||
When settings.scraper_proxy_url resolves to the avito URL (fallback path),
|
||||
the dict shape must be {"http": url, "https": url}.
|
||||
"""
|
||||
from app.services.scrape_pipeline import _avito_proxies
|
||||
|
||||
avito_url = "http://mobile-proxy.space:9090"
|
||||
with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)):
|
||||
result = _avito_proxies()
|
||||
|
||||
assert result == {"http": avito_url, "https": avito_url}
|
||||
Loading…
Add table
Reference in a new issue