bot-backend
e5f4e37415
Снимает egress-часть #639: cian-скрейперы ходили с datacenter-IP (без proxies) → даже валидные куки не фетчили с прод-бокса. Прокидываем тот же mobile-proxy, что у avito (#623/#805). - config: generic scraper_proxy_url (env SCRAPER_PROXY_URL → fallback AVITO_PROXY_URL → None/direct). Прод .env.runtime с AVITO_PROXY_URL работает без изменений env. - avito.py + scrape_pipeline._avito_proxies(): на settings.scraper_proxy_url (поведение идентично). - cian.py / cian_detail.fetch_detail / cian_session.verify_session: proxies= {http,https} из settings.scraper_proxy_url; пусто → direct (dev no-op). Cookie-refresh DMIR_AUTH (#639) — НЕ тронут (needs-human). 7 тестов (precedence + dict-shape + fallback), ruff clean.
102 lines
3.7 KiB
Python
102 lines
3.7 KiB
Python
"""Tests for scraper proxy wiring (#806).
|
|
|
|
Covers:
|
|
- scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL
|
|
- AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
|
|
- None/empty → direct connection (no proxies dict)
|
|
- _avito_proxies() helper returns correct dict shape
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
|
|
def _mock_settings(scraper_proxy_url: str | None) -> SimpleNamespace:
|
|
"""Minimal settings stand-in with only the fields the proxy helpers read."""
|
|
return SimpleNamespace(scraper_proxy_url=scraper_proxy_url)
|
|
|
|
|
|
# ── Settings.scraper_proxy_url property ──────────────────────────────────────
|
|
# Test the property logic directly via inline replica — avoids instantiating
|
|
# pydantic-settings (requires DATABASE_URL) for pure unit tests of the rule.
|
|
|
|
|
|
def _resolve_scraper_proxy_url(
|
|
scraper_proxy_url_env: str | None, avito_proxy_url: str | None
|
|
) -> str | None:
|
|
"""Inline replica of Settings.scraper_proxy_url property logic."""
|
|
return scraper_proxy_url_env or avito_proxy_url
|
|
|
|
|
|
def test_scraper_proxy_url_uses_scraper_env_when_set():
|
|
result = _resolve_scraper_proxy_url(
|
|
scraper_proxy_url_env="http://scraper-proxy:1234",
|
|
avito_proxy_url="http://avito-proxy:5678",
|
|
)
|
|
assert result == "http://scraper-proxy:1234"
|
|
|
|
|
|
def test_scraper_proxy_url_falls_back_to_avito_proxy_url():
|
|
"""AVITO_PROXY_URL used when SCRAPER_PROXY_URL absent — zero env change on prod."""
|
|
result = _resolve_scraper_proxy_url(
|
|
scraper_proxy_url_env=None,
|
|
avito_proxy_url="http://avito-proxy:5678",
|
|
)
|
|
assert result == "http://avito-proxy:5678"
|
|
|
|
|
|
def test_scraper_proxy_url_none_when_both_absent():
|
|
result = _resolve_scraper_proxy_url(scraper_proxy_url_env=None, avito_proxy_url=None)
|
|
assert result is None
|
|
|
|
|
|
def test_scraper_proxy_url_empty_string_falls_through_to_avito():
|
|
"""Empty string is falsy — falls through to avito_proxy_url."""
|
|
result = _resolve_scraper_proxy_url(
|
|
scraper_proxy_url_env="",
|
|
avito_proxy_url="http://avito-proxy:5678",
|
|
)
|
|
assert result == "http://avito-proxy:5678"
|
|
|
|
|
|
# ── _avito_proxies() dict shape ───────────────────────────────────────────────
|
|
|
|
|
|
def test_avito_proxies_returns_correct_dict_shape():
|
|
from app.services.scrape_pipeline import _avito_proxies
|
|
|
|
proxy_url = "http://user:pass@proxy.example.com:8080"
|
|
with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)):
|
|
result = _avito_proxies()
|
|
|
|
assert result == {"http": proxy_url, "https": proxy_url}
|
|
|
|
|
|
def test_avito_proxies_returns_none_when_no_proxy():
|
|
from app.services.scrape_pipeline import _avito_proxies
|
|
|
|
with patch("app.services.scrape_pipeline.settings", _mock_settings(None)):
|
|
result = _avito_proxies()
|
|
|
|
assert result is None
|
|
|
|
|
|
def test_avito_proxies_fallback_via_avito_proxy_url():
|
|
"""_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback.
|
|
|
|
When settings.scraper_proxy_url resolves to the avito URL (fallback path),
|
|
the dict shape must be {"http": url, "https": url}.
|
|
"""
|
|
from app.services.scrape_pipeline import _avito_proxies
|
|
|
|
avito_url = "http://mobile-proxy.space:9090"
|
|
with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)):
|
|
result = _avito_proxies()
|
|
|
|
assert result == {"http": avito_url, "https": avito_url}
|