142 lines
5.4 KiB
Python
142 lines
5.4 KiB
Python
"""Tests for scraper proxy wiring (#806).
|
|
|
|
Covers:
|
|
- scraper_proxy_url property: SCRAPER_PROXY_URL takes precedence over AVITO_PROXY_URL
|
|
- AVITO_PROXY_URL fallback works when SCRAPER_PROXY_URL is absent
|
|
- None/empty → direct connection (no proxies dict)
|
|
- _avito_proxies() helper returns correct dict shape
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
|
|
def _mock_settings(scraper_proxy_url: str | None) -> SimpleNamespace:
|
|
"""Minimal settings stand-in with only the fields the proxy helpers read."""
|
|
return SimpleNamespace(scraper_proxy_url=scraper_proxy_url)
|
|
|
|
|
|
# ── Settings.scraper_proxy_url property ──────────────────────────────────────
|
|
# Test the property logic directly via inline replica — avoids instantiating
|
|
# pydantic-settings (requires DATABASE_URL) for pure unit tests of the rule.
|
|
|
|
|
|
def _resolve_scraper_proxy_url(
|
|
scraper_proxy_url_env: str | None, avito_proxy_url: str | None
|
|
) -> str | None:
|
|
"""Inline replica of Settings.scraper_proxy_url property logic."""
|
|
return scraper_proxy_url_env or avito_proxy_url
|
|
|
|
|
|
def test_scraper_proxy_url_uses_scraper_env_when_set():
|
|
result = _resolve_scraper_proxy_url(
|
|
scraper_proxy_url_env="http://scraper-proxy:1234",
|
|
avito_proxy_url="http://avito-proxy:5678",
|
|
)
|
|
assert result == "http://scraper-proxy:1234"
|
|
|
|
|
|
def test_scraper_proxy_url_falls_back_to_avito_proxy_url():
|
|
"""AVITO_PROXY_URL used when SCRAPER_PROXY_URL absent — zero env change on prod."""
|
|
result = _resolve_scraper_proxy_url(
|
|
scraper_proxy_url_env=None,
|
|
avito_proxy_url="http://avito-proxy:5678",
|
|
)
|
|
assert result == "http://avito-proxy:5678"
|
|
|
|
|
|
def test_scraper_proxy_url_none_when_both_absent():
|
|
result = _resolve_scraper_proxy_url(scraper_proxy_url_env=None, avito_proxy_url=None)
|
|
assert result is None
|
|
|
|
|
|
def test_scraper_proxy_url_empty_string_falls_through_to_avito():
|
|
"""Empty string is falsy — falls through to avito_proxy_url."""
|
|
result = _resolve_scraper_proxy_url(
|
|
scraper_proxy_url_env="",
|
|
avito_proxy_url="http://avito-proxy:5678",
|
|
)
|
|
assert result == "http://avito-proxy:5678"
|
|
|
|
|
|
# ── _avito_proxies() dict shape ───────────────────────────────────────────────
|
|
|
|
|
|
def test_avito_proxies_returns_correct_dict_shape():
|
|
from app.services.scrape_pipeline import _avito_proxies
|
|
|
|
proxy_url = "http://user:pass@proxy.example.com:8080"
|
|
with patch("app.services.scrape_pipeline.settings", _mock_settings(proxy_url)):
|
|
result = _avito_proxies()
|
|
|
|
assert result == {"http": proxy_url, "https": proxy_url}
|
|
|
|
|
|
def test_avito_proxies_returns_none_when_no_proxy():
|
|
from app.services.scrape_pipeline import _avito_proxies
|
|
|
|
with patch("app.services.scrape_pipeline.settings", _mock_settings(None)):
|
|
result = _avito_proxies()
|
|
|
|
assert result is None
|
|
|
|
|
|
def test_avito_proxies_fallback_via_avito_proxy_url():
|
|
"""_avito_proxies() uses AVITO_PROXY_URL via scraper_proxy_url fallback.
|
|
|
|
When settings.scraper_proxy_url resolves to the avito URL (fallback path),
|
|
the dict shape must be {"http": url, "https": url}.
|
|
"""
|
|
from app.services.scrape_pipeline import _avito_proxies
|
|
|
|
avito_url = "http://mobile-proxy.space:9090"
|
|
with patch("app.services.scrape_pipeline.settings", _mock_settings(avito_url)):
|
|
result = _avito_proxies()
|
|
|
|
assert result == {"http": avito_url, "https": avito_url}
|
|
|
|
|
|
# ── Real pydantic env-binding (regression for #806 fixup) ─────────────────────
|
|
# The inline-replica tests above prove the OR-logic but NOT that the field
|
|
# actually binds to env SCRAPER_PROXY_URL. `validation_alias` is what makes that
|
|
# work — without it pydantic-settings reads SCRAPER_PROXY_URL_ENV (the field
|
|
# name), so setting SCRAPER_PROXY_URL would be a silent no-op. These tests
|
|
# construct a real Settings and assert end-to-end env binding.
|
|
|
|
|
|
def _fresh_settings():
|
|
from app.core.config import Settings
|
|
|
|
return Settings(_env_file=None)
|
|
|
|
|
|
def test_settings_binds_scraper_proxy_url_env(monkeypatch):
|
|
monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234")
|
|
monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
|
|
assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234"
|
|
|
|
|
|
def test_settings_scraper_env_overrides_avito(monkeypatch):
|
|
monkeypatch.setenv("SCRAPER_PROXY_URL", "http://scraper-proxy:1234")
|
|
monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678")
|
|
assert _fresh_settings().scraper_proxy_url == "http://scraper-proxy:1234"
|
|
|
|
|
|
def test_settings_falls_back_to_avito_env(monkeypatch):
|
|
monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
|
|
monkeypatch.setenv("AVITO_PROXY_URL", "http://avito-proxy:5678")
|
|
assert _fresh_settings().scraper_proxy_url == "http://avito-proxy:5678"
|
|
|
|
|
|
def test_settings_ignores_misnamed_scraper_proxy_url_env(monkeypatch):
|
|
"""Regression: the old wrong env name SCRAPER_PROXY_URL_ENV must NOT bind."""
|
|
monkeypatch.delenv("SCRAPER_PROXY_URL", raising=False)
|
|
monkeypatch.delenv("AVITO_PROXY_URL", raising=False)
|
|
monkeypatch.setenv("SCRAPER_PROXY_URL_ENV", "http://wrong-name:1111")
|
|
assert _fresh_settings().scraper_proxy_url is None
|