a95c38bdcc
All checks were successful
CI / changes (pull_request) Successful in 7s
CI Trade-In / changes (pull_request) Successful in 8s
CI / backend-tests (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 2m40s
104 lines
4 KiB
Python
104 lines
4 KiB
Python
"""#2616: POST /api/v1/admin/scrape (source=avito) прокидывает proxy_provider в AvitoScraper.
|
||
|
||
Root cause (issue #2613/#2616): `AvitoScraper(config, delay_provider=get_scraper_delay)` в
|
||
`scrape_around` не передавал `proxy_provider=proxy_provider`, хотя переменная уже вычислена
|
||
в scope и передаётся в CianScraper/YandexRealtyScraper на соседних строках. Браузерный путь
|
||
Avito (SCRAPER_FETCH_MODE=browser) в результате всегда шёл через мёртвый `BROWSER_PROXY_AVITO`
|
||
env-прокси вместо пула.
|
||
|
||
Offline: db/AvitoScraper/save_listings мокаются, NO live network/DB.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from typing import Any, ClassVar
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
import pytest
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
|
||
@pytest.fixture
|
||
def client() -> TestClient:
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
def fake_db() -> Any:
|
||
yield MagicMock()
|
||
|
||
app.dependency_overrides[get_db] = fake_db
|
||
return TestClient(app)
|
||
|
||
|
||
class _RecordingScraper:
|
||
"""Recording async-context-manager: пишет init-kwargs в общий список, fetch_around
|
||
возвращает пустой список лотов (save_listings ниже мокнут, до БД не доходим)."""
|
||
|
||
calls: ClassVar[list[dict[str, Any]]] = []
|
||
|
||
def __init__(self, config: Any, **kwargs: Any) -> None:
|
||
type(self).calls.append({"config": config, **kwargs})
|
||
|
||
async def __aenter__(self) -> _RecordingScraper:
|
||
return self
|
||
|
||
async def __aexit__(self, *_args: object) -> None:
|
||
return None
|
||
|
||
async def fetch_around(self, *_args: Any, **_kwargs: Any) -> list[Any]:
|
||
return []
|
||
|
||
|
||
def test_scrape_avito_passes_proxy_provider(client: TestClient) -> None:
|
||
"""source=['avito'] → AvitoScraper(config, ..., proxy_provider=<sentinel>).
|
||
|
||
Falsification: если admin.py перестанет передавать proxy_provider= в AvitoScraper(...),
|
||
calls[-1] не будет содержать ключ 'proxy_provider' со значением sentinel (либо будет
|
||
None) — assert падает на VALUE (или KeyError на .get None-fallback), не на TypeError
|
||
(recorder принимает **kwargs, сигнатуру не проверяет).
|
||
"""
|
||
sentinel = object()
|
||
_RecordingScraper.calls = []
|
||
|
||
with (
|
||
patch("app.api.v1.admin.AvitoScraper", _RecordingScraper),
|
||
patch("app.api.v1.admin._kit_proxy_provider", return_value=sentinel),
|
||
patch("app.api.v1.admin.save_listings", return_value=(0, 0)),
|
||
):
|
||
r = client.post(
|
||
"/api/v1/admin/scrape",
|
||
json={"lat": 56.8332, "lon": 60.5944, "radius_m": 1000, "sources": ["avito"]},
|
||
)
|
||
|
||
assert r.status_code == 200
|
||
assert len(_RecordingScraper.calls) == 1
|
||
assert _RecordingScraper.calls[0].get("proxy_provider") is sentinel
|
||
|
||
|
||
def test_scrape_avito_no_pool_default_none(client: TestClient) -> None:
|
||
"""_kit_proxy_provider() → None (pool off, ship-dark default) → AvitoScraper получает
|
||
|
||
proxy_provider=None — поведение прежнее.
|
||
"""
|
||
_RecordingScraper.calls = []
|
||
|
||
with (
|
||
patch("app.api.v1.admin.AvitoScraper", _RecordingScraper),
|
||
patch("app.api.v1.admin._kit_proxy_provider", return_value=None),
|
||
patch("app.api.v1.admin.save_listings", return_value=(0, 0)),
|
||
):
|
||
r = client.post(
|
||
"/api/v1/admin/scrape",
|
||
json={"lat": 56.8332, "lon": 60.5944, "radius_m": 1000, "sources": ["avito"]},
|
||
)
|
||
|
||
assert r.status_code == 200
|
||
assert len(_RecordingScraper.calls) == 1
|
||
assert _RecordingScraper.calls[0].get("proxy_provider") is None
|