570c987e78
Re-review feedback (PR #260 head 8191a85):
1. tests/scrapers/test_nspd_bulk_client.py:360 — test referenced bulk_mod._SEMAPHORE
which was renamed to _SEMAPHORE_LIMIT in previous commit. AttributeError на
полном pytest run. Заменено: assert bulk_mod._SEMAPHORE_LIMIT == 3 (smoke).
max_concurrent <= 3 уже валидирует throttling — capacity check был лишним.
2. app/services/scrapers/nspd_client.py docstrings (lines 483, 506) — обновлены
ссылки NSPDBulkClient._SEMAPHORE(3) → NSPDBulkClient._sem (per-instance,
capacity=3) чтобы соответствовать новой архитектуре.
498 lines
19 KiB
Python
498 lines
19 KiB
Python
"""Тесты для NSPDBulkClient (nspd_bulk_client.py) — PR 2/5 issue #168.
|
||
|
||
Структура:
|
||
- Unit тесты (без сети) — параметры, семафор, parse-логика
|
||
- E2E тесты (@pytest.mark.slow + @pytest.mark.skip по умолчанию) —
|
||
реальные запросы к NSPD, CI не дёргает
|
||
|
||
Запуск e2e вручную:
|
||
cd backend && uv run pytest tests/scrapers/test_nspd_bulk_client.py -m slow -s -v
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.schemas.nspd_bulk import NSPDBulkFeature, ObjectsListing, QuarterSnapshot
|
||
from app.scrapers.nspd_bulk_client import (
|
||
NSPDBulkClient,
|
||
NspdBulkError,
|
||
NspdBulkWafError,
|
||
)
|
||
|
||
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.fixture
|
||
def sample_quarter_response() -> dict[str, Any]:
|
||
"""Минимальный валидный ответ search_by_quarter из NSPD.
|
||
|
||
Real NSPD shape (verified live on 66:41:0303161 in #177):
|
||
- `properties.category` (НЕ categoryId)
|
||
- `meta` лежит на top-level (не в data.meta)
|
||
"""
|
||
return {
|
||
"data": {
|
||
"type": "FeatureCollection",
|
||
"features": [
|
||
{
|
||
"id": 1001,
|
||
"type": "Feature",
|
||
"geometry": {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[
|
||
[6700000.0, 7700000.0],
|
||
[6701000.0, 7700000.0],
|
||
[6701000.0, 7701000.0],
|
||
[6700000.0, 7701000.0],
|
||
[6700000.0, 7700000.0],
|
||
],
|
||
],
|
||
},
|
||
"properties": {
|
||
"category": 36368,
|
||
"categoryName": "Земельные участки ЕГРН",
|
||
"options": {
|
||
"cad_num": "66:41:0303161:123",
|
||
"area": 500.0,
|
||
},
|
||
},
|
||
},
|
||
{
|
||
"id": 2002,
|
||
"type": "Feature",
|
||
"geometry": {"type": "Point", "coordinates": [6700500.0, 7700500.0]},
|
||
"properties": {
|
||
"category": 36369,
|
||
"categoryName": "Здания",
|
||
"options": {
|
||
"cad_num": "66:41:0303161:456:1",
|
||
"floors": "5",
|
||
},
|
||
},
|
||
},
|
||
],
|
||
},
|
||
# meta — top-level, primary path в search_by_quarter
|
||
"meta": [
|
||
{"categoryId": 36368, "totalCount": 170},
|
||
{"categoryId": 36369, "totalCount": 256},
|
||
{"categoryId": 36383, "totalCount": 75},
|
||
{"categoryId": 37163, "totalCount": 5877},
|
||
{"categoryId": 39663, "totalCount": 1},
|
||
],
|
||
}
|
||
|
||
|
||
@pytest.fixture
|
||
def sample_wms_response() -> dict[str, Any]:
|
||
"""Минимальный WMS GetFeatureInfo response."""
|
||
return {
|
||
"type": "FeatureCollection",
|
||
"features": [
|
||
{
|
||
"id": 3003,
|
||
"type": "Feature",
|
||
"geometry": {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[
|
||
[6700100.0, 7700100.0],
|
||
[6700900.0, 7700100.0],
|
||
[6700900.0, 7700900.0],
|
||
[6700100.0, 7700900.0],
|
||
[6700100.0, 7700100.0],
|
||
],
|
||
],
|
||
},
|
||
"properties": {
|
||
"category": 36368,
|
||
"options": {"cad_num": "66:41:0303161:789"},
|
||
},
|
||
}
|
||
],
|
||
}
|
||
|
||
|
||
@pytest.fixture
|
||
def sample_tab_group_response() -> dict[str, Any]:
|
||
"""Минимальный tab-group-data response для здания."""
|
||
return {
|
||
"title": "Список объектов",
|
||
"object": [
|
||
{"title": "Помещения (количество)", "value": ["264"]},
|
||
{"title": "Машино-места (количество)", "value": ["4"]},
|
||
{
|
||
"title": "Помещения (список)",
|
||
"value": ["66:41:0303161:100:1", "66:41:0303161:100:2"],
|
||
},
|
||
{
|
||
"title": "Машино-места (список)",
|
||
"value": ["66:41:0303161:100:265", "66:41:0303161:100:266"],
|
||
},
|
||
],
|
||
}
|
||
|
||
|
||
# ── Unit tests: schemas ───────────────────────────────────────────────────────
|
||
|
||
|
||
def test_nspd_bulk_feature_parse_basic() -> None:
|
||
"""NSPDBulkFeature парсит id, geometry, properties (primary path: properties.category)."""
|
||
raw = {
|
||
"id": 42,
|
||
"type": "Feature",
|
||
"geometry": {"type": "Point", "coordinates": [6700000.0, 7700000.0]},
|
||
"properties": {
|
||
"category": 36368,
|
||
"options": {"cad_num": "66:41:0303161:1", "area": 300.0},
|
||
},
|
||
}
|
||
feat = NSPDBulkFeature.model_validate(raw)
|
||
assert feat.id == 42
|
||
assert feat.category_id == 36368
|
||
assert feat.cad_num == "66:41:0303161:1"
|
||
assert feat.options.area == 300.0
|
||
|
||
|
||
def test_nspd_bulk_feature_extra_fields_preserved() -> None:
|
||
"""extra='allow' → неизвестные поля сохраняются."""
|
||
raw = {
|
||
"id": 1,
|
||
"geometry": None,
|
||
"properties": {},
|
||
"unknown_future_field": "value",
|
||
}
|
||
feat = NSPDBulkFeature.model_validate(raw)
|
||
assert feat.model_extra["unknown_future_field"] == "value" # type: ignore[index]
|
||
|
||
|
||
def test_nspd_options_effective_cad_num_prefers_cad_num() -> None:
|
||
"""effective_cad_num берёт cad_num если оба заданы."""
|
||
from app.schemas.nspd_bulk import NSPDOptions
|
||
|
||
opts = NSPDOptions(cad_num="66:41:0303161:1", cad_number="66:41:0303161:2")
|
||
assert opts.effective_cad_num == "66:41:0303161:1"
|
||
|
||
|
||
def test_nspd_options_fallback_to_cad_number() -> None:
|
||
"""effective_cad_num fallback → cad_number если cad_num is None."""
|
||
from app.schemas.nspd_bulk import NSPDOptions
|
||
|
||
opts = NSPDOptions(cad_number="66:41:0303161:2")
|
||
assert opts.effective_cad_num == "66:41:0303161:2"
|
||
|
||
|
||
def test_quarter_snapshot_overflow_categories() -> None:
|
||
"""overflow_categories = категории с totalCount > 20."""
|
||
snap = QuarterSnapshot(
|
||
quarter="66:41:0303161",
|
||
fetched_at="2026-05-15T00:00:00+00:00",
|
||
meta_counts={36368: 170, 36369: 256, 36383: 5, 39663: 1},
|
||
)
|
||
overflow = snap.overflow_categories
|
||
assert 36368 in overflow
|
||
assert 36369 in overflow
|
||
assert 36383 not in overflow # 5 <= 20
|
||
assert 39663 not in overflow # 1 <= 20
|
||
|
||
|
||
def test_objects_listing_defaults() -> None:
|
||
"""ObjectsListing дефолтные значения."""
|
||
listing = ObjectsListing(objdoc_id=12345)
|
||
assert listing.flats_count == 0
|
||
assert listing.parking_count == 0
|
||
assert listing.flats_cad_nums == []
|
||
|
||
|
||
# ── Unit tests: client parse logic (mocked HTTP) ─────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_search_by_quarter_parses_features(
|
||
sample_quarter_response: dict[str, Any],
|
||
) -> None:
|
||
"""search_by_quarter корректно парсит features и meta_counts."""
|
||
async with NSPDBulkClient() as client:
|
||
with patch.object(client, "_get_json", new=AsyncMock(return_value=sample_quarter_response)):
|
||
snap = await client.search_by_quarter("66:41:0303161")
|
||
|
||
assert snap.quarter == "66:41:0303161"
|
||
assert len(snap.features) == 2
|
||
assert snap.meta_counts[36368] == 170
|
||
assert snap.meta_counts[36369] == 256
|
||
assert snap.meta_counts[37163] == 5877
|
||
# overflow: категории с >20
|
||
overflow = snap.overflow_categories
|
||
assert 36368 in overflow
|
||
assert 36369 in overflow
|
||
assert 36383 in overflow
|
||
assert 37163 in overflow
|
||
assert 39663 not in overflow # totalCount=1 → no overflow
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_search_by_quarter_empty_on_404() -> None:
|
||
"""search_by_quarter возвращает пустой snapshot при 404."""
|
||
async with NSPDBulkClient() as client:
|
||
with patch.object(
|
||
client,
|
||
"_get_json",
|
||
new=AsyncMock(side_effect=NspdBulkError("HTTP 404: not found")),
|
||
):
|
||
snap = await client.search_by_quarter("66:41:9999999")
|
||
|
||
assert snap.features == []
|
||
assert snap.meta_counts == {}
|
||
assert snap.quarter == "66:41:9999999"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_search_by_quarter_raises_on_waf() -> None:
|
||
"""search_by_quarter пробрасывает NspdBulkWafError (403) без retry."""
|
||
async with NSPDBulkClient() as client:
|
||
with patch.object(
|
||
client,
|
||
"_get_json",
|
||
new=AsyncMock(side_effect=NspdBulkWafError("HTTP 403 WAF")),
|
||
):
|
||
with pytest.raises(NspdBulkWafError):
|
||
await client.search_by_quarter("66:41:0303161")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_wms_feature_info_parses_features(
|
||
sample_wms_response: dict[str, Any],
|
||
) -> None:
|
||
"""wms_feature_info парсит features из GeoJSON response."""
|
||
async with NSPDBulkClient() as client:
|
||
with patch.object(client, "_get_json", new=AsyncMock(return_value=sample_wms_response)):
|
||
features = await client.wms_feature_info(
|
||
layer_id=36368,
|
||
bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0),
|
||
click_xy=(256, 256),
|
||
)
|
||
|
||
assert len(features) == 1
|
||
assert features[0].category_id == 36368
|
||
assert features[0].cad_num == "66:41:0303161:789"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_wms_feature_info_empty_response() -> None:
|
||
"""wms_feature_info возвращает [] при пустом response."""
|
||
async with NSPDBulkClient() as client:
|
||
with patch.object(client, "_get_json", new=AsyncMock(return_value={"features": []})):
|
||
features = await client.wms_feature_info(
|
||
layer_id=36368,
|
||
bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0),
|
||
click_xy=(256, 256),
|
||
)
|
||
assert features == []
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_list_objects_in_building_parses(
|
||
sample_tab_group_response: dict[str, Any],
|
||
) -> None:
|
||
"""list_objects_in_building парсит counts и cad_nums."""
|
||
async with NSPDBulkClient() as client:
|
||
with patch.object(
|
||
client, "_get_json", new=AsyncMock(return_value=sample_tab_group_response)
|
||
):
|
||
listing = await client.list_objects_in_building(objdoc_id=42065602)
|
||
|
||
assert listing.objdoc_id == 42065602
|
||
assert listing.flats_count == 264
|
||
assert listing.parking_count == 4
|
||
assert "66:41:0303161:100:1" in listing.flats_cad_nums
|
||
assert "66:41:0303161:100:265" in listing.parking_cad_nums
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_rate_limit_semaphore_max_3_concurrent() -> None:
|
||
"""Не более 3 одновременных запросов через per-instance self._sem (PR #260).
|
||
|
||
Мокируем httpx.AsyncClient.get (нижний уровень) с задержкой, чтобы
|
||
реальный семафор работал. Считаем max in-flight внутри семафора.
|
||
"""
|
||
import httpx
|
||
|
||
import app.scrapers.nspd_bulk_client as bulk_mod
|
||
|
||
max_concurrent = 0
|
||
current = 0
|
||
lock = asyncio.Lock()
|
||
call_count = 0
|
||
|
||
async def slow_get(*args: Any, **kwargs: Any) -> httpx.Response:
|
||
nonlocal max_concurrent, current, call_count
|
||
# Фиксируем вход — уже внутри семафора (httpx.get вызывается после async with self._sem)
|
||
async with lock:
|
||
current += 1
|
||
call_count += 1
|
||
if current > max_concurrent:
|
||
max_concurrent = current
|
||
await asyncio.sleep(0.02) # симулируем I/O задержку
|
||
async with lock:
|
||
current -= 1
|
||
return httpx.Response(
|
||
200,
|
||
json={"data": {"features": [], "meta": []}},
|
||
request=MagicMock(),
|
||
)
|
||
|
||
async with NSPDBulkClient() as client:
|
||
assert client._client is not None
|
||
with patch.object(client._client, "get", new=AsyncMock(side_effect=slow_get)):
|
||
tasks = [client.search_by_quarter(f"66:41:{i:07d}") for i in range(6)]
|
||
await asyncio.gather(*tasks)
|
||
|
||
# self._sem(3) → не более 3 одновременно внутри slow_get
|
||
assert max_concurrent <= 3, f"Expected ≤3 concurrent, got {max_concurrent}"
|
||
assert call_count == 6 # все 6 вызовов прошли
|
||
# NB: per-instance self._sem cleanup проверяется через max_concurrent <= 3 выше.
|
||
# Module-level _SEMAPHORE удалён в PR #260 (cross-loop binding fix); smoke на capacity:
|
||
assert bulk_mod._SEMAPHORE_LIMIT == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_http_403_raises_waf_no_retry() -> None:
|
||
"""403 → NspdBulkWafError немедленно, без retry."""
|
||
import httpx
|
||
|
||
call_count = 0
|
||
|
||
async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
return httpx.Response(403, text="Forbidden", request=MagicMock())
|
||
|
||
async with NSPDBulkClient() as client:
|
||
assert client._client is not None
|
||
with patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)):
|
||
with pytest.raises(NspdBulkWafError):
|
||
await client._get_json("https://nspd.gov.ru/test")
|
||
|
||
assert call_count == 1 # ровно 1 попытка, нет retry
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_http_429_backoff_exhausted() -> None:
|
||
"""429 → exponential backoff 3 retries → NspdBulkRateLimitError."""
|
||
import httpx
|
||
|
||
from app.scrapers.nspd_bulk_client import NspdBulkRateLimitError
|
||
|
||
call_count = 0
|
||
|
||
async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
return httpx.Response(429, text="Too Many Requests", request=MagicMock())
|
||
|
||
# Патчим sleep чтобы тест не ждал 2+4+8 секунд
|
||
async with NSPDBulkClient(max_retries=3) as client:
|
||
assert client._client is not None
|
||
with (
|
||
patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)),
|
||
patch("app.scrapers.nspd_bulk_client.asyncio.sleep", new=AsyncMock()),
|
||
):
|
||
with pytest.raises(NspdBulkRateLimitError):
|
||
await client._get_json("https://nspd.gov.ru/test")
|
||
|
||
# 1 оригинальный + 3 retry = 4 вызова
|
||
assert call_count == 4
|
||
|
||
|
||
# ── E2E tests (network, skipped by default) ───────────────────────────────────
|
||
|
||
pytestmark_slow = pytest.mark.slow
|
||
|
||
|
||
@pytest.mark.slow
|
||
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
|
||
@pytest.mark.asyncio
|
||
async def test_search_by_quarter_real_ekb() -> None:
|
||
"""E2E: поиск по реальному кварталу ЕКБ — 66:41:0303161.
|
||
|
||
Проверяет:
|
||
- features > 100 (квартал непустой)
|
||
- meta содержит ключи 36368/36369/36383 с totalCount > 0
|
||
- overflow_categories не пуст (значит totalCount > 20 для хотя бы одной)
|
||
- headers корректны (нет 403/429)
|
||
"""
|
||
async with NSPDBulkClient() as client:
|
||
snap = await client.search_by_quarter("66:41:0303161")
|
||
|
||
assert len(snap.features) > 0, "Ожидали хотя бы 1 feature"
|
||
# Meta должна содержать основные категории
|
||
assert 36368 in snap.meta_counts, "Нет categoryId 36368 (ЗУ ЕГРН) в meta"
|
||
assert 36369 in snap.meta_counts, "Нет categoryId 36369 (Здания) в meta"
|
||
assert snap.meta_counts[36368] > 0, "ЗУ ЕГРН totalCount должен быть > 0"
|
||
assert snap.meta_counts[36369] > 0, "Здания totalCount должен быть > 0"
|
||
# В этом квартале >170 ЗУ и >256 зданий → overflow
|
||
assert len(snap.overflow_categories) > 0, "Ожидали overflow_categories непустой"
|
||
# total_meta_count > 100 (квартал богатый данными)
|
||
assert snap.total_meta_count > 100
|
||
|
||
|
||
@pytest.mark.slow
|
||
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
|
||
@pytest.mark.asyncio
|
||
async def test_search_by_quarter_empty() -> None:
|
||
"""E2E: несуществующий квартал → пустой snapshot."""
|
||
async with NSPDBulkClient() as client:
|
||
snap = await client.search_by_quarter("66:41:9999999")
|
||
|
||
# Должно вернуть пустой snapshot (не raise)
|
||
assert snap.features == []
|
||
assert snap.quarter == "66:41:9999999"
|
||
|
||
|
||
@pytest.mark.slow
|
||
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
|
||
@pytest.mark.asyncio
|
||
async def test_wms_feature_info_real() -> None:
|
||
"""E2E: WMS GetFeatureInfo реальный bbox из ЕКБ.
|
||
|
||
Использует bbox центра квартала 66:41:0303161 в EPSG:3857.
|
||
"""
|
||
# Центральная точка квартала 66:41:0303161 в 3857 (примерно)
|
||
# ЕКБ центр ≈ 60.6°E, 56.84°N → 3857 ≈ (6748000, 7772000)
|
||
center_x = 6748000.0
|
||
center_y = 7772000.0
|
||
half = 500.0 # 500м bbox
|
||
bbox = (center_x - half, center_y - half, center_x + half, center_y + half)
|
||
|
||
async with NSPDBulkClient() as client:
|
||
features = await client.wms_feature_info(
|
||
layer_id=36368, # ЗУ ЕГРН
|
||
bbox=bbox,
|
||
click_xy=(256, 256),
|
||
width=512,
|
||
height=512,
|
||
)
|
||
|
||
# Хотя бы 1 feature в bbox центра ЕКБ квартала
|
||
assert len(features) >= 1, f"Ожидали ≥1 feature, получили {len(features)}"
|
||
|
||
|
||
@pytest.mark.slow
|
||
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
|
||
@pytest.mark.asyncio
|
||
async def test_list_objects_in_building_real() -> None:
|
||
"""E2E: реальный objdocId=42065602 → >150 flats_cad_nums."""
|
||
async with NSPDBulkClient() as client:
|
||
listing = await client.list_objects_in_building(objdoc_id=42065602)
|
||
|
||
assert listing.objdoc_id == 42065602
|
||
assert listing.flats_count > 150, f"Ожидали >150 помещений, получили {listing.flats_count}"
|
||
assert (
|
||
len(listing.flats_cad_nums) > 150
|
||
), f"Ожидали >150 cad_nums помещений, получили {len(listing.flats_cad_nums)}"
|