5da66bbc93
* feat(cadastre): bulk_harvest_quarter Celery task + grid-walker + saga state (#168 PR3/5) Add bulk cadastre harvest pipeline: - services/cadastre/bulk_harvest.py: async harvest_quarter() orchestrator (4 phases) + 7 upsert helpers (parcels/buildings/constructions/oncs/enks/zouit/quarter_stats) using CAST(:x AS jsonb) pattern, begin_nested() SAVEPOINT per grid-walk upsert - services/cadastre/grid_geometry.py: quarter_bbox_3857 (PostGIS ST_Extent) + generate_grid_click_points (15x15 = 225 sub-bbox grid) - workers/tasks/scrape_cadastre.py: bulk_harvest_quarter_task (acks_late=True, dont_autoretry_for NspdBulkWafError), enqueue_cadastre_harvest, cleanup_zombies - api/v1/admin_cadastre.py: 5 endpoints behind AdminTokenAuth — create/list/get/cancel/resume - Tests: 13 service unit + 8 API tests * fixup(cadastre): drop importorskip (PR2 merged) — imports now top-level (#168 PR3) * fixup(cadastre): tile_size unified param + register slow marker (#168 PR3) Blocker #1: rename tile_width/tile_height → tile_size in generate_grid_click_points. Callers in bulk_harvest.py and test_cadastre_bulk.py used tile_size; def had tile_width+tile_height → TypeError at runtime. Blocker #2 (10 failures in test_admin_cadastre.py) was side-effect of #1: TypeError at import chain (app.main → admin_cadastre → bulk_harvest) broke FastAPI app load → dependency_overrides AttributeError. Now resolved. Also register `slow` pytest marker in pyproject.toml to suppress PytestUnknownMarkWarning. * fixup(cadastre): use importlib.util.find_spec instead of import (#168 PR3) Root cause Blocker #2: `import app.workers.tasks.scrape_cadastre` at module top-level REBOUND `app` from FastAPI instance (line 14) to the Python package. All subsequent `app.dependency_overrides[get_db] = ...` failed with AttributeError because `app` was now the namespace module. Fix: probe module availability with importlib.util.find_spec — does not import or rebind names. FastAPI `app` instance stays intact. All 28 cadastre tests pass locally. --------- Co-authored-by: lekss361 <claudestars@proton.me>
306 lines
10 KiB
Python
306 lines
10 KiB
Python
"""Тесты для admin_cadastre.py API endpoints.
|
||
|
||
Использует FastAPI TestClient с mock DB и mock Celery tasks.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
# scrape_cadastre зависит от app.scrapers.nspd_bulk_client (PR 2/5).
|
||
# Используем importlib.util.find_spec вместо прямого import — иначе
|
||
# `import app.workers...` пересоздаёт `app` как Python package и
|
||
# перебивает FastAPI instance, привязанный в строке выше → AttributeError
|
||
# на app.dependency_overrides.
|
||
import importlib.util
|
||
from typing import Any
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.main import app
|
||
|
||
_SCRAPE_CADASTRE_AVAILABLE = (
|
||
importlib.util.find_spec("app.workers.tasks.scrape_cadastre") is not None
|
||
)
|
||
|
||
requires_scrape_cadastre = pytest.mark.skipif(
|
||
not _SCRAPE_CADASTRE_AVAILABLE,
|
||
reason=(
|
||
"app.scrapers.nspd_bulk_client отсутствует — нужно смержить main (PR #170). "
|
||
"Команда: git rebase origin/main"
|
||
),
|
||
)
|
||
|
||
# Токен для тестов (не реальный)
|
||
ADMIN_TOKEN = "test-admin-token"
|
||
ADMIN_HEADERS = {"X-Admin-Token": ADMIN_TOKEN}
|
||
|
||
|
||
def _make_mock_db(quarters: list[str] | None = None, job_row: dict[str, Any] | None = None):
|
||
"""Создать mock db session с преднастроенными ответами."""
|
||
db = MagicMock()
|
||
|
||
# Результат для _resolve_quarters (SELECT cad_number FROM cad_quarters_geom)
|
||
quarter_rows = [(q,) for q in (quarters or ["66:41:0303161", "66:41:0303162"])]
|
||
|
||
# Mock execute chain для разных запросов
|
||
def mock_execute(stmt, params=None):
|
||
stmt_str = str(stmt)
|
||
result = MagicMock()
|
||
|
||
if "cad_quarters_geom" in stmt_str and "cad_number" in stmt_str:
|
||
# _resolve_quarters query
|
||
result.all.return_value = quarter_rows
|
||
elif "cadastre_jobs" in stmt_str and "RETURNING job_id" in stmt_str:
|
||
# _create_job INSERT
|
||
result.scalar_one.return_value = 42
|
||
elif "cadastre_jobs" in stmt_str and "SELECT" in stmt_str:
|
||
# get/list jobs
|
||
result.mappings.return_value.all.return_value = [job_row] if job_row else []
|
||
result.mappings.return_value.first.return_value = job_row
|
||
elif "RETURNING job_id" in stmt_str:
|
||
# cancel/resume
|
||
result.first.return_value = (42,)
|
||
else:
|
||
result.all.return_value = []
|
||
result.mappings.return_value.all.return_value = []
|
||
result.mappings.return_value.first.return_value = None
|
||
result.scalar_one.return_value = 1
|
||
|
||
return result
|
||
|
||
db.execute = mock_execute
|
||
db.commit = MagicMock()
|
||
return db
|
||
|
||
|
||
def _mock_get_db(db):
|
||
"""FastAPI dependency override factory."""
|
||
|
||
def _get_db_override():
|
||
yield db
|
||
|
||
return _get_db_override
|
||
|
||
|
||
def _make_sample_job_row() -> dict[str, Any]:
|
||
import datetime
|
||
|
||
return {
|
||
"job_id": 42,
|
||
"name": "cadastre_pilot",
|
||
"job_kind": "pilot",
|
||
"scope": "pilot",
|
||
"status": "queued",
|
||
"triggered_by": "manual",
|
||
"rate_ms": 333,
|
||
"created_at": datetime.datetime(2026, 5, 15, 10, 0, 0),
|
||
"started_at": None,
|
||
"finished_at": None,
|
||
"heartbeat_at": None,
|
||
"targets_total": 50,
|
||
"targets_done": 0,
|
||
"targets_failed": 0,
|
||
"targets_skipped": 0,
|
||
"requests_count": 0,
|
||
"waf_blocked_count": 0,
|
||
"error": None,
|
||
"phase_state": None,
|
||
"raw_targets": {"quarters": ["66:41:0303161"]},
|
||
}
|
||
|
||
|
||
# ── Tests ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
@requires_scrape_cadastre
|
||
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
|
||
def test_create_job_pilot_returns_job_id() -> None:
|
||
"""POST /cadastre/jobs scope=pilot → job_id + targets_total."""
|
||
quarters_50 = [f"66:41:{i:07d}" for i in range(50)]
|
||
db = _make_mock_db(quarters=quarters_50)
|
||
|
||
from app.core.db import get_db
|
||
|
||
app.dependency_overrides[get_db] = _mock_get_db(db)
|
||
|
||
try:
|
||
with patch("app.workers.tasks.scrape_cadastre.enqueue_cadastre_harvest") as mock_task:
|
||
mock_task.apply_async = MagicMock()
|
||
client = TestClient(app)
|
||
response = client.post(
|
||
"/api/v1/admin/cadastre/jobs",
|
||
json={"scope": "pilot"},
|
||
headers=ADMIN_HEADERS,
|
||
)
|
||
|
||
assert response.status_code == 200, response.text
|
||
body = response.json()
|
||
assert "job_id" in body
|
||
assert body["scope"] == "pilot"
|
||
assert body["targets_total"] == 50
|
||
assert "estimate_minutes" in body
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
@requires_scrape_cadastre
|
||
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
|
||
def test_create_job_manual_list() -> None:
|
||
"""POST /cadastre/jobs scope=manual_list с явным списком."""
|
||
db = _make_mock_db()
|
||
|
||
from app.core.db import get_db
|
||
|
||
app.dependency_overrides[get_db] = _mock_get_db(db)
|
||
|
||
try:
|
||
with patch("app.workers.tasks.scrape_cadastre.enqueue_cadastre_harvest") as mock_task:
|
||
mock_task.apply_async = MagicMock()
|
||
client = TestClient(app)
|
||
response = client.post(
|
||
"/api/v1/admin/cadastre/jobs",
|
||
json={
|
||
"scope": "manual_list",
|
||
"quarters": ["66:41:0303161", "66:41:0303162"],
|
||
},
|
||
headers=ADMIN_HEADERS,
|
||
)
|
||
|
||
assert response.status_code == 200, response.text
|
||
body = response.json()
|
||
assert body["targets_total"] == 2
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
@requires_scrape_cadastre
|
||
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
|
||
def test_create_job_manual_list_empty_quarters_returns_400() -> None:
|
||
"""scope=manual_list без quarters → 400."""
|
||
db = _make_mock_db()
|
||
|
||
from app.core.db import get_db
|
||
|
||
app.dependency_overrides[get_db] = _mock_get_db(db)
|
||
|
||
try:
|
||
client = TestClient(app)
|
||
response = client.post(
|
||
"/api/v1/admin/cadastre/jobs",
|
||
json={"scope": "manual_list"},
|
||
headers=ADMIN_HEADERS,
|
||
)
|
||
assert response.status_code == 400
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
|
||
def test_list_jobs_returns_list() -> None:
|
||
"""GET /cadastre/jobs → список jobs."""
|
||
db = _make_mock_db(job_row=_make_sample_job_row())
|
||
|
||
# Переопределяем execute чтобы list query вернул список
|
||
job_row = _make_sample_job_row()
|
||
mock_result = MagicMock()
|
||
mock_result.mappings.return_value.all.return_value = [job_row]
|
||
db.execute = MagicMock(return_value=mock_result)
|
||
|
||
from app.core.db import get_db
|
||
|
||
app.dependency_overrides[get_db] = _mock_get_db(db)
|
||
|
||
try:
|
||
client = TestClient(app)
|
||
response = client.get("/api/v1/admin/cadastre/jobs", headers=ADMIN_HEADERS)
|
||
assert response.status_code == 200, response.text
|
||
body = response.json()
|
||
assert isinstance(body, list)
|
||
assert len(body) == 1
|
||
assert body[0]["job_id"] == 42
|
||
assert body[0]["scope"] == "pilot"
|
||
assert "progress_pct" in body[0]
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
|
||
def test_get_job_not_found_returns_404() -> None:
|
||
"""GET /cadastre/jobs/9999 → 404."""
|
||
db = MagicMock()
|
||
mock_result = MagicMock()
|
||
mock_result.mappings.return_value.first.return_value = None
|
||
db.execute = MagicMock(return_value=mock_result)
|
||
|
||
from app.core.db import get_db
|
||
|
||
app.dependency_overrides[get_db] = _mock_get_db(db)
|
||
|
||
try:
|
||
client = TestClient(app)
|
||
response = client.get("/api/v1/admin/cadastre/jobs/9999", headers=ADMIN_HEADERS)
|
||
assert response.status_code == 404
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
|
||
def test_cancel_job_success() -> None:
|
||
"""POST /cadastre/jobs/42/cancel → {cancelled: true}."""
|
||
db = MagicMock()
|
||
mock_result = MagicMock()
|
||
mock_result.first.return_value = (42,)
|
||
db.execute = MagicMock(return_value=mock_result)
|
||
db.commit = MagicMock()
|
||
|
||
from app.core.db import get_db
|
||
|
||
app.dependency_overrides[get_db] = _mock_get_db(db)
|
||
|
||
try:
|
||
client = TestClient(app)
|
||
response = client.post(
|
||
"/api/v1/admin/cadastre/jobs/42/cancel",
|
||
headers=ADMIN_HEADERS,
|
||
)
|
||
assert response.status_code == 200, response.text
|
||
body = response.json()
|
||
assert body["cancelled"] is True
|
||
assert body["job_id"] == 42
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
@patch("app.core.config.settings.scrape_admin_token", ADMIN_TOKEN)
|
||
def test_cancel_job_not_found_returns_404() -> None:
|
||
"""POST /cadastre/jobs/9999/cancel когда job не найден → 404."""
|
||
db = MagicMock()
|
||
mock_result = MagicMock()
|
||
mock_result.first.return_value = None # job не найден / уже cancelled
|
||
db.execute = MagicMock(return_value=mock_result)
|
||
db.commit = MagicMock()
|
||
|
||
from app.core.db import get_db
|
||
|
||
app.dependency_overrides[get_db] = _mock_get_db(db)
|
||
|
||
try:
|
||
client = TestClient(app)
|
||
response = client.post(
|
||
"/api/v1/admin/cadastre/jobs/9999/cancel",
|
||
headers=ADMIN_HEADERS,
|
||
)
|
||
assert response.status_code == 404
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
def test_create_job_no_token_returns_401() -> None:
|
||
"""Без X-Admin-Token → 401/503."""
|
||
client = TestClient(app)
|
||
response = client.post(
|
||
"/api/v1/admin/cadastre/jobs",
|
||
json={"scope": "pilot"},
|
||
)
|
||
assert response.status_code in (401, 503)
|