gendesign/backend/tests/workers/test_rosstat_macro_sync.py
Light1YT 2da8a285b3
All checks were successful
CI / changes (push) Successful in 6s
CI / frontend-tests (push) Has been skipped
CI / changes (pull_request) Successful in 6s
CI / frontend-tests (pull_request) Has been skipped
CI / backend-tests (push) Successful in 6m18s
CI / backend-tests (pull_request) Successful in 6m33s
feat(macro): Росстат open-data macro scraper -> macro_indicator (#946)
EPIC2 macro-ingest: региональный (Свердловская обл. ОКТМО 65 / ЕКБ) скрейпер
Росстата в унифицированную macro_indicator (м.123). Зеркалит форму cbr_macro:
pure-парсеры + тонкий httpx, Celery-таска с SAVEPOINT-per-row upsert
(CAST not ::, ON CONFLICT по PK indicator_type/region/obs_date).

ЕМИСС/fedstat.ru (SDMX) за WAF (hard-403 на всех путях из dev-IP) -> ИПЦ/доходы/
СМР-цены пока недоступны (документировано в коде). rosstat.gov.ru/opendata
(стандарт 4.0) WAF-free -> приземляет демографию population_total
(sverdl 4.32M / ekb 1.41M, §7.11).

- services/scrapers/rosstat_emiss.py — fetch+parse open-data (registry->meta->data CSV)
- workers/tasks/rosstat_macro_sync.py — Celery upsert (source=rosstat)
- beat: rosstat-macro-sync-monthly; include в celery_app
- site_finder/macro.py: population_total -> region-aware default sverdl (additive)
- tests: 18 offline-тестов парсера + контракта таски

Без миграции (вписано в существующую схему), без новых зависимостей.

Refs #946.
2026-06-07 18:07:56 +05:00

126 lines
4.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Тесты Celery-таски rosstat_macro_sync (mock session + mock fetch, без живой сети).
Покрывают: контракт upsert-SQL (source='rosstat', ON CONFLICT, CAST not ::),
счётчики fetched/upserted, проброс ошибки fetch'а, регистрацию в beat (ежемесячно).
"""
from __future__ import annotations
from datetime import date
from decimal import Decimal
from typing import Any
from unittest.mock import MagicMock, patch
def _make_mock_db() -> tuple[MagicMock, list[dict[str, Any]]]:
"""Mock DB session с поддержкой db.begin_nested() context manager (SAVEPOINT).
Возвращает (db, captured_params) — captured содержит params каждого
db.execute(..., params).
"""
captured: list[dict[str, Any]] = []
def execute_router(stmt: Any, params: Any = None) -> MagicMock:
if params is not None:
captured.append(params)
return MagicMock()
db = MagicMock()
db.execute = execute_router
db.begin_nested.return_value.__enter__ = MagicMock(return_value=None)
db.begin_nested.return_value.__exit__ = MagicMock(return_value=False)
return db, captured
def test_upsert_sql_contract() -> None:
"""UPSERT_ROSSTAT_SQL содержит обязательные литералы контракта macro_indicator
и НЕ использует :x::type (psycopg v3)."""
from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_SQL
sql = str(UPSERT_ROSSTAT_SQL)
upper = sql.upper()
assert "INTO MACRO_INDICATOR" in upper
assert "'rosstat'" in sql
assert "'yearly'" in sql
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
assert "VALUE = EXCLUDED.VALUE" in upper
assert "UPDATED_AT = NOW()" in upper
# psycopg v3: CAST(:x AS type), НИКОГДА :x::type
assert "CAST(:ITYPE AS TEXT)" in upper
assert "CAST(:D AS DATE)" in upper
assert "CAST(:V AS NUMERIC)" in upper
assert "::" not in sql
def test_task_upserts_each_row_with_correct_params() -> None:
"""Каждая MacroRow → один execute с bind-параметрами itype/region/d/v/unit/comment."""
from app.services.scrapers.rosstat_emiss import MacroRow
from app.workers.tasks import rosstat_macro_sync as task_mod
db, captured = _make_mock_db()
fake_rows = [
MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c1"),
MacroRow("population_total", "ekb", date(2014, 1, 1), Decimal("1412346"), "чел", "c2"),
]
with (
patch.object(task_mod, "SessionLocal", return_value=db),
patch.object(task_mod, "fetch_all_datasets", return_value=fake_rows) as mock_fetch,
):
result = task_mod.rosstat_macro_sync.run()
mock_fetch.assert_called_once()
assert result["fetched"] == 2
assert result["upserted"] == 2
assert captured == [
{
"itype": "population_total",
"region": "sverdl",
"d": date(2014, 1, 1),
"v": Decimal("4320677"),
"unit": "чел",
"comment": "c1",
},
{
"itype": "population_total",
"region": "ekb",
"d": date(2014, 1, 1),
"v": Decimal("1412346"),
"unit": "чел",
"comment": "c2",
},
]
db.commit.assert_called()
db.close.assert_called_once()
def test_task_reraises_fetch_failure() -> None:
"""Ошибка fetch'а пробрасывается (surfaces в Celery/GlitchTip), DB закрывается."""
from app.workers.tasks import rosstat_macro_sync as task_mod
db, _ = _make_mock_db()
with (
patch.object(task_mod, "SessionLocal", return_value=db),
patch.object(task_mod, "fetch_all_datasets", side_effect=RuntimeError("rosstat down")),
):
try:
task_mod.rosstat_macro_sync.run()
raise AssertionError("expected RuntimeError to propagate")
except RuntimeError as e:
assert "rosstat down" in str(e)
db.close.assert_called_once()
def test_task_registered_in_beat_monthly() -> None:
"""Задача зарегистрирована в beat как ежемесячная (1-е число)."""
from app.workers.beat_schedule import build_beat_schedule
schedule = build_beat_schedule()
assert "rosstat-macro-sync-monthly" in schedule
entry = schedule["rosstat-macro-sync-monthly"]
assert entry["task"] == "tasks.rosstat_macro_sync.rosstat_macro_sync"
# crontab day_of_month = 1 (1-е число месяца)
assert 1 in entry["schedule"].day_of_month