"""Тесты Celery-таски rosstat_macro_sync (mock session + mock fetch, без живой сети). Покрывают: контракт upsert-SQL обоих источников (source='rosstat'/'emiss', ON CONFLICT, CAST not ::), счётчики fetched/upserted (open-data + ЕМИСС), per-source guard (сбой одного источника НЕ мешает другому, но пробрасывается), регистрацию в beat (ежемесячно). """ from __future__ import annotations from datetime import date from decimal import Decimal from typing import Any from unittest.mock import MagicMock, patch def _make_mock_db() -> tuple[MagicMock, list[dict[str, Any]]]: """Mock DB session с поддержкой db.begin_nested() context manager (SAVEPOINT). Возвращает (db, captured_params) — captured содержит params каждого db.execute(..., params). """ captured: list[dict[str, Any]] = [] def execute_router(stmt: Any, params: Any = None) -> MagicMock: if params is not None: captured.append(params) return MagicMock() db = MagicMock() db.execute = execute_router db.begin_nested.return_value.__enter__ = MagicMock(return_value=None) db.begin_nested.return_value.__exit__ = MagicMock(return_value=False) return db, captured def test_upsert_sql_contract() -> None: """UPSERT_ROSSTAT_SQL содержит обязательные литералы контракта macro_indicator и НЕ использует :x::type (psycopg v3).""" from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_SQL sql = str(UPSERT_ROSSTAT_SQL) upper = sql.upper() assert "INTO MACRO_INDICATOR" in upper assert "'rosstat'" in sql assert "'yearly'" in sql assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper assert "VALUE = EXCLUDED.VALUE" in upper assert "UPDATED_AT = NOW()" in upper # psycopg v3: CAST(:x AS type), НИКОГДА :x::type assert "CAST(:ITYPE AS TEXT)" in upper assert "CAST(:D AS DATE)" in upper assert "CAST(:V AS NUMERIC)" in upper assert "::" not in sql def test_task_upserts_each_row_with_correct_params() -> None: """Каждая MacroRow → один execute с bind-параметрами itype/region/d/v/unit/comment.""" from app.services.scrapers.rosstat_emiss import MacroRow from app.workers.tasks import rosstat_macro_sync as task_mod db, captured = _make_mock_db() fake_rows = [ MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c1"), MacroRow("population_total", "ekb", date(2014, 1, 1), Decimal("1412346"), "чел", "c2"), ] with ( patch.object(task_mod, "SessionLocal", return_value=db), patch.object(task_mod, "fetch_all_datasets", return_value=fake_rows) as mock_fetch, patch.object(task_mod, "fetch_all_emiss", return_value=[]) as mock_emiss, ): result = task_mod.rosstat_macro_sync.run() mock_fetch.assert_called_once() mock_emiss.assert_called_once() assert result["fetched"] == 2 assert result["upserted"] == 2 assert result["rosstat"] == {"fetched": 2, "upserted": 2} assert result["emiss"] == {"fetched": 0, "upserted": 0} assert captured == [ { "itype": "population_total", "region": "sverdl", "d": date(2014, 1, 1), "v": Decimal("4320677"), "unit": "чел", "comment": "c1", }, { "itype": "population_total", "region": "ekb", "d": date(2014, 1, 1), "v": Decimal("1412346"), "unit": "чел", "comment": "c2", }, ] db.commit.assert_called() db.close.assert_called_once() def test_task_reraises_fetch_failure() -> None: """Ошибка open-data fetch'а пробрасывается (surfaces в Celery/GlitchTip), DB закрывается. Even with the per-source guard, ошибка ПОСЛЕ попытки обоих источников re-raise'ится. """ from app.workers.tasks import rosstat_macro_sync as task_mod db, _ = _make_mock_db() with ( patch.object(task_mod, "SessionLocal", return_value=db), patch.object(task_mod, "fetch_all_datasets", side_effect=RuntimeError("rosstat down")), patch.object(task_mod, "fetch_all_emiss", return_value=[]), ): try: task_mod.rosstat_macro_sync.run() raise AssertionError("expected RuntimeError to propagate") except RuntimeError as e: assert "rosstat down" in str(e) db.close.assert_called_once() def test_task_emiss_failure_does_not_block_opendata() -> None: """Per-source guard: сбой ЕМИСС НЕ мешает open-data апсертнуться, но ошибка пробрасывается.""" from app.services.scrapers.rosstat_emiss import MacroRow from app.workers.tasks import rosstat_macro_sync as task_mod db, captured = _make_mock_db() rosstat_rows = [ MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c"), ] with ( patch.object(task_mod, "SessionLocal", return_value=db), patch.object(task_mod, "fetch_all_datasets", return_value=rosstat_rows), patch.object(task_mod, "fetch_all_emiss", side_effect=RuntimeError("emiss down")), ): try: task_mod.rosstat_macro_sync.run() raise AssertionError("expected RuntimeError to propagate") except RuntimeError as e: assert "emiss down" in str(e) # open-data всё равно апсертнулся ДО проброса emiss-ошибки assert any(p.get("itype") == "population_total" for p in captured) db.close.assert_called_once() def test_task_upserts_emiss_rows_with_correct_params() -> None: """Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/v/freq/unit/comment.""" from app.services.scrapers.rosstat_emiss import EmissRow from app.workers.tasks import rosstat_macro_sync as task_mod db, captured = _make_mock_db() emiss_rows = [ EmissRow( "income_per_capita", "sverdl", date(2024, 1, 1), Decimal("54006"), "руб", "quarterly", "c", ), ] with ( patch.object(task_mod, "SessionLocal", return_value=db), patch.object(task_mod, "fetch_all_datasets", return_value=[]), patch.object(task_mod, "fetch_all_emiss", return_value=emiss_rows), ): result = task_mod.rosstat_macro_sync.run() assert result["emiss"] == {"fetched": 1, "upserted": 1} assert captured == [ { "itype": "income_per_capita", "region": "sverdl", "d": date(2024, 1, 1), "v": Decimal("54006"), "freq": "quarterly", "unit": "руб", "comment": "c", }, ] def test_upsert_emiss_sql_contract() -> None: """UPSERT_EMISS_SQL: source='emiss', frequency параметризован, ON CONFLICT, CAST not ::.""" from app.workers.tasks.rosstat_macro_sync import UPSERT_EMISS_SQL sql = str(UPSERT_EMISS_SQL) upper = sql.upper() assert "INTO MACRO_INDICATOR" in upper assert "'emiss'" in sql assert "CAST(:FREQ AS TEXT)" in upper assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper assert "CAST(:ITYPE AS TEXT)" in upper assert "CAST(:V AS NUMERIC)" in upper assert "::" not in sql def test_task_registered_in_beat_monthly() -> None: """Задача зарегистрирована в beat как ежемесячная (1-е число).""" from app.workers.beat_schedule import build_beat_schedule schedule = build_beat_schedule() assert "rosstat-macro-sync-monthly" in schedule entry = schedule["rosstat-macro-sync-monthly"] assert entry["task"] == "tasks.rosstat_macro_sync.rosstat_macro_sync" # crontab day_of_month = 1 (1-е число месяца) assert 1 in entry["schedule"].day_of_month