gendesign/backend/tests/workers/test_rosstat_macro_sync.py

"""Тесты Celery-таски rosstat_macro_sync (mock session + mock fetch, без живой сети).

Покрывают: контракт upsert-SQL обоих источников (source='rosstat'/'emiss', ON CONFLICT,
CAST not ::), счётчики fetched/upserted (open-data + ЕМИСС), per-source guard (сбой
одного источника НЕ мешает другому, но пробрасывается), регистрацию в beat (ежемесячно).
"""

from __future__ import annotations

from datetime import date
from decimal import Decimal
from typing import Any
from unittest.mock import MagicMock, patch


def _make_mock_db() -> tuple[MagicMock, list[dict[str, Any]]]:
    """Mock DB session с поддержкой db.begin_nested() context manager (SAVEPOINT).

    Возвращает (db, captured_params) — captured содержит params каждого
    db.execute(..., params).
    """
    captured: list[dict[str, Any]] = []

    def execute_router(stmt: Any, params: Any = None) -> MagicMock:
        if params is not None:
            captured.append(params)
        return MagicMock()

    db = MagicMock()
    db.execute = execute_router
    db.begin_nested.return_value.__enter__ = MagicMock(return_value=None)
    db.begin_nested.return_value.__exit__ = MagicMock(return_value=False)
    return db, captured


def test_upsert_sql_contract() -> None:
    """UPSERT_ROSSTAT_SQL содержит обязательные литералы контракта macro_indicator
    и НЕ использует :x::type (psycopg v3). PK включает period_type (migration 163)."""
    from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_SQL

    sql = str(UPSERT_ROSSTAT_SQL)
    upper = sql.upper()

    assert "INTO MACRO_INDICATOR" in upper
    assert "'rosstat'" in sql
    assert "'yearly'" in sql
    assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
    assert "VALUE = EXCLUDED.VALUE" in upper
    assert "UPDATED_AT = NOW()" in upper
    # psycopg v3: CAST(:x AS type), НИКОГДА :x::type
    assert "CAST(:ITYPE AS TEXT)" in upper
    assert "CAST(:D AS DATE)" in upper
    assert "CAST(:V AS NUMERIC)" in upper
    assert "::" not in sql
    # не-ЕМИСС строки используют period_type='unknown' (литерал в SQL)
    assert "'unknown'" in sql


def test_task_upserts_each_row_with_correct_params() -> None:
    """Каждая MacroRow → один execute с bind-параметрами itype/region/d/v/unit/comment."""
    from app.services.scrapers.rosstat_emiss import MacroRow
    from app.workers.tasks import rosstat_macro_sync as task_mod

    db, captured = _make_mock_db()
    fake_rows = [
        MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c1"),
        MacroRow("population_total", "ekb", date(2014, 1, 1), Decimal("1412346"), "чел", "c2"),
    ]

    with (
        patch.object(task_mod, "SessionLocal", return_value=db),
        patch.object(task_mod, "fetch_all_datasets", return_value=fake_rows) as mock_fetch,
        patch.object(task_mod, "fetch_all_emiss", return_value=[]) as mock_emiss,
        patch.object(task_mod, "fetch_construction_index", return_value=[]) as mock_constr,
    ):
        result = task_mod.rosstat_macro_sync.run()

    mock_fetch.assert_called_once()
    mock_emiss.assert_called_once()
    mock_constr.assert_called_once()
    assert result["fetched"] == 2
    assert result["upserted"] == 2
    assert result["rosstat"] == {"fetched": 2, "upserted": 2}
    assert result["emiss"] == {"fetched": 0, "upserted": 0}
    assert result["construction"] == {"fetched": 0, "upserted": 0}
    assert captured == [
        {
            "itype": "population_total",
            "region": "sverdl",
            "d": date(2014, 1, 1),
            "v": Decimal("4320677"),
            "unit": "чел",
            "comment": "c1",
        },
        {
            "itype": "population_total",
            "region": "ekb",
            "d": date(2014, 1, 1),
            "v": Decimal("1412346"),
            "unit": "чел",
            "comment": "c2",
        },
    ]
    db.commit.assert_called()
    db.close.assert_called_once()


def test_task_reraises_fetch_failure() -> None:
    """Ошибка open-data fetch'а пробрасывается (surfaces в Celery/GlitchTip), DB закрывается.

    Even with the per-source guard, ошибка ПОСЛЕ попытки обоих источников re-raise'ится.
    """
    from app.workers.tasks import rosstat_macro_sync as task_mod

    db, _ = _make_mock_db()

    with (
        patch.object(task_mod, "SessionLocal", return_value=db),
        patch.object(task_mod, "fetch_all_datasets", side_effect=RuntimeError("rosstat down")),
        patch.object(task_mod, "fetch_all_emiss", return_value=[]),
        patch.object(task_mod, "fetch_construction_index", return_value=[]),
    ):
        try:
            task_mod.rosstat_macro_sync.run()
            raise AssertionError("expected RuntimeError to propagate")
        except RuntimeError as e:
            assert "rosstat down" in str(e)

    db.close.assert_called_once()


def test_task_emiss_failure_does_not_block_opendata() -> None:
    """Per-source guard: сбой ЕМИСС НЕ мешает open-data апсертнуться, но ошибка пробрасывается."""
    from app.services.scrapers.rosstat_emiss import MacroRow
    from app.workers.tasks import rosstat_macro_sync as task_mod

    db, captured = _make_mock_db()
    rosstat_rows = [
        MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c"),
    ]

    with (
        patch.object(task_mod, "SessionLocal", return_value=db),
        patch.object(task_mod, "fetch_all_datasets", return_value=rosstat_rows),
        patch.object(task_mod, "fetch_all_emiss", side_effect=RuntimeError("emiss down")),
        patch.object(task_mod, "fetch_construction_index", return_value=[]),
    ):
        try:
            task_mod.rosstat_macro_sync.run()
            raise AssertionError("expected RuntimeError to propagate")
        except RuntimeError as e:
            assert "emiss down" in str(e)

    # open-data всё равно апсертнулся ДО проброса emiss-ошибки
    assert any(p.get("itype") == "population_total" for p in captured)
    db.close.assert_called_once()


def test_task_upserts_emiss_rows_with_correct_params() -> None:
    """Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/period_type/v/freq/unit/comment.

    period_type передаётся из r.period_type (часть нового PK, migration 163).
    """
    from app.services.scrapers.rosstat_emiss import EmissRow
    from app.workers.tasks import rosstat_macro_sync as task_mod

    db, captured = _make_mock_db()
    emiss_rows = [
        EmissRow(
            "income_per_capita",
            "sverdl",
            date(2024, 1, 1),
            Decimal("54006"),
            "руб",
            "quarterly",
            "c",
            period_type="quarter",
        ),
    ]

    with (
        patch.object(task_mod, "SessionLocal", return_value=db),
        patch.object(task_mod, "fetch_all_datasets", return_value=[]),
        patch.object(task_mod, "fetch_all_emiss", return_value=emiss_rows),
        patch.object(task_mod, "fetch_construction_index", return_value=[]),
    ):
        result = task_mod.rosstat_macro_sync.run()

    assert result["emiss"] == {"fetched": 1, "upserted": 1}
    assert captured == [
        {
            "itype": "income_per_capita",
            "region": "sverdl",
            "d": date(2024, 1, 1),
            "period_type": "quarter",
            "v": Decimal("54006"),
            "freq": "quarterly",
            "unit": "руб",
            "comment": "c",
        },
    ]


def test_upsert_emiss_sql_contract() -> None:
    """UPSERT_EMISS_SQL: source='emiss', frequency+period_type параметризованы,
    ON CONFLICT включает period_type (migration 163), CAST not ::."""
    from app.workers.tasks.rosstat_macro_sync import UPSERT_EMISS_SQL

    sql = str(UPSERT_EMISS_SQL)
    upper = sql.upper()
    assert "INTO MACRO_INDICATOR" in upper
    assert "'emiss'" in sql
    assert "CAST(:FREQ AS TEXT)" in upper
    assert "CAST(:PERIOD_TYPE AS TEXT)" in upper
    assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
    assert "CAST(:ITYPE AS TEXT)" in upper
    assert "CAST(:V AS NUMERIC)" in upper
    assert "::" not in sql


def test_upsert_rosstat_monthly_sql_contract() -> None:
    """UPSERT_ROSSTAT_MONTHLY_SQL: source='rosstat', frequency='monthly', period_type='unknown',
    ON CONFLICT включает period_type (migration 163), CAST not ::."""
    from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_MONTHLY_SQL

    sql = str(UPSERT_ROSSTAT_MONTHLY_SQL)
    upper = sql.upper()
    assert "INTO MACRO_INDICATOR" in upper
    assert "'rosstat'" in sql
    assert "'monthly'" in sql
    assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE, PERIOD_TYPE) DO UPDATE" in upper
    assert "CAST(:ITYPE AS TEXT)" in upper
    assert "CAST(:D AS DATE)" in upper
    assert "CAST(:V AS NUMERIC)" in upper
    assert "::" not in sql
    assert "'unknown'" in sql


def test_task_upserts_construction_rows_with_correct_params() -> None:
    """Каждая construction MacroRow → execute UPSERT_ROSSTAT_MONTHLY_SQL c bind-параметрами."""
    from app.services.scrapers.rosstat_emiss import MacroRow
    from app.workers.tasks import rosstat_macro_sync as task_mod

    db, captured = _make_mock_db()
    constr_rows = [
        MacroRow(
            "construction_price_index",
            "rf",
            date(2025, 1, 1),
            Decimal("100.8"),
            "%",
            "smr",
        ),
    ]

    with (
        patch.object(task_mod, "SessionLocal", return_value=db),
        patch.object(task_mod, "fetch_all_datasets", return_value=[]),
        patch.object(task_mod, "fetch_all_emiss", return_value=[]),
        patch.object(task_mod, "fetch_construction_index", return_value=constr_rows),
    ):
        result = task_mod.rosstat_macro_sync.run()

    assert result["construction"] == {"fetched": 1, "upserted": 1}
    assert result["fetched"] == 1
    assert captured == [
        {
            "itype": "construction_price_index",
            "region": "rf",
            "d": date(2025, 1, 1),
            "v": Decimal("100.8"),
            "unit": "%",
            "comment": "smr",
        },
    ]


def test_task_construction_failure_does_not_block_others() -> None:
    """Per-source guard: сбой СМР-источника НЕ мешает open-data/ЕМИСС, ошибка пробрасывается."""
    from app.services.scrapers.rosstat_emiss import MacroRow
    from app.workers.tasks import rosstat_macro_sync as task_mod

    db, captured = _make_mock_db()
    rosstat_rows = [
        MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c"),
    ]

    with (
        patch.object(task_mod, "SessionLocal", return_value=db),
        patch.object(task_mod, "fetch_all_datasets", return_value=rosstat_rows),
        patch.object(task_mod, "fetch_all_emiss", return_value=[]),
        patch.object(
            task_mod, "fetch_construction_index", side_effect=RuntimeError("rosstat xlsx down")
        ),
    ):
        try:
            task_mod.rosstat_macro_sync.run()
            raise AssertionError("expected RuntimeError to propagate")
        except RuntimeError as e:
            assert "rosstat xlsx down" in str(e)

    # open-data всё равно апсертнулся ДО проброса construction-ошибки
    assert any(p.get("itype") == "population_total" for p in captured)
    db.close.assert_called_once()


def test_task_registered_in_beat_monthly() -> None:
    """Задача зарегистрирована в beat как ежемесячная (1-е число)."""
    from app.workers.beat_schedule import build_beat_schedule

    schedule = build_beat_schedule()
    assert "rosstat-macro-sync-monthly" in schedule
    entry = schedule["rosstat-macro-sync-monthly"]
    assert entry["task"] == "tasks.rosstat_macro_sync.rosstat_macro_sync"
    # crontab day_of_month = 1 (1-е число месяца)
    assert 1 in entry["schedule"].day_of_month