Light1YT
f9e045028c
All checks were successful
CI / changes (push) Successful in 6s
CI / changes (pull_request) Successful in 7s
CI / frontend-tests (pull_request) Has been skipped
CI / backend-tests (pull_request) Successful in 6m27s
Deploy / build-frontend (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / backend-tests (push) Successful in 6m29s
Deploy / changes (push) Successful in 6s
Deploy / build-backend (push) Successful in 1m43s
Deploy / build-worker (push) Successful in 3m3s
Deploy / deploy (push) Successful in 1m15s
Closes the last #946 gap (СМР). Fetches «Индексы цен производителей на строительную продукцию по РФ» from the OPEN Rosstat xlsx (rosstat.gov.ru/storage/mediabank/Invest_ind_stroitel_MM-YYYY.xlsx), parses the monthly «к предыдущему месяцу» section → macro_indicator (indicator_type='construction_price_index', region='rf', monthly, %). The reCAPTCHA-gated fedstat dataGrid.do path is DELIBERATELY NOT used — the open Rosstat source needs no captcha. EMISS id=31108 documented as a prod-only SDMX alternative (not wired). URL is date-derived with bounded look-back (filename changes monthly; _fetch_construction_latest walks current month back to 6, first 200 wins; year-boundary handled, exhaustion raises, 404 breaks fast — no hang). Wired into rosstat_macro_sync with per-source try/except (can't break demography/income) + SAVEPOINT-per-row. None-not-0 on blank months. 33 tests (incl look-back + year-boundary + graceful), ruff clean. Note: no construction-cost channel in macro_coefficient yet (demand-side only) — series is ingested/available for future cost-side use + display. Refs #946
296 lines
12 KiB
Python
296 lines
12 KiB
Python
"""Тесты Celery-таски rosstat_macro_sync (mock session + mock fetch, без живой сети).
|
||
|
||
Покрывают: контракт upsert-SQL обоих источников (source='rosstat'/'emiss', ON CONFLICT,
|
||
CAST not ::), счётчики fetched/upserted (open-data + ЕМИСС), per-source guard (сбой
|
||
одного источника НЕ мешает другому, но пробрасывается), регистрацию в beat (ежемесячно).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import date
|
||
from decimal import Decimal
|
||
from typing import Any
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
|
||
def _make_mock_db() -> tuple[MagicMock, list[dict[str, Any]]]:
|
||
"""Mock DB session с поддержкой db.begin_nested() context manager (SAVEPOINT).
|
||
|
||
Возвращает (db, captured_params) — captured содержит params каждого
|
||
db.execute(..., params).
|
||
"""
|
||
captured: list[dict[str, Any]] = []
|
||
|
||
def execute_router(stmt: Any, params: Any = None) -> MagicMock:
|
||
if params is not None:
|
||
captured.append(params)
|
||
return MagicMock()
|
||
|
||
db = MagicMock()
|
||
db.execute = execute_router
|
||
db.begin_nested.return_value.__enter__ = MagicMock(return_value=None)
|
||
db.begin_nested.return_value.__exit__ = MagicMock(return_value=False)
|
||
return db, captured
|
||
|
||
|
||
def test_upsert_sql_contract() -> None:
|
||
"""UPSERT_ROSSTAT_SQL содержит обязательные литералы контракта macro_indicator
|
||
и НЕ использует :x::type (psycopg v3)."""
|
||
from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_SQL
|
||
|
||
sql = str(UPSERT_ROSSTAT_SQL)
|
||
upper = sql.upper()
|
||
|
||
assert "INTO MACRO_INDICATOR" in upper
|
||
assert "'rosstat'" in sql
|
||
assert "'yearly'" in sql
|
||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
|
||
assert "VALUE = EXCLUDED.VALUE" in upper
|
||
assert "UPDATED_AT = NOW()" in upper
|
||
# psycopg v3: CAST(:x AS type), НИКОГДА :x::type
|
||
assert "CAST(:ITYPE AS TEXT)" in upper
|
||
assert "CAST(:D AS DATE)" in upper
|
||
assert "CAST(:V AS NUMERIC)" in upper
|
||
assert "::" not in sql
|
||
|
||
|
||
def test_task_upserts_each_row_with_correct_params() -> None:
|
||
"""Каждая MacroRow → один execute с bind-параметрами itype/region/d/v/unit/comment."""
|
||
from app.services.scrapers.rosstat_emiss import MacroRow
|
||
from app.workers.tasks import rosstat_macro_sync as task_mod
|
||
|
||
db, captured = _make_mock_db()
|
||
fake_rows = [
|
||
MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c1"),
|
||
MacroRow("population_total", "ekb", date(2014, 1, 1), Decimal("1412346"), "чел", "c2"),
|
||
]
|
||
|
||
with (
|
||
patch.object(task_mod, "SessionLocal", return_value=db),
|
||
patch.object(task_mod, "fetch_all_datasets", return_value=fake_rows) as mock_fetch,
|
||
patch.object(task_mod, "fetch_all_emiss", return_value=[]) as mock_emiss,
|
||
patch.object(task_mod, "fetch_construction_index", return_value=[]) as mock_constr,
|
||
):
|
||
result = task_mod.rosstat_macro_sync.run()
|
||
|
||
mock_fetch.assert_called_once()
|
||
mock_emiss.assert_called_once()
|
||
mock_constr.assert_called_once()
|
||
assert result["fetched"] == 2
|
||
assert result["upserted"] == 2
|
||
assert result["rosstat"] == {"fetched": 2, "upserted": 2}
|
||
assert result["emiss"] == {"fetched": 0, "upserted": 0}
|
||
assert result["construction"] == {"fetched": 0, "upserted": 0}
|
||
assert captured == [
|
||
{
|
||
"itype": "population_total",
|
||
"region": "sverdl",
|
||
"d": date(2014, 1, 1),
|
||
"v": Decimal("4320677"),
|
||
"unit": "чел",
|
||
"comment": "c1",
|
||
},
|
||
{
|
||
"itype": "population_total",
|
||
"region": "ekb",
|
||
"d": date(2014, 1, 1),
|
||
"v": Decimal("1412346"),
|
||
"unit": "чел",
|
||
"comment": "c2",
|
||
},
|
||
]
|
||
db.commit.assert_called()
|
||
db.close.assert_called_once()
|
||
|
||
|
||
def test_task_reraises_fetch_failure() -> None:
|
||
"""Ошибка open-data fetch'а пробрасывается (surfaces в Celery/GlitchTip), DB закрывается.
|
||
|
||
Even with the per-source guard, ошибка ПОСЛЕ попытки обоих источников re-raise'ится.
|
||
"""
|
||
from app.workers.tasks import rosstat_macro_sync as task_mod
|
||
|
||
db, _ = _make_mock_db()
|
||
|
||
with (
|
||
patch.object(task_mod, "SessionLocal", return_value=db),
|
||
patch.object(task_mod, "fetch_all_datasets", side_effect=RuntimeError("rosstat down")),
|
||
patch.object(task_mod, "fetch_all_emiss", return_value=[]),
|
||
patch.object(task_mod, "fetch_construction_index", return_value=[]),
|
||
):
|
||
try:
|
||
task_mod.rosstat_macro_sync.run()
|
||
raise AssertionError("expected RuntimeError to propagate")
|
||
except RuntimeError as e:
|
||
assert "rosstat down" in str(e)
|
||
|
||
db.close.assert_called_once()
|
||
|
||
|
||
def test_task_emiss_failure_does_not_block_opendata() -> None:
|
||
"""Per-source guard: сбой ЕМИСС НЕ мешает open-data апсертнуться, но ошибка пробрасывается."""
|
||
from app.services.scrapers.rosstat_emiss import MacroRow
|
||
from app.workers.tasks import rosstat_macro_sync as task_mod
|
||
|
||
db, captured = _make_mock_db()
|
||
rosstat_rows = [
|
||
MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c"),
|
||
]
|
||
|
||
with (
|
||
patch.object(task_mod, "SessionLocal", return_value=db),
|
||
patch.object(task_mod, "fetch_all_datasets", return_value=rosstat_rows),
|
||
patch.object(task_mod, "fetch_all_emiss", side_effect=RuntimeError("emiss down")),
|
||
patch.object(task_mod, "fetch_construction_index", return_value=[]),
|
||
):
|
||
try:
|
||
task_mod.rosstat_macro_sync.run()
|
||
raise AssertionError("expected RuntimeError to propagate")
|
||
except RuntimeError as e:
|
||
assert "emiss down" in str(e)
|
||
|
||
# open-data всё равно апсертнулся ДО проброса emiss-ошибки
|
||
assert any(p.get("itype") == "population_total" for p in captured)
|
||
db.close.assert_called_once()
|
||
|
||
|
||
def test_task_upserts_emiss_rows_with_correct_params() -> None:
|
||
"""Каждая EmissRow → execute UPSERT_EMISS_SQL с itype/region/d/v/freq/unit/comment."""
|
||
from app.services.scrapers.rosstat_emiss import EmissRow
|
||
from app.workers.tasks import rosstat_macro_sync as task_mod
|
||
|
||
db, captured = _make_mock_db()
|
||
emiss_rows = [
|
||
EmissRow(
|
||
"income_per_capita", "sverdl", date(2024, 1, 1),
|
||
Decimal("54006"), "руб", "quarterly", "c",
|
||
),
|
||
]
|
||
|
||
with (
|
||
patch.object(task_mod, "SessionLocal", return_value=db),
|
||
patch.object(task_mod, "fetch_all_datasets", return_value=[]),
|
||
patch.object(task_mod, "fetch_all_emiss", return_value=emiss_rows),
|
||
patch.object(task_mod, "fetch_construction_index", return_value=[]),
|
||
):
|
||
result = task_mod.rosstat_macro_sync.run()
|
||
|
||
assert result["emiss"] == {"fetched": 1, "upserted": 1}
|
||
assert captured == [
|
||
{
|
||
"itype": "income_per_capita",
|
||
"region": "sverdl",
|
||
"d": date(2024, 1, 1),
|
||
"v": Decimal("54006"),
|
||
"freq": "quarterly",
|
||
"unit": "руб",
|
||
"comment": "c",
|
||
},
|
||
]
|
||
|
||
|
||
def test_upsert_emiss_sql_contract() -> None:
|
||
"""UPSERT_EMISS_SQL: source='emiss', frequency параметризован, ON CONFLICT, CAST not ::."""
|
||
from app.workers.tasks.rosstat_macro_sync import UPSERT_EMISS_SQL
|
||
|
||
sql = str(UPSERT_EMISS_SQL)
|
||
upper = sql.upper()
|
||
assert "INTO MACRO_INDICATOR" in upper
|
||
assert "'emiss'" in sql
|
||
assert "CAST(:FREQ AS TEXT)" in upper
|
||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
|
||
assert "CAST(:ITYPE AS TEXT)" in upper
|
||
assert "CAST(:V AS NUMERIC)" in upper
|
||
assert "::" not in sql
|
||
|
||
|
||
def test_upsert_rosstat_monthly_sql_contract() -> None:
|
||
"""UPSERT_ROSSTAT_MONTHLY_SQL: source='rosstat', frequency='monthly', CAST not ::."""
|
||
from app.workers.tasks.rosstat_macro_sync import UPSERT_ROSSTAT_MONTHLY_SQL
|
||
|
||
sql = str(UPSERT_ROSSTAT_MONTHLY_SQL)
|
||
upper = sql.upper()
|
||
assert "INTO MACRO_INDICATOR" in upper
|
||
assert "'rosstat'" in sql
|
||
assert "'monthly'" in sql
|
||
assert "ON CONFLICT (INDICATOR_TYPE, REGION, OBS_DATE) DO UPDATE" in upper
|
||
assert "CAST(:ITYPE AS TEXT)" in upper
|
||
assert "CAST(:D AS DATE)" in upper
|
||
assert "CAST(:V AS NUMERIC)" in upper
|
||
assert "::" not in sql
|
||
|
||
|
||
def test_task_upserts_construction_rows_with_correct_params() -> None:
|
||
"""Каждая construction MacroRow → execute UPSERT_ROSSTAT_MONTHLY_SQL c bind-параметрами."""
|
||
from app.services.scrapers.rosstat_emiss import MacroRow
|
||
from app.workers.tasks import rosstat_macro_sync as task_mod
|
||
|
||
db, captured = _make_mock_db()
|
||
constr_rows = [
|
||
MacroRow(
|
||
"construction_price_index", "rf", date(2025, 1, 1),
|
||
Decimal("100.8"), "%", "smr",
|
||
),
|
||
]
|
||
|
||
with (
|
||
patch.object(task_mod, "SessionLocal", return_value=db),
|
||
patch.object(task_mod, "fetch_all_datasets", return_value=[]),
|
||
patch.object(task_mod, "fetch_all_emiss", return_value=[]),
|
||
patch.object(task_mod, "fetch_construction_index", return_value=constr_rows),
|
||
):
|
||
result = task_mod.rosstat_macro_sync.run()
|
||
|
||
assert result["construction"] == {"fetched": 1, "upserted": 1}
|
||
assert result["fetched"] == 1
|
||
assert captured == [
|
||
{
|
||
"itype": "construction_price_index",
|
||
"region": "rf",
|
||
"d": date(2025, 1, 1),
|
||
"v": Decimal("100.8"),
|
||
"unit": "%",
|
||
"comment": "smr",
|
||
},
|
||
]
|
||
|
||
|
||
def test_task_construction_failure_does_not_block_others() -> None:
|
||
"""Per-source guard: сбой СМР-источника НЕ мешает open-data/ЕМИСС, ошибка пробрасывается."""
|
||
from app.services.scrapers.rosstat_emiss import MacroRow
|
||
from app.workers.tasks import rosstat_macro_sync as task_mod
|
||
|
||
db, captured = _make_mock_db()
|
||
rosstat_rows = [
|
||
MacroRow("population_total", "sverdl", date(2014, 1, 1), Decimal("4320677"), "чел", "c"),
|
||
]
|
||
|
||
with (
|
||
patch.object(task_mod, "SessionLocal", return_value=db),
|
||
patch.object(task_mod, "fetch_all_datasets", return_value=rosstat_rows),
|
||
patch.object(task_mod, "fetch_all_emiss", return_value=[]),
|
||
patch.object(
|
||
task_mod, "fetch_construction_index", side_effect=RuntimeError("rosstat xlsx down")
|
||
),
|
||
):
|
||
try:
|
||
task_mod.rosstat_macro_sync.run()
|
||
raise AssertionError("expected RuntimeError to propagate")
|
||
except RuntimeError as e:
|
||
assert "rosstat xlsx down" in str(e)
|
||
|
||
# open-data всё равно апсертнулся ДО проброса construction-ошибки
|
||
assert any(p.get("itype") == "population_total" for p in captured)
|
||
db.close.assert_called_once()
|
||
|
||
|
||
def test_task_registered_in_beat_monthly() -> None:
|
||
"""Задача зарегистрирована в beat как ежемесячная (1-е число)."""
|
||
from app.workers.beat_schedule import build_beat_schedule
|
||
|
||
schedule = build_beat_schedule()
|
||
assert "rosstat-macro-sync-monthly" in schedule
|
||
entry = schedule["rosstat-macro-sync-monthly"]
|
||
assert entry["task"] == "tasks.rosstat_macro_sync.rosstat_macro_sync"
|
||
# crontab day_of_month = 1 (1-е число месяца)
|
||
assert 1 in entry["schedule"].day_of_month
|