"""Тесты Celery-таски supply_layers_refresh (#950 EPIC 6 Step5). Чистые тесты (mock session + mock сервиса, без живой сети/БД). Покрывают: • контракт UPSERT-SQL: точный ON CONFLICT target (uq_supply_layers_logical column list), DO UPDATE по ВСЕМ мутабельным колонкам (database-expert: иначе stale), ключевые колонки НЕ обновляются, CAST(:x AS type) не :x::type (psycopg v3); • district-enumeration: UNION objective_lots + domrf_kn_objects (region_cd=66), DISTINCT non-NULL, форма запроса; • coherent single snapshot_date: один date.today() прокинут КАЖДОЙ строке; • per-row SAVEPOINT (db.begin_nested) на каждой строке; • счётчики upserted/skipped; resilience: сбой compute одного района не валит батч. """ from __future__ import annotations import os from datetime import date from typing import Any from unittest.mock import MagicMock, call, patch os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from app.services.site_finder.supply_layers import SupplyLayerRow def _make_mock_db( district_rows: list[dict[str, Any]], ) -> tuple[MagicMock, list[dict[str, Any]], list[Any]]: """Mock DB session с поддержкой db.begin_nested() (SAVEPOINT). Первый db.execute (без params, либо с region_cd) трактуем как districts-запрос → возвращаем district_rows через .mappings().all(). Остальные execute (UPSERT) — захватываем params в captured_upserts. Returns: (db, captured_upserts, executed_sql) — captured_upserts содержит params каждого UPSERT-execute; executed_sql — все stmt-объекты (для проверки SQL). """ captured_upserts: list[dict[str, Any]] = [] executed_sql: list[Any] = [] state = {"districts_served": False} def execute_router(stmt: Any, params: Any = None) -> MagicMock: executed_sql.append(stmt) result = MagicMock() # districts-запрос: первый execute (region_cd-only params или до UPSERT'ов) if not state["districts_served"] and "supply_layers" not in str(stmt).lower(): state["districts_served"] = True result.mappings.return_value.all.return_value = district_rows return result if params is not None: captured_upserts.append(params) return result db = MagicMock() db.execute = execute_router db.begin_nested.return_value.__enter__ = MagicMock(return_value=None) db.begin_nested.return_value.__exit__ = MagicMock(return_value=False) return db, captured_upserts, executed_sql def _row( *, layer: int, district: str, source: str, confidence: str = "high", units: int | None = 10, area: float | None = 500.0, obj_class: str | None = None, dev_group_name: str | None = None, expected: date | None = None, snapshot_date: date | None = None, ) -> SupplyLayerRow: return SupplyLayerRow( layer=layer, # type: ignore[arg-type] district_name=district, complex_id=None, obj_class=obj_class, dev_group_name=dev_group_name, units_estimate=units, area_estimate=area, expected_online_date=expected, source=source, # type: ignore[arg-type] confidence=confidence, # type: ignore[arg-type] method="test", snapshot_date=snapshot_date, ) # ── UPSERT-SQL контракт ────────────────────────────────────────────────────── def test_upsert_sql_conflict_target_and_updated_columns() -> None: """ON CONFLICT таргетит точный логический ключ (#970: + dev_group_name); DO UPDATE обновляет ВСЕ мутабельные колонки + computed_at; ключевые колонки НЕ в SET; нет :x::type.""" from app.workers.tasks.supply_layers_refresh import _UPSERT_SQL sql = str(_UPSERT_SQL) upper = sql.upper() # нормализуем whitespace — ON CONFLICT column-list перенесён на неск. строк (м.128). flat = " ".join(upper.split()) assert "INSERT INTO SUPPLY_LAYERS" in upper # точный ON CONFLICT target = uq_supply_layers_logical column-list (м.128, с dev_group_name) assert ( "ON CONFLICT ( LAYER, DISTRICT_NAME, COMPLEX_ID, OBJ_CLASS, " "DEV_GROUP_NAME, SOURCE, SNAPSHOT_DATE )" in flat ), f"ON CONFLICT target mismatch (must include dev_group_name): {flat}" # dev_group_name присутствует в INSERT column-list и в VALUES как CAST(...AS text) assert "DEV_GROUP_NAME," in upper assert "CAST(:DEV_GROUP_NAME AS TEXT)" in upper assert "DO UPDATE SET" in upper # ВСЕ мутабельные колонки обновляются (database-expert: иначе stale-данные) for col in ( "UNITS_ESTIMATE = EXCLUDED.UNITS_ESTIMATE", "AREA_ESTIMATE = EXCLUDED.AREA_ESTIMATE", "EXPECTED_ONLINE_DATE = EXCLUDED.EXPECTED_ONLINE_DATE", "CONFIDENCE = EXCLUDED.CONFIDENCE", "METHOD = EXCLUDED.METHOD", "COMPUTED_AT = NOW()", ): assert col in upper, f"missing updated column: {col}" # ключевые колонки НЕ обновляются в SET-части (после DO UPDATE SET) — dev_group_name # теперь тоже КЛЮЧ (#970) → не должен апдейтиться. set_clause = upper.split("DO UPDATE SET", 1)[1] for key_col in ( "LAYER = EXCLUDED", "DISTRICT_NAME = EXCLUDED", "COMPLEX_ID = EXCLUDED", "OBJ_CLASS = EXCLUDED", "DEV_GROUP_NAME = EXCLUDED", "SOURCE = EXCLUDED", "SNAPSHOT_DATE = EXCLUDED", ): assert key_col not in set_clause, f"key column must not be updated: {key_col}" # psycopg v3: CAST(:x AS type), НИКОГДА :x::type assert "::" not in sql assert "CAST(:LAYER AS SMALLINT)" in upper assert "CAST(:SNAPSHOT_DATE AS DATE)" in upper def test_no_double_colon_cast_in_sql() -> None: """psycopg v3: ни один bind не использует :name::type (recurring bug class).""" import re from app.workers.tasks.supply_layers_refresh import _DISTRICTS_SQL, _UPSERT_SQL for stmt in (_UPSERT_SQL, _DISTRICTS_SQL): sql = str(stmt) assert not re.search(r":[a-z_]+::[a-z]", sql), f"found :x::type cast in: {sql}" # ── district enumeration ───────────────────────────────────────────────────── def test_districts_query_unions_both_sources() -> None: """Запрос районов = UNION objective_lots.district + domrf_kn_objects.district_name (region_cd=66), DISTINCT non-NULL — не хардкодит имена.""" from app.workers.tasks.supply_layers_refresh import _DISTRICTS_SQL sql = str(_DISTRICTS_SQL) upper = sql.upper() assert "FROM OBJECTIVE_LOTS" in upper assert "FROM DOMRF_KN_OBJECTS" in upper assert "UNION" in upper assert "DISTINCT" in upper assert "IS NOT NULL" in upper # region-фильтр domrf через CAST(:x AS integer) assert "REGION_CD = CAST(:REGION_CD AS INTEGER)" in upper def test_list_districts_filters_null_and_returns_names() -> None: """_list_districts отдаёт только non-NULL имена; на пустом источнике → [].""" from app.workers.tasks import supply_layers_refresh as task_mod db, _, _ = _make_mock_db( [{"district_name": "Кировский"}, {"district_name": None}, {"district_name": "Чкаловский"}] ) assert task_mod._list_districts(db) == ["Кировский", "Чкаловский"] db_empty, _, _ = _make_mock_db([]) assert task_mod._list_districts(db_empty) == [] def test_list_districts_graceful_on_query_error() -> None: """Сбой districts-запроса → [] (батч no-op, не падает).""" from app.workers.tasks import supply_layers_refresh as task_mod db = MagicMock() db.execute.side_effect = RuntimeError("db down") assert task_mod._list_districts(db) == [] # ── per-row UPSERT / SAVEPOINT ─────────────────────────────────────────────── def test_upsert_rows_uses_savepoint_per_row() -> None: """Каждая строка UPSERT-ится внутри db.begin_nested() (SAVEPOINT, backend.md).""" from app.workers.tasks import supply_layers_refresh as task_mod db, captured, _ = _make_mock_db([]) # пометим districts как обслуженные, чтобы execute шёл в UPSERT-ветку db.execute("SELECT FROM supply_layers warmup", None) snap = date(2026, 6, 3) rows = [ _row(layer=1, district="Кировский", source="objective", snapshot_date=snap), _row(layer=2, district="Кировский", source="domrf_declared", snapshot_date=snap), ] upserted, skipped = task_mod._upsert_rows(db, rows) assert upserted == 2 assert skipped == 0 # begin_nested вызван по разу на строку assert db.begin_nested.call_count == 2 # params каждого UPSERT — это row.as_dict() assert captured == [rows[0].as_dict(), rows[1].as_dict()] def test_upsert_rows_savepoint_isolates_bad_row() -> None: """Битая строка (execute бросает) → skipped++, остальные продолжают (SAVEPOINT).""" from app.workers.tasks import supply_layers_refresh as task_mod captured: list[dict[str, Any]] = [] calls = {"n": 0} def execute_router(stmt: Any, params: Any = None) -> MagicMock: calls["n"] += 1 if calls["n"] == 2: # вторая строка — битая raise RuntimeError("CHECK violation") captured.append(params) return MagicMock() db = MagicMock() db.execute = execute_router db.begin_nested.return_value.__enter__ = MagicMock(return_value=None) db.begin_nested.return_value.__exit__ = MagicMock(return_value=False) rows = [ _row(layer=1, district="A", source="objective", snapshot_date=date(2026, 6, 3)), _row(layer=2, district="B", source="domrf_declared", snapshot_date=date(2026, 6, 3)), _row(layer=3, district="C", source="domrf_multiphase", snapshot_date=date(2026, 6, 3)), ] upserted, skipped = task_mod._upsert_rows(db, rows) assert upserted == 2 assert skipped == 1 # первая и третья строки прошли, вторая — нет assert [p["district_name"] for p in captured] == ["A", "C"] def test_upsert_rows_l3_same_district_diff_dev_group_distinct_keys() -> None: """#970 (REGRESSION GUARD, worker-level): две L3-строки ОДНОГО района с РАЗНЫМИ dev_group_name проходят как ДВЕ строки с РАЗЛИЧНЫМИ ключами UPSERT — не схлопываются. Без dev_group_name в ключе (complex_id/obj_class у L3 = NULL) обе дали бы один ключ → ON CONFLICT перезатёр бы их (~95.6% потерь L3 на prod). Проверяем, что param-dict каждой строки несёт свой dev_group_name и ключевые кортежи различны. """ from app.workers.tasks import supply_layers_refresh as task_mod db, captured, _ = _make_mock_db([]) db.execute("SELECT FROM supply_layers warmup", None) # пометить districts обслуженными snap = date(2026, 6, 3) rows = [ _row( layer=3, district="Академический", source="domrf_multiphase", confidence="low", dev_group_name="Брусника", snapshot_date=snap, ), _row( layer=3, district="Академический", source="domrf_multiphase", confidence="low", dev_group_name="Атомстройкомплекс", snapshot_date=snap, ), ] upserted, skipped = task_mod._upsert_rows(db, rows) assert upserted == 2 assert skipped == 0 # оба dev_group дошли до UPSERT-params (binding передаёт dev_group_name) assert {p["dev_group_name"] for p in captured} == {"Брусника", "Атомстройкомплекс"} # ключевые кортежи (под uq_supply_layers_logical м.128) различны → нет коллапса def _key(p: dict[str, Any]) -> tuple: return ( p["layer"], p["district_name"], p["complex_id"], p["obj_class"], p["dev_group_name"], p["source"], p["snapshot_date"], ) keys = [_key(p) for p in captured] assert len(set(keys)) == 2, f"L3 dev_group keys collapsed at worker level: {keys}" # ── end-to-end task: coherent snapshot, counters, resilience ───────────────── def test_task_coherent_single_snapshot_date() -> None: """run_date = date.today() ОДИН раз → передан snapshot_date КАЖДОЙ строке всех районов; ВСЕ UPSERT-params несут один и тот же snapshot_date.""" from app.workers.tasks import supply_layers_refresh as task_mod db, captured, _ = _make_mock_db( [{"district_name": "Кировский"}, {"district_name": "Чкаловский"}] ) today = date(2026, 6, 3) def fake_compute(_db: Any, *, district: str, snapshot_date: date) -> list[SupplyLayerRow]: # сервис обязан получить наш единый snapshot_date assert snapshot_date == today return [_row(layer=1, district=district, source="objective", snapshot_date=snapshot_date)] with ( patch.object(task_mod, "SessionLocal", return_value=db), patch.object(task_mod, "date") as mock_date, patch.object(task_mod, "compute_all_layers", side_effect=fake_compute) as mock_compute, ): mock_date.today.return_value = today result = task_mod.supply_layers_refresh.run() # compute вызван на каждый район с единым snapshot_date assert mock_compute.call_count == 2 for c in mock_compute.call_args_list: assert c == call(db, district=c.kwargs["district"], snapshot_date=today) # все UPSERT-params несут один snapshot_date assert captured, "expected at least one upsert" assert {p["snapshot_date"] for p in captured} == {today} assert result["snapshot_date"] == today.isoformat() assert result["districts"] == 2 assert result["rows"] == 2 assert result["upserted"] == 2 assert result["skipped"] == 0 db.commit.assert_called_once() db.close.assert_called_once() def test_task_resilient_to_one_district_failure() -> None: """Сбой compute_all_layers одного района не валит батч: остальные районы обрабатываются, commit вызывается.""" from app.workers.tasks import supply_layers_refresh as task_mod db, captured, _ = _make_mock_db([{"district_name": "Good"}, {"district_name": "Bad"}]) def fake_compute(_db: Any, *, district: str, snapshot_date: date) -> list[SupplyLayerRow]: if district == "Bad": raise RuntimeError("service blew up") return [_row(layer=1, district=district, source="objective", snapshot_date=snapshot_date)] with ( patch.object(task_mod, "SessionLocal", return_value=db), patch.object(task_mod, "compute_all_layers", side_effect=fake_compute), ): result = task_mod.supply_layers_refresh.run() # только "Good" дал строку; "Bad" проглочен (logged) и пропущен assert result["districts"] == 2 assert result["rows"] == 1 assert result["upserted"] == 1 assert [p["district_name"] for p in captured] == ["Good"] db.commit.assert_called_once() db.close.assert_called_once() def test_task_closes_db_on_infra_error() -> None: """Инфраструктурный сбой (напр. districts-enum) пробрасывается, db закрывается.""" from app.workers.tasks import supply_layers_refresh as task_mod db = MagicMock() db.execute.side_effect = None with ( patch.object(task_mod, "SessionLocal", return_value=db), patch.object(task_mod, "_list_districts", side_effect=RuntimeError("boom")), ): try: task_mod.supply_layers_refresh.run() raise AssertionError("expected RuntimeError to propagate") except RuntimeError as e: assert "boom" in str(e) db.close.assert_called_once() # ── beat registration ──────────────────────────────────────────────────────── def test_task_registered_in_beat_weekly_monday() -> None: """Задача зарегистрирована в beat как еженедельная (понедельник).""" from app.workers.beat_schedule import build_beat_schedule schedule = build_beat_schedule() assert "supply-layers-refresh-weekly" in schedule entry = schedule["supply-layers-refresh-weekly"] assert entry["task"] == "tasks.supply_layers_refresh.supply_layers_refresh" # crontab day_of_week = понедельник (1) assert 1 in entry["schedule"].day_of_week