Light1YT
b00e338667
В WAF-ветке `nspd_geo` worker спит до 240с (4 мин при consecutive_waf=8),
heartbeat коммитится ДО сна. Старый `cleanup_zombies` (`* * * * *`) с
порогом `INTERVAL '2 minutes'` для status IN ('running', 'paused')
гарантированно ре-enqueue'ил **живой** WAF-job:
- Два worker'а параллельно долбили забаненный NSPD-сервис, углубляя бан.
- SELECT pending без `FOR UPDATE SKIP LOCKED` → дубли запросов.
- Counter'ы done/failed двух инстансов затираются друг другом.
Бонус-баг: status='paused' (после 8 WAF подряд) воскресал через минуту,
аннулируя WAF-защиту.
Patch: разделил пороги через module-level константы и параметризованный
SQL (CAST(:x AS interval) per psycopg v3 rule):
- `_ZOMBIE_RUNNING_THRESHOLD = "6 minutes"` — выше max WAF backoff (4 мин).
- `_ZOMBIE_PAUSED_THRESHOLD = "30 minutes"` — реальная WAF-пауза.
Beat-расписание (1 мин tick) не тронуто. WAF-логика / heartbeat-in-loop
(вариант B) / FOR UPDATE SKIP LOCKED (вариант «идеальный») вне scope.
4 новых юнит-теста (test_nspd_geo.py): psycopg v3 guard, оба порога
биндятся, два предиката вместо одного, paused > running.
18/18 nspd_geo тестов зелёные. ruff clean.
Closes #1215
416 lines
17 KiB
Python
416 lines
17 KiB
Python
"""Тесты для backend/app/workers/tasks/nspd_geo.py — on-demand НСПД-ингест.
|
||
|
||
Покрывает 3 prod-confirmed бага «пользователь вводит новый/произвольный реальный участок»:
|
||
- Finding 1: _save_parcel / _save_quarter должны ST_Multi() geom (колонки MultiPolygon,
|
||
migration 93 / 58) — иначе "Geometry type (Polygon) does not match column type (MultiPolygon)".
|
||
- Finding 2: _save_building пишет в широкую схему cad_buildings (migration 92):
|
||
quarter_cad_number (не quarter_cad_num), source NOT NULL, floors через _safe_int.
|
||
|
||
Mock DB через unittest.mock — без реального PostgreSQL / NSPD сети.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
from unittest.mock import MagicMock
|
||
|
||
from app.workers.tasks import nspd_geo as nspd_geo_mod
|
||
from app.workers.tasks.nspd_geo import (
|
||
_ZOMBIE_PAUSED_THRESHOLD,
|
||
_ZOMBIE_RUNNING_THRESHOLD,
|
||
_save_building,
|
||
_save_parcel,
|
||
_save_quarter,
|
||
cleanup_zombies,
|
||
)
|
||
|
||
# ── Фикстуры payload (форма ответа _persist_target: {"data": {"features": [...]}}) ──
|
||
|
||
|
||
def _parcel_payload(cad_num: str = "66:41:0610029:83") -> dict[str, Any]:
|
||
"""NSPD-ответ для ЗУ (одноконтурный Polygon — триггер бага type-mismatch)."""
|
||
return {
|
||
"data": {
|
||
"features": [
|
||
{
|
||
"type": "Feature",
|
||
"geometry": {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[
|
||
[60.6, 56.8],
|
||
[60.61, 56.8],
|
||
[60.61, 56.81],
|
||
[60.6, 56.81],
|
||
[60.6, 56.8],
|
||
]
|
||
],
|
||
},
|
||
"properties": {"options": {"cad_num": cad_num}},
|
||
}
|
||
]
|
||
}
|
||
}
|
||
|
||
|
||
def _quarter_payload(cad_num: str = "66:41:0610029") -> dict[str, Any]:
|
||
"""NSPD-ответ для квартала (одноконтурный Polygon в EPSG:3857)."""
|
||
return {
|
||
"data": {
|
||
"features": [
|
||
{
|
||
"type": "Feature",
|
||
"geometry": {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[
|
||
[6090000.0, 7590000.0],
|
||
[6090200.0, 7590000.0],
|
||
[6090200.0, 7590200.0],
|
||
[6090000.0, 7590200.0],
|
||
[6090000.0, 7590000.0],
|
||
]
|
||
],
|
||
},
|
||
"properties": {"options": {}},
|
||
}
|
||
]
|
||
}
|
||
}
|
||
|
||
|
||
def _building_payload(
|
||
cad_num: str = "66:41:0610029:83",
|
||
floors: Any = "5-7",
|
||
quarter: str | None = "66:41:0610029",
|
||
) -> dict[str, Any]:
|
||
"""NSPD-ответ для здания. floors по умолчанию range-строка "5-7" (триггер бага floors INT)."""
|
||
opts: dict[str, Any] = {
|
||
"cad_num": cad_num,
|
||
"purpose": "Жилое",
|
||
"building_name": "Тестовое здание",
|
||
"readable_address": "г Екатеринбург, ул Тестовая, д 1",
|
||
"area": "1234.56",
|
||
"floors": floors,
|
||
}
|
||
if quarter is not None:
|
||
opts["quarter_cad_number"] = quarter
|
||
return {
|
||
"data": {
|
||
"features": [
|
||
{
|
||
"type": "Feature",
|
||
"geometry": {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[
|
||
[6090000.0, 7590000.0],
|
||
[6090050.0, 7590000.0],
|
||
[6090050.0, 7590050.0],
|
||
[6090000.0, 7590050.0],
|
||
[6090000.0, 7590000.0],
|
||
]
|
||
],
|
||
},
|
||
"properties": {"options": opts},
|
||
}
|
||
]
|
||
}
|
||
}
|
||
|
||
|
||
def _capturing_db() -> tuple[MagicMock, list[tuple[str, Any]]]:
|
||
"""Mock Session, фиксирующий (sql_text, params) каждого db.execute."""
|
||
captured: list[tuple[str, Any]] = []
|
||
db = MagicMock()
|
||
db.execute = lambda stmt, params=None: captured.append((str(stmt), params)) or MagicMock()
|
||
return db, captured
|
||
|
||
|
||
def _strip_sql_comments(sql: str) -> str:
|
||
"""Убрать `-- ...` комментарии — чтобы assert'ить ИСПОЛНЯЕМЫЙ SQL, не текст комментов.
|
||
|
||
Комментарии легитимно упоминают ST_Transform/ST_Multi, поэтому проверяем выражения
|
||
только в реальном SQL.
|
||
"""
|
||
lines = []
|
||
for line in sql.splitlines():
|
||
idx = line.find("--")
|
||
lines.append(line if idx == -1 else line[:idx])
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ── Finding 1: ST_Multi для MultiPolygon-колонок ─────────────────────────────
|
||
|
||
|
||
def test_save_parcel_sql_wraps_geom_in_st_multi() -> None:
|
||
"""cad_parcels.geom = MultiPolygon (migration 93) → SQL должен ST_Multi() обернуть geom.
|
||
|
||
Иначе одноконтурный Polygon из rosreestr2coord → "Geometry type (Polygon) does not
|
||
match column type (MultiPolygon)" → on-demand ингест участка крашится.
|
||
"""
|
||
db, captured = _capturing_db()
|
||
n = _save_parcel(db, _parcel_payload(), "66:41:0610029:83")
|
||
|
||
assert n == 1
|
||
assert len(captured) == 1
|
||
sql, _params = captured[0]
|
||
assert "ST_Multi(" in sql, "SQL _save_parcel должен содержать ST_Multi для MultiPolygon schema"
|
||
|
||
|
||
def test_save_parcel_sql_no_transform_parcel_is_4326() -> None:
|
||
"""parcel GeoJSON уже WGS84 (rosreestr2coord) — только ST_SetSRID(...,4326), без ST_Transform.
|
||
|
||
ST_Transform 3857→4326 на 4326-геометрии схлопнул бы участок в точку у (0,0).
|
||
"""
|
||
db, captured = _capturing_db()
|
||
_save_parcel(db, _parcel_payload(), "66:41:0610029:83")
|
||
|
||
sql, _params = captured[0]
|
||
executable = _strip_sql_comments(sql)
|
||
assert "ST_Transform" not in executable, "parcel уже 4326 — ST_Transform схлопнет в точку"
|
||
assert "ST_SetSRID(ST_GeomFromGeoJSON(:g), 4326)" in executable
|
||
|
||
|
||
def test_save_quarter_sql_wraps_geom_in_st_multi() -> None:
|
||
"""cad_quarters_geom.geom = MultiPolygon NOT NULL (migration 58) → ST_Multi() обязателен."""
|
||
db, captured = _capturing_db()
|
||
n = _save_quarter(db, _quarter_payload(), "66:41:0610029")
|
||
|
||
assert n == 1
|
||
assert len(captured) == 1
|
||
sql, _params = captured[0]
|
||
assert "ST_Multi(" in sql, "SQL _save_quarter должен содержать ST_Multi для MultiPolygon schema"
|
||
|
||
|
||
def test_save_quarter_sql_keeps_3857_to_4326_transform() -> None:
|
||
"""quarter GeoJSON в Mercator (3857) → ST_Transform к 4326 сохраняется (mirror bulk path)."""
|
||
db, captured = _capturing_db()
|
||
_save_quarter(db, _quarter_payload(), "66:41:0610029")
|
||
|
||
sql, _params = captured[0]
|
||
assert "ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:g), 3857), 4326)" in sql
|
||
|
||
|
||
def test_save_parcel_empty_features_returns_zero() -> None:
|
||
"""Пустой features → 0, без db.execute."""
|
||
db, captured = _capturing_db()
|
||
assert _save_parcel(db, {"data": {"features": []}}, "66:41:0610029:83") == 0
|
||
assert captured == []
|
||
|
||
|
||
# ── Finding 2: _save_building → широкая схема cad_buildings (migration 92) ─────
|
||
|
||
|
||
def test_save_building_uses_quarter_cad_number_column() -> None:
|
||
"""migration 92: колонка quarter_cad_number (НЕ quarter_cad_num) → иначе UndefinedColumn."""
|
||
db, captured = _capturing_db()
|
||
n = _save_building(db, _building_payload(), "66:41:0610029:83")
|
||
|
||
assert n == 1
|
||
sql, params = captured[0]
|
||
assert "quarter_cad_number" in sql
|
||
# старое имя колонки не должно встречаться (как отдельное слово quarter_cad_num)
|
||
assert "quarter_cad_num " not in sql
|
||
assert "quarter_cad_num," not in sql
|
||
assert "quarter_cad_num=" not in sql
|
||
assert params["qcad"] == "66:41:0610029"
|
||
|
||
|
||
def test_save_building_sets_source_not_null() -> None:
|
||
"""cad_buildings.source NOT NULL без дефолта (migration 92) → INSERT должен биндить source."""
|
||
db, captured = _capturing_db()
|
||
_save_building(db, _building_payload(), "66:41:0610029:83")
|
||
|
||
sql, params = captured[0]
|
||
assert ":source" in sql
|
||
assert params["source"] == "nspd"
|
||
|
||
|
||
def test_save_building_safe_int_floors_range_string() -> None:
|
||
"""floors INT: NSPD range "5-7" → _safe_int → None (иначе InvalidTextRepresentation).
|
||
|
||
Сырой "5-7" в floors INT крашил бы INSERT.
|
||
"""
|
||
db, captured = _capturing_db()
|
||
_save_building(db, _building_payload(floors="5-7"), "66:41:0610029:83")
|
||
|
||
_sql, params = captured[0]
|
||
assert params["floors"] is None # "5-7" не парсится → None, не сырая строка
|
||
|
||
|
||
def test_save_building_safe_int_floors_numeric() -> None:
|
||
"""floors "9" → int 9 (валидное число парсится)."""
|
||
db, captured = _capturing_db()
|
||
_save_building(db, _building_payload(floors="9"), "66:41:0610029:83")
|
||
|
||
_sql, params = captured[0]
|
||
assert params["floors"] == 9
|
||
|
||
|
||
def test_save_building_derives_quarter_when_missing() -> None:
|
||
"""Если NSPD не дал quarter_cad_number → выводим из cad_num (первые 3 сегмента)."""
|
||
db, captured = _capturing_db()
|
||
_save_building(db, _building_payload(quarter=None), "66:41:0610029:83")
|
||
|
||
_sql, params = captured[0]
|
||
assert params["qcad"] == "66:41:0610029"
|
||
|
||
|
||
def test_save_building_geom_transform_no_st_multi() -> None:
|
||
"""cad_buildings.geom = GEOMETRY(Geometry, 4326) полиморфный → ST_Multi НЕ нужен,
|
||
transform 3857→4326 сохраняется (mirror upsert_building)."""
|
||
db, captured = _capturing_db()
|
||
_save_building(db, _building_payload(), "66:41:0610029:83")
|
||
|
||
sql, _params = captured[0]
|
||
assert "ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:g), 3857), 4326)" in sql
|
||
assert "ST_Multi(" not in sql # полиморфная колонка — coerce не нужен
|
||
|
||
|
||
def test_save_building_skips_feature_with_other_cad_num() -> None:
|
||
"""NSPD может вернуть здания всего квартала — берём только запрошенный cad_num."""
|
||
db, captured = _capturing_db()
|
||
payload = _building_payload(cad_num="66:41:0610029:99") # чужой cad_num
|
||
n = _save_building(db, payload, "66:41:0610029:83")
|
||
|
||
assert n == 0
|
||
assert captured == []
|
||
|
||
|
||
def test_save_building_area_safe_numeric() -> None:
|
||
"""area NUMERIC: строковое "1234.56" → float через _safe_numeric."""
|
||
db, captured = _capturing_db()
|
||
_save_building(db, _building_payload(), "66:41:0610029:83")
|
||
|
||
_sql, params = captured[0]
|
||
assert params["area"] == 1234.56
|
||
|
||
|
||
# ── psycopg v3 CAST-trap guard (backend.md) ─────────────────────────────────
|
||
|
||
|
||
def test_save_sql_uses_cast_not_double_colon() -> None:
|
||
"""SQL _save_* использует CAST(:x AS jsonb), не :x::jsonb (SQLAlchemy psycopg v3 trap)."""
|
||
db, captured = _capturing_db()
|
||
_save_parcel(db, _parcel_payload(), "66:41:0610029:83")
|
||
_save_quarter(db, _quarter_payload(), "66:41:0610029")
|
||
_save_building(db, _building_payload(), "66:41:0610029:83")
|
||
|
||
for sql, _params in captured:
|
||
assert "::jsonb" not in sql, f"SQL содержит ::jsonb — нужен CAST(:x AS jsonb): {sql[:80]}"
|
||
assert "CAST(:props AS jsonb)" in sql
|
||
|
||
|
||
# ── cleanup_zombies thresholds (issue #1215: WAF backoff vs zombie race) ────
|
||
|
||
|
||
class _FakeRowsResult:
|
||
"""sqlalchemy-like Result, который запоминает (sql, params) и возвращает 0 rows.
|
||
|
||
cleanup_zombies использует cain `.mappings().all()` — мокаем под него.
|
||
"""
|
||
|
||
def __init__(self, captured: list[tuple[str, dict[str, Any]]], stmt: Any, params: Any) -> None:
|
||
captured.append((str(stmt), params or {}))
|
||
self._stmt = stmt
|
||
self._params = params
|
||
|
||
def mappings(self) -> _FakeRowsResult:
|
||
return self
|
||
|
||
def all(self) -> list[dict[str, Any]]:
|
||
return []
|
||
|
||
|
||
def _capturing_session_factory() -> tuple[MagicMock, list[tuple[str, dict[str, Any]]]]:
|
||
"""Fake SessionLocal() — возвращает session, чьи execute-calls фиксируются."""
|
||
captured: list[tuple[str, dict[str, Any]]] = []
|
||
session = MagicMock()
|
||
session.execute = lambda stmt, params=None: _FakeRowsResult(captured, stmt, params)
|
||
session.commit = MagicMock()
|
||
session.rollback = MagicMock()
|
||
session.close = MagicMock()
|
||
|
||
factory = MagicMock(return_value=session)
|
||
return factory, captured
|
||
|
||
|
||
def test_cleanup_zombies_sql_uses_cast_as_interval_not_double_colon(
|
||
monkeypatch: Any,
|
||
) -> None:
|
||
"""psycopg v3: только CAST(:x AS interval), запрещён `:x::interval`."""
|
||
factory, captured = _capturing_session_factory()
|
||
monkeypatch.setattr(nspd_geo_mod, "SessionLocal", factory)
|
||
|
||
cleanup_zombies()
|
||
|
||
assert len(captured) == 1, "cleanup_zombies должен выполнить ровно один UPDATE"
|
||
sql, _params = captured[0]
|
||
assert "CAST(:running_threshold AS interval)" in sql
|
||
assert "CAST(:paused_threshold AS interval)" in sql
|
||
# SQLAlchemy + psycopg v3 ломаются на `:name::type` (backend.md)
|
||
assert ":running_threshold::" not in sql
|
||
assert ":paused_threshold::" not in sql
|
||
|
||
|
||
def test_cleanup_zombies_binds_running_and_paused_thresholds(monkeypatch: Any) -> None:
|
||
"""Issue #1215: running порог > max WAF backoff (240s = 4 min), paused — большой cooldown.
|
||
|
||
Контракт thresholds задокументирован в module-level константах.
|
||
"""
|
||
factory, captured = _capturing_session_factory()
|
||
monkeypatch.setattr(nspd_geo_mod, "SessionLocal", factory)
|
||
|
||
cleanup_zombies()
|
||
|
||
_sql, params = captured[0]
|
||
assert params["running_threshold"] == _ZOMBIE_RUNNING_THRESHOLD
|
||
assert params["paused_threshold"] == _ZOMBIE_PAUSED_THRESHOLD
|
||
# Контракт: running threshold выше старых 2 мин (которые ре-enqueue'или живые WAF-jobs)
|
||
# и выше max WAF backoff (WAF_BACKOFF_BASE_S * 8 = 240s = 4 мин).
|
||
assert _ZOMBIE_RUNNING_THRESHOLD == "6 minutes"
|
||
# Контракт: paused threshold большой (WAF-пауза должна реально удержаться).
|
||
assert _ZOMBIE_PAUSED_THRESHOLD == "30 minutes"
|
||
|
||
|
||
def test_cleanup_zombies_splits_running_and_paused_in_where_clause(
|
||
monkeypatch: Any,
|
||
) -> None:
|
||
"""SQL разделяет 'running' и 'paused' с разными порогами — не общий IN-фильтр."""
|
||
factory, captured = _capturing_session_factory()
|
||
monkeypatch.setattr(nspd_geo_mod, "SessionLocal", factory)
|
||
|
||
cleanup_zombies()
|
||
|
||
sql, _params = captured[0]
|
||
executable = _strip_sql_comments(sql)
|
||
# Старая ветка с общим INTERVAL '2 minutes' для обоих статусов удалена
|
||
assert "INTERVAL '2 minutes'" not in executable
|
||
assert "status IN ('running', 'paused')" not in executable
|
||
# Новая ветка: status='running' и status='paused' в отдельных предикатах
|
||
assert "status = 'running'" in executable
|
||
assert "status = 'paused'" in executable
|
||
# Каждый из двух предикатов сравнивает heartbeat_at с NOW() - threshold
|
||
assert executable.count("heartbeat_at") >= 2
|
||
|
||
|
||
def test_cleanup_zombies_paused_threshold_strictly_greater_than_running(
|
||
monkeypatch: Any,
|
||
) -> None:
|
||
"""paused cooldown должен быть СТРОГО больше running, иначе пауза аннулируется минутным beat."""
|
||
factory, captured = _capturing_session_factory()
|
||
monkeypatch.setattr(nspd_geo_mod, "SessionLocal", factory)
|
||
|
||
cleanup_zombies()
|
||
|
||
# 6 vs 30 — числовое сравнение префиксов как guard от случайного "6 minutes" == "30 minutes"
|
||
running_min = int(_ZOMBIE_RUNNING_THRESHOLD.split()[0])
|
||
paused_min = int(_ZOMBIE_PAUSED_THRESHOLD.split()[0])
|
||
assert paused_min > running_min, (
|
||
"paused threshold должен быть строго больше running, "
|
||
"иначе WAF-пауза не удержится дольше running-cooldown"
|
||
)
|
||
# Sanity: SQL действительно отправлен с двумя разными порогами
|
||
_sql, params = captured[0]
|
||
assert params["running_threshold"] != params["paused_threshold"]
|