From 2b439521cd4a0915cccfb9d4e9f7bdc1ae3173f0 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sun, 7 Jun 2026 12:39:47 +0300 Subject: [PATCH] =?UTF-8?q?feat(sf):=20gknspecial=5Fzone=20=D0=98=D0=A0?= =?UTF-8?q?=D0=94-harvest=20=E2=86=92=20ird=5Foverlays=20(#1063/#1064/#106?= =?UTF-8?q?5/#1066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/scrapers/ekb_geoportal_client.py | 15 + .../site_finder/ird_overlay_lookup.py | 8 +- backend/app/workers/beat_schedule.py | 10 + backend/app/workers/celery_app.py | 1 + .../app/workers/tasks/gknspecial_harvest.py | 287 +++++++++++++++++ .../tests/services/test_ird_overlay_lookup.py | 21 ++ .../workers/tasks/test_gknspecial_harvest.py | 304 ++++++++++++++++++ 7 files changed, 644 insertions(+), 2 deletions(-) create mode 100644 backend/app/workers/tasks/gknspecial_harvest.py create mode 100644 backend/tests/workers/tasks/test_gknspecial_harvest.py diff --git a/backend/app/services/scrapers/ekb_geoportal_client.py b/backend/app/services/scrapers/ekb_geoportal_client.py index 28341332..f884d121 100644 --- a/backend/app/services/scrapers/ekb_geoportal_client.py +++ b/backend/app/services/scrapers/ekb_geoportal_client.py @@ -299,6 +299,21 @@ class EKBGeoportalClient: card = self._urban_card(terzone_key, keys.get("GknParcel")) return parse_zone_regulation(card) + def features_by_cql(self, layer: str, cql_filter: str) -> list[EKBFeature]: + """Фичи слоя по произвольному CQL-фильтру (напр. ``full_name LIKE 'X%'``). + + Тонкая обёртка над ``_resolve_type_name`` + ``_get_feature``. Используется + для harvest ЗОУИТ-слоя по префиксу full_name (#1063/#1064/#1065/#1066). + + Args: + layer: ключ WFS_LAYERS ('zouit', ...) или сырой typeName. + cql_filter: CQL-выражение (WFS 2.0 OGC CQL). Кириллица в одиночных + LIKE-выражениях работает. Составные AND с кириллицей могут давать + HTTP-ошибку геопортала — фильтруй client-side. + """ + type_name = self._resolve_type_name(layer) + return self._get_feature(type_name, cql_filter) + __all__ = [ "EKB_GEOPORTAL_BASE", diff --git a/backend/app/services/site_finder/ird_overlay_lookup.py b/backend/app/services/site_finder/ird_overlay_lookup.py index a8308909..33cc9b76 100644 --- a/backend/app/services/site_finder/ird_overlay_lookup.py +++ b/backend/app/services/site_finder/ird_overlay_lookup.py @@ -29,10 +29,11 @@ IRD_EMPTY_RESULT: dict[str, Any] = {"ird_overlaps": [], "ird_by_kind": {}} # ИРД-overlay по участку: ST_Intersects через GIST idx_ird_overlays_geom. geom уже EPSG:4326 # (harvest стор ST_Transform 3857→4326). Участок передаётся как WKT в 4326. CAST psycopg v3. +# content добавлен для геопортальных ЗОУИТ-слоёв (gknspecial_harvest #1063/#1064/#1065/#1066). _IRD_OVERLAP_SQL = text( """ SELECT layer_kind, reg_numb_border, category, subcategory, - zone_index, zone_type_name + zone_index, zone_type_name, content FROM ird_overlays WHERE ST_Intersects(geom, ST_GeomFromText(CAST(:parcel_wkt AS text), 4326)) ORDER BY layer_kind, reg_numb_border @@ -49,7 +50,9 @@ def parcel_ird_overlaps(db: Session, parcel_wkt: str | None) -> dict[str, Any]: Returns: {'ird_overlaps': [{layer_kind, reg_numb_border, category, subcategory, - zone_index, zone_type_name}], 'ird_by_kind': {layer_kind: count}}. + zone_index, zone_type_name, content}], 'ird_by_kind': {layer_kind: count}}. + content — JSONB-dict с ограничениями/реквизитами зоны (геопортальные слои: + gknspecial_harvest #1063/#1064/#1065/#1066); None для НСПД-слоёв до резолва D9. Пусто при отсутствии участка/таблицы/пересечений (analyze не падает). """ if not parcel_wkt: @@ -73,6 +76,7 @@ def parcel_ird_overlaps(db: Session, parcel_wkt: str | None) -> dict[str, Any]: "subcategory": r["subcategory"], "zone_index": r["zone_index"], "zone_type_name": r["zone_type_name"], + "content": r["content"], } ) by_kind[kind] = by_kind.get(kind, 0) + 1 diff --git a/backend/app/workers/beat_schedule.py b/backend/app/workers/beat_schedule.py index e48bd7dd..4c6031b2 100644 --- a/backend/app/workers/beat_schedule.py +++ b/backend/app/workers/beat_schedule.py @@ -332,6 +332,16 @@ def build_beat_schedule() -> dict: "options": {"queue": "celery"}, } + # ЗОУИТ-harvest ЕКБ-геопортала (#1063/#1064/#1065/#1066, #1067 GG-форсайт): WFS-фетч + # gknspecial_zone по full_name-префиксам → ird_overlays (layer_kind=servitude/water_*/ + # airfield/okn_protection_zone/zrz). Понедельник 05:30 МСК — после ird-harvest (05:00), + # до supply-layers (06:00). Геометрия зон стабильна, еженедельно достаточно. + schedule["gknspecial-harvest-weekly"] = { + "task": "tasks.gknspecial_harvest.harvest_gknspecial_zones", + "schedule": _parse_cron("30 5 * * mon"), + "options": {"queue": "celery"}, + } + # Opportunity-ЗУ harvest (#1086): grid-walk НСПД opportunity-слоёв по кварталам ЕКБ → # ird_overlays (layer_kind='opportunity_*'). 1-е число месяца 05:30 МСК — реже ИРД (weekly), # opportunity-земля меняется медленнее ограничений. Тот же движок, что ird-harvest. diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index 3a8ecf8e..bde69e1d 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -64,6 +64,7 @@ celery_app = Celery( "app.workers.tasks.supply_layers_refresh", "app.workers.tasks.forecast", "app.workers.tasks.ird_harvest", + "app.workers.tasks.gknspecial_harvest", "app.workers.tasks.opportunity_harvest", "app.workers.tasks.planning_harvest", "app.workers.tasks.zone_regulation_refresh", diff --git a/backend/app/workers/tasks/gknspecial_harvest.py b/backend/app/workers/tasks/gknspecial_harvest.py new file mode 100644 index 00000000..dd1234d6 --- /dev/null +++ b/backend/app/workers/tasks/gknspecial_harvest.py @@ -0,0 +1,287 @@ +"""Celery task: harvest ЗОУИТ ЕКБ-геопортала (gknspecial_zone) → ``ird_overlays`` (м.132). + +#1067 GG-форсайт: закрывает ИРД-пункты: + #1063 — сервитут (Зона публичного сервитута) + #1064 — водоохранные зоны (Водоохранная зона / Прибрежная защитная полоса / + Зона санитарной охраны / Санитарно-защитная зона) + #1065 — приаэродромная территория (Приаэродромная территория — ЕГРН) + #1066 — ОКН-зоны/ЗРЗ (Зона охраны объекта культурного / Зона регулирования застройки) + +Источник: ЕКБ-геопортал, WFS-слой ``portal:portal_geo_gknspecial_zone`` (ключ 'zouit'). +Геометрия уже в EPSG:4326 — ST_Transform НЕ нужен. + +Sentinel source_layer_id'ы (вне диапазона НСПД): 990001–990008 + 990099 (unmatched). + +Conventions (зеркало ird_harvest.py): + • SessionLocal() + try/finally close. + • SAVEPOINT per-row (with db.begin_nested():) — одна битая фича не валит батч. + • CAST(:x AS type) — НИКОГДА :x::type (psycopg v3). + • ON CONFLICT (source_layer_id, geom_data_id) DO UPDATE. + • content_* пишем мы же (геопортал как источник); content НЕ трогается НСПД-harvest'ом. + • Кириллица в combined-CQL WFS ломает запрос → фильтр zone_state_v client-side. + +Расписание: beat_schedule.py → gknspecial-harvest-weekly (пн 05:30 МСК). +""" + +from __future__ import annotations + +import json +import logging +import re +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.db import SessionLocal +from app.services.scrapers.ekb_geoportal_client import EKBGeoportalClient +from app.workers.celery_app import celery_app + +logger = logging.getLogger(__name__) + +# ── Mapping: full_name-префикс → (layer_kind, sentinel source_layer_id) ───── +# Sentinels вне диапазона НСПД (990001+), чтобы не конфликтовать с реальными layer_id. +_PREFIX_MAP: list[tuple[str, str, int]] = [ + ("Зона публичного сервитута", "servitude", 990001), # #1063 + ("Водоохранная зона", "water_protection", 990002), # #1064 + ("Прибрежная защитная полоса", "coastal_strip", 990003), # #1064 + ("Зона санитарной охраны", "water_sanitary", 990004), # #1064 + ("Санитарно-защитная зона", "sanitary_protection", 990005), # #1064 + ("Зона охраны объекта культурного", "okn_protection_zone", 990006), # #1066 + ("Зона регулирования застройки", "zrz", 990007), # #1066 + ("Приаэродромная территория", "airfield", 990008), # #1065 +] + +# Sentinel для фич, чей full_name не попал ни под один префикс (defensive). +_UNMATCHED_LAYER_KIND = "gknspecial_unmatched" +_UNMATCHED_SOURCE_LAYER_ID = 990099 + +# Regex для извлечения reg_numb_border из full_name (формат «66:41-6.4125»). +_REG_NUMB_RE = re.compile(r"\d{2}:\d{2}-\d+\.\d+") + +# UPSERT: геометрия уже EPSG:4326 → ST_SetSRID(ST_GeomFromGeoJSON, 4326), без ST_Transform. +# content_* поля обновляются в DO UPDATE (геопортал — их источник). +_UPSERT_SQL = text( + """ + INSERT INTO ird_overlays ( + source_layer_id, layer_kind, geom_data_id, reg_numb_border, + category, subcategory, geom, raw_props, + zone_type_name, content, content_source, content_fetched_at, fetched_at + ) VALUES ( + CAST(:source_layer_id AS integer), + CAST(:layer_kind AS text), + CAST(:geom_data_id AS bigint), + CAST(:reg_numb_border AS text), + CAST(:category AS integer), + CAST(:subcategory AS integer), + ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geojson AS text)), 4326), + CAST(:raw_props AS jsonb), + CAST(:zone_type_name AS text), + CAST(:content AS jsonb), + CAST(:content_source AS text), + CAST(:content_fetched_at AS timestamptz), + CAST(:fetched_at AS timestamptz) + ) + ON CONFLICT (source_layer_id, geom_data_id) DO UPDATE SET + layer_kind = EXCLUDED.layer_kind, + reg_numb_border = EXCLUDED.reg_numb_border, + geom = EXCLUDED.geom, + raw_props = EXCLUDED.raw_props, + zone_type_name = EXCLUDED.zone_type_name, + content = EXCLUDED.content, + content_source = EXCLUDED.content_source, + content_fetched_at = EXCLUDED.content_fetched_at, + fetched_at = EXCLUDED.fetched_at + """ +) + + +def _classify_full_name(full_name: str) -> tuple[str, int] | None: + """Сопоставить full_name с записью _PREFIX_MAP → (layer_kind, source_layer_id) или None.""" + for prefix, layer_kind, source_layer_id in _PREFIX_MAP: + if full_name.startswith(prefix): + return layer_kind, source_layer_id + return None + + +def _extract_reg_numb(full_name: str) -> str | None: + """Извлечь reg_numb_border из full_name по паттерну \\d{2}:\\d{2}-\\d+\\.\\d+.""" + m = _REG_NUMB_RE.search(full_name) + return m.group(0) if m else None + + +def _extract_geom_data_id(feature_id: str | None) -> int | None: + """Trailing int из WFS feature_id (формат 'portal_geo_gknspecial_zone.NNNN') или None.""" + if not feature_id: + return None + parts = feature_id.split(".") + if len(parts) < 2: + return None + try: + return int(parts[-1]) + except (ValueError, TypeError): + return None + + +def _build_content(props: dict[str, Any]) -> str: + """Собрать content-jsonb из свойств фичи (только присутствующие ключи, ensure_ascii=False).""" + mapping: dict[str, str] = { + "restrictions": "gkn_specialz_contentrestrictions", + "protected_object": "gkn_specialz_protectedobject", + "authority_decision": "authority_decision", + "document": "gkn_document", + "registration_date": "registration_date", + "area": "area", + "zone_state": "zone_state_v", + } + content: dict[str, Any] = {} + for out_key, prop_key in mapping.items(): + val = props.get(prop_key) + if val is not None: + content[out_key] = val + return json.dumps(content, ensure_ascii=False) + + +def _upsert_feature( + db: Session, + *, + source_layer_id: int, + layer_kind: str, + geom_data_id: int, + props: dict[str, Any], + geom: dict[str, Any], + fetched_at: str, +) -> bool: + """UPSERT одной фичи в ird_overlays под SAVEPOINT. True если записана.""" + full_name: str = str(props.get("full_name") or "") + reg_numb = _extract_reg_numb(full_name) + content_json = _build_content(props) + raw_props_json = json.dumps(props, ensure_ascii=False) + try: + with db.begin_nested(): + db.execute( + _UPSERT_SQL, + { + "source_layer_id": source_layer_id, + "layer_kind": layer_kind, + "geom_data_id": geom_data_id, + "reg_numb_border": reg_numb, + "category": None, + "subcategory": None, + "geojson": json.dumps(geom), + "raw_props": raw_props_json, + "zone_type_name": full_name or None, + "content": content_json, + "content_source": "ekb_geoportal", + "content_fetched_at": fetched_at, + "fetched_at": fetched_at, + }, + ) + return True + except Exception as exc: + logger.warning( + "gknspecial_harvest: upsert failed layer=%s geom_data_id=%s: %s", + layer_kind, + geom_data_id, + exc, + ) + return False + + +@celery_app.task(name="tasks.gknspecial_harvest.harvest_gknspecial_zones") +def harvest_gknspecial_zones() -> dict[str, int]: + """Harvest ЗОУИТ-слоя ЕКБ-геопортала (gknspecial_zone) → ird_overlays. + + Для каждого full_name-префикса запрашивает WFS с CQL ``full_name LIKE 'X%'``, + фильтрует архивные записи client-side (zone_state_v != 'Действующий'), UPSERT-ит + в ird_overlays под SAVEPOINT per-row. + + Фичи, чей full_name не попал ни под один известный префикс, пишутся под + layer_kind='gknspecial_unmatched' (source_layer_id=990099) — defensive, чтобы + новые типы зон не терялись молча. + + Returns: + {'types': N_prefixes_fetched, 'features': M_upserted} + """ + fetched_at = datetime.now(UTC).isoformat() + client = EKBGeoportalClient() + db = SessionLocal() + n_types = 0 + n_features = 0 + try: + for prefix, expected_layer_kind, _ in _PREFIX_MAP: + cql = f"full_name LIKE '{prefix}%'" + try: + feats = client.features_by_cql("zouit", cql) + except Exception as exc: + logger.warning( + "gknspecial_harvest: WFS failed prefix=%r layer_kind=%s: %s", + prefix, + expected_layer_kind, + exc, + ) + continue + n_types += 1 + n_written = 0 + for feat in feats: + props = feat.properties or {} + # Фильтр активности client-side: пропускаем архивные зоны. + # Если поля нет вообще — не скипаем (неизвестное = считаем активным). + zone_state = props.get("zone_state_v") + if zone_state is not None and zone_state != "Действующий": + continue + geom = feat.geometry + if not geom: + logger.debug( + "gknspecial_harvest: нет геометрии layer=%s feature_id=%s, skip", + expected_layer_kind, + feat.feature_id, + ) + continue + geom_data_id = _extract_geom_data_id(feat.feature_id) + if geom_data_id is None: + logger.debug( + "gknspecial_harvest: нет geom_data_id feature_id=%s, skip", + feat.feature_id, + ) + continue + # Роутинг: классифицируем по full_name (единый источник истины). + # Defensive: WFS мог вернуть фичу с неожиданным full_name — пишем в unmatched. + full_name: str = str(props.get("full_name") or "") + cls = _classify_full_name(full_name) + if cls is None: + logger.warning( + "gknspecial_harvest: unmatched full_name=%r geom_data_id=%s, " + "пишем в unmatched", + full_name, + geom_data_id, + ) + actual_layer_kind = _UNMATCHED_LAYER_KIND + actual_source_layer_id = _UNMATCHED_SOURCE_LAYER_ID + else: + actual_layer_kind, actual_source_layer_id = cls + if _upsert_feature( + db, + source_layer_id=actual_source_layer_id, + layer_kind=actual_layer_kind, + geom_data_id=geom_data_id, + props=props, + geom=geom, + fetched_at=fetched_at, + ): + n_written += 1 + n_features += 1 + logger.info( + "gknspecial_harvest: prefix=%r layer_kind=%s fetched=%d written=%d", + prefix, + expected_layer_kind, + len(feats), + n_written, + ) + + db.commit() + logger.info("gknspecial_harvest: готово, types=%d features=%d", n_types, n_features) + return {"types": n_types, "features": n_features} + finally: + db.close() diff --git a/backend/tests/services/test_ird_overlay_lookup.py b/backend/tests/services/test_ird_overlay_lookup.py index 13eb8022..e5a0a945 100644 --- a/backend/tests/services/test_ird_overlay_lookup.py +++ b/backend/tests/services/test_ird_overlay_lookup.py @@ -45,6 +45,7 @@ def test_groups_overlaps_by_kind() -> None: "subcategory": 20, "zone_index": None, "zone_type_name": None, + "content": None, }, { "layer_kind": "zouit_okn", @@ -53,6 +54,7 @@ def test_groups_overlaps_by_kind() -> None: "subcategory": 20, "zone_index": None, "zone_type_name": None, + "content": None, }, { "layer_kind": "territorial_zones", @@ -61,6 +63,7 @@ def test_groups_overlaps_by_kind() -> None: "subcategory": 1, "zone_index": "Ц-1", "zone_type_name": None, + "content": None, }, ] res = parcel_ird_overlaps(_FakeDB(rows), _WKT) @@ -71,6 +74,24 @@ def test_groups_overlaps_by_kind() -> None: assert tz["zone_index"] == "Ц-1" +def test_content_passthrough() -> None: + """content из gknspecial_harvest пробрасывается в каждый overlap-dict.""" + content_val = {"restrictions": "запрещена жилая застройка", "zone_state": "Действующий"} + rows = [ + { + "layer_kind": "water_protection", + "reg_numb_border": "66:41-6.4125", + "category": None, + "subcategory": None, + "zone_index": None, + "zone_type_name": "Водоохранная зона р. Исеть", + "content": content_val, + }, + ] + res = parcel_ird_overlaps(_FakeDB(rows), _WKT) + assert res["ird_overlaps"][0]["content"] == content_val + + def test_empty_when_no_wkt() -> None: res = parcel_ird_overlaps(_FakeDB([{"layer_kind": "x"}]), None) assert res == {"ird_overlaps": [], "ird_by_kind": {}} diff --git a/backend/tests/workers/tasks/test_gknspecial_harvest.py b/backend/tests/workers/tasks/test_gknspecial_harvest.py new file mode 100644 index 00000000..8a70e78e --- /dev/null +++ b/backend/tests/workers/tasks/test_gknspecial_harvest.py @@ -0,0 +1,304 @@ +"""Тесты harvest_gknspecial_zones (#1063/#1064/#1065/#1066, #1067 GG-форсайт). + +Чистые тесты без живого геопортала / БД: + • classify: full_name → корректный (layer_kind, source_layer_id) по каждому префиксу. + • reg_numb_border regex: извлечение из full_name, мусор → None. + • geom_data_id: 'portal_geo_gknspecial_zone.12345' → 12345; без точки/нечисло → None. + • активность: zone_state_v != 'Действующий' → skip; поле отсутствует → не skip. + • upsert idempotency: mock-клиент + mock-SessionLocal, два прогона → строки не дублируются. +""" + +from __future__ import annotations + +import os +from typing import Any +from unittest.mock import MagicMock, patch + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from app.workers.tasks.gknspecial_harvest import ( + _UNMATCHED_LAYER_KIND, + _UNMATCHED_SOURCE_LAYER_ID, + _classify_full_name, + _extract_geom_data_id, + _extract_reg_numb, + harvest_gknspecial_zones, +) + +# ── classify ───────────────────────────────────────────────────────────────── + + +def test_classify_servitude() -> None: + cls = _classify_full_name("Зона публичного сервитута 66:41-6.100") + assert cls == ("servitude", 990001) + + +def test_classify_water_protection() -> None: + assert _classify_full_name("Водоохранная зона р. Исеть") == ("water_protection", 990002) + + +def test_classify_coastal_strip() -> None: + assert _classify_full_name("Прибрежная защитная полоса р.Исеть") == ("coastal_strip", 990003) + + +def test_classify_water_sanitary() -> None: + assert _classify_full_name("Зона санитарной охраны ВЗУ") == ("water_sanitary", 990004) + + +def test_classify_sanitary_protection() -> None: + assert _classify_full_name("Санитарно-защитная зона завода") == ("sanitary_protection", 990005) + + +def test_classify_okn_protection_zone() -> None: + assert _classify_full_name("Зона охраны объекта культурного наследия") == ( + "okn_protection_zone", + 990006, + ) + + +def test_classify_zrz() -> None: + assert _classify_full_name("Зона регулирования застройки") == ("zrz", 990007) + + +def test_classify_airfield() -> None: + assert _classify_full_name("Приаэродромная территория") == ("airfield", 990008) + + +def test_classify_airfield_with_suffix() -> None: + assert _classify_full_name("Приаэродромная территория аэропорта Кольцово") == ( + "airfield", + 990008, + ) + + +def test_classify_unmatched_returns_none() -> None: + assert _classify_full_name("Неизвестная зона нового типа") is None + + +def test_classify_empty_returns_none() -> None: + assert _classify_full_name("") is None + + +# ── reg_numb_border regex ───────────────────────────────────────────────────── + + +def test_reg_numb_extracts_from_full_name() -> None: + assert _extract_reg_numb("Водоохранная зона 66:41-6.4125 р. Исеть") == "66:41-6.4125" + + +def test_reg_numb_extracts_with_multiple_digits() -> None: + assert _extract_reg_numb("Зона охраны 66:01-12.300") == "66:01-12.300" + + +def test_reg_numb_returns_none_for_garbage() -> None: + assert _extract_reg_numb("Нет номера границы здесь") is None + + +def test_reg_numb_returns_none_for_empty() -> None: + assert _extract_reg_numb("") is None + + +# ── geom_data_id ───────────────────────────────────────────────────────────── + + +def test_geom_data_id_normal() -> None: + assert _extract_geom_data_id("portal_geo_gknspecial_zone.12345") == 12345 + + +def test_geom_data_id_large_number() -> None: + assert _extract_geom_data_id("portal_geo_gknspecial_zone.9999999") == 9999999 + + +def test_geom_data_id_no_dot() -> None: + assert _extract_geom_data_id("portal_geo_gknspecial_zone") is None + + +def test_geom_data_id_non_numeric() -> None: + assert _extract_geom_data_id("portal_geo_gknspecial_zone.abc") is None + + +def test_geom_data_id_none_input() -> None: + assert _extract_geom_data_id(None) is None + + +def test_geom_data_id_empty_string() -> None: + assert _extract_geom_data_id("") is None + + +# ── zone_state_v фильтр активности ─────────────────────────────────────────── + + +def _make_feature( + feature_id: str = "portal_geo_gknspecial_zone.1", + full_name: str = "Водоохранная зона", + zone_state_v: str | None = "Действующий", + geom: dict[str, Any] | None = None, +) -> MagicMock: + """Фейковый EKBFeature для тестов.""" + from app.services.scrapers.ekb_geoportal_client import EKBFeature + + props: dict[str, Any] = {"full_name": full_name} + if zone_state_v is not None: + props["zone_state_v"] = zone_state_v + if geom is None: + geom = {"type": "Polygon", "coordinates": [[[60.5, 56.8], [60.6, 56.8], [60.5, 56.8]]]} + return EKBFeature(feature_id=feature_id, geometry=geom, properties=props) + + +def _run_task_with_feats(feats_per_prefix: list[list[Any]]) -> dict[str, int]: + """Запустить harvest_gknspecial_zones с мок-клиентом и мок-БД. + + feats_per_prefix: список из 8 элементов (по числу _PREFIX_MAP), каждый — список EKBFeature. + """ + mock_db = MagicMock() + mock_db.begin_nested.return_value.__enter__ = MagicMock(return_value=None) + mock_db.begin_nested.return_value.__exit__ = MagicMock(return_value=False) + + # Счётчик вызовов features_by_cql → разные фичи для разных префиксов + call_idx: list[int] = [0] + + def fake_features_by_cql(layer: str, cql_filter: str) -> list[Any]: + idx = call_idx[0] + call_idx[0] += 1 + if idx < len(feats_per_prefix): + return feats_per_prefix[idx] + return [] + + with ( + patch("app.workers.tasks.gknspecial_harvest.EKBGeoportalClient") as mock_client_cls, + patch("app.workers.tasks.gknspecial_harvest.SessionLocal", return_value=mock_db), + ): + mock_client = MagicMock() + mock_client.features_by_cql.side_effect = fake_features_by_cql + mock_client_cls.return_value = mock_client + return harvest_gknspecial_zones() + + +def test_skip_archived_zone_state() -> None: + """zone_state_v='Архивный' → фича не пишется.""" + feat = _make_feature(zone_state_v="Архивный") + # 8 префиксов: первый с архивной фичей, остальные пустые + feats_per_prefix = [[feat]] + [[] for _ in range(7)] + result = _run_task_with_feats(feats_per_prefix) + assert result["features"] == 0 + + +def test_include_feature_without_zone_state() -> None: + """Нет поля zone_state_v → фича не скипается (неизвестное = активное).""" + feat = _make_feature(zone_state_v=None) + feats_per_prefix = [[feat]] + [[] for _ in range(7)] + result = _run_task_with_feats(feats_per_prefix) + assert result["features"] == 1 + + +def test_include_active_zone_state() -> None: + """zone_state_v='Действующий' → фича пишется.""" + feat = _make_feature(zone_state_v="Действующий") + feats_per_prefix = [[feat]] + [[] for _ in range(7)] + result = _run_task_with_feats(feats_per_prefix) + assert result["features"] == 1 + + +# ── upsert idempotency ─────────────────────────────────────────────────────── + + +def _run_single_feat_task(feat: Any, execute_calls: list[Any]) -> None: + """Вспомогательная — один прогон harvest_gknspecial_zones с одной фичей в первом префиксе.""" + feats_per_prefix = [[feat]] + [[] for _ in range(7)] + call_idx: list[int] = [0] + + def fake_by_cql(layer: str, cql_filter: str) -> list[Any]: + idx = call_idx[0] + call_idx[0] += 1 + return feats_per_prefix[idx] if idx < len(feats_per_prefix) else [] + + mock_db = MagicMock() + mock_db.begin_nested.return_value.__enter__ = MagicMock(return_value=None) + mock_db.begin_nested.return_value.__exit__ = MagicMock(return_value=False) + mock_db.execute.side_effect = lambda *a, **kw: execute_calls.append(a) + + with ( + patch("app.workers.tasks.gknspecial_harvest.EKBGeoportalClient") as mock_client_cls, + patch("app.workers.tasks.gknspecial_harvest.SessionLocal", return_value=mock_db), + ): + mock_client = MagicMock() + mock_client.features_by_cql.side_effect = fake_by_cql + mock_client_cls.return_value = mock_client + harvest_gknspecial_zones() + + +def test_upsert_idempotency_no_duplicates() -> None: + """Два прогона с одной и той же фичей → db.execute дважды, но ON CONFLICT — не дублируем.""" + feat = _make_feature( + feature_id="portal_geo_gknspecial_zone.777", + full_name="Водоохранная зона 66:41-6.100", + zone_state_v="Действующий", + ) + execute_calls: list[Any] = [] + _run_single_feat_task(feat, execute_calls) + _run_single_feat_task(feat, execute_calls) + + # Оба прогона должны были вызвать db.execute по одному разу (за каждый run по одной фиче). + # ON CONFLICT в SQL — гарантирует что строка не дублируется на уровне БД. + # Здесь проверяем: execute вызывался ровно дважды (1 фича × 2 прогона). + assert len(execute_calls) == 2 + + +def test_returns_types_and_features_counts() -> None: + """Возврат {'types': N, 'features': M}.""" + feat1 = _make_feature(feature_id="portal_geo_gknspecial_zone.1", full_name="Водоохранная зона") + feat2 = _make_feature(feature_id="portal_geo_gknspecial_zone.2", full_name="Водоохранная зона") + # 2 фичи в первом префиксе, остальные пустые + feats_per_prefix = [[feat1, feat2]] + [[] for _ in range(7)] + result = _run_task_with_feats(feats_per_prefix) + # types = 8 (все 8 префиксов запрошены успешно) + assert result["types"] == 8 + assert result["features"] == 2 + + +def test_unmatched_feature_written_to_sentinel() -> None: + """Фича с full_name не из _PREFIX_MAP → пишется в unmatched (source_layer_id=990099).""" + feat = _make_feature( + feature_id="portal_geo_gknspecial_zone.999", + full_name="Неизвестная зона нового типа", + zone_state_v=None, + ) + feats_per_prefix = [[feat]] + [[] for _ in range(7)] + + written_params: list[dict[str, Any]] = [] + + def make_db() -> MagicMock: + mock_db = MagicMock() + mock_db.begin_nested.return_value.__enter__ = MagicMock(return_value=None) + mock_db.begin_nested.return_value.__exit__ = MagicMock(return_value=False) + + def capture_execute(sql: Any, params: dict[str, Any] | None = None) -> None: + if params: + written_params.append(params) + + mock_db.execute.side_effect = capture_execute + return mock_db + + call_idx: list[int] = [0] + + def fake_by_cql(layer: str, cql_filter: str) -> list[Any]: + idx = call_idx[0] + call_idx[0] += 1 + return feats_per_prefix[idx] if idx < len(feats_per_prefix) else [] + + with ( + patch("app.workers.tasks.gknspecial_harvest.EKBGeoportalClient") as mock_client_cls, + patch( + "app.workers.tasks.gknspecial_harvest.SessionLocal", + return_value=make_db(), + ), + ): + mock_client = MagicMock() + mock_client.features_by_cql.side_effect = fake_by_cql + mock_client_cls.return_value = mock_client + result = harvest_gknspecial_zones() + + assert result["features"] == 1 + assert len(written_params) == 1 + assert written_params[0]["source_layer_id"] == _UNMATCHED_SOURCE_LAYER_ID + assert written_params[0]["layer_kind"] == _UNMATCHED_LAYER_KIND -- 2.45.3