feat(sf): opportunity-ЗУ harvest → ird_overlays (future-supply, #1086)

backend/app/services/scrapers/nspd_client.py

@@ -105,6 +105,7 @@ LAYERS: dict[str, int] = {
     "scheme_parcels": 37294,  # ЗУ по схеме расположения
     "free_parcels": 37298,  # ЗУ свободные от прав
     "future_parcels": 36473,  # ЗУ по проекту межевания
+    "krt_territories": 37430,  # территории ККР (комплексное развитие) — opportunity (#1086)
     "okn_territory": 875840,
     "cadastral_cost_heatmap": 37236,
     "cadastral_cost_per_m2_heatmap": 37758,

backend/app/workers/beat_schedule.py

@@ -332,4 +332,13 @@ def build_beat_schedule() -> dict:
         "options": {"queue": "celery"},
     }
 
+    # Opportunity-ЗУ harvest (#1086): grid-walk НСПД opportunity-слоёв по кварталам ЕКБ →
+    # ird_overlays (layer_kind='opportunity_*'). 1-е число месяца 05:30 МСК — реже ИРД (weekly),
+    # opportunity-земля меняется медленнее ограничений. Тот же движок, что ird-harvest.
+    schedule["opportunity-harvest-monthly"] = {
+        "task": "tasks.opportunity_harvest.harvest_opportunity_overlays",
+        "schedule": _parse_cron("30 5 1 * *"),
+        "options": {"queue": "celery"},
+    }
+
     return schedule

backend/app/workers/celery_app.py

@@ -64,6 +64,7 @@ celery_app = Celery(
         "app.workers.tasks.supply_layers_refresh",
         "app.workers.tasks.forecast",
         "app.workers.tasks.ird_harvest",
+        "app.workers.tasks.opportunity_harvest",
     ],
 )
 celery_app.conf.timezone = "Europe/Moscow"

backend/app/workers/tasks/opportunity_harvest.py

@@ -0,0 +1,105 @@
+"""Celery task: harvest opportunity-ЗУ (future-supply) из НСПД → ``ird_overlays`` (#1086).
+
+Future-supply / land opportunity — свободная и выставляемая под застройку земля. Источник —
+НСПД (федеральный, единообразный, vault research/NSPD_IRD_Layers_Automation_Recon_Jun06). В центре
+города такие ЗУ редки (точечный probe даёт 0), но по агломерации ЕКБ присутствуют → нужен
+СПЛОШНОЙ grid-walk по кварталам, как в ИРД-harvest (#1067 B5).
+
+DIRECT-сигналы «земля под застройку» (ID в nspd_client.LAYERS):
+  37299 ЗУ на аукционе · 37294 ЗУ по схеме расположения · 37298 ЗУ свободные от прав ·
+  36473 ЗУ по проекту межевания · 37430 территории ККР (комплексное развитие).
+
+Переиспользует ДВИЖОК ИРД-harvest (один grid-walk + UPSERT-engine, не дублируем): тот же
+``ird_overlays`` (м.132), тот же ``_upsert_feature`` (ST_Transform 3857→4326, SAVEPOINT per-row,
+ON CONFLICT по (source_layer_id, geom_data_id)), те же кварталы ЕКБ (``_ekb_quarters``). Отличие —
+набор слоёв (opportunity вместо ограничений) и ``layer_kind='opportunity_*'`` для фильтра в analyze.
+
+Расписание — ежемесячно (beat_schedule.py: opportunity-harvest-monthly): opportunity-земля
+меняется медленнее ограничений, еженедельный прогон избыточен.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+
+from app.core.db import SessionLocal
+from app.services.scrapers.nspd_client import LAYERS, NSPDClient, _geojson_bbox_3857
+from app.workers.celery_app import celery_app
+from app.workers.tasks.ird_harvest import _ekb_quarters, _upsert_feature
+
+logger = logging.getLogger(__name__)
+
+# Opportunity-слои: ключ nspd_client.LAYERS → layer_kind в ird_overlays ('opportunity_*' для
+# фильтра в analyze «рядом свободная/аукционная земля»). НЕ ограничения — отдельный бакет.
+OPPORTUNITY_LAYER_KINDS: dict[str, str] = {
+    "auction_parcels": "opportunity_auction",
+    "scheme_parcels": "opportunity_scheme",
+    "free_parcels": "opportunity_free",
+    "future_parcels": "opportunity_future",
+    "krt_territories": "opportunity_krt",
+}
+
+
+@celery_app.task(name="tasks.opportunity_harvest.harvest_opportunity_overlays")
+def harvest_opportunity_overlays(quarters: list[str] | None = None) -> dict[str, int]:
+    """Harvest opportunity-ЗУ НСПД по кварталам ЕКБ → ird_overlays (layer_kind='opportunity_*').
+
+    Args:
+        quarters: явный список cad-номеров кварталов ('66:41:NNNNNNN'). None → distinct
+            кварталы ЕКБ из cad_parcels (``_ekb_quarters``).
+
+    Returns:
+        {'quarters': N, 'features': M} — обработано кварталов / записано opportunity-фич.
+    """
+    fetched_at = datetime.now(UTC).isoformat()
+    client = NSPDClient()
+    db = SessionLocal()
+    n_quarters = 0
+    n_features = 0
+    try:
+        quarter_list = quarters if quarters is not None else _ekb_quarters(db)
+        logger.info("opportunity_harvest: старт, кварталов=%d", len(quarter_list))
+        for quarter in quarter_list:
+            try:
+                qres = client.search_by_cad(quarter, thematic_id=2)
+                qfeat = qres.first
+                if not qfeat or not qfeat.geometry:
+                    logger.info("opportunity_harvest: квартал %s пуст в НСПД, скип", quarter)
+                    continue
+                bbox = _geojson_bbox_3857(qfeat.geometry)
+                if bbox is None:
+                    continue
+                n_quarters += 1
+                for layer_key, layer_kind in OPPORTUNITY_LAYER_KINDS.items():
+                    layer_id = LAYERS.get(layer_key)
+                    if layer_id is None:
+                        logger.warning("opportunity_harvest: нет layer_id для %s", layer_key)
+                        continue
+                    try:
+                        feats = client.get_features_in_bbox_grid(layer_id, bbox)
+                    except Exception as exc:
+                        logger.warning(
+                            "opportunity_harvest: grid-walk failed quarter=%s layer=%s: %s",
+                            quarter,
+                            layer_key,
+                            exc,
+                        )
+                        continue
+                    for feat in feats:
+                        if _upsert_feature(
+                            db,
+                            layer_id=layer_id,
+                            layer_kind=layer_kind,
+                            feature=feat,
+                            fetched_at=fetched_at,
+                        ):
+                            n_features += 1
+            except Exception as exc:
+                logger.warning("opportunity_harvest: квартал %s failed: %s", quarter, exc)
+                continue
+        db.commit()
+        logger.info("opportunity_harvest: готово, кварталов=%d фич=%d", n_quarters, n_features)
+        return {"quarters": n_quarters, "features": n_features}
+    finally:
+        db.close()

backend/tests/workers/test_opportunity_harvest.py

@@ -0,0 +1,118 @@
+"""Тесты harvest_opportunity_overlays (#1086) — НСПД opportunity-harvest → ird_overlays.
+
+Сеть/БД не дёргаются: мокаем NSPDClient + SessionLocal, проверяем grid-walk по opportunity-слоям,
+layer_kind='opportunity_*', skip пустого квартала, commit/close. Движок общий с ird_harvest.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Any
+
+from app.workers.tasks import opportunity_harvest
+
+
+class _Feat:
+    def __init__(self, geometry: dict[str, Any] | None, properties: dict[str, Any]) -> None:
+        self.geometry = geometry
+        self.properties = properties
+
+
+class _SearchRes:
+    def __init__(self, feat: _Feat | None) -> None:
+        self.first = feat
+
+
+class _FakeClient:
+    """NSPDClient-двойник: квартал → полигон (3857), grid-walk → 1 фича на слой."""
+
+    def __init__(self, *a: Any, **kw: Any) -> None:
+        self.grid_calls: list[int] = []
+
+    def search_by_cad(self, cad: str, thematic_id: int = 1) -> _SearchRes:
+        ring = [[0, 0], [100, 0], [100, 100], [0, 100], [0, 0]]
+        geom = {"type": "Polygon", "coordinates": [ring]}
+        return _SearchRes(_Feat(geom, {"cad": cad}))
+
+    def get_features_in_bbox_grid(self, layer_id: int, bbox: Any, **kw: Any) -> list[_Feat]:
+        self.grid_calls.append(layer_id)
+        return [
+            _Feat(
+                {"type": "Polygon", "coordinates": [[[1, 1], [2, 1], [2, 2], [1, 1]]]},
+                {
+                    "geom_data_id": 2000 + layer_id,
+                    "reg_numb_border": "66:41:0204016:10",
+                    "category": 36940,
+                    "subcategory": 5,
+                },
+            )
+        ]
+
+
+class _FakeDB:
+    """Session-двойник: пишет execute-вызовы, begin_nested как no-op CM."""
+
+    def __init__(self) -> None:
+        self.executed: list[tuple[Any, dict[str, Any] | None]] = []
+        self.committed = False
+        self.closed = False
+
+    @contextmanager
+    def begin_nested(self) -> Any:
+        yield
+
+    def execute(self, sql: Any, params: dict[str, Any] | None = None) -> Any:
+        self.executed.append((sql, params))
+        return None
+
+    def commit(self) -> None:
+        self.committed = True
+
+    def close(self) -> None:
+        self.closed = True
+
+
+def _patch(monkeypatch: Any, db: _FakeDB) -> _FakeClient:
+    client = _FakeClient()
+    monkeypatch.setattr(opportunity_harvest, "NSPDClient", lambda *a, **kw: client)
+    monkeypatch.setattr(opportunity_harvest, "SessionLocal", lambda: db)
+    return client
+
+
+def test_harvest_explicit_quarters_all_opportunity_layers(monkeypatch: Any) -> None:
+    """1 квартал × N opportunity-слоёв → upsert по фиче на слой; layer_kind='opportunity_*'."""
+    db = _FakeDB()
+    client = _patch(monkeypatch, db)
+    n_layers = len(opportunity_harvest.OPPORTUNITY_LAYER_KINDS)
+
+    result = opportunity_harvest.harvest_opportunity_overlays(quarters=["66:41:0204016"])
+
+    assert result == {"quarters": 1, "features": n_layers}
+    assert len(client.grid_calls) == n_layers
+    assert len(db.executed) == n_layers
+    assert db.committed is True
+    assert db.closed is True
+    kinds = {params["layer_kind"] for _, params in db.executed if params}
+    assert kinds == set(opportunity_harvest.OPPORTUNITY_LAYER_KINDS.values())
+    assert all(k.startswith("opportunity_") for k in kinds)
+
+
+def test_harvest_skips_empty_quarter(monkeypatch: Any) -> None:
+    """Квартал без геометрии в НСПД → 0 кварталов/фич, без падения."""
+    db = _FakeDB()
+    client = _patch(monkeypatch, db)
+    monkeypatch.setattr(client, "search_by_cad", lambda cad, thematic_id=1: _SearchRes(None))
+
+    result = opportunity_harvest.harvest_opportunity_overlays(quarters=["66:41:9999999"])
+
+    assert result == {"quarters": 0, "features": 0}
+    assert db.executed == []
+    assert db.committed is True
+
+
+def test_krt_layer_registered() -> None:
+    """ККР (37430) добавлен в LAYERS и в opportunity-набор (#1086)."""
+    from app.services.scrapers.nspd_client import LAYERS
+
+    assert LAYERS["krt_territories"] == 37430
+    assert "krt_territories" in opportunity_harvest.OPPORTUNITY_LAYER_KINDS