From b1530467f7a887965245f992fc7c299d59b552c0 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 6 Jun 2026 19:32:45 +0000 Subject: [PATCH] =?UTF-8?q?feat(sf):=20opportunity-=D0=97=D0=A3=20harvest?= =?UTF-8?q?=20=E2=86=92=20ird=5Foverlays=20(future-supply,=20#1100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit opportunity_harvest.py — grid-walk НСПД opportunity-слоёв (ЗУ аукцион/схема/свободные/межевание/ККР) по кварталам ЕКБ → ird_overlays с layer_kind='opportunity_*'. Переиспользует движок ird_harvest (#1090): _upsert_feature/_ekb_quarters. +krt_territories=37430 в LAYERS. Beat monthly (1-е 05:30). Self-contained, прод-прогон gated B6. Closes #1086. Co-authored-by: lekss361 Co-committed-by: lekss361 --- backend/app/services/scrapers/nspd_client.py | 1 + backend/app/workers/beat_schedule.py | 9 ++ backend/app/workers/celery_app.py | 1 + .../app/workers/tasks/opportunity_harvest.py | 105 ++++++++++++++++ .../tests/workers/test_opportunity_harvest.py | 118 ++++++++++++++++++ 5 files changed, 234 insertions(+) create mode 100644 backend/app/workers/tasks/opportunity_harvest.py create mode 100644 backend/tests/workers/test_opportunity_harvest.py diff --git a/backend/app/services/scrapers/nspd_client.py b/backend/app/services/scrapers/nspd_client.py index 5be0efc7..3c868305 100644 --- a/backend/app/services/scrapers/nspd_client.py +++ b/backend/app/services/scrapers/nspd_client.py @@ -105,6 +105,7 @@ LAYERS: dict[str, int] = { "scheme_parcels": 37294, # ЗУ по схеме расположения "free_parcels": 37298, # ЗУ свободные от прав "future_parcels": 36473, # ЗУ по проекту межевания + "krt_territories": 37430, # территории ККР (комплексное развитие) — opportunity (#1086) "okn_territory": 875840, "cadastral_cost_heatmap": 37236, "cadastral_cost_per_m2_heatmap": 37758, diff --git a/backend/app/workers/beat_schedule.py b/backend/app/workers/beat_schedule.py index b332fd2c..6af8dedd 100644 --- a/backend/app/workers/beat_schedule.py +++ b/backend/app/workers/beat_schedule.py @@ -332,4 +332,13 @@ def build_beat_schedule() -> dict: "options": {"queue": "celery"}, } + # Opportunity-ЗУ harvest (#1086): grid-walk НСПД opportunity-слоёв по кварталам ЕКБ → + # ird_overlays (layer_kind='opportunity_*'). 1-е число месяца 05:30 МСК — реже ИРД (weekly), + # opportunity-земля меняется медленнее ограничений. Тот же движок, что ird-harvest. + schedule["opportunity-harvest-monthly"] = { + "task": "tasks.opportunity_harvest.harvest_opportunity_overlays", + "schedule": _parse_cron("30 5 1 * *"), + "options": {"queue": "celery"}, + } + return schedule diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index cd51c76a..849917f1 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -64,6 +64,7 @@ celery_app = Celery( "app.workers.tasks.supply_layers_refresh", "app.workers.tasks.forecast", "app.workers.tasks.ird_harvest", + "app.workers.tasks.opportunity_harvest", ], ) celery_app.conf.timezone = "Europe/Moscow" diff --git a/backend/app/workers/tasks/opportunity_harvest.py b/backend/app/workers/tasks/opportunity_harvest.py new file mode 100644 index 00000000..392dece9 --- /dev/null +++ b/backend/app/workers/tasks/opportunity_harvest.py @@ -0,0 +1,105 @@ +"""Celery task: harvest opportunity-ЗУ (future-supply) из НСПД → ``ird_overlays`` (#1086). + +Future-supply / land opportunity — свободная и выставляемая под застройку земля. Источник — +НСПД (федеральный, единообразный, vault research/NSPD_IRD_Layers_Automation_Recon_Jun06). В центре +города такие ЗУ редки (точечный probe даёт 0), но по агломерации ЕКБ присутствуют → нужен +СПЛОШНОЙ grid-walk по кварталам, как в ИРД-harvest (#1067 B5). + +DIRECT-сигналы «земля под застройку» (ID в nspd_client.LAYERS): + 37299 ЗУ на аукционе · 37294 ЗУ по схеме расположения · 37298 ЗУ свободные от прав · + 36473 ЗУ по проекту межевания · 37430 территории ККР (комплексное развитие). + +Переиспользует ДВИЖОК ИРД-harvest (один grid-walk + UPSERT-engine, не дублируем): тот же +``ird_overlays`` (м.132), тот же ``_upsert_feature`` (ST_Transform 3857→4326, SAVEPOINT per-row, +ON CONFLICT по (source_layer_id, geom_data_id)), те же кварталы ЕКБ (``_ekb_quarters``). Отличие — +набор слоёв (opportunity вместо ограничений) и ``layer_kind='opportunity_*'`` для фильтра в analyze. + +Расписание — ежемесячно (beat_schedule.py: opportunity-harvest-monthly): opportunity-земля +меняется медленнее ограничений, еженедельный прогон избыточен. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime + +from app.core.db import SessionLocal +from app.services.scrapers.nspd_client import LAYERS, NSPDClient, _geojson_bbox_3857 +from app.workers.celery_app import celery_app +from app.workers.tasks.ird_harvest import _ekb_quarters, _upsert_feature + +logger = logging.getLogger(__name__) + +# Opportunity-слои: ключ nspd_client.LAYERS → layer_kind в ird_overlays ('opportunity_*' для +# фильтра в analyze «рядом свободная/аукционная земля»). НЕ ограничения — отдельный бакет. +OPPORTUNITY_LAYER_KINDS: dict[str, str] = { + "auction_parcels": "opportunity_auction", + "scheme_parcels": "opportunity_scheme", + "free_parcels": "opportunity_free", + "future_parcels": "opportunity_future", + "krt_territories": "opportunity_krt", +} + + +@celery_app.task(name="tasks.opportunity_harvest.harvest_opportunity_overlays") +def harvest_opportunity_overlays(quarters: list[str] | None = None) -> dict[str, int]: + """Harvest opportunity-ЗУ НСПД по кварталам ЕКБ → ird_overlays (layer_kind='opportunity_*'). + + Args: + quarters: явный список cad-номеров кварталов ('66:41:NNNNNNN'). None → distinct + кварталы ЕКБ из cad_parcels (``_ekb_quarters``). + + Returns: + {'quarters': N, 'features': M} — обработано кварталов / записано opportunity-фич. + """ + fetched_at = datetime.now(UTC).isoformat() + client = NSPDClient() + db = SessionLocal() + n_quarters = 0 + n_features = 0 + try: + quarter_list = quarters if quarters is not None else _ekb_quarters(db) + logger.info("opportunity_harvest: старт, кварталов=%d", len(quarter_list)) + for quarter in quarter_list: + try: + qres = client.search_by_cad(quarter, thematic_id=2) + qfeat = qres.first + if not qfeat or not qfeat.geometry: + logger.info("opportunity_harvest: квартал %s пуст в НСПД, скип", quarter) + continue + bbox = _geojson_bbox_3857(qfeat.geometry) + if bbox is None: + continue + n_quarters += 1 + for layer_key, layer_kind in OPPORTUNITY_LAYER_KINDS.items(): + layer_id = LAYERS.get(layer_key) + if layer_id is None: + logger.warning("opportunity_harvest: нет layer_id для %s", layer_key) + continue + try: + feats = client.get_features_in_bbox_grid(layer_id, bbox) + except Exception as exc: + logger.warning( + "opportunity_harvest: grid-walk failed quarter=%s layer=%s: %s", + quarter, + layer_key, + exc, + ) + continue + for feat in feats: + if _upsert_feature( + db, + layer_id=layer_id, + layer_kind=layer_kind, + feature=feat, + fetched_at=fetched_at, + ): + n_features += 1 + except Exception as exc: + logger.warning("opportunity_harvest: квартал %s failed: %s", quarter, exc) + continue + db.commit() + logger.info("opportunity_harvest: готово, кварталов=%d фич=%d", n_quarters, n_features) + return {"quarters": n_quarters, "features": n_features} + finally: + db.close() diff --git a/backend/tests/workers/test_opportunity_harvest.py b/backend/tests/workers/test_opportunity_harvest.py new file mode 100644 index 00000000..b6547ff4 --- /dev/null +++ b/backend/tests/workers/test_opportunity_harvest.py @@ -0,0 +1,118 @@ +"""Тесты harvest_opportunity_overlays (#1086) — НСПД opportunity-harvest → ird_overlays. + +Сеть/БД не дёргаются: мокаем NSPDClient + SessionLocal, проверяем grid-walk по opportunity-слоям, +layer_kind='opportunity_*', skip пустого квартала, commit/close. Движок общий с ird_harvest. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import Any + +from app.workers.tasks import opportunity_harvest + + +class _Feat: + def __init__(self, geometry: dict[str, Any] | None, properties: dict[str, Any]) -> None: + self.geometry = geometry + self.properties = properties + + +class _SearchRes: + def __init__(self, feat: _Feat | None) -> None: + self.first = feat + + +class _FakeClient: + """NSPDClient-двойник: квартал → полигон (3857), grid-walk → 1 фича на слой.""" + + def __init__(self, *a: Any, **kw: Any) -> None: + self.grid_calls: list[int] = [] + + def search_by_cad(self, cad: str, thematic_id: int = 1) -> _SearchRes: + ring = [[0, 0], [100, 0], [100, 100], [0, 100], [0, 0]] + geom = {"type": "Polygon", "coordinates": [ring]} + return _SearchRes(_Feat(geom, {"cad": cad})) + + def get_features_in_bbox_grid(self, layer_id: int, bbox: Any, **kw: Any) -> list[_Feat]: + self.grid_calls.append(layer_id) + return [ + _Feat( + {"type": "Polygon", "coordinates": [[[1, 1], [2, 1], [2, 2], [1, 1]]]}, + { + "geom_data_id": 2000 + layer_id, + "reg_numb_border": "66:41:0204016:10", + "category": 36940, + "subcategory": 5, + }, + ) + ] + + +class _FakeDB: + """Session-двойник: пишет execute-вызовы, begin_nested как no-op CM.""" + + def __init__(self) -> None: + self.executed: list[tuple[Any, dict[str, Any] | None]] = [] + self.committed = False + self.closed = False + + @contextmanager + def begin_nested(self) -> Any: + yield + + def execute(self, sql: Any, params: dict[str, Any] | None = None) -> Any: + self.executed.append((sql, params)) + return None + + def commit(self) -> None: + self.committed = True + + def close(self) -> None: + self.closed = True + + +def _patch(monkeypatch: Any, db: _FakeDB) -> _FakeClient: + client = _FakeClient() + monkeypatch.setattr(opportunity_harvest, "NSPDClient", lambda *a, **kw: client) + monkeypatch.setattr(opportunity_harvest, "SessionLocal", lambda: db) + return client + + +def test_harvest_explicit_quarters_all_opportunity_layers(monkeypatch: Any) -> None: + """1 квартал × N opportunity-слоёв → upsert по фиче на слой; layer_kind='opportunity_*'.""" + db = _FakeDB() + client = _patch(monkeypatch, db) + n_layers = len(opportunity_harvest.OPPORTUNITY_LAYER_KINDS) + + result = opportunity_harvest.harvest_opportunity_overlays(quarters=["66:41:0204016"]) + + assert result == {"quarters": 1, "features": n_layers} + assert len(client.grid_calls) == n_layers + assert len(db.executed) == n_layers + assert db.committed is True + assert db.closed is True + kinds = {params["layer_kind"] for _, params in db.executed if params} + assert kinds == set(opportunity_harvest.OPPORTUNITY_LAYER_KINDS.values()) + assert all(k.startswith("opportunity_") for k in kinds) + + +def test_harvest_skips_empty_quarter(monkeypatch: Any) -> None: + """Квартал без геометрии в НСПД → 0 кварталов/фич, без падения.""" + db = _FakeDB() + client = _patch(monkeypatch, db) + monkeypatch.setattr(client, "search_by_cad", lambda cad, thematic_id=1: _SearchRes(None)) + + result = opportunity_harvest.harvest_opportunity_overlays(quarters=["66:41:9999999"]) + + assert result == {"quarters": 0, "features": 0} + assert db.executed == [] + assert db.committed is True + + +def test_krt_layer_registered() -> None: + """ККР (37430) добавлен в LAYERS и в opportunity-набор (#1086).""" + from app.services.scrapers.nspd_client import LAYERS + + assert LAYERS["krt_territories"] == 37430 + assert "krt_territories" in opportunity_harvest.OPPORTUNITY_LAYER_KINDS