From 99ca7280044e703e313347a0df53c122650060d7 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 6 Jun 2026 23:59:54 +0300 Subject: [PATCH] =?UTF-8?q?feat(sf):=20=D0=BF=D1=80=D0=BE=D0=B3=D1=80?= =?UTF-8?q?=D0=B5=D0=B2=20zone=5Fregulation=5Fcache=20=D0=BF=D0=BE=20?= =?UTF-8?q?=D1=82=D0=B5=D1=80=D0=B7=D0=BE=D0=BD=D0=B0=D0=BC=20=D0=95=D0=9A?= =?UTF-8?q?=D0=91=20(#1059/C8b)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/workers/beat_schedule.py | 9 ++ backend/app/workers/celery_app.py | 1 + .../workers/tasks/zone_regulation_refresh.py | 92 +++++++++++++++ .../workers/test_zone_regulation_refresh.py | 110 ++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 backend/app/workers/tasks/zone_regulation_refresh.py create mode 100644 backend/tests/workers/test_zone_regulation_refresh.py diff --git a/backend/app/workers/beat_schedule.py b/backend/app/workers/beat_schedule.py index a3057f27..e48bd7dd 100644 --- a/backend/app/workers/beat_schedule.py +++ b/backend/app/workers/beat_schedule.py @@ -350,4 +350,13 @@ def build_beat_schedule() -> dict: "options": {"queue": "celery"}, } + # Прогрев кэша ПЗЗ-регламента зон ЕКБ (#1059/C8b): WFS-перечисление терзон → urbanCard → + # zone_regulation_cache. 1-е число месяца 06:30 МСК — после planning (06:00). Регламент + # меняется только при правке ПЗЗ, ежемесячно достаточно. Наполняет кэш C8b проактивно. + schedule["zone-regulation-refresh-monthly"] = { + "task": "tasks.zone_regulation_refresh.refresh_zone_regulations", + "schedule": _parse_cron("30 6 1 * *"), + "options": {"queue": "celery"}, + } + return schedule diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index 6d9aba2b..3a8ecf8e 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -66,6 +66,7 @@ celery_app = Celery( "app.workers.tasks.ird_harvest", "app.workers.tasks.opportunity_harvest", "app.workers.tasks.planning_harvest", + "app.workers.tasks.zone_regulation_refresh", ], ) celery_app.conf.timezone = "Europe/Moscow" diff --git a/backend/app/workers/tasks/zone_regulation_refresh.py b/backend/app/workers/tasks/zone_regulation_refresh.py new file mode 100644 index 00000000..6ffa6b3b --- /dev/null +++ b/backend/app/workers/tasks/zone_regulation_refresh.py @@ -0,0 +1,92 @@ +"""Celery task: прогрев кэша ПЗЗ-регламента зон ЕКБ → ``zone_regulation_cache`` (#1059 / C8b). + +Регламент ПЗЗ идентичен для всех участков одной терзоны (~100 зон на ЕКБ). C8b (#1099) умеет +резолвить регламент по точке (`zone_regulation_at` → urbanCard) и кэшировать по `zone_index`, но +кэш наполняется лениво — первый запрос участка в незакэшированной зоне платит за live-urbanCard. + +Эта задача наполняет кэш ПРОАКТИВНО: WFS-перечисление всех терзон ЕКБ → по одной репрезентативной +точке на уникальный `urban_index` → `zone_regulation_at` → `upsert_zone_regulation`. После прогона +analyze отдаёт `ird.zone_regulation` из кэша (быстро), без live-геопортала в hot-пути. + +Дедуп по `urban_index`: регламент одинаков для всех полигонов зоны → резолвим индекс один раз. +Репрезентативная точка (`shapely.representative_point`) гарантированно внутри полигона (в отличие +от centroid для вогнутых) → searchByGeom находит терзону. + +Mirror conventions (planning_harvest.py / backend.md): SessionLocal()+try/finally; per-zone +try/except (сбой одной зоны не валит прогон); commit в конце. Расписание — ежемесячно (регламент +меняется только при правке ПЗЗ). +""" + +from __future__ import annotations + +import logging +from typing import Any + +from shapely.geometry import shape + +from app.core.db import SessionLocal +from app.services.scrapers.ekb_geoportal_client import EKBGeoportalClient +from app.services.site_finder.zone_regulation import upsert_zone_regulation +from app.workers.celery_app import celery_app + +logger = logging.getLogger(__name__) + +# bbox агломерации ЕКБ (minlon, minlat, maxlon, maxlat) EPSG:4326 — как в planning_harvest. +EKB_AGGLOMERATION_BBOX: tuple[float, float, float, float] = (60.0, 56.6, 61.1, 57.1) + + +def _representative_point(geometry: dict[str, Any] | None) -> tuple[float, float] | None: + """Точка (lon, lat) гарантированно внутри полигона зоны. None при невалидной геометрии.""" + if not geometry: + return None + try: + pt = shape(geometry).representative_point() + return (float(pt.x), float(pt.y)) + except Exception as exc: # битая геометрия фичи — пропускаем зону + logger.warning("zone_regulation_refresh: bad geometry, skip: %s", exc) + return None + + +@celery_app.task(name="tasks.zone_regulation_refresh.refresh_zone_regulations") +def refresh_zone_regulations( + bbox: tuple[float, float, float, float] | None = None, +) -> dict[str, int]: + """Прогрев кэша регламента по терзонам ЕКБ. + + Args: + bbox: (minlon, minlat, maxlon, maxlat) EPSG:4326. None → bbox агломерации ЕКБ. + + Returns: + {'zones': N, 'cached': M} — уникальных индексов зон / успешно закэшировано регламентов. + """ + target_bbox = bbox if bbox is not None else EKB_AGGLOMERATION_BBOX + client = EKBGeoportalClient() + db = SessionLocal() + seen: set[str] = set() + n_cached = 0 + try: + feats = client.features_in_bbox("territorial_zone", target_bbox) + logger.info("zone_regulation_refresh: терзон в bbox=%d", len(feats)) + for feat in feats: + idx = feat.properties.get("urban_index") + if not idx or str(idx) in seen: + continue + seen.add(str(idx)) + point = _representative_point(feat.geometry) + if point is None: + continue + lon, lat = point + try: + reg = client.zone_regulation_at(lon, lat) + except Exception as exc: + logger.warning("zone_regulation_refresh: resolve failed idx=%s: %s", idx, exc) + continue + if reg is None or not reg.zone_index: + continue + if upsert_zone_regulation(db, reg) is not None: + n_cached += 1 + db.commit() + logger.info("zone_regulation_refresh: зон=%d закэшировано=%d", len(seen), n_cached) + return {"zones": len(seen), "cached": n_cached} + finally: + db.close() diff --git a/backend/tests/workers/test_zone_regulation_refresh.py b/backend/tests/workers/test_zone_regulation_refresh.py new file mode 100644 index 00000000..0a3ea239 --- /dev/null +++ b/backend/tests/workers/test_zone_regulation_refresh.py @@ -0,0 +1,110 @@ +"""Тесты refresh_zone_regulations (#1059/C8b) — прогрев кэша регламента зон ЕКБ. + +Сеть/БД не дёргаются: мокаем EKBGeoportalClient + SessionLocal + upsert_zone_regulation. +Проверяем дедуп по urban_index, представительную точку, skip пустых/битых, счётчики. +""" + +from __future__ import annotations + +from typing import Any + +from app.services.scrapers.ekb_geoportal_client import ZoneRegulation +from app.workers.tasks import zone_regulation_refresh as zrr + + +class _Feat: + def __init__(self, geometry: dict[str, Any] | None, properties: dict[str, Any]) -> None: + self.geometry = geometry + self.properties = properties + + +def _poly() -> dict[str, Any]: + return {"type": "Polygon", "coordinates": [[[60, 56], [60.1, 56], [60.1, 56.1], [60, 56]]]} + + +def _reg(idx: str) -> ZoneRegulation: + return ZoneRegulation( + zone_index=idx, + zone_full_name=f"{idx} зона", + main_vri=[], + conditional_vri=[], + auxiliary_vri=[], + limit_params=[], + ) + + +class _FakeClient: + def __init__(self, feats: list[_Feat]) -> None: + self._feats = feats + self.resolved: list[tuple[float, float]] = [] + + def features_in_bbox(self, layer: str, bbox: Any) -> list[_Feat]: + return self._feats + + def zone_regulation_at(self, lon: float, lat: float) -> ZoneRegulation: + self.resolved.append((lon, lat)) + return _reg("Ц-1") + + +class _DB: + def __init__(self) -> None: + self.committed = False + self.closed = False + + def commit(self) -> None: + self.committed = True + + def close(self) -> None: + self.closed = True + + +def _patch(monkeypatch: Any, feats: list[_Feat]) -> tuple[_FakeClient, list[Any]]: + client = _FakeClient(feats) + upserts: list[Any] = [] + monkeypatch.setattr(zrr, "EKBGeoportalClient", lambda *a, **kw: client) + monkeypatch.setattr(zrr, "SessionLocal", lambda: _DB()) + monkeypatch.setattr( + zrr, "upsert_zone_regulation", lambda db, reg, **kw: upserts.append(reg) or {} + ) + return client, upserts + + +def test_dedup_by_urban_index(monkeypatch: Any) -> None: + """Две фичи одного индекса → резолвим/кэшируем зону один раз.""" + feats = [ + _Feat(_poly(), {"urban_index": "Ц-1"}), + _Feat(_poly(), {"urban_index": "Ц-1"}), + _Feat(_poly(), {"urban_index": "Ж-2"}), + ] + client, upserts = _patch(monkeypatch, feats) + + result = zrr.refresh_zone_regulations() + + assert result == {"zones": 2, "cached": 2} + assert len(client.resolved) == 2 # уникальных индексов + assert len(upserts) == 2 + + +def test_skips_blank_index_and_bad_geometry(monkeypatch: Any) -> None: + feats = [ + _Feat(_poly(), {"urban_index": ""}), # пустой индекс + _Feat(None, {"urban_index": "Ц-3"}), # нет геометрии + ] + client, upserts = _patch(monkeypatch, feats) + + result = zrr.refresh_zone_regulations() + + assert result == {"zones": 1, "cached": 0} # Ц-3 в seen, но geom=None → не резолвится + assert client.resolved == [] + assert upserts == [] + + +def test_representative_point_inside_polygon() -> None: + lon, lat = zrr._representative_point(_poly()) # type: ignore[misc] + assert 60.0 <= lon <= 60.1 + assert 56.0 <= lat <= 56.1 + + +def test_representative_point_none_on_bad_geom() -> None: + assert zrr._representative_point(None) is None + assert zrr._representative_point({"type": "Polygon", "coordinates": "garbage"}) is None -- 2.45.3