bot-backend
14f3ef2019
Co-authored-by: bot-backend <bot-backend@gendsgn.local> Co-committed-by: bot-backend <bot-backend@gendsgn.local>
1602 lines
75 KiB
Python
1602 lines
75 KiB
Python
"""Оркестратор bulk cadastre harvest — pure-async, без Celery декораторов.
|
||
|
||
Вызывается из Celery-таска (scrape_cadastre.py) через asyncio.run().
|
||
|
||
Фазы harvest_quarter:
|
||
Phase 1: search_by_quarter → upsert первые 20 per категории
|
||
Phase 2: определить overflow-категории (meta > 20)
|
||
Phase 3: grid_walk для overflow 36368 (ЗУ) и 36369 (Здания)
|
||
Phase 4: upsert_quarter_stats из meta квартального snapshot (36381)
|
||
|
||
Idempotent: все upsert'ы — ON CONFLICT DO UPDATE.
|
||
Resumable: phase_state в cadastre_jobs показывает прогресс.
|
||
|
||
Питфоллы (см. CLAUDE.md):
|
||
- CAST(:x AS jsonb) не :x::jsonb (SQLAlchemy парсит :: как параметр)
|
||
- psycopg v3 only (не psycopg2)
|
||
- SAVEPOINT через db.begin_nested() вокруг каждого upsert
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
from collections.abc import Callable
|
||
from dataclasses import dataclass, field
|
||
from typing import Any
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.schemas.nspd_bulk import NSPDBulkFeature, QuarterSnapshot
|
||
from app.scrapers.nspd_bulk_client import NSPDBulkClient, NspdBulkServerError
|
||
from app.services.cadastre.grid_geometry import generate_grid_click_points, quarter_bbox_3857
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Категории NSPD для routing
|
||
CAT_PARCEL = 36368 # ЗУ ЕГРН
|
||
CAT_BUILDING = 36369 # Здания ЕГРН
|
||
CAT_CONSTRUCTION = 36383 # Сооружения
|
||
CAT_ONC = 36384 # ОНС
|
||
CAT_ENK = 39663 # ЕНК
|
||
CAT_QUARTER_STATS = 36381 # Квартальная статистика
|
||
CAT_ZOUIT = frozenset({36940, 469039, 469040, 469042}) # ЗОУИТ
|
||
|
||
# Категории для grid-walk overflow
|
||
GRID_WALK_CATS = frozenset({CAT_PARCEL, CAT_BUILDING})
|
||
|
||
# Категории с выделенным thematicSearchId в NSPD API (curl-probe 2026-05-15).
|
||
# Для них отдельный per-category вызов даёт ДРУГОЙ результат, чем базовый thematic=1:
|
||
# - ЗОУИТ: thematicSearchId=5 → до 20 per subtype × 4 subtypes
|
||
# - ЕНК: thematicSearchId=15 → до 20 ЕНК объектов
|
||
# ЗУ/Здания/Сооружения/ОНС не имеют dedicated thematicSearchId — отдельный вызов
|
||
# вернёт те же 20 что и базовый, поэтому per-category probe для них бессмыслен.
|
||
PER_CATEGORY_PROBE_CATS = frozenset({*CAT_ZOUIT, CAT_ENK})
|
||
|
||
|
||
# ── Result type ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
@dataclass
|
||
class HarvestResult:
|
||
"""Итог harvest одного квартала."""
|
||
|
||
quarter: str
|
||
parcels_upserted: int = 0
|
||
buildings_upserted: int = 0
|
||
constructions_upserted: int = 0
|
||
oncs_upserted: int = 0
|
||
enks_upserted: int = 0
|
||
zouit_upserted: int = 0
|
||
snapshot_requests: int = 0 # search_by_quarter (Phase 1) + per-cat probes (Phase 1.5)
|
||
grid_walk_requests: int = 0 # wms_feature_info (Phase 2-3)
|
||
# Issue #252: layer_id'ы, чей grid-walk целиком упал на server-side 500/
|
||
# ServiceException (все cells сбойнули). Квартал НЕ валится — слой
|
||
# пропускается, факт фиксируется в cadastre_jobs.phase_state.harvest_meta.
|
||
failed_layers: list[int] = field(default_factory=list)
|
||
phase_state: dict[str, Any] | None = None
|
||
|
||
@property
|
||
def total_requests(self) -> int:
|
||
"""Все NSPD HTTP requests для этого квартала (snapshot + grid-walk)."""
|
||
return self.snapshot_requests + self.grid_walk_requests
|
||
|
||
|
||
# ── Основной оркестратор ─────────────────────────────────────────────────────
|
||
|
||
|
||
async def harvest_quarter(
|
||
db: Session,
|
||
client: NSPDBulkClient,
|
||
quarter: str,
|
||
job_id: int,
|
||
update_progress: Callable[[dict[str, Any]], None],
|
||
) -> HarvestResult:
|
||
"""Собрать все NSPD данные для квартала в cad_* таблицы.
|
||
|
||
Идемпотентен: повторный вызов не создаёт дубликаты (ON CONFLICT DO UPDATE).
|
||
Возобновляем: проверяет job.phase_state, пропускает завершённые фазы.
|
||
|
||
Args:
|
||
db: SQLAlchemy session. Caller должен управлять commit/rollback.
|
||
client: NSPDBulkClient (async context manager, уже открытый).
|
||
quarter: кадастровый номер квартала.
|
||
job_id: ID записи в cadastre_jobs для обновления прогресса.
|
||
update_progress: callback для записи phase_state в cadastre_jobs.
|
||
|
||
Returns:
|
||
HarvestResult с счётчиками upserted объектов.
|
||
|
||
Raises:
|
||
NspdBulkWafError: при WAF 403 — caller не retry.
|
||
NspdBulkRateLimitError: при исчерпании retries — caller может retry.
|
||
Exception: прочие ошибки — propagate.
|
||
"""
|
||
result = HarvestResult(quarter=quarter)
|
||
|
||
# NB: `cadastre_jobs.phase_state` шарится между ВСЕМИ parallel Celery workers одного job.
|
||
# Раньше тут был early-exit `if phase_state.phase == 'done': return` — но он триггерился
|
||
# как только ПЕРВЫЙ quarter записывал phase=done через update_progress callback, и все
|
||
# остальные workers (читая ту же shared row) бейлились без работы. Эффект: 95% quarters
|
||
# early-exit'или, coverage 1.6% parcels (pilot v7: 5 NSPD req для 50 quarters).
|
||
# Idempotency обеспечивается ON CONFLICT DO UPDATE в каждом upsert_* — re-enqueued task
|
||
# просто перепишет те же rows, не создавая дубликатов.
|
||
|
||
# ── Phase 1: search_by_quarter snapshot ──────────────────────────────────
|
||
update_progress({"phase": "snapshot_started", "quarter": quarter})
|
||
|
||
snapshot: QuarterSnapshot = await client.search_by_quarter(quarter)
|
||
result.snapshot_requests += 1
|
||
|
||
snapshot_stats = upsert_features(db, snapshot.features, source="search")
|
||
result.parcels_upserted += snapshot_stats["parcels"]
|
||
result.buildings_upserted += snapshot_stats["buildings"]
|
||
result.constructions_upserted += snapshot_stats["constructions"]
|
||
result.oncs_upserted += snapshot_stats["oncs"]
|
||
result.enks_upserted += snapshot_stats["enks"]
|
||
result.zouit_upserted += snapshot_stats["zouit"]
|
||
db.commit()
|
||
|
||
parcels_meta = snapshot.meta_counts.get(CAT_PARCEL, 0)
|
||
buildings_meta = snapshot.meta_counts.get(CAT_BUILDING, 0)
|
||
|
||
update_progress(
|
||
{
|
||
"phase": "snapshot_done",
|
||
"quarter": quarter,
|
||
"parcels_total_meta": parcels_meta,
|
||
"buildings_total_meta": buildings_meta,
|
||
"overflow_cats": snapshot.overflow_categories,
|
||
}
|
||
)
|
||
|
||
# ── Phase 1.5: per-category snapshot для ЗОУИТ и ЕНК ────────────────────
|
||
# ЗОУИТ (36940/469039/469040/469042) и ЕНК (39663) имеют dedicated thematicSearchId
|
||
# в NSPD API (5 и 15 соответственно) и не всегда попадают в базовый thematic=1 snapshot
|
||
# если totalCount в их категории > 20.
|
||
# Для ЗУ/Зданий/Сооружений/ОНС — dedicated thematicSearchId нет: отдельный вызов
|
||
# вернёт те же 20 что и базовый snapshot, поэтому их пропускаем.
|
||
# Если category ∈ GRID_WALK_CATS и bbox валиден — grid-walk даёт лучшее покрытие (225 точек).
|
||
for cat_id in PER_CATEGORY_PROBE_CATS:
|
||
meta_total = snapshot.meta_counts.get(cat_id, 0)
|
||
if meta_total == 0:
|
||
# NSPD не вернул meta для этой категории — объектов нет в квартале
|
||
continue
|
||
|
||
# Проверяем: нужен ли per-category probe?
|
||
# Условия skip: category ∈ GRID_WALK_CATS И bbox валиден → grid-walk сделает это лучше.
|
||
# PER_CATEGORY_PROBE_CATS ∩ GRID_WALK_CATS = ∅ (ZOUIT/ENK не в GRID_WALK_CATS),
|
||
# но явная проверка оставлена как safety guard.
|
||
if cat_id in GRID_WALK_CATS and quarter_bbox_3857(db, quarter) is not None:
|
||
logger.debug(
|
||
"harvest_quarter: skip per-cat probe cat=%d quarter=%s — grid-walk preferred",
|
||
cat_id,
|
||
quarter,
|
||
)
|
||
continue
|
||
|
||
update_progress({"phase": f"per_cat_probe_{cat_id}_started", "quarter": quarter})
|
||
|
||
try:
|
||
cat_snapshot = await client.search_by_quarter(quarter, category_id=cat_id)
|
||
result.snapshot_requests += 1
|
||
except Exception as e:
|
||
logger.warning(
|
||
"harvest_quarter: per-cat probe failed cat=%d quarter=%s: %s",
|
||
cat_id,
|
||
quarter,
|
||
e,
|
||
)
|
||
continue
|
||
|
||
if not cat_snapshot.features:
|
||
continue
|
||
|
||
cat_stats = upsert_features(db, cat_snapshot.features, source="search_per_cat")
|
||
result.parcels_upserted += cat_stats["parcels"]
|
||
result.buildings_upserted += cat_stats["buildings"]
|
||
result.constructions_upserted += cat_stats["constructions"]
|
||
result.oncs_upserted += cat_stats["oncs"]
|
||
result.enks_upserted += cat_stats["enks"]
|
||
result.zouit_upserted += cat_stats["zouit"]
|
||
db.commit()
|
||
|
||
new_total = sum(cat_stats.values())
|
||
logger.info(
|
||
"harvest_quarter per-cat probe: quarter=%s cat=%d new_features=%d",
|
||
quarter,
|
||
cat_id,
|
||
new_total,
|
||
)
|
||
|
||
# ── Phase 2-3: grid-walk для overflow категорий ───────────────────────────
|
||
overflow_cats = [c for c in snapshot.overflow_categories if c in GRID_WALK_CATS]
|
||
|
||
for cat_id in overflow_cats:
|
||
cat_label = "parcels" if cat_id == CAT_PARCEL else "buildings"
|
||
update_progress({"phase": f"grid_walk_{cat_label}_started", "quarter": quarter})
|
||
|
||
discovered, n_requests, layer_failed = await _grid_walk_category(
|
||
db=db,
|
||
client=client,
|
||
quarter=quarter,
|
||
layer_id=cat_id,
|
||
update_progress=update_progress,
|
||
)
|
||
result.grid_walk_requests += n_requests
|
||
|
||
if cat_id == CAT_PARCEL:
|
||
result.parcels_upserted += discovered
|
||
else:
|
||
result.buildings_upserted += discovered
|
||
|
||
# Issue #252: per-layer skip — слой целиком сбойнул на server-side 500/
|
||
# ServiceException. Фиксируем в harvest_meta (cadastre_jobs.phase_state),
|
||
# квартал продолжаем (остальные слои и фазы отрабатывают штатно).
|
||
done_progress: dict[str, Any] = {
|
||
"phase": f"grid_walk_{cat_label}_done",
|
||
"quarter": quarter,
|
||
f"{cat_label}_grid_discovered": discovered,
|
||
}
|
||
if layer_failed:
|
||
result.failed_layers.append(cat_id)
|
||
done_progress["harvest_meta"] = {f"layer_{cat_id}_failed": True}
|
||
|
||
db.commit()
|
||
update_progress(done_progress)
|
||
|
||
# ── Phase 4: quarter stats + auto-heal geom из snapshot ─────────────────
|
||
# Bug #1583: auto-heal geom выполняем ДО Phase 2.5 (territorial_zones). Иначе
|
||
# кварталы с broken/NULL geom дают quarter_bbox_3857() == None → ПЗЗ молча
|
||
# пропускаются, а на следующем harvest квартал отсекается skip_fresh_hours.
|
||
# Чиним geom здесь → Phase 2.5 ниже получит валидный bbox в этом же прогоне.
|
||
stats_features = [f for f in snapshot.features if f.category_id == CAT_QUARTER_STATS]
|
||
if stats_features:
|
||
upsert_quarter_stats(db, quarter, stats_features[0])
|
||
# NSPD изредка возвращает edge-case GeoJSON (self-intersecting, unclosed ring) —
|
||
# ST_GeomFromGeoJSON упадёт в PostgreSQL exception. Savepoint изолирует failure:
|
||
# rollback'нёт только geom UPDATE, успешный upsert_quarter_stats сохранится.
|
||
try:
|
||
with db.begin_nested():
|
||
if upsert_quarter_geom_from_feature(db, quarter, stats_features[0]):
|
||
logger.info("harvest_quarter: auto-healed cad_quarters_geom for %s", quarter)
|
||
except Exception as e:
|
||
logger.warning("harvest_quarter: geom auto-heal failed for %s: %s", quarter, e)
|
||
db.commit()
|
||
|
||
# ── Phase 2.5: grid-walk для territorial_zones (ПЗЗ, layer 875838) ────────
|
||
# Выполняем после основного grid-walk (Phase 2-3) И после Phase 4 geom
|
||
# auto-heal (см. Bug #1583) — так broken-geom кварталы, починенные выше,
|
||
# получают валидный bbox и ПЗЗ собираются в том же прогоне. Требует bbox квартала.
|
||
quarter_bbox = quarter_bbox_3857(db, quarter)
|
||
if quarter_bbox is not None:
|
||
update_progress({"phase": "territorial_zones_started", "quarter": quarter})
|
||
try:
|
||
tz_features = await client.get_territorial_zones_in_bbox(quarter_bbox)
|
||
tz_count = _save_territorial_zones(db, quarter, tz_features)
|
||
logger.info(
|
||
"harvest_quarter: territorial_zones quarter=%s upserted=%d", quarter, tz_count
|
||
)
|
||
except Exception as e:
|
||
logger.warning("harvest_quarter: territorial_zones failed quarter=%s: %s", quarter, e)
|
||
|
||
# Issue #252: финальный phase_state несёт АГРЕГИРОВАННЫЙ harvest_meta по всем
|
||
# сбойным слоям. progress_cb мержит phase_state через JSONB `||` (shallow) —
|
||
# per-layer done-апдейты перетёрли бы harvest_meta друг друга, поэтому в
|
||
# терминальном 'done' собираем полный список → итоговая строка корректна.
|
||
done_state: dict[str, Any] = {"phase": "done", "quarter": quarter}
|
||
if result.failed_layers:
|
||
done_state["harvest_meta"] = {f"layer_{lid}_failed": True for lid in result.failed_layers}
|
||
update_progress(done_state)
|
||
result.phase_state = done_state
|
||
|
||
logger.info(
|
||
"harvest_quarter done: quarter=%s parcels=%d buildings=%d "
|
||
"constructions=%d zouit=%d grid_requests=%d failed_layers=%s",
|
||
quarter,
|
||
result.parcels_upserted,
|
||
result.buildings_upserted,
|
||
result.constructions_upserted,
|
||
result.zouit_upserted,
|
||
result.grid_walk_requests,
|
||
result.failed_layers or None,
|
||
)
|
||
return result
|
||
|
||
|
||
# ── Grid-walk для одной категории ────────────────────────────────────────────
|
||
|
||
|
||
async def _grid_walk_category(
|
||
db: Session,
|
||
client: NSPDBulkClient,
|
||
quarter: str,
|
||
layer_id: int,
|
||
grid_size: int = 15,
|
||
tile_size: int = 512,
|
||
update_progress: Callable[[dict[str, Any]], None] | None = None,
|
||
heartbeat_every: int = 50,
|
||
) -> tuple[int, int, bool]:
|
||
"""Grid-walk layer_id в bbox квартала.
|
||
|
||
Генерирует grid_size×grid_size ячеек, для каждой делает wms_feature_info.
|
||
Дедуплицирует по cad_num, upsert'ит новые объекты.
|
||
|
||
Args:
|
||
update_progress: Fix I — callback для heartbeat внутри grid-walk loop.
|
||
225 cells × ~70ms = ~15-75s без heartbeat, zombie-cleanup ложно
|
||
помечал worker как мёртвого. Вызываем каждые heartbeat_every cells.
|
||
|
||
Returns:
|
||
(upserted_count, requests_count, layer_failed).
|
||
|
||
layer_failed=True — issue #252 — когда ВСЕ выполненные запросы слоя
|
||
упали на server-side 500/ServiceException и ни один не прошёл. Это
|
||
отличает «слой реально пуст в квартале» (0 discovered, 0 ошибок) от
|
||
«слой временно недоступен на стороне NSPD» (0 discovered потому что
|
||
каждый cell отдал 500). Вызывающий фиксирует флаг в harvest_meta —
|
||
квартал НЕ валится, один сбойный слой просто пропускается.
|
||
"""
|
||
bbox = quarter_bbox_3857(db, quarter)
|
||
if bbox is None:
|
||
logger.warning(
|
||
"_grid_walk_category: bbox для квартала %s не найден, пропускаем layer=%d",
|
||
quarter,
|
||
layer_id,
|
||
)
|
||
return 0, 0, False
|
||
|
||
grid_points = generate_grid_click_points(bbox, grid_size=grid_size, tile_size=tile_size)
|
||
|
||
# Bug #1584: считаем discovered ТОЛЬКО для таблицы запрошенного layer_id, а не
|
||
# sum(stats.values()). Иначе skipped/чужекатегорийные features завышают счётчик,
|
||
# ошибочно приписываясь к parcels/buildings вызывающим (harvest_quarter:230-233).
|
||
layer_count_key = "parcels" if layer_id == CAT_PARCEL else "buildings"
|
||
|
||
discovered_cads: set[str] = set()
|
||
upserted = 0
|
||
requests = 0
|
||
server_errors = 0 # cells, упавшие на 5xx/ServiceException (issue #252)
|
||
ok_cells = 0 # cells, вернувшие ответ (пусть и пустой) — слой жив
|
||
|
||
for idx, (cell_bbox, click_xy) in enumerate(grid_points):
|
||
# Fix I: heartbeat каждые N cells — обновляет cadastre_jobs.heartbeat_at
|
||
# чтобы cleanup_zombies (5min stale check) не cancel'нул живой worker.
|
||
if update_progress is not None and idx > 0 and idx % heartbeat_every == 0:
|
||
update_progress(
|
||
{
|
||
"phase": f"grid_walk_{layer_id}_cell_{idx}",
|
||
"quarter": quarter,
|
||
"grid_cells_done": idx,
|
||
"grid_cells_total": len(grid_points),
|
||
}
|
||
)
|
||
|
||
try:
|
||
features = await client.wms_feature_info(
|
||
layer_id=layer_id,
|
||
bbox=cell_bbox,
|
||
click_xy=click_xy,
|
||
width=tile_size,
|
||
height=tile_size,
|
||
)
|
||
requests += 1
|
||
ok_cells += 1
|
||
except NspdBulkServerError as e:
|
||
# Issue #252: 5xx / WMS ServiceException — transient server-side noise
|
||
# NSPD (десятки cells per quarter). logger.debug чтобы не засорять prod
|
||
# логи. Считаем отдельно: если ВСЕ cells слоя упали так и ни один не
|
||
# прошёл — поднимем layer_failed (см. return) → harvest_meta-флаг.
|
||
logger.debug(
|
||
"_grid_walk_category: server error layer=%d quarter=%s cell=%d: %s",
|
||
layer_id,
|
||
quarter,
|
||
idx,
|
||
e,
|
||
)
|
||
requests += 1
|
||
server_errors += 1
|
||
continue
|
||
except Exception as e:
|
||
# Прочие (сетевые / parse) ошибки одного cell — тоже не валим квартал,
|
||
# но это НЕ server-side 500 → не учитываем в server_errors (иначе сеть
|
||
# ложно triggers layer_failed). Слой жив, просто этот cell не дошёл.
|
||
logger.debug(
|
||
"_grid_walk_category: wms_feature_info error layer=%d quarter=%s cell=%d: %s",
|
||
layer_id,
|
||
quarter,
|
||
idx,
|
||
e,
|
||
)
|
||
requests += 1
|
||
continue
|
||
|
||
for feature in features:
|
||
cad = feature.cad_num
|
||
if not cad or cad in discovered_cads:
|
||
continue
|
||
discovered_cads.add(cad)
|
||
# Upsert в соответствующую таблицу
|
||
try:
|
||
with db.begin_nested():
|
||
stats = upsert_features(db, [feature], source="wms_grid_walk")
|
||
upserted += stats[layer_count_key]
|
||
except Exception as e:
|
||
logger.warning(
|
||
"_grid_walk_category: upsert failed cad=%s layer=%d: %s",
|
||
cad,
|
||
layer_id,
|
||
e,
|
||
)
|
||
|
||
# Issue #252: слой считаем «сбойным» только если БЫЛИ server-side ошибки И
|
||
# ни один cell не прошёл успешно. ok_cells>0 → слой жив (0 discovered значит
|
||
# реально пусто). server_errors==0 → штатный пустой/полный обход.
|
||
layer_failed = server_errors > 0 and ok_cells == 0
|
||
if layer_failed:
|
||
logger.warning(
|
||
"_grid_walk_category: layer=%d quarter=%s ПОЛНОСТЬЮ сбойный — "
|
||
"%d/%d cells вернули 5xx/ServiceException, 0 успешных. Слой пропущен.",
|
||
layer_id,
|
||
quarter,
|
||
server_errors,
|
||
len(grid_points),
|
||
)
|
||
|
||
return upserted, requests, layer_failed
|
||
|
||
|
||
# ── Geom backfill для участков с geom IS NULL (issue #200) ────────────────────
|
||
|
||
|
||
@dataclass
|
||
class GeomBackfillResult:
|
||
"""Итог докачки geom для участков с geom IS NULL (issue #200)."""
|
||
|
||
quarters_scanned: int = 0
|
||
parcels_targeted: int = 0 # участки без geom на входе
|
||
parcels_healed: int = 0 # получили полигон после grid-walk
|
||
parcels_marked_unavailable: int = 0 # geom так и не нашёлся → флаг
|
||
grid_walk_requests: int = 0
|
||
|
||
|
||
async def backfill_parcel_geom(
|
||
db: Session,
|
||
client: NSPDBulkClient,
|
||
*,
|
||
limit: int = 500,
|
||
update_heartbeat: Callable[[dict[str, Any]], None] | None = None,
|
||
) -> GeomBackfillResult:
|
||
"""Докачать geom для участков cad_parcels с geom IS NULL (issue #200).
|
||
|
||
Алгоритм:
|
||
1. Выбрать до `limit` участков `geom IS NULL AND NOT geom_unavailable`.
|
||
2. Сгруппировать по кварталу (3-сегментный cad). NSPD search-эндпоинт
|
||
(Phase 1) отдаёт эти участки без полигона; реальную геометрию
|
||
возвращает grid-walk слоя 36368 (WMS GetFeatureInfo по bbox квартала).
|
||
3. Для каждого квартала вызвать _grid_walk_category(36368) — он upsert'ит
|
||
полигоны через ON CONFLICT ... geom = COALESCE(EXCLUDED.geom, ...).
|
||
4. Участки, которые ПОСЛЕ grid-walk всё ещё geom IS NULL, пометить
|
||
geom_unavailable=TRUE — geom систематически недоступен (кросс-региональные
|
||
артефакты поиска без полигона в слое), чтобы не ретраить вечно.
|
||
|
||
Идемпотентен: повторный вызов берёт следующую порцию (healed уходят из
|
||
выборки — у них geom уже не NULL; unavailable исключены флагом).
|
||
|
||
Args:
|
||
limit: максимум участков за один прогон (батч против WAF-burst).
|
||
update_heartbeat: callback для heartbeat (как в harvest_quarter) — задача
|
||
может идти минуты при многих кварталах.
|
||
|
||
Returns:
|
||
GeomBackfillResult со счётчиками.
|
||
"""
|
||
result = GeomBackfillResult()
|
||
|
||
rows = (
|
||
db.execute(
|
||
text(
|
||
"SELECT cad_num FROM cad_parcels "
|
||
"WHERE geom IS NULL AND geom_unavailable = FALSE "
|
||
"ORDER BY cad_num LIMIT :lim"
|
||
),
|
||
{"lim": limit},
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
if not rows:
|
||
logger.info("backfill_parcel_geom: нет участков с geom IS NULL — нечего докачивать")
|
||
return result
|
||
|
||
result.parcels_targeted = len(rows)
|
||
|
||
# Группируем по кварталу (первые 3 colon-сегмента). Участки без валидного
|
||
# 3-сегментного префикса (мусорные cad) — сразу в unavailable, grid-walk
|
||
# для них невозможен (нет квартала → нет bbox).
|
||
by_quarter: dict[str, list[str]] = {}
|
||
orphan_cads: list[str] = []
|
||
for cad in rows:
|
||
parts = cad.split(":")
|
||
if len(parts) >= 3 and all(parts[:3]):
|
||
by_quarter.setdefault(":".join(parts[:3]), []).append(cad)
|
||
else:
|
||
orphan_cads.append(cad)
|
||
|
||
if orphan_cads:
|
||
result.parcels_marked_unavailable += _mark_geom_unavailable(db, orphan_cads)
|
||
db.commit()
|
||
|
||
for quarter, cads in by_quarter.items():
|
||
result.quarters_scanned += 1
|
||
if update_heartbeat is not None:
|
||
update_heartbeat(
|
||
{
|
||
"phase": "geom_backfill",
|
||
"quarter": quarter,
|
||
"quarters_scanned": result.quarters_scanned,
|
||
"quarters_total": len(by_quarter),
|
||
}
|
||
)
|
||
|
||
try:
|
||
_discovered, n_requests, _failed = await _grid_walk_category(
|
||
db=db,
|
||
client=client,
|
||
quarter=quarter,
|
||
layer_id=CAT_PARCEL,
|
||
update_progress=update_heartbeat,
|
||
)
|
||
result.grid_walk_requests += n_requests
|
||
db.commit()
|
||
except Exception as e:
|
||
# Один сбойный квартал не валит весь backfill — лог + продолжаем.
|
||
# (WAF 403 пробросится из client и прервёт прогон — это ожидаемо,
|
||
# caller-task ловит и не ретраит, как в bulk_harvest.)
|
||
logger.warning(
|
||
"backfill_parcel_geom: grid-walk failed quarter=%s: %s", quarter, e
|
||
)
|
||
db.rollback()
|
||
continue
|
||
|
||
# Какие из целевых участков квартала всё ещё без geom?
|
||
still_null = (
|
||
db.execute(
|
||
text(
|
||
"SELECT cad_num FROM cad_parcels "
|
||
"WHERE cad_num = ANY(CAST(:cads AS text[])) AND geom IS NULL"
|
||
),
|
||
{"cads": cads},
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
healed_here = len(cads) - len(still_null)
|
||
result.parcels_healed += healed_here
|
||
|
||
if still_null:
|
||
result.parcels_marked_unavailable += _mark_geom_unavailable(db, list(still_null))
|
||
db.commit()
|
||
|
||
logger.info(
|
||
"backfill_parcel_geom: quarter=%s targeted=%d healed=%d unavailable=%d",
|
||
quarter,
|
||
len(cads),
|
||
healed_here,
|
||
len(still_null),
|
||
)
|
||
|
||
logger.info(
|
||
"backfill_parcel_geom done: quarters=%d targeted=%d healed=%d unavailable=%d requests=%d",
|
||
result.quarters_scanned,
|
||
result.parcels_targeted,
|
||
result.parcels_healed,
|
||
result.parcels_marked_unavailable,
|
||
result.grid_walk_requests,
|
||
)
|
||
return result
|
||
|
||
|
||
def _mark_geom_unavailable(db: Session, cad_nums: list[str]) -> int:
|
||
"""Пометить участки geom_unavailable=TRUE (issue #200). Возвращает rowcount.
|
||
|
||
Параметризованный UPDATE по списку cad_num (ANY(text[])) — не f-string SQL.
|
||
"""
|
||
if not cad_nums:
|
||
return 0
|
||
res = db.execute(
|
||
text(
|
||
"UPDATE cad_parcels SET geom_unavailable = TRUE, updated_at = NOW() "
|
||
"WHERE cad_num = ANY(CAST(:cads AS text[])) "
|
||
"AND geom IS NULL AND geom_unavailable = FALSE"
|
||
),
|
||
{"cads": cad_nums},
|
||
)
|
||
return res.rowcount or 0
|
||
|
||
|
||
# ── Routing dispatcher ───────────────────────────────────────────────────────
|
||
|
||
|
||
def upsert_features(
|
||
db: Session,
|
||
features: list[NSPDBulkFeature],
|
||
source: str = "search",
|
||
) -> dict[str, int]:
|
||
"""Маршрутизировать features по category_id в специфичные upsert-функции.
|
||
|
||
Returns:
|
||
Словарь {table: count} с количеством upserted строк.
|
||
"""
|
||
counts: dict[str, int] = {
|
||
"parcels": 0,
|
||
"buildings": 0,
|
||
"constructions": 0,
|
||
"oncs": 0,
|
||
"enks": 0,
|
||
"zouit": 0,
|
||
"skipped": 0,
|
||
}
|
||
|
||
for feature in features:
|
||
cat = feature.category_id
|
||
# Per-feature SAVEPOINT (backend.md SAVEPOINT rule): один битый feature (edge-case
|
||
# GeoJSON / NOT NULL violation / type mismatch) НЕ должен ронять весь snapshot-tx
|
||
# квартала. Зеркалит _grid_walk_category (~368) и _save_territorial_zones (~1234):
|
||
# begin_nested() rollback'ит только этот feature, остальные сохраняются. Счётчик
|
||
# инкрементим ТОЛЬКО при успехе (внутри блока) → counts остаются точными.
|
||
try:
|
||
with db.begin_nested():
|
||
if cat == CAT_PARCEL:
|
||
upsert_parcel(db, feature, source=source)
|
||
counts["parcels"] += 1
|
||
elif cat == CAT_BUILDING:
|
||
upsert_building(db, feature, source=source)
|
||
counts["buildings"] += 1
|
||
elif cat == CAT_CONSTRUCTION:
|
||
upsert_construction(db, feature, source=source)
|
||
counts["constructions"] += 1
|
||
elif cat == CAT_ONC:
|
||
upsert_onc(db, feature, source=source)
|
||
counts["oncs"] += 1
|
||
elif cat == CAT_ENK:
|
||
upsert_enk(db, feature, source=source)
|
||
counts["enks"] += 1
|
||
elif cat in CAT_ZOUIT:
|
||
upsert_zouit(db, feature, source=source)
|
||
counts["zouit"] += 1
|
||
elif cat == CAT_QUARTER_STATS:
|
||
pass # обрабатывается отдельно в upsert_quarter_stats
|
||
else:
|
||
if cat is not None:
|
||
logger.debug("upsert_features: неизвестный category_id=%s, skip", cat)
|
||
counts["skipped"] += 1
|
||
except Exception as e:
|
||
# Логируем + skip (не re-raise) — один битый feature не отравляет квартал.
|
||
logger.warning(
|
||
"upsert_features: feature failed (cat=%s, id=%s): %s", cat, feature.id, e
|
||
)
|
||
|
||
return counts
|
||
|
||
|
||
# ── cad_parcels upsert ───────────────────────────────────────────────────────
|
||
|
||
|
||
def upsert_parcel(db: Session, feature: NSPDBulkFeature, source: str = "search") -> None:
|
||
"""Upsert ЗУ ЕГРН в cad_parcels. ON CONFLICT (cad_num) DO UPDATE."""
|
||
opts = feature.options
|
||
cad_num = opts.effective_cad_num
|
||
if not cad_num:
|
||
logger.debug("upsert_parcel: пропуск — нет cad_num в feature id=%s", feature.id)
|
||
return
|
||
|
||
raw_opts = feature.properties.get("options") or {}
|
||
# cad_parcels.geom — geometry(Polygon, 4326). NSPD изредка возвращает Point
|
||
# для участков без detailed boundary — INSERT fail'нит на schema check.
|
||
# Сохраняем raw_props (есть координаты), geom=NULL. Не блокирует Phase 1.
|
||
geom = feature.geometry or {}
|
||
geom_type = geom.get("type") if isinstance(geom, dict) else None
|
||
if geom_type in ("Polygon", "MultiPolygon"):
|
||
geom_json: str | None = json.dumps(feature.geometry)
|
||
else:
|
||
if geom_type:
|
||
logger.info(
|
||
"upsert_parcel: cad_num=%s имеет geometry type=%s (не Polygon) — geom=NULL",
|
||
cad_num,
|
||
geom_type,
|
||
)
|
||
geom_json = None
|
||
quarter_cad = opts.quarter_cad_number or _derive_quarter_cad(cad_num)
|
||
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cad_parcels (
|
||
cad_num, quarter_cad_number, category_id,
|
||
land_record_area, specified_area, declared_area,
|
||
land_record_category_type, land_record_subtype, land_record_type,
|
||
permitted_use_established_by_document,
|
||
cost_value, cost_index,
|
||
ownership_type, right_type,
|
||
readable_address, status, previously_posted,
|
||
registration_date, subcategory, cadastral_districts_code,
|
||
geom, raw_props, source, fetched_at, updated_at
|
||
) VALUES (
|
||
:cad_num, :quarter_cad_number, 36368,
|
||
:land_record_area, :specified_area, :declared_area,
|
||
:land_record_category_type, :land_record_subtype, :land_record_type,
|
||
:permitted_use,
|
||
:cost_value, :cost_index,
|
||
:ownership_type, :right_type,
|
||
:readable_address, :status, :previously_posted,
|
||
CAST(:registration_date AS date),
|
||
:subcategory, :cadastral_districts_code,
|
||
-- ST_Multi: cad_parcels.geom = MULTIPOLYGON (migration 93).
|
||
-- NSPD возвращает Polygon ИЛИ MultiPolygon (Многоконтурные участки) —
|
||
-- оба нужно сохранить, поэтому ST_Multi() coerce'ит Polygon → MultiPolygon.
|
||
CASE WHEN CAST(:geom AS text) IS NOT NULL THEN
|
||
ST_Multi(
|
||
ST_Transform(
|
||
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857),
|
||
4326
|
||
)
|
||
)
|
||
ELSE NULL END,
|
||
CAST(:raw_props AS jsonb), :source, NOW(), NOW()
|
||
)
|
||
ON CONFLICT (cad_num) DO UPDATE SET
|
||
quarter_cad_number = EXCLUDED.quarter_cad_number,
|
||
land_record_area = EXCLUDED.land_record_area,
|
||
specified_area = EXCLUDED.specified_area,
|
||
declared_area = EXCLUDED.declared_area,
|
||
land_record_category_type = EXCLUDED.land_record_category_type,
|
||
land_record_subtype = EXCLUDED.land_record_subtype,
|
||
land_record_type = EXCLUDED.land_record_type,
|
||
permitted_use_established_by_document =
|
||
EXCLUDED.permitted_use_established_by_document,
|
||
cost_value = EXCLUDED.cost_value,
|
||
cost_index = EXCLUDED.cost_index,
|
||
ownership_type = EXCLUDED.ownership_type,
|
||
right_type = EXCLUDED.right_type,
|
||
readable_address = EXCLUDED.readable_address,
|
||
status = EXCLUDED.status,
|
||
previously_posted = EXCLUDED.previously_posted,
|
||
registration_date = EXCLUDED.registration_date,
|
||
subcategory = EXCLUDED.subcategory,
|
||
cadastral_districts_code = EXCLUDED.cadastral_districts_code,
|
||
geom = COALESCE(EXCLUDED.geom, cad_parcels.geom),
|
||
raw_props = EXCLUDED.raw_props,
|
||
source = EXCLUDED.source,
|
||
updated_at = NOW()
|
||
"""),
|
||
{
|
||
"cad_num": cad_num,
|
||
"quarter_cad_number": quarter_cad,
|
||
"land_record_area": _safe_numeric(raw_opts.get("land_record_area")),
|
||
"specified_area": _safe_numeric(raw_opts.get("specified_area")),
|
||
"declared_area": _safe_numeric(raw_opts.get("declared_area")),
|
||
"land_record_category_type": raw_opts.get("land_record_category_type"),
|
||
"land_record_subtype": raw_opts.get("land_record_subtype"),
|
||
"land_record_type": raw_opts.get("land_record_type"),
|
||
"permitted_use": raw_opts.get("permitted_use_established_by_document"),
|
||
"cost_value": _safe_numeric(raw_opts.get("cost_value")),
|
||
"cost_index": _safe_numeric(raw_opts.get("cost_index")),
|
||
"ownership_type": raw_opts.get("ownership_type"),
|
||
"right_type": raw_opts.get("right_type"),
|
||
"readable_address": raw_opts.get("readable_address"),
|
||
"status": raw_opts.get("status"),
|
||
"previously_posted": raw_opts.get("previously_posted"),
|
||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||
"subcategory": _safe_int(raw_opts.get("subcategory")),
|
||
"cadastral_districts_code": _safe_int(raw_opts.get("cadastral_districts_code")),
|
||
"geom": geom_json,
|
||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||
"source": source,
|
||
},
|
||
)
|
||
|
||
|
||
# ── cad_buildings upsert ─────────────────────────────────────────────────────
|
||
|
||
|
||
def upsert_building(db: Session, feature: NSPDBulkFeature, source: str = "search") -> None:
|
||
"""Upsert здание ЕГРН в cad_buildings. ON CONFLICT (cad_num) DO UPDATE."""
|
||
opts = feature.options
|
||
cad_num = opts.effective_cad_num
|
||
if not cad_num:
|
||
logger.debug("upsert_building: пропуск — нет cad_num в feature id=%s", feature.id)
|
||
return
|
||
|
||
raw_opts = feature.properties.get("options") or {}
|
||
geom_json = json.dumps(feature.geometry) if feature.geometry else None
|
||
quarter_cad = opts.quarter_cad_number or _derive_quarter_cad(cad_num)
|
||
|
||
# floors из NSPD может быть str "5" или int 5
|
||
floors_val = _safe_int(raw_opts.get("floors"))
|
||
ugnd_floors_val = _safe_int(raw_opts.get("underground_floors"))
|
||
|
||
# cultural_heritage — строка → bool
|
||
ch_str = raw_opts.get("cultural_heritage")
|
||
cultural_heritage_bool = bool(ch_str) if ch_str is not None else None
|
||
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cad_buildings (
|
||
cad_num, quarter_cad_number,
|
||
objdoc_id, registers_id,
|
||
purpose, building_name,
|
||
floors, underground_floors,
|
||
area, build_record_area,
|
||
cost_value, cost_index,
|
||
year_built, year_commisioning,
|
||
build_record_registration_date, registration_date,
|
||
ownership_type,
|
||
cultural_heritage_object, cultural_heritage_val,
|
||
readable_address, status, common_data_status,
|
||
type, build_record_type_value,
|
||
geom, raw_props, source, fetched_at, updated_at
|
||
) VALUES (
|
||
:cad_num, :quarter_cad_number,
|
||
:objdoc_id, :registers_id,
|
||
:purpose, :building_name,
|
||
:floors, :underground_floors,
|
||
:area, :build_record_area,
|
||
:cost_value, :cost_index,
|
||
:year_built, :year_commisioning,
|
||
CAST(:build_record_registration_date AS date),
|
||
CAST(:registration_date AS date),
|
||
:ownership_type,
|
||
:cultural_heritage_object, :cultural_heritage_val,
|
||
:readable_address, :status, :common_data_status,
|
||
:type, :build_record_type_value,
|
||
CASE WHEN CAST(:geom AS text) IS NOT NULL THEN
|
||
ST_Transform(
|
||
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857),
|
||
4326
|
||
)
|
||
ELSE NULL END,
|
||
CAST(:raw_props AS jsonb), :source, NOW(), NOW()
|
||
)
|
||
ON CONFLICT (cad_num) DO UPDATE SET
|
||
quarter_cad_number = EXCLUDED.quarter_cad_number,
|
||
objdoc_id = COALESCE(EXCLUDED.objdoc_id, cad_buildings.objdoc_id),
|
||
registers_id = COALESCE(EXCLUDED.registers_id, cad_buildings.registers_id),
|
||
purpose = EXCLUDED.purpose,
|
||
building_name = EXCLUDED.building_name,
|
||
floors = EXCLUDED.floors,
|
||
underground_floors = EXCLUDED.underground_floors,
|
||
area = EXCLUDED.area,
|
||
build_record_area = EXCLUDED.build_record_area,
|
||
cost_value = EXCLUDED.cost_value,
|
||
cost_index = EXCLUDED.cost_index,
|
||
year_built = EXCLUDED.year_built,
|
||
year_commisioning = EXCLUDED.year_commisioning,
|
||
build_record_registration_date = EXCLUDED.build_record_registration_date,
|
||
registration_date = EXCLUDED.registration_date,
|
||
ownership_type = EXCLUDED.ownership_type,
|
||
cultural_heritage_object = EXCLUDED.cultural_heritage_object,
|
||
cultural_heritage_val = EXCLUDED.cultural_heritage_val,
|
||
readable_address = EXCLUDED.readable_address,
|
||
status = EXCLUDED.status,
|
||
common_data_status = EXCLUDED.common_data_status,
|
||
type = EXCLUDED.type,
|
||
build_record_type_value = EXCLUDED.build_record_type_value,
|
||
geom = COALESCE(EXCLUDED.geom, cad_buildings.geom),
|
||
raw_props = EXCLUDED.raw_props,
|
||
source = EXCLUDED.source,
|
||
updated_at = NOW()
|
||
"""),
|
||
{
|
||
"cad_num": cad_num,
|
||
"quarter_cad_number": quarter_cad,
|
||
"objdoc_id": _safe_int(raw_opts.get("objdoc_id")),
|
||
"registers_id": _safe_int(raw_opts.get("registers_id")),
|
||
"purpose": raw_opts.get("purpose"),
|
||
"building_name": raw_opts.get("building_name") or raw_opts.get("name"),
|
||
"floors": floors_val,
|
||
"underground_floors": ugnd_floors_val,
|
||
"area": _safe_numeric(raw_opts.get("area")),
|
||
"build_record_area": _safe_numeric(raw_opts.get("build_record_area")),
|
||
"cost_value": _safe_numeric(raw_opts.get("cost_value")),
|
||
"cost_index": _safe_numeric(raw_opts.get("cost_index")),
|
||
"year_built": _safe_int(raw_opts.get("year_built")),
|
||
"year_commisioning": _safe_int(raw_opts.get("year_commisioning")),
|
||
"build_record_registration_date": _parse_nspd_date(
|
||
raw_opts.get("build_record_registration_date")
|
||
),
|
||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||
"ownership_type": raw_opts.get("ownership_type"),
|
||
"cultural_heritage_object": cultural_heritage_bool,
|
||
"cultural_heritage_val": ch_str,
|
||
"readable_address": raw_opts.get("readable_address"),
|
||
"status": raw_opts.get("status"),
|
||
"common_data_status": raw_opts.get("common_data_status"),
|
||
"type": raw_opts.get("obj_type") or raw_opts.get("type"),
|
||
"build_record_type_value": (
|
||
raw_opts.get("build_record_type_value") or raw_opts.get("build_record_type")
|
||
),
|
||
"geom": geom_json,
|
||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||
"source": source,
|
||
},
|
||
)
|
||
|
||
|
||
# ── cad_constructions upsert ─────────────────────────────────────────────────
|
||
|
||
|
||
def upsert_construction(db: Session, feature: NSPDBulkFeature, source: str = "search") -> None:
|
||
"""Upsert сооружение в cad_constructions. ON CONFLICT (cad_num) DO UPDATE."""
|
||
raw_props = feature.properties
|
||
raw_opts = raw_props.get("options") or {}
|
||
cad_num = raw_opts.get("cad_num") or raw_opts.get("cad_number")
|
||
if not cad_num:
|
||
logger.debug("upsert_construction: пропуск — нет cad_num, feature id=%s", feature.id)
|
||
return
|
||
|
||
cat_id = feature.category_id
|
||
# Определить тип: 36383 = сооружение, у NSPD есть 2 варианта
|
||
is_egrn = bool(raw_opts.get("is_egrn") or raw_opts.get("registers_id"))
|
||
category_name = raw_props.get("categoryName") or str(cat_id or "")
|
||
quarter_cad = raw_opts.get("quarter_cad_number") or _derive_quarter_cad(cad_num)
|
||
geom_json = json.dumps(feature.geometry) if feature.geometry else None
|
||
|
||
# params_* поля из NSPD characteristics
|
||
params = raw_opts.get("params") or {}
|
||
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cad_constructions (
|
||
cad_num, quarter_cad_number,
|
||
category_name, is_egrn,
|
||
params_purpose, params_name,
|
||
params_area, params_built_up_area,
|
||
params_year_built, params_year_commisioning,
|
||
params_height, params_volume, params_floors, params_underground_floors,
|
||
object_type_value, facility_cad_number,
|
||
cost_value, cost_index,
|
||
ownership_type, right_type,
|
||
readable_address, address_readable_address,
|
||
status, common_data_status,
|
||
registration_date, registers_id, united_cad_number,
|
||
geom, raw_props, source, fetched_at, updated_at
|
||
) VALUES (
|
||
:cad_num, :quarter_cad_number,
|
||
:category_name, :is_egrn,
|
||
:params_purpose, :params_name,
|
||
:params_area, :params_built_up_area,
|
||
:params_year_built, :params_year_commisioning,
|
||
:params_height, :params_volume, :params_floors, :params_underground_floors,
|
||
:object_type_value, :facility_cad_number,
|
||
:cost_value, :cost_index,
|
||
:ownership_type, :right_type,
|
||
:readable_address, :address_readable_address,
|
||
:status, :common_data_status,
|
||
CAST(:registration_date AS date), :registers_id, :united_cad_number,
|
||
CASE WHEN CAST(:geom AS text) IS NOT NULL THEN
|
||
ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857), 4326)
|
||
ELSE NULL END,
|
||
CAST(:raw_props AS jsonb), :source, NOW(), NOW()
|
||
)
|
||
ON CONFLICT (cad_num) DO UPDATE SET
|
||
category_name = EXCLUDED.category_name,
|
||
is_egrn = EXCLUDED.is_egrn,
|
||
params_purpose = EXCLUDED.params_purpose,
|
||
params_name = EXCLUDED.params_name,
|
||
params_area = EXCLUDED.params_area,
|
||
params_built_up_area = EXCLUDED.params_built_up_area,
|
||
params_year_built = EXCLUDED.params_year_built,
|
||
params_year_commisioning = EXCLUDED.params_year_commisioning,
|
||
params_height = EXCLUDED.params_height,
|
||
params_volume = EXCLUDED.params_volume,
|
||
params_floors = EXCLUDED.params_floors,
|
||
params_underground_floors = EXCLUDED.params_underground_floors,
|
||
object_type_value = EXCLUDED.object_type_value,
|
||
facility_cad_number = EXCLUDED.facility_cad_number,
|
||
cost_value = EXCLUDED.cost_value,
|
||
cost_index = EXCLUDED.cost_index,
|
||
ownership_type = EXCLUDED.ownership_type,
|
||
right_type = EXCLUDED.right_type,
|
||
readable_address = EXCLUDED.readable_address,
|
||
address_readable_address = EXCLUDED.address_readable_address,
|
||
status = EXCLUDED.status,
|
||
common_data_status = EXCLUDED.common_data_status,
|
||
registration_date = EXCLUDED.registration_date,
|
||
registers_id = EXCLUDED.registers_id,
|
||
united_cad_number = EXCLUDED.united_cad_number,
|
||
geom = COALESCE(EXCLUDED.geom, cad_constructions.geom),
|
||
raw_props = EXCLUDED.raw_props,
|
||
source = EXCLUDED.source,
|
||
updated_at = NOW()
|
||
"""),
|
||
{
|
||
"cad_num": cad_num,
|
||
"quarter_cad_number": quarter_cad,
|
||
"category_name": category_name,
|
||
"is_egrn": is_egrn,
|
||
"params_purpose": params.get("purpose") or raw_opts.get("purpose"),
|
||
"params_name": params.get("name") or raw_opts.get("building_name"),
|
||
"params_area": _safe_numeric(params.get("area") or raw_opts.get("area")),
|
||
"params_built_up_area": _safe_numeric(raw_opts.get("built_up_area")),
|
||
"params_year_built": _safe_int(params.get("year_built") or raw_opts.get("year_built")),
|
||
"params_year_commisioning": _safe_int(raw_opts.get("year_commisioning")),
|
||
"params_height": _safe_numeric(raw_opts.get("height")),
|
||
"params_volume": _safe_numeric(raw_opts.get("volume")),
|
||
"params_floors": _safe_int(raw_opts.get("floors")),
|
||
"params_underground_floors": _safe_int(raw_opts.get("underground_floors")),
|
||
"object_type_value": raw_opts.get("object_type_value"),
|
||
"facility_cad_number": raw_opts.get("facility_cad_number"),
|
||
"cost_value": _safe_numeric(raw_opts.get("cost_value")),
|
||
"cost_index": _safe_numeric(raw_opts.get("cost_index")),
|
||
"ownership_type": raw_opts.get("ownership_type"),
|
||
"right_type": raw_opts.get("right_type"),
|
||
"readable_address": raw_opts.get("readable_address"),
|
||
"address_readable_address": raw_opts.get("address_readable_address"),
|
||
"status": raw_opts.get("status"),
|
||
"common_data_status": raw_opts.get("common_data_status"),
|
||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||
"registers_id": _safe_int(raw_opts.get("registers_id")),
|
||
"united_cad_number": raw_opts.get("united_cad_number"),
|
||
"geom": geom_json,
|
||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||
"source": source,
|
||
},
|
||
)
|
||
|
||
|
||
# ── cad_oncs upsert ──────────────────────────────────────────────────────────
|
||
|
||
|
||
def upsert_onc(db: Session, feature: NSPDBulkFeature, source: str = "search") -> None:
|
||
"""Upsert ОНС в cad_oncs. ON CONFLICT (cad_num) DO UPDATE."""
|
||
raw_opts = feature.properties.get("options") or {}
|
||
cad_num = raw_opts.get("cad_num") or raw_opts.get("cad_number")
|
||
if not cad_num:
|
||
logger.debug("upsert_onc: пропуск — нет cad_num, feature id=%s", feature.id)
|
||
return
|
||
|
||
quarter_cad = raw_opts.get("quarter_cad_number") or _derive_quarter_cad(cad_num)
|
||
geom_json = json.dumps(feature.geometry) if feature.geometry else None
|
||
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cad_oncs (
|
||
cad_num, quarter_cad_number,
|
||
purpose, area, build_record_area,
|
||
cost_value, status, readable_address,
|
||
geom, raw_props, source, fetched_at, updated_at
|
||
) VALUES (
|
||
:cad_num, :quarter_cad_number,
|
||
:purpose, :area, :build_record_area,
|
||
:cost_value, :status, :readable_address,
|
||
CASE WHEN CAST(:geom AS text) IS NOT NULL THEN
|
||
ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857), 4326)
|
||
ELSE NULL END,
|
||
CAST(:raw_props AS jsonb), :source, NOW(), NOW()
|
||
)
|
||
ON CONFLICT (cad_num) DO UPDATE SET
|
||
quarter_cad_number = EXCLUDED.quarter_cad_number,
|
||
purpose = EXCLUDED.purpose,
|
||
area = EXCLUDED.area,
|
||
build_record_area = EXCLUDED.build_record_area,
|
||
cost_value = EXCLUDED.cost_value,
|
||
status = EXCLUDED.status,
|
||
readable_address = EXCLUDED.readable_address,
|
||
geom = COALESCE(EXCLUDED.geom, cad_oncs.geom),
|
||
raw_props = EXCLUDED.raw_props,
|
||
source = EXCLUDED.source,
|
||
updated_at = NOW()
|
||
"""),
|
||
{
|
||
"cad_num": cad_num,
|
||
"quarter_cad_number": quarter_cad,
|
||
"purpose": raw_opts.get("purpose"),
|
||
"area": _safe_numeric(raw_opts.get("area")),
|
||
"build_record_area": _safe_numeric(raw_opts.get("build_record_area")),
|
||
"cost_value": _safe_numeric(raw_opts.get("cost_value")),
|
||
"status": raw_opts.get("status"),
|
||
"readable_address": raw_opts.get("readable_address"),
|
||
"geom": geom_json,
|
||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||
"source": source,
|
||
},
|
||
)
|
||
|
||
|
||
# ── cad_enk upsert ───────────────────────────────────────────────────────────
|
||
|
||
|
||
def upsert_enk(db: Session, feature: NSPDBulkFeature, source: str = "search") -> None:
|
||
"""Upsert ЕНК в cad_enk. ON CONFLICT (cad_num) DO UPDATE."""
|
||
raw_opts = feature.properties.get("options") or {}
|
||
cad_num = raw_opts.get("cad_num") or raw_opts.get("cad_number")
|
||
if not cad_num:
|
||
logger.debug("upsert_enk: пропуск — нет cad_num, feature id=%s", feature.id)
|
||
return
|
||
|
||
quarter_cad = raw_opts.get("quarter_cad_number") or _derive_quarter_cad(cad_num)
|
||
geom_json = json.dumps(feature.geometry) if feature.geometry else None
|
||
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cad_enk (
|
||
cad_num, quarter_cad_number,
|
||
name, purpose, type,
|
||
cost_value, facility_cad_number,
|
||
readable_address, registration_date, registers_id,
|
||
geom, raw_props, fetched_at, updated_at
|
||
) VALUES (
|
||
:cad_num, :quarter_cad_number,
|
||
:name, :purpose, :type,
|
||
:cost_value, :facility_cad_number,
|
||
:readable_address, CAST(:registration_date AS date), :registers_id,
|
||
CASE WHEN CAST(:geom AS text) IS NOT NULL THEN
|
||
ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857), 4326)
|
||
ELSE NULL END,
|
||
CAST(:raw_props AS jsonb), NOW(), NOW()
|
||
)
|
||
ON CONFLICT (cad_num) DO UPDATE SET
|
||
quarter_cad_number = EXCLUDED.quarter_cad_number,
|
||
name = EXCLUDED.name,
|
||
purpose = EXCLUDED.purpose,
|
||
type = EXCLUDED.type,
|
||
cost_value = EXCLUDED.cost_value,
|
||
facility_cad_number = EXCLUDED.facility_cad_number,
|
||
readable_address = EXCLUDED.readable_address,
|
||
registration_date = EXCLUDED.registration_date,
|
||
registers_id = EXCLUDED.registers_id,
|
||
geom = COALESCE(EXCLUDED.geom, cad_enk.geom),
|
||
raw_props = EXCLUDED.raw_props,
|
||
updated_at = NOW()
|
||
"""),
|
||
{
|
||
"cad_num": cad_num,
|
||
"quarter_cad_number": quarter_cad,
|
||
"name": raw_opts.get("name"),
|
||
"purpose": raw_opts.get("purpose"),
|
||
"type": raw_opts.get("type") or raw_opts.get("object_type"),
|
||
"cost_value": _safe_numeric(raw_opts.get("cost_value")),
|
||
"facility_cad_number": raw_opts.get("facility_cad_number"),
|
||
"readable_address": raw_opts.get("readable_address"),
|
||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||
"registers_id": _safe_int(raw_opts.get("registers_id")),
|
||
"geom": geom_json,
|
||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||
},
|
||
)
|
||
|
||
|
||
# ── cad_zouit upsert ─────────────────────────────────────────────────────────
|
||
|
||
|
||
def upsert_zouit(db: Session, feature: NSPDBulkFeature, source: str = "search") -> None:
|
||
"""Upsert ЗОУИТ в cad_zouit.
|
||
|
||
Deduplicate по (reg_numb_border, category_id).
|
||
Geom — MultiPolygon; если пришёл Polygon — NSPD иногда так делает.
|
||
"""
|
||
raw_opts = feature.properties.get("options") or {}
|
||
cat_id = feature.category_id
|
||
reg_numb_border = raw_opts.get("reg_numb_border") or raw_opts.get("registration_number")
|
||
type_zone = raw_opts.get("type_zone") or raw_opts.get("typeZone") or ""
|
||
|
||
if not reg_numb_border or cat_id is None:
|
||
logger.debug(
|
||
"upsert_zouit: пропуск — нет reg_numb_border или category_id, feature id=%s",
|
||
feature.id,
|
||
)
|
||
return
|
||
|
||
# cad_zouit.geom — geometry(MultiPolygon, 4326). Polygon допустим (ST_Multi
|
||
# обернёт в SQL), но Point/LineString → INSERT fail. Фильтруем как в upsert_parcel.
|
||
geom = feature.geometry or {}
|
||
geom_type = geom.get("type") if isinstance(geom, dict) else None
|
||
if geom_type in ("Polygon", "MultiPolygon"):
|
||
geom_json: str | None = json.dumps(feature.geometry)
|
||
else:
|
||
if geom_type:
|
||
logger.info(
|
||
"upsert_zouit: reg=%s имеет geometry type=%s (не Polygon) — geom=NULL",
|
||
reg_numb_border,
|
||
geom_type,
|
||
)
|
||
geom_json = None
|
||
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cad_zouit (
|
||
reg_numb_border, category_id, category_name,
|
||
type_zone, type_boundary_value, subcategory,
|
||
cadastral_district,
|
||
content_restrict_encumbrances, name_by_doc,
|
||
legal_act_name, legal_act_date, legal_act_number,
|
||
legal_act_issuer, registration_date,
|
||
old_account_number,
|
||
geom, raw_props, source, fetched_at, updated_at
|
||
) VALUES (
|
||
:reg_numb_border, :category_id, :category_name,
|
||
:type_zone, :type_boundary_value, :subcategory,
|
||
:cadastral_district,
|
||
:content_restrict_encumbrances, :name_by_doc,
|
||
:legal_act_name, CAST(:legal_act_date AS date), :legal_act_number,
|
||
:legal_act_issuer, CAST(:registration_date AS date),
|
||
:old_account_number,
|
||
CASE WHEN CAST(:geom AS text) IS NOT NULL THEN
|
||
ST_Multi(
|
||
ST_Transform(
|
||
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857),
|
||
4326
|
||
)
|
||
)
|
||
ELSE NULL END,
|
||
CAST(:raw_props AS jsonb), :source, NOW(), NOW()
|
||
)
|
||
ON CONFLICT (reg_numb_border, category_id) DO UPDATE SET
|
||
category_name = EXCLUDED.category_name,
|
||
type_zone = EXCLUDED.type_zone,
|
||
type_boundary_value = EXCLUDED.type_boundary_value,
|
||
subcategory = EXCLUDED.subcategory,
|
||
cadastral_district = EXCLUDED.cadastral_district,
|
||
content_restrict_encumbrances = EXCLUDED.content_restrict_encumbrances,
|
||
name_by_doc = EXCLUDED.name_by_doc,
|
||
legal_act_name = EXCLUDED.legal_act_name,
|
||
legal_act_date = EXCLUDED.legal_act_date,
|
||
legal_act_number = EXCLUDED.legal_act_number,
|
||
legal_act_issuer = EXCLUDED.legal_act_issuer,
|
||
registration_date = EXCLUDED.registration_date,
|
||
old_account_number = EXCLUDED.old_account_number,
|
||
geom = COALESCE(EXCLUDED.geom, cad_zouit.geom),
|
||
raw_props = EXCLUDED.raw_props,
|
||
source = EXCLUDED.source,
|
||
updated_at = NOW()
|
||
"""),
|
||
{
|
||
"reg_numb_border": reg_numb_border,
|
||
"category_id": cat_id,
|
||
"category_name": feature.properties.get("categoryName") or str(cat_id),
|
||
"type_zone": type_zone,
|
||
"type_boundary_value": raw_opts.get("type_boundary_value"),
|
||
"subcategory": _safe_int(raw_opts.get("subcategory")),
|
||
"cadastral_district": raw_opts.get("cadastral_district"),
|
||
"content_restrict_encumbrances": raw_opts.get("content_restrict_encumbrances"),
|
||
"name_by_doc": raw_opts.get("name_by_doc"),
|
||
"legal_act_name": raw_opts.get("legal_act_name"),
|
||
"legal_act_date": _parse_nspd_date(raw_opts.get("legal_act_date")),
|
||
"legal_act_number": raw_opts.get("legal_act_number"),
|
||
"legal_act_issuer": raw_opts.get("legal_act_issuer"),
|
||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||
"old_account_number": raw_opts.get("old_account_number"),
|
||
"geom": geom_json,
|
||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||
"source": source,
|
||
},
|
||
)
|
||
|
||
|
||
# ── cad_quarters_geom auto-heal ─────────────────────────────────────────────
|
||
|
||
|
||
def upsert_quarter_geom_from_feature(
|
||
db: Session,
|
||
quarter: str,
|
||
feature: NSPDBulkFeature,
|
||
min_width_m: float = 100.0,
|
||
max_width_m: float = 10000.0,
|
||
) -> bool:
|
||
"""Auto-heal cad_quarters_geom используя CAT_QUARTER_STATS feature.
|
||
|
||
NSPD возвращает геометрию квартала в CAT_QUARTER_STATS feature в EPSG:3857.
|
||
Трансформируем в EPSG:4326 на стороне DB (ST_Transform), сохраняем как ST_Multi.
|
||
|
||
Sanity-check выполняется внутри SQL: UPDATE происходит только если:
|
||
- Новая geom является валидным полигоном шириной [min_width_m, max_width_m] в EPSG:3857
|
||
- Существующая geom отсутствует (NULL) ИЛИ сломана (ширина вне допустимого диапазона)
|
||
|
||
Это предотвращает перезапись здоровой geom сломанной и не upserт сломанной обратно.
|
||
|
||
Args:
|
||
db: SQLAlchemy session (без commit — caller коммитит вместе с quarter_stats).
|
||
quarter: кадастровый номер квартала (cad_number).
|
||
feature: NSPDBulkFeature с category_id=CAT_QUARTER_STATS.
|
||
min_width_m: минимальная ширина bbox в метрах (EPSG:3857) — по умолчанию 100м.
|
||
max_width_m: максимальная ширина bbox в метрах (EPSG:3857) — по умолчанию 10000м.
|
||
|
||
Returns:
|
||
True если geom была обновлена, False если пропущено (нет геометрии / broken / уже healthy).
|
||
"""
|
||
geom = feature.geometry
|
||
if not geom or geom.get("type") not in ("Polygon", "MultiPolygon"):
|
||
return False
|
||
|
||
geom_json = json.dumps(geom)
|
||
|
||
result = db.execute(
|
||
text("""
|
||
WITH new_geom AS (
|
||
SELECT ST_Multi(
|
||
ST_Transform(
|
||
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857),
|
||
4326
|
||
)
|
||
) AS g
|
||
),
|
||
new_geom_3857 AS (
|
||
SELECT ST_Transform(g, 3857) AS g3 FROM new_geom
|
||
),
|
||
width_check AS (
|
||
SELECT
|
||
(ST_XMax(g3) - ST_XMin(g3)) AS w
|
||
FROM new_geom_3857
|
||
)
|
||
UPDATE cad_quarters_geom cqg
|
||
SET
|
||
geom = (SELECT g FROM new_geom),
|
||
source = 'quarter_stats_autoheal',
|
||
fetched_at = NOW()
|
||
FROM width_check
|
||
WHERE cqg.cad_number = :quarter
|
||
AND width_check.w BETWEEN :min_w AND :max_w
|
||
AND (
|
||
cqg.geom IS NULL
|
||
OR (
|
||
ST_XMax(ST_Transform(cqg.geom, 3857))
|
||
- ST_XMin(ST_Transform(cqg.geom, 3857))
|
||
NOT BETWEEN :min_w AND :max_w
|
||
)
|
||
)
|
||
RETURNING cqg.cad_number
|
||
"""),
|
||
{
|
||
"geom": geom_json,
|
||
"quarter": quarter,
|
||
"min_w": min_width_m,
|
||
"max_w": max_width_m,
|
||
},
|
||
)
|
||
updated = result.scalar()
|
||
return updated is not None
|
||
|
||
|
||
# ── cad_quarter_stats upsert ─────────────────────────────────────────────────
|
||
|
||
|
||
def upsert_quarter_stats(
|
||
db: Session,
|
||
quarter: str,
|
||
feature: NSPDBulkFeature,
|
||
) -> None:
|
||
"""Upsert квартальную статистику из feature 36381 в cad_quarter_stats.
|
||
|
||
cad_quarter_stats имеет FK на cad_quarters_geom(cad_number).
|
||
Если квартала ещё нет в cad_quarters_geom — пропускаем тихо
|
||
(INSERT не нарушит FK, но строка не появится; при следующем
|
||
проходе после добавления квартала — сработает).
|
||
"""
|
||
raw_opts = feature.properties.get("options") or {}
|
||
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cad_quarter_stats (
|
||
cad_number,
|
||
cnt_land, cnt_oks,
|
||
cnt_land_geom, cnt_land_not_geom,
|
||
cnt_oks_area, cnt_oks_geom,
|
||
cnt_enk, cnt_pik,
|
||
cost_value_total, cost_value_total_geom,
|
||
sum_land_area, sum_land_geom_area,
|
||
date_cr, real_srid,
|
||
raw_props, fetched_at, updated_at
|
||
)
|
||
SELECT
|
||
:cad_number,
|
||
:cnt_land, :cnt_oks,
|
||
:cnt_land_geom, :cnt_land_not_geom,
|
||
:cnt_oks_area, :cnt_oks_geom,
|
||
:cnt_enk, :cnt_pik,
|
||
:cost_value_total, :cost_value_total_geom,
|
||
:sum_land_area, :sum_land_geom_area,
|
||
CAST(:date_cr AS date), :real_srid,
|
||
CAST(:raw_props AS jsonb), NOW(), NOW()
|
||
WHERE EXISTS (
|
||
SELECT 1 FROM cad_quarters_geom WHERE cad_number = :cad_number
|
||
)
|
||
ON CONFLICT (cad_number) DO UPDATE SET
|
||
cnt_land = EXCLUDED.cnt_land,
|
||
cnt_oks = EXCLUDED.cnt_oks,
|
||
cnt_land_geom = EXCLUDED.cnt_land_geom,
|
||
cnt_land_not_geom = EXCLUDED.cnt_land_not_geom,
|
||
cnt_oks_area = EXCLUDED.cnt_oks_area,
|
||
cnt_oks_geom = EXCLUDED.cnt_oks_geom,
|
||
cnt_enk = EXCLUDED.cnt_enk,
|
||
cnt_pik = EXCLUDED.cnt_pik,
|
||
cost_value_total = EXCLUDED.cost_value_total,
|
||
cost_value_total_geom = EXCLUDED.cost_value_total_geom,
|
||
sum_land_area = EXCLUDED.sum_land_area,
|
||
sum_land_geom_area = EXCLUDED.sum_land_geom_area,
|
||
date_cr = EXCLUDED.date_cr,
|
||
real_srid = EXCLUDED.real_srid,
|
||
raw_props = EXCLUDED.raw_props,
|
||
updated_at = NOW()
|
||
"""),
|
||
{
|
||
"cad_number": quarter,
|
||
"cnt_land": _safe_int(raw_opts.get("cnt_land")),
|
||
"cnt_oks": _safe_int(raw_opts.get("cnt_oks")),
|
||
"cnt_land_geom": _safe_int(raw_opts.get("cnt_land_geom")),
|
||
"cnt_land_not_geom": _safe_int(raw_opts.get("cnt_land_not_geom")),
|
||
"cnt_oks_area": _safe_int(raw_opts.get("cnt_oks_area")),
|
||
"cnt_oks_geom": _safe_int(raw_opts.get("cnt_oks_geom")),
|
||
"cnt_enk": _safe_int(raw_opts.get("cnt_enk")),
|
||
"cnt_pik": _safe_int(raw_opts.get("cnt_pik")),
|
||
"cost_value_total": _safe_numeric(raw_opts.get("cost_value_total")),
|
||
"cost_value_total_geom": _safe_numeric(raw_opts.get("cost_value_total_geom")),
|
||
"sum_land_area": _safe_numeric(raw_opts.get("sum_land_area")),
|
||
"sum_land_geom_area": _safe_numeric(raw_opts.get("sum_land_geom_area")),
|
||
"date_cr": _parse_nspd_date(raw_opts.get("date_cr")),
|
||
"real_srid": _safe_int(raw_opts.get("real_srid")),
|
||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||
},
|
||
)
|
||
|
||
|
||
# ── cad_territorial_zones upsert ────────────────────────────────────────────
|
||
|
||
|
||
def _save_territorial_zones(db: Session, quarter_cad: str, features: list[dict]) -> int:
|
||
"""UPSERT territorial_zones features в cad_territorial_zones по zone_id.
|
||
|
||
Args:
|
||
db: SQLAlchemy session.
|
||
quarter_cad: кадастровый номер квартала (3 сегмента).
|
||
features: list of raw feature dicts от get_features_in_bbox_grid.
|
||
|
||
Returns:
|
||
Количество успешно upserted строк.
|
||
"""
|
||
inserted = 0
|
||
for f in features:
|
||
props: dict = f.get("properties") or {}
|
||
geom = f.get("geometry")
|
||
geom_geojson: str | None = json.dumps(geom) if geom else None
|
||
|
||
# zone_id — NSPD feature id или стабильный fallback на основе md5 от properties.
|
||
# md5 гарантирует идемпотентность между runs (счётчик inserted сбрасывается).
|
||
_raw_id = props.get("id") or props.get("zone_id") or f.get("id")
|
||
if _raw_id:
|
||
zone_id = str(_raw_id)
|
||
else:
|
||
_props_hash = hashlib.md5(
|
||
json.dumps(props, sort_keys=True).encode("utf-8")
|
||
).hexdigest()[:12]
|
||
zone_id = f"{quarter_cad}_{_props_hash}"
|
||
zone_code = (
|
||
props.get("zone_code") or props.get("zone_index") or props.get("reg_numb_border")
|
||
)
|
||
zone_name = props.get("zone_name") or props.get("zone_type_name") or props.get("type_zone")
|
||
permitted_use = props.get("permitted_use") or props.get("vri")
|
||
|
||
try:
|
||
# begin_nested() требует активной outer-транзакции для SAVEPOINT.
|
||
# SQLAlchemy Session (autobegin=True) автоматически начинает tx при первом
|
||
# db.execute() в этом loop — outer tx гарантирована.
|
||
with db.begin_nested():
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO cad_territorial_zones
|
||
(quarter_cad, zone_id, zone_code, zone_name,
|
||
permitted_use, raw_props, geom)
|
||
VALUES (
|
||
CAST(:quarter_cad AS text),
|
||
CAST(:zone_id AS text),
|
||
CAST(:zone_code AS text),
|
||
CAST(:zone_name AS text),
|
||
CAST(:permitted_use AS text),
|
||
CAST(:raw_props AS jsonb),
|
||
CASE WHEN CAST(:geom AS text) IS NOT NULL
|
||
THEN ST_Transform(
|
||
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom AS text)), 3857),
|
||
4326
|
||
)::geography
|
||
ELSE NULL
|
||
END
|
||
)
|
||
ON CONFLICT (zone_id) DO UPDATE SET
|
||
zone_code = EXCLUDED.zone_code,
|
||
zone_name = EXCLUDED.zone_name,
|
||
permitted_use = EXCLUDED.permitted_use,
|
||
raw_props = EXCLUDED.raw_props,
|
||
geom = EXCLUDED.geom,
|
||
fetched_at = NOW()
|
||
"""),
|
||
{
|
||
"quarter_cad": quarter_cad,
|
||
"zone_id": zone_id,
|
||
"zone_code": zone_code,
|
||
"zone_name": zone_name,
|
||
"permitted_use": permitted_use,
|
||
"raw_props": json.dumps(props, ensure_ascii=False),
|
||
"geom": geom_geojson,
|
||
},
|
||
)
|
||
inserted += 1
|
||
except Exception as e:
|
||
logger.warning(
|
||
"_save_territorial_zones: upsert failed zone_id=%s quarter=%s: %s",
|
||
zone_id,
|
||
quarter_cad,
|
||
e,
|
||
)
|
||
db.commit()
|
||
return inserted
|
||
|
||
|
||
# ── Утилиты ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _derive_quarter_cad(cad_num: str) -> str:
|
||
"""Вывести номер квартала из кад. номера: '66:41:0303161:1' → '66:41:0303161'."""
|
||
parts = cad_num.split(":")
|
||
if len(parts) >= 3:
|
||
return ":".join(parts[:3])
|
||
return cad_num
|
||
|
||
|
||
def _safe_numeric(value: Any) -> float | None:
|
||
"""Конвертировать значение в float или None (безопасно)."""
|
||
if value is None:
|
||
return None
|
||
try:
|
||
return float(value)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def _safe_int(value: Any) -> int | None:
|
||
"""Конвертировать значение в int или None (безопасно)."""
|
||
if value is None:
|
||
return None
|
||
try:
|
||
return int(value)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def _parse_nspd_date(value: Any) -> str | None:
|
||
"""Конвертировать NSPD date в ISO YYYY-MM-DD string.
|
||
|
||
NSPD возвращает даты в смешанных форматах:
|
||
- '13.03.2023' (DD.MM.YYYY — для Сооружения/ОНС/ЕНК)
|
||
- '2023-03-13' (ISO — для ЗУ/Здания)
|
||
- '2023-03-13 00:00:00' (ISO with time)
|
||
- None / empty string
|
||
|
||
PostgreSQL DATE cast принимает ISO. Без conversion DD.MM.YYYY → DatetimeFieldOverflow
|
||
crash в upsert_*. Также валидируем что результат — actually-parseable date чтобы
|
||
invalid input не уехал в SQL.
|
||
"""
|
||
import datetime as _dt
|
||
|
||
if value is None or value == "":
|
||
return None
|
||
if not isinstance(value, str):
|
||
return None
|
||
s = value.strip()
|
||
candidate: str | None = None
|
||
# ISO format already — pass through (PG accept YYYY-MM-DD)
|
||
if len(s) >= 10 and s[4] == "-" and s[7] == "-":
|
||
candidate = s[:10]
|
||
# DD.MM.YYYY → YYYY-MM-DD
|
||
elif len(s) == 10 and s[2] == "." and s[5] == ".":
|
||
parts = s.split(".")
|
||
if len(parts) == 3 and all(p.isdigit() for p in parts):
|
||
candidate = f"{parts[2]}-{parts[1]}-{parts[0]}"
|
||
if candidate is None:
|
||
return None
|
||
# Validate: actually parseable as date (catches '2023-13-45', '32.13.2023', etc)
|
||
try:
|
||
_dt.date.fromisoformat(candidate)
|
||
return candidate
|
||
except ValueError:
|
||
return None
|