* fix(cadastre): NSPD response parse — properties.category + top-level meta (#168) * fix(cadastre): test fixtures use real NSPD shape — properties.category + top-level meta (#168) Bot review of #177 flagged that fixtures still used legacy properties.categoryId + data.meta — meaning CI exercised only fallback branches, not primary paths where the actual bug lived. Pattern of 3 consecutive "live-only" parse fixes (#175 verify, #176 headers, #177 parse) confirmed need for test coverage. Changes: - test_nspd_bulk_client.py: sample_quarter_response → properties.category + meta moved to top-level (real NSPD shape verified live) - test_cadastre_bulk.py: 7 fixtures categoryId → category (regex replace) - test_nspd_bulk_feature_parse_basic: primary path now exercised Plus schema hardening per bot review: - NSPDBulkFeature.category_id: `is not None` check (not truthy `or`) to avoid edge case category=0; int() wrapped in try/except so non-numeric ID (e.g. "ЗУ") doesn't crash upsert_features loop. 35/35 tests pass locally. --------- Co-authored-by: lekss361 <claudestars@proton.me>
151 lines
5.6 KiB
Python
151 lines
5.6 KiB
Python
"""Pydantic schemas для NSPD bulk cadastre ingest (PR 2/5, issue #168).
|
||
|
||
Отдельный модуль от legacy NSPDFeature dataclass (nspd_client.py) —
|
||
там frozen dataclass для on-demand flow, тут Pydantic для bulk pipeline
|
||
с extra='allow' чтобы не потерять неизвестные поля → raw_props JSONB.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
|
||
from pydantic import BaseModel, ConfigDict
|
||
|
||
|
||
class NSPDOptions(BaseModel):
|
||
"""Опции кадастрового объекта из NSPD.
|
||
|
||
Все поля Optional — разные categoryId возвращают разные subsets.
|
||
extra='allow' → любые незнакомые поля сохраняются (forward-compatible).
|
||
"""
|
||
|
||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||
|
||
cad_num: str | None = None
|
||
cad_number: str | None = None # некоторые категории используют cad_number
|
||
quarter_cad_number: str | None = None
|
||
parent_cad_number: str | None = None
|
||
area: float | None = None
|
||
cost_value: float | None = None
|
||
cost_index: float | None = None
|
||
purpose: str | None = None
|
||
floors: str | int | None = None
|
||
year_built: str | int | None = None
|
||
readable_address: str | None = None
|
||
ownership_type: str | None = None
|
||
right_type: str | None = None
|
||
status: str | None = None
|
||
land_record_area: float | None = None
|
||
specified_area: float | None = None
|
||
declared_area: float | None = None
|
||
permitted_use_established_by_document: str | None = None
|
||
category_type: str | None = None
|
||
build_record_type: str | None = None
|
||
build_record_type_value: str | None = None
|
||
objdoc_id: int | None = None
|
||
registers_id: int | None = None
|
||
okato: str | None = None
|
||
kladr: str | None = None
|
||
|
||
@property
|
||
def effective_cad_num(self) -> str | None:
|
||
"""Нормализованный кад. номер: cad_num или cad_number (разные категории)."""
|
||
return self.cad_num or self.cad_number
|
||
|
||
|
||
class NSPDBulkFeature(BaseModel):
|
||
"""GeoJSON Feature из NSPD bulk endpoints.
|
||
|
||
Pydantic-версия (не dataclass) — для удобной сериализации + extra='allow'.
|
||
geometry — GeoJSON-dict в EPSG:3857 (transform на стороне Celery-таска).
|
||
"""
|
||
|
||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||
|
||
id: int | str | None = None
|
||
type: str = "Feature"
|
||
geometry: dict[str, Any] | None = None
|
||
properties: dict[str, Any] = {}
|
||
|
||
@property
|
||
def category_id(self) -> int | None:
|
||
"""category из properties (ключ для routing к target table).
|
||
|
||
NSPD response shape: `properties.category` (НЕ `categoryId`).
|
||
Field name был неверно interpretirovan в initial PR2 implementation
|
||
based on HAR audit; в #177 поправили оба пути.
|
||
|
||
Использует `is not None` (НЕ `or`) — иначе `category=0` truthy-trapped
|
||
бы skipped (хотя observed NSPD IDs все positive, paranoia не вредна).
|
||
int() wrapped в try/except — если NSPD когда-то вернёт string ID
|
||
("ЗУ"), мы не падаем посреди upsert_features loop.
|
||
"""
|
||
val = self.properties.get("category")
|
||
if val is None:
|
||
val = self.properties.get("categoryId")
|
||
if val is None:
|
||
return None
|
||
try:
|
||
return int(val)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
@property
|
||
def options(self) -> NSPDOptions:
|
||
"""Распарсить options из properties.options если есть, иначе пустой."""
|
||
opts = self.properties.get("options") or {}
|
||
return NSPDOptions.model_validate(opts)
|
||
|
||
@property
|
||
def cad_num(self) -> str | None:
|
||
"""Shortcut: cad_num из options (часто нужен в caller)."""
|
||
return self.options.effective_cad_num
|
||
|
||
|
||
class QuarterSnapshot(BaseModel):
|
||
"""Результат search_by_quarter — snapshot данных для одного квартала.
|
||
|
||
Первые 20 объектов на категорию из NSPD search endpoint.
|
||
overflow_categories — категории требующие grid-walk (totalCount > 20).
|
||
"""
|
||
|
||
model_config = ConfigDict(extra="allow")
|
||
|
||
quarter: str
|
||
fetched_at: str # ISO datetime string
|
||
features: list[NSPDBulkFeature] = []
|
||
meta_counts: dict[int, int] = {} # {categoryId: totalCount}
|
||
|
||
@property
|
||
def overflow_categories(self) -> list[int]:
|
||
"""Категории где totalCount > 20 — нужен grid-walk для полноты."""
|
||
return [cat_id for cat_id, count in self.meta_counts.items() if count > 20]
|
||
|
||
@property
|
||
def total_meta_count(self) -> int:
|
||
"""Суммарное количество объектов по meta (до cap)."""
|
||
return sum(self.meta_counts.values())
|
||
|
||
|
||
class ObjectsListing(BaseModel):
|
||
"""Результат list_objects_in_building — список помещений и машино-мест здания.
|
||
|
||
Получается через /api/geoportal/v1/tab-group-data?tabClass=objectsList.
|
||
Q3 deferred — код готов, но в bulk_harvest_quarter MVP не используется.
|
||
"""
|
||
|
||
model_config = ConfigDict(extra="allow")
|
||
|
||
objdoc_id: int
|
||
flats_count: int = 0
|
||
parking_count: int = 0
|
||
flats_cad_nums: list[str] = []
|
||
parking_cad_nums: list[str] = []
|
||
|
||
|
||
__all__ = [
|
||
"NSPDBulkFeature",
|
||
"NSPDOptions",
|
||
"ObjectsListing",
|
||
"QuarterSnapshot",
|
||
]
|