gendesign/backend/app/schemas/nspd_bulk.py
lekss361 99e2210919
fix(cadastre): NSPD response parse — properties.category + top-level meta (#168) (#177)
* fix(cadastre): NSPD response parse — properties.category + top-level meta (#168)

* fix(cadastre): test fixtures use real NSPD shape — properties.category + top-level meta (#168)

Bot review of #177 flagged that fixtures still used legacy properties.categoryId
+ data.meta — meaning CI exercised only fallback branches, not primary paths
where the actual bug lived. Pattern of 3 consecutive "live-only" parse fixes
(#175 verify, #176 headers, #177 parse) confirmed need for test coverage.

Changes:
- test_nspd_bulk_client.py: sample_quarter_response → properties.category +
  meta moved to top-level (real NSPD shape verified live)
- test_cadastre_bulk.py: 7 fixtures categoryId → category (regex replace)
- test_nspd_bulk_feature_parse_basic: primary path now exercised

Plus schema hardening per bot review:
- NSPDBulkFeature.category_id: `is not None` check (not truthy `or`)
  to avoid edge case category=0; int() wrapped in try/except so
  non-numeric ID (e.g. "ЗУ") doesn't crash upsert_features loop.

35/35 tests pass locally.

---------

Co-authored-by: lekss361 <claudestars@proton.me>
2026-05-15 15:40:01 +03:00

151 lines
5.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Pydantic schemas для NSPD bulk cadastre ingest (PR 2/5, issue #168).
Отдельный модуль от legacy NSPDFeature dataclass (nspd_client.py) —
там frozen dataclass для on-demand flow, тут Pydantic для bulk pipeline
с extra='allow' чтобы не потерять неизвестные поля → raw_props JSONB.
"""
from __future__ import annotations
from typing import Any
from pydantic import BaseModel, ConfigDict
class NSPDOptions(BaseModel):
"""Опции кадастрового объекта из NSPD.
Все поля Optional — разные categoryId возвращают разные subsets.
extra='allow' → любые незнакомые поля сохраняются (forward-compatible).
"""
model_config = ConfigDict(extra="allow", populate_by_name=True)
cad_num: str | None = None
cad_number: str | None = None # некоторые категории используют cad_number
quarter_cad_number: str | None = None
parent_cad_number: str | None = None
area: float | None = None
cost_value: float | None = None
cost_index: float | None = None
purpose: str | None = None
floors: str | int | None = None
year_built: str | int | None = None
readable_address: str | None = None
ownership_type: str | None = None
right_type: str | None = None
status: str | None = None
land_record_area: float | None = None
specified_area: float | None = None
declared_area: float | None = None
permitted_use_established_by_document: str | None = None
category_type: str | None = None
build_record_type: str | None = None
build_record_type_value: str | None = None
objdoc_id: int | None = None
registers_id: int | None = None
okato: str | None = None
kladr: str | None = None
@property
def effective_cad_num(self) -> str | None:
"""Нормализованный кад. номер: cad_num или cad_number (разные категории)."""
return self.cad_num or self.cad_number
class NSPDBulkFeature(BaseModel):
"""GeoJSON Feature из NSPD bulk endpoints.
Pydantic-версия (не dataclass) — для удобной сериализации + extra='allow'.
geometry — GeoJSON-dict в EPSG:3857 (transform на стороне Celery-таска).
"""
model_config = ConfigDict(extra="allow", populate_by_name=True)
id: int | str | None = None
type: str = "Feature"
geometry: dict[str, Any] | None = None
properties: dict[str, Any] = {}
@property
def category_id(self) -> int | None:
"""category из properties (ключ для routing к target table).
NSPD response shape: `properties.category` (НЕ `categoryId`).
Field name был неверно interpretirovan в initial PR2 implementation
based on HAR audit; в #177 поправили оба пути.
Использует `is not None` (НЕ `or`) — иначе `category=0` truthy-trapped
бы skipped (хотя observed NSPD IDs все positive, paranoia не вредна).
int() wrapped в try/except — если NSPD когда-то вернёт string ID
("ЗУ"), мы не падаем посреди upsert_features loop.
"""
val = self.properties.get("category")
if val is None:
val = self.properties.get("categoryId")
if val is None:
return None
try:
return int(val)
except (TypeError, ValueError):
return None
@property
def options(self) -> NSPDOptions:
"""Распарсить options из properties.options если есть, иначе пустой."""
opts = self.properties.get("options") or {}
return NSPDOptions.model_validate(opts)
@property
def cad_num(self) -> str | None:
"""Shortcut: cad_num из options (часто нужен в caller)."""
return self.options.effective_cad_num
class QuarterSnapshot(BaseModel):
"""Результат search_by_quarter — snapshot данных для одного квартала.
Первые 20 объектов на категорию из NSPD search endpoint.
overflow_categories — категории требующие grid-walk (totalCount > 20).
"""
model_config = ConfigDict(extra="allow")
quarter: str
fetched_at: str # ISO datetime string
features: list[NSPDBulkFeature] = []
meta_counts: dict[int, int] = {} # {categoryId: totalCount}
@property
def overflow_categories(self) -> list[int]:
"""Категории где totalCount > 20 — нужен grid-walk для полноты."""
return [cat_id for cat_id, count in self.meta_counts.items() if count > 20]
@property
def total_meta_count(self) -> int:
"""Суммарное количество объектов по meta (до cap)."""
return sum(self.meta_counts.values())
class ObjectsListing(BaseModel):
"""Результат list_objects_in_building — список помещений и машино-мест здания.
Получается через /api/geoportal/v1/tab-group-data?tabClass=objectsList.
Q3 deferred — код готов, но в bulk_harvest_quarter MVP не используется.
"""
model_config = ConfigDict(extra="allow")
objdoc_id: int
flats_count: int = 0
parking_count: int = 0
flats_cad_nums: list[str] = []
parking_cad_nums: list[str] = []
__all__ = [
"NSPDBulkFeature",
"NSPDOptions",
"ObjectsListing",
"QuarterSnapshot",
]