fix(cadastre): NSPD response parse — properties.category + top-level meta (#168) #177

Merged
lekss361 merged 2 commits from fix/nspd-response-parse into main 2026-05-15 12:40:01 +00:00
4 changed files with 50 additions and 25 deletions

View file

@ -69,9 +69,26 @@ class NSPDBulkFeature(BaseModel):
@property @property
def category_id(self) -> int | None: def category_id(self) -> int | None:
"""categoryId из properties (ключ для routing к target table).""" """category из properties (ключ для routing к target table).
val = self.properties.get("categoryId")
return int(val) if val is not None else None NSPD response shape: `properties.category` (НЕ `categoryId`).
Field name был неверно interpretirovan в initial PR2 implementation
based on HAR audit; в #177 поправили оба пути.
Использует `is not None` (НЕ `or`) иначе `category=0` truthy-trapped
бы skipped (хотя observed NSPD IDs все positive, paranoia не вредна).
int() wrapped в try/except если NSPD когда-то вернёт string ID
("ЗУ"), мы не падаем посреди upsert_features loop.
"""
val = self.properties.get("category")
if val is None:
val = self.properties.get("categoryId")
if val is None:
return None
try:
return int(val)
except (TypeError, ValueError):
return None
@property @property
def options(self) -> NSPDOptions: def options(self) -> NSPDOptions:

View file

@ -242,7 +242,9 @@ class NSPDBulkClient:
return QuarterSnapshot(quarter=quarter, fetched_at=fetched_at) return QuarterSnapshot(quarter=quarter, fetched_at=fetched_at)
raw_features: list[dict[str, Any]] = data_block.get("features") or [] raw_features: list[dict[str, Any]] = data_block.get("features") or []
raw_meta: list[dict[str, Any]] = data_block.get("meta") or [] # meta лежит на TOP level: {"data": {features}, "meta": [...]}
# fallback на data.meta для legacy shape (just in case)
raw_meta: list[dict[str, Any]] = payload.get("meta") or data_block.get("meta") or []
features = [NSPDBulkFeature.model_validate(f) for f in raw_features] features = [NSPDBulkFeature.model_validate(f) for f in raw_features]

View file

@ -29,7 +29,12 @@ from app.scrapers.nspd_bulk_client import (
@pytest.fixture @pytest.fixture
def sample_quarter_response() -> dict[str, Any]: def sample_quarter_response() -> dict[str, Any]:
"""Минимальный валидный ответ search_by_quarter из NSPD.""" """Минимальный валидный ответ search_by_quarter из NSPD.
Real NSPD shape (verified live on 66:41:0303161 in #177):
- `properties.category` (НЕ categoryId)
- `meta` лежит на top-level (не в data.meta)
"""
return { return {
"data": { "data": {
"type": "FeatureCollection", "type": "FeatureCollection",
@ -50,7 +55,7 @@ def sample_quarter_response() -> dict[str, Any]:
], ],
}, },
"properties": { "properties": {
"categoryId": 36368, "category": 36368,
"categoryName": "Земельные участки ЕГРН", "categoryName": "Земельные участки ЕГРН",
"options": { "options": {
"cad_num": "66:41:0303161:123", "cad_num": "66:41:0303161:123",
@ -63,7 +68,7 @@ def sample_quarter_response() -> dict[str, Any]:
"type": "Feature", "type": "Feature",
"geometry": {"type": "Point", "coordinates": [6700500.0, 7700500.0]}, "geometry": {"type": "Point", "coordinates": [6700500.0, 7700500.0]},
"properties": { "properties": {
"categoryId": 36369, "category": 36369,
"categoryName": "Здания", "categoryName": "Здания",
"options": { "options": {
"cad_num": "66:41:0303161:456:1", "cad_num": "66:41:0303161:456:1",
@ -72,14 +77,15 @@ def sample_quarter_response() -> dict[str, Any]:
}, },
}, },
], ],
"meta": [ },
{"categoryId": 36368, "totalCount": 170}, # meta — top-level, primary path в search_by_quarter
{"categoryId": 36369, "totalCount": 256}, "meta": [
{"categoryId": 36383, "totalCount": 75}, {"categoryId": 36368, "totalCount": 170},
{"categoryId": 37163, "totalCount": 5877}, {"categoryId": 36369, "totalCount": 256},
{"categoryId": 39663, "totalCount": 1}, {"categoryId": 36383, "totalCount": 75},
], {"categoryId": 37163, "totalCount": 5877},
} {"categoryId": 39663, "totalCount": 1},
],
} }
@ -105,7 +111,7 @@ def sample_wms_response() -> dict[str, Any]:
], ],
}, },
"properties": { "properties": {
"categoryId": 36368, "category": 36368,
"options": {"cad_num": "66:41:0303161:789"}, "options": {"cad_num": "66:41:0303161:789"},
}, },
} }
@ -137,13 +143,13 @@ def sample_tab_group_response() -> dict[str, Any]:
def test_nspd_bulk_feature_parse_basic() -> None: def test_nspd_bulk_feature_parse_basic() -> None:
"""NSPDBulkFeature парсит id, geometry, properties.""" """NSPDBulkFeature парсит id, geometry, properties (primary path: properties.category)."""
raw = { raw = {
"id": 42, "id": 42,
"type": "Feature", "type": "Feature",
"geometry": {"type": "Point", "coordinates": [6700000.0, 7700000.0]}, "geometry": {"type": "Point", "coordinates": [6700000.0, 7700000.0]},
"properties": { "properties": {
"categoryId": 36368, "category": 36368,
"options": {"cad_num": "66:41:0303161:1", "area": 300.0}, "options": {"cad_num": "66:41:0303161:1", "area": 300.0},
}, },
} }

View file

@ -42,7 +42,7 @@ def _make_parcel_feature(cad_num: str = "66:41:0303161:1") -> NSPDBulkFeature:
], ],
}, },
"properties": { "properties": {
"categoryId": 36368, "category": 36368,
"options": { "options": {
"cad_num": cad_num, "cad_num": cad_num,
"quarter_cad_number": "66:41:0303161", "quarter_cad_number": "66:41:0303161",
@ -76,7 +76,7 @@ def _make_building_feature(cad_num: str = "66:41:0303161:1:1") -> NSPDBulkFeatur
], ],
}, },
"properties": { "properties": {
"categoryId": 36369, "category": 36369,
"options": { "options": {
"cad_num": cad_num, "cad_num": cad_num,
"quarter_cad_number": "66:41:0303161", "quarter_cad_number": "66:41:0303161",
@ -112,7 +112,7 @@ def _make_zouit_feature(reg_numb: str = "66-0.1-1.1-1234") -> NSPDBulkFeature:
], ],
}, },
"properties": { "properties": {
"categoryId": 36940, "category": 36940,
"categoryName": "ЗОУИТ", "categoryName": "ЗОУИТ",
"options": { "options": {
"reg_numb_border": reg_numb, "reg_numb_border": reg_numb,
@ -237,7 +237,7 @@ def test_upsert_features_unknown_category_counts_as_skipped() -> None:
{ {
"type": "Feature", "type": "Feature",
"geometry": None, "geometry": None,
"properties": {"categoryId": 99999, "options": {}}, "properties": {"category": 99999, "options": {}},
} }
) )
db = MagicMock() db = MagicMock()
@ -251,7 +251,7 @@ def test_upsert_features_quarter_stats_not_counted() -> None:
{ {
"type": "Feature", "type": "Feature",
"geometry": None, "geometry": None,
"properties": {"categoryId": 36381, "options": {"cnt_land": 5}}, "properties": {"category": 36381, "options": {"cnt_land": 5}},
} }
) )
db = MagicMock() db = MagicMock()
@ -458,7 +458,7 @@ def test_zouit_dedup_different_category_ids() -> None:
"type": "Feature", "type": "Feature",
"geometry": None, "geometry": None,
"properties": { "properties": {
"categoryId": 36940, "category": 36940,
"categoryName": "ЗОУИТ 1", "categoryName": "ЗОУИТ 1",
"options": {"reg_numb_border": "66-0.1-1.1-1234", "type_zone": "тип А"}, "options": {"reg_numb_border": "66-0.1-1.1-1234", "type_zone": "тип А"},
}, },
@ -469,7 +469,7 @@ def test_zouit_dedup_different_category_ids() -> None:
"type": "Feature", "type": "Feature",
"geometry": None, "geometry": None,
"properties": { "properties": {
"categoryId": 469039, "category": 469039,
"categoryName": "ЗОУИТ 2", "categoryName": "ЗОУИТ 2",
"options": {"reg_numb_border": "66-0.1-1.1-1234", "type_zone": "тип Б"}, "options": {"reg_numb_border": "66-0.1-1.1-1234", "type_zone": "тип Б"},
}, },