fix(cadastre): parse NSPD DD.MM.YYYY dates (Fix K — job 18 root cause)
Job 18 (manual_list recovery 11 quarters) crashed every quarter with: psycopg.errors.DatetimeFieldOverflow: date/time field value out of range: '13.03.2023' NSPD returns registration_date in mixed formats: - DD.MM.YYYY for Сооружения / ОНС / ЕНК / ЗОУИТ - YYYY-MM-DD ISO for ЗУ / Здания - YYYY-MM-DD HH:MM:SS with time PG CAST(:date AS date) accepts ISO only — DD.MM.YYYY crashes. Add _parse_nspd_date helper: handles both formats + None/empty. Apply in upsert_construction, upsert_onc, upsert_enk, upsert_zouit, upsert_parcel for registration_date / build_record_registration_date / legal_act_date. 3 new unit tests for the helper (43 tests total).
This commit is contained in:
lekss361
parent
ae04dcb4b9
commit
726b15ad55
2 changed files with 63 additions and 7 deletions
|
|
@ -523,7 +523,7 @@ def upsert_parcel(db: Session, feature: NSPDBulkFeature, source: str = "search")
|
|||
"readable_address": raw_opts.get("readable_address"),
|
||||
"status": raw_opts.get("status"),
|
||||
"previously_posted": raw_opts.get("previously_posted"),
|
||||
"registration_date": raw_opts.get("registration_date"),
|
||||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||||
"subcategory": _safe_int(raw_opts.get("subcategory")),
|
||||
"cadastral_districts_code": _safe_int(raw_opts.get("cadastral_districts_code")),
|
||||
"geom": geom_json,
|
||||
|
|
@ -638,8 +638,10 @@ def upsert_building(db: Session, feature: NSPDBulkFeature, source: str = "search
|
|||
"cost_index": _safe_numeric(raw_opts.get("cost_index")),
|
||||
"year_built": _safe_int(raw_opts.get("year_built")),
|
||||
"year_commisioning": _safe_int(raw_opts.get("year_commisioning")),
|
||||
"build_record_registration_date": raw_opts.get("build_record_registration_date"),
|
||||
"registration_date": raw_opts.get("registration_date"),
|
||||
"build_record_registration_date": _parse_nspd_date(
|
||||
raw_opts.get("build_record_registration_date")
|
||||
),
|
||||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||||
"ownership_type": raw_opts.get("ownership_type"),
|
||||
"cultural_heritage_object": cultural_heritage_bool,
|
||||
"cultural_heritage_val": ch_str,
|
||||
|
|
@ -769,7 +771,7 @@ def upsert_construction(db: Session, feature: NSPDBulkFeature, source: str = "se
|
|||
"address_readable_address": raw_opts.get("address_readable_address"),
|
||||
"status": raw_opts.get("status"),
|
||||
"common_data_status": raw_opts.get("common_data_status"),
|
||||
"registration_date": raw_opts.get("registration_date"),
|
||||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||||
"registers_id": _safe_int(raw_opts.get("registers_id")),
|
||||
"united_cad_number": raw_opts.get("united_cad_number"),
|
||||
"geom": geom_json,
|
||||
|
|
@ -893,7 +895,7 @@ def upsert_enk(db: Session, feature: NSPDBulkFeature, source: str = "search") ->
|
|||
"cost_value": _safe_numeric(raw_opts.get("cost_value")),
|
||||
"facility_cad_number": raw_opts.get("facility_cad_number"),
|
||||
"readable_address": raw_opts.get("readable_address"),
|
||||
"registration_date": raw_opts.get("registration_date"),
|
||||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||||
"registers_id": _safe_int(raw_opts.get("registers_id")),
|
||||
"geom": geom_json,
|
||||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||||
|
|
@ -996,10 +998,10 @@ def upsert_zouit(db: Session, feature: NSPDBulkFeature, source: str = "search")
|
|||
"content_restrict_encumbrances": raw_opts.get("content_restrict_encumbrances"),
|
||||
"name_by_doc": raw_opts.get("name_by_doc"),
|
||||
"legal_act_name": raw_opts.get("legal_act_name"),
|
||||
"legal_act_date": raw_opts.get("legal_act_date"),
|
||||
"legal_act_date": _parse_nspd_date(raw_opts.get("legal_act_date")),
|
||||
"legal_act_number": raw_opts.get("legal_act_number"),
|
||||
"legal_act_issuer": raw_opts.get("legal_act_issuer"),
|
||||
"registration_date": raw_opts.get("registration_date"),
|
||||
"registration_date": _parse_nspd_date(raw_opts.get("registration_date")),
|
||||
"old_account_number": raw_opts.get("old_account_number"),
|
||||
"geom": geom_json,
|
||||
"raw_props": json.dumps(raw_opts, ensure_ascii=False),
|
||||
|
|
@ -1203,3 +1205,31 @@ def _safe_int(value: Any) -> int | None:
|
|||
return int(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_nspd_date(value: Any) -> str | None:
|
||||
"""Конвертировать NSPD date в ISO YYYY-MM-DD string.
|
||||
|
||||
NSPD возвращает даты в смешанных форматах:
|
||||
- '13.03.2023' (DD.MM.YYYY — для Сооружения/ОНС/ЕНК)
|
||||
- '2023-03-13' (ISO — для ЗУ/Здания)
|
||||
- '2023-03-13 00:00:00' (ISO with time)
|
||||
- None / empty string
|
||||
|
||||
PostgreSQL DATE cast принимает ISO. Без conversion DD.MM.YYYY → DatetimeFieldOverflow
|
||||
crash в upsert_*.
|
||||
"""
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
s = value.strip()
|
||||
# ISO format already — pass through (PG accept YYYY-MM-DD)
|
||||
if len(s) >= 10 and s[4] == "-" and s[7] == "-":
|
||||
return s[:10]
|
||||
# DD.MM.YYYY → YYYY-MM-DD
|
||||
if len(s) == 10 and s[2] == "." and s[5] == ".":
|
||||
parts = s.split(".")
|
||||
if len(parts) == 3 and all(p.isdigit() for p in parts):
|
||||
return f"{parts[2]}-{parts[1]}-{parts[0]}"
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -168,6 +168,32 @@ def test_safe_int_none_returns_none() -> None:
|
|||
assert _safe_int(None) is None
|
||||
|
||||
|
||||
def test_parse_nspd_date_handles_dd_mm_yyyy() -> None:
|
||||
"""NSPD Сооружения/ОНС/ЕНК отдают даты в DD.MM.YYYY → конвертируем в ISO."""
|
||||
from app.services.cadastre.bulk_harvest import _parse_nspd_date
|
||||
|
||||
assert _parse_nspd_date("13.03.2023") == "2023-03-13"
|
||||
assert _parse_nspd_date("24.02.2022") == "2022-02-24"
|
||||
|
||||
|
||||
def test_parse_nspd_date_passes_through_iso() -> None:
|
||||
"""ISO формат (YYYY-MM-DD) уже валидный — pass через."""
|
||||
from app.services.cadastre.bulk_harvest import _parse_nspd_date
|
||||
|
||||
assert _parse_nspd_date("2023-03-13") == "2023-03-13"
|
||||
assert _parse_nspd_date("2010-08-12 00:00:00") == "2010-08-12"
|
||||
|
||||
|
||||
def test_parse_nspd_date_none_and_invalid() -> None:
|
||||
"""None / empty string / garbage → None."""
|
||||
from app.services.cadastre.bulk_harvest import _parse_nspd_date
|
||||
|
||||
assert _parse_nspd_date(None) is None
|
||||
assert _parse_nspd_date("") is None
|
||||
assert _parse_nspd_date("nonsense") is None
|
||||
assert _parse_nspd_date(123) is None # not str
|
||||
|
||||
|
||||
# ── Grid geometry ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue