db1ff5b1fc
- data/sql/99_nspd_entities_denorm.sql: 2 tables (cad_num PK) + 6 indexes (quarter_cad, GIST geom, partial WHERE purpose ILIKE '%многокв%') - nspd_denorm.py: denorm_parcel_feature / denorm_building_feature / denorm_dump - SAVEPOINT per row via with db.begin_nested() - ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom), 3857), 4326) - json.dumps(geom, ensure_ascii=False) for psycopg bind - All CAST(:x AS type) — no ❌:type - nspd_sync.py: inline denorm after _upsert_dump in harvest_quarter (non-fatal) - nspd_denorm_backfill.py: Celery one-shot task (per-quarter commit) - admin_etl.py: POST /api/v1/admin/etl/nspd-denorm-backfill (AdminTokenAuth) - 19 tests: coerce helpers, denorm parcel/building, aggregate counts Foundation for downstream features (МКД neighbors lookup, parcel attrs). Part of #94
96 lines
3.7 KiB
Python
96 lines
3.7 KiB
Python
"""One-shot backfill: denormalize all existing nspd_quarter_dumps → nspd_parcels/buildings.
|
||
|
||
Запускается вручную через admin endpoint POST /api/v1/admin/etl/nspd-denorm-backfill
|
||
или через Celery CLI:
|
||
celery -A app.workers.celery_app call nspd_denorm.backfill_all_dumps
|
||
|
||
Идемпотентен: повторный запуск обновляет строки через ON CONFLICT DO UPDATE.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
|
||
from celery import shared_task
|
||
from sqlalchemy import text
|
||
|
||
from app.core.db import SessionLocal
|
||
from app.services.scrapers.nspd_denorm import denorm_dump
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
@shared_task(
|
||
name="nspd_denorm.backfill_all_dumps",
|
||
bind=True,
|
||
soft_time_limit=3600, # 1 час — много кварталов × ~10ms per parcel
|
||
max_retries=0, # One-shot, не retry
|
||
)
|
||
def backfill_all_dumps(self: object, *, limit: int | None = None) -> dict[str, int]: # type: ignore[misc]
|
||
"""Backfill all existing nspd_quarter_dumps → nspd_parcels / nspd_buildings.
|
||
|
||
Читает все строки nspd_quarter_dumps (или limit строк), и для каждой вызывает
|
||
denorm_dump. Возвращает агрегированные счётчики по всем кварталам.
|
||
|
||
Args:
|
||
limit: если задан — обработать только первые N кварталов (для тестирования).
|
||
|
||
Returns:
|
||
dict {
|
||
"parcels": суммарно вставлено/обновлено parcel строк,
|
||
"buildings": суммарно вставлено/обновлено building строк,
|
||
"errors": суммарно пропусков (нет cad_num, DB ошибки),
|
||
"quarters_processed": количество обработанных кварталов,
|
||
}
|
||
"""
|
||
totals: dict[str, int] = {
|
||
"parcels": 0,
|
||
"buildings": 0,
|
||
"errors": 0,
|
||
"quarters_processed": 0,
|
||
}
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
sql = (
|
||
"SELECT quarter_cad, features_json "
|
||
"FROM nspd_quarter_dumps "
|
||
"WHERE total_features > 0 "
|
||
"ORDER BY quarter_cad"
|
||
)
|
||
if limit is not None:
|
||
# Используем параметр через text() чтобы избежать SQL-injection.
|
||
rows = db.execute(
|
||
text(sql + " LIMIT CAST(:lim AS integer)"), {"lim": int(limit)}
|
||
).fetchall()
|
||
else:
|
||
rows = db.execute(text(sql)).fetchall()
|
||
|
||
logger.info("backfill_all_dumps: starting, quarters=%d limit=%s", len(rows), limit)
|
||
|
||
for row in rows:
|
||
quarter_cad: str = row[0]
|
||
features_json = row[1]
|
||
# features_json — уже list[dict] через psycopg v3 JSON decoding.
|
||
features: list[dict] = features_json if isinstance(features_json, list) else []
|
||
|
||
try:
|
||
counts = denorm_dump(db, quarter_cad=quarter_cad, features=features)
|
||
totals["parcels"] += counts["parcels"]
|
||
totals["buildings"] += counts["buildings"]
|
||
totals["errors"] += counts["errors"]
|
||
totals["quarters_processed"] += 1
|
||
except Exception as e:
|
||
logger.warning("backfill_all_dumps: quarter=%s failed: %s", quarter_cad, e)
|
||
totals["errors"] += 1
|
||
# Продолжаем с остальными кварталами
|
||
try:
|
||
db.rollback()
|
||
except Exception:
|
||
pass
|
||
|
||
logger.info("backfill_all_dumps: done totals=%s", totals)
|
||
finally:
|
||
db.close()
|
||
|
||
return totals
|