"""Celery task: monthly refresh ekburg permits xlsx (Issue #105). Запускается через beat (1-е число каждого месяца в 05:00 МСК). Добавить в beat_schedule через job_settings или hardcoded entry (Phase 2 followup). """ from __future__ import annotations import json import logging from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import SessionLocal from app.services.scrapers.ekburg_permits import ( EKBURG_PERMITS_URLS, EkburgPermitsClient, PermitRow, msk66_to_wgs84, ) from app.workers.celery_app import celery_app logger = logging.getLogger(__name__) # Размер батча для backfill_geom (SAVEPOINT per batch) _BACKFILL_BATCH_SIZE = 200 def _upsert_permit(db: Session, row: PermitRow) -> None: """UPSERT одной строки разрешения в ekburg_construction_permits. Если geocoded_lat/lon заполнены — записывает geom как ST_SetSRID(ST_MakePoint(lon,lat),4326). """ if row.geocoded_lon is not None and row.geocoded_lat is not None: geom_expr = ( "ST_SetSRID(ST_MakePoint(CAST(:geocoded_lon AS double precision)," " CAST(:geocoded_lat AS double precision)), 4326)" ) else: geom_expr = "NULL" db.execute( text(f""" INSERT INTO ekburg_construction_permits ( permit_type, permit_number, issue_date, expiry_date, developer_inn, developer_name, object_name, object_type, construction_address, cadastral_number, total_area_sqm, living_area_sqm, living_area_fact_sqm, rve_number, rve_date, raw_coord_x, raw_coord_y, geocoded_lat, geocoded_lon, geom, source_year, source_url, raw_row ) VALUES ( :permit_type, :permit_number, :issue_date, :expiry_date, :developer_inn, :developer_name, :object_name, :object_type, :construction_address, :cadastral_number, :total_area_sqm, :living_area_sqm, :living_area_fact_sqm, :rve_number, :rve_date, :raw_coord_x, :raw_coord_y, :geocoded_lat, :geocoded_lon, {geom_expr}, :source_year, :source_url, CAST(:raw_row AS jsonb) ) ON CONFLICT (permit_type, permit_number) DO UPDATE SET issue_date = EXCLUDED.issue_date, expiry_date = EXCLUDED.expiry_date, developer_inn = EXCLUDED.developer_inn, developer_name = EXCLUDED.developer_name, object_name = EXCLUDED.object_name, object_type = EXCLUDED.object_type, construction_address = EXCLUDED.construction_address, cadastral_number = EXCLUDED.cadastral_number, total_area_sqm = EXCLUDED.total_area_sqm, living_area_sqm = EXCLUDED.living_area_sqm, living_area_fact_sqm = EXCLUDED.living_area_fact_sqm, rve_number = EXCLUDED.rve_number, rve_date = EXCLUDED.rve_date, raw_coord_x = EXCLUDED.raw_coord_x, raw_coord_y = EXCLUDED.raw_coord_y, geocoded_lat = EXCLUDED.geocoded_lat, geocoded_lon = EXCLUDED.geocoded_lon, geom = EXCLUDED.geom, raw_row = EXCLUDED.raw_row, fetched_at = NOW() """), { "permit_type": row.permit_type, "permit_number": row.permit_number, "issue_date": row.issue_date, "expiry_date": row.expiry_date, "developer_inn": row.developer_inn, "developer_name": row.developer_name, "object_name": row.object_name, "object_type": row.object_type, "construction_address": row.construction_address, "cadastral_number": row.cadastral_number, "total_area_sqm": row.total_area_sqm, "living_area_sqm": row.living_area_sqm, "living_area_fact_sqm": row.living_area_fact_sqm, "rve_number": row.rve_number, "rve_date": row.rve_date, "raw_coord_x": row.raw_coord_x, "raw_coord_y": row.raw_coord_y, "geocoded_lat": row.geocoded_lat, "geocoded_lon": row.geocoded_lon, "source_year": row.source_year, "source_url": row.source_url, "raw_row": json.dumps(row.raw_row, ensure_ascii=False), }, ) def backfill_geom(db: Session | None = None) -> dict[str, int]: """Бэкфилл geom/geocoded_lat/lon для существующих строк с raw_coord_x/y но без geom. Выполняется разово (admin-trigger или параметр refresh). Использует батч-UPDATE с SAVEPOINT per batch для устойчивости. Возвращает {"updated": N, "skipped": N, "errors": N}. """ own_session = db is None if own_session: db = SessionLocal() updated = 0 skipped = 0 errors = 0 try: # Выбираем строки с координатами, у которых geom пуст rows = db.execute( # type: ignore[union-attr] text(""" SELECT id, raw_coord_x, raw_coord_y FROM ekburg_construction_permits WHERE raw_coord_x IS NOT NULL AND raw_coord_y IS NOT NULL AND geom IS NULL ORDER BY id """) ).fetchall() logger.info("ekburg_permits backfill_geom: %d строк для обработки", len(rows)) batch: list[dict] = [] def _flush_batch(b: list[dict]) -> tuple[int, int]: u = e = 0 try: with db.begin_nested(): # type: ignore[union-attr] for params in b: db.execute( # type: ignore[union-attr] text(""" UPDATE ekburg_construction_permits SET geocoded_lat = CAST(:lat AS double precision), geocoded_lon = CAST(:lon AS double precision), geom = ST_SetSRID( ST_MakePoint( CAST(:lon AS double precision), CAST(:lat AS double precision) ), 4326 ), fetched_at = NOW() WHERE id = CAST(:id AS bigint) """), params, ) u += 1 except Exception as exc: logger.warning("backfill_geom: batch failed: %s", exc) e += len(b) u = 0 return u, e for row in rows: coords = msk66_to_wgs84(row.raw_coord_x, row.raw_coord_y) if coords is None: skipped += 1 continue lon, lat = coords batch.append({"id": row.id, "lon": lon, "lat": lat}) if len(batch) >= _BACKFILL_BATCH_SIZE: u, e = _flush_batch(batch) updated += u errors += e batch = [] if batch: u, e = _flush_batch(batch) updated += u errors += e db.commit() # type: ignore[union-attr] except Exception as exc: logger.error("backfill_geom: unexpected error: %s", exc) raise finally: if own_session: db.close() # type: ignore[union-attr] logger.info( "ekburg_permits backfill_geom done: updated=%d skipped=%d errors=%d", updated, skipped, errors, ) return {"updated": updated, "skipped": skipped, "errors": errors} @celery_app.task(name="tasks.ekburg_permits_sync.refresh_year", queue="celery") def refresh_year(year: int) -> dict[str, int]: """Скачать + распарсить + upsert РНС/РВЭ за один год. Возвращает {"inserted": N, "errors": N}. """ inserted = 0 errors = 0 url = EKBURG_PERMITS_URLS.get(year) if not url: logger.error("ekburg_permits_sync: no URL for year=%d", year) return {"inserted": 0, "errors": 1} with EkburgPermitsClient() as client: try: content = client.download_xlsx(year) except Exception as exc: logger.error("ekburg_permits_sync: download failed year=%d: %s", year, exc) return {"inserted": 0, "errors": 1} with SessionLocal() as db: for row in client.parse_xlsx(content, year, url): try: with db.begin_nested(): _upsert_permit(db, row) inserted += 1 except Exception as exc: logger.warning( "ekburg_permits_sync: upsert failed %s/%s year=%d: %s", row.permit_type, row.permit_number, year, exc, ) errors += 1 db.commit() logger.info( "ekburg_permits_sync: year=%d done inserted=%d errors=%d", year, inserted, errors, ) return {"inserted": inserted, "errors": errors} @celery_app.task(name="tasks.ekburg_permits_sync.refresh_all", queue="celery") def refresh_all() -> dict[str, dict[str, int]]: """Обновить все 5 лет (2022-2026). Планируется через Celery beat ежемесячно.""" results: dict[str, dict[str, int]] = {} for year in sorted(EKBURG_PERMITS_URLS.keys()): logger.info("ekburg_permits_sync: starting year=%d", year) results[str(year)] = refresh_year(year) logger.info("ekburg_permits_sync: refresh_all done: %s", results) return results @celery_app.task(name="tasks.ekburg_permits_sync.backfill_geom_task", queue="celery") def backfill_geom_task() -> dict[str, int]: """Разовый бэкфилл geom для уже загруженных строк без geom. Запускать вручную через admin или celery CLI: celery -A app.workers.celery_app call tasks.ekburg_permits_sync.backfill_geom_task """ return backfill_geom()