"""Celery task: monthly refresh ekburg permits xlsx (Issue #105). Запускается через beat (1-е число каждого месяца в 05:00 МСК). Добавить в beat_schedule через job_settings или hardcoded entry (Phase 2 followup). """ from __future__ import annotations import json import logging from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import SessionLocal from app.services.scrapers.ekburg_permits import ( EKBURG_PERMITS_URLS, EkburgPermitsClient, PermitRow, ) from app.workers.celery_app import celery_app logger = logging.getLogger(__name__) def _upsert_permit(db: Session, row: PermitRow) -> None: """UPSERT одной строки разрешения в ekburg_construction_permits.""" db.execute( text(""" INSERT INTO ekburg_construction_permits ( permit_type, permit_number, issue_date, expiry_date, developer_inn, developer_name, object_name, object_type, construction_address, cadastral_number, total_area_sqm, living_area_sqm, living_area_fact_sqm, rve_number, rve_date, raw_coord_x, raw_coord_y, source_year, source_url, raw_row ) VALUES ( :permit_type, :permit_number, :issue_date, :expiry_date, :developer_inn, :developer_name, :object_name, :object_type, :construction_address, :cadastral_number, :total_area_sqm, :living_area_sqm, :living_area_fact_sqm, :rve_number, :rve_date, :raw_coord_x, :raw_coord_y, :source_year, :source_url, CAST(:raw_row AS jsonb) ) ON CONFLICT (permit_type, permit_number) DO UPDATE SET issue_date = EXCLUDED.issue_date, expiry_date = EXCLUDED.expiry_date, developer_inn = EXCLUDED.developer_inn, developer_name = EXCLUDED.developer_name, object_name = EXCLUDED.object_name, object_type = EXCLUDED.object_type, construction_address = EXCLUDED.construction_address, cadastral_number = EXCLUDED.cadastral_number, total_area_sqm = EXCLUDED.total_area_sqm, living_area_sqm = EXCLUDED.living_area_sqm, living_area_fact_sqm = EXCLUDED.living_area_fact_sqm, rve_number = EXCLUDED.rve_number, rve_date = EXCLUDED.rve_date, raw_coord_x = EXCLUDED.raw_coord_x, raw_coord_y = EXCLUDED.raw_coord_y, raw_row = EXCLUDED.raw_row, fetched_at = NOW() """), { "permit_type": row.permit_type, "permit_number": row.permit_number, "issue_date": row.issue_date, "expiry_date": row.expiry_date, "developer_inn": row.developer_inn, "developer_name": row.developer_name, "object_name": row.object_name, "object_type": row.object_type, "construction_address": row.construction_address, "cadastral_number": row.cadastral_number, "total_area_sqm": row.total_area_sqm, "living_area_sqm": row.living_area_sqm, "living_area_fact_sqm": row.living_area_fact_sqm, "rve_number": row.rve_number, "rve_date": row.rve_date, "raw_coord_x": row.raw_coord_x, "raw_coord_y": row.raw_coord_y, "source_year": row.source_year, "source_url": row.source_url, "raw_row": json.dumps(row.raw_row, ensure_ascii=False), }, ) @celery_app.task(name="tasks.ekburg_permits_sync.refresh_year", queue="celery") def refresh_year(year: int) -> dict[str, int]: """Скачать + распарсить + upsert РНС/РВЭ за один год. Возвращает {"inserted": N, "errors": N}. """ inserted = 0 errors = 0 url = EKBURG_PERMITS_URLS.get(year) if not url: logger.error("ekburg_permits_sync: no URL for year=%d", year) return {"inserted": 0, "errors": 1} with EkburgPermitsClient() as client: try: content = client.download_xlsx(year) except Exception as exc: logger.error("ekburg_permits_sync: download failed year=%d: %s", year, exc) return {"inserted": 0, "errors": 1} with SessionLocal() as db: for row in client.parse_xlsx(content, year, url): try: with db.begin_nested(): _upsert_permit(db, row) inserted += 1 except Exception as exc: logger.warning( "ekburg_permits_sync: upsert failed %s/%s year=%d: %s", row.permit_type, row.permit_number, year, exc, ) errors += 1 db.commit() logger.info( "ekburg_permits_sync: year=%d done inserted=%d errors=%d", year, inserted, errors, ) return {"inserted": inserted, "errors": errors} @celery_app.task(name="tasks.ekburg_permits_sync.refresh_all", queue="celery") def refresh_all() -> dict[str, dict[str, int]]: """Обновить все 5 лет (2022-2026). Планируется через Celery beat ежемесячно.""" results: dict[str, dict[str, int]] = {} for year in sorted(EKBURG_PERMITS_URLS.keys()): logger.info("ekburg_permits_sync: starting year=%d", year) results[str(year)] = refresh_year(year) logger.info("ekburg_permits_sync: refresh_all done: %s", results) return results