gendesign/backend/app/workers/tasks/ekburg_permits_sync.py

"""Celery task: monthly refresh ekburg permits xlsx (Issue #105).

Запускается через beat (1-е число каждого месяца в 05:00 МСК).
Добавить в beat_schedule через job_settings или hardcoded entry (Phase 2 followup).
"""

from __future__ import annotations

import json
import logging

from sqlalchemy import text
from sqlalchemy.orm import Session

from app.core.db import SessionLocal
from app.services.scrapers.ekburg_permits import (
    EKBURG_PERMITS_URLS,
    EkburgPermitsClient,
    PermitRow,
)
from app.workers.celery_app import celery_app

logger = logging.getLogger(__name__)


def _upsert_permit(db: Session, row: PermitRow) -> None:
    """UPSERT одной строки разрешения в ekburg_construction_permits."""
    db.execute(
        text("""
            INSERT INTO ekburg_construction_permits (
                permit_type, permit_number,
                issue_date, expiry_date,
                developer_inn, developer_name,
                object_name, object_type,
                construction_address, cadastral_number,
                total_area_sqm, living_area_sqm, living_area_fact_sqm,
                rve_number, rve_date,
                raw_coord_x, raw_coord_y,
                source_year, source_url, raw_row
            )
            VALUES (
                :permit_type, :permit_number,
                :issue_date, :expiry_date,
                :developer_inn, :developer_name,
                :object_name, :object_type,
                :construction_address, :cadastral_number,
                :total_area_sqm, :living_area_sqm, :living_area_fact_sqm,
                :rve_number, :rve_date,
                :raw_coord_x, :raw_coord_y,
                :source_year, :source_url, CAST(:raw_row AS jsonb)
            )
            ON CONFLICT (permit_type, permit_number) DO UPDATE SET
                issue_date           = EXCLUDED.issue_date,
                expiry_date          = EXCLUDED.expiry_date,
                developer_inn        = EXCLUDED.developer_inn,
                developer_name       = EXCLUDED.developer_name,
                object_name          = EXCLUDED.object_name,
                object_type          = EXCLUDED.object_type,
                construction_address = EXCLUDED.construction_address,
                cadastral_number     = EXCLUDED.cadastral_number,
                total_area_sqm       = EXCLUDED.total_area_sqm,
                living_area_sqm      = EXCLUDED.living_area_sqm,
                living_area_fact_sqm = EXCLUDED.living_area_fact_sqm,
                rve_number           = EXCLUDED.rve_number,
                rve_date             = EXCLUDED.rve_date,
                raw_coord_x          = EXCLUDED.raw_coord_x,
                raw_coord_y          = EXCLUDED.raw_coord_y,
                raw_row              = EXCLUDED.raw_row,
                fetched_at           = NOW()
        """),
        {
            "permit_type": row.permit_type,
            "permit_number": row.permit_number,
            "issue_date": row.issue_date,
            "expiry_date": row.expiry_date,
            "developer_inn": row.developer_inn,
            "developer_name": row.developer_name,
            "object_name": row.object_name,
            "object_type": row.object_type,
            "construction_address": row.construction_address,
            "cadastral_number": row.cadastral_number,
            "total_area_sqm": row.total_area_sqm,
            "living_area_sqm": row.living_area_sqm,
            "living_area_fact_sqm": row.living_area_fact_sqm,
            "rve_number": row.rve_number,
            "rve_date": row.rve_date,
            "raw_coord_x": row.raw_coord_x,
            "raw_coord_y": row.raw_coord_y,
            "source_year": row.source_year,
            "source_url": row.source_url,
            "raw_row": json.dumps(row.raw_row, ensure_ascii=False),
        },
    )


@celery_app.task(name="tasks.ekburg_permits_sync.refresh_year", queue="celery")
def refresh_year(year: int) -> dict[str, int]:
    """Скачать + распарсить + upsert РНС/РВЭ за один год.

    Возвращает {"inserted": N, "errors": N}.
    """
    inserted = 0
    errors = 0

    url = EKBURG_PERMITS_URLS.get(year)
    if not url:
        logger.error("ekburg_permits_sync: no URL for year=%d", year)
        return {"inserted": 0, "errors": 1}

    with EkburgPermitsClient() as client:
        try:
            content = client.download_xlsx(year)
        except Exception as exc:
            logger.error("ekburg_permits_sync: download failed year=%d: %s", year, exc)
            return {"inserted": 0, "errors": 1}

        with SessionLocal() as db:
            for row in client.parse_xlsx(content, year, url):
                try:
                    with db.begin_nested():
                        _upsert_permit(db, row)
                    inserted += 1
                except Exception as exc:
                    logger.warning(
                        "ekburg_permits_sync: upsert failed %s/%s year=%d: %s",
                        row.permit_type,
                        row.permit_number,
                        year,
                        exc,
                    )
                    errors += 1
            db.commit()

    logger.info(
        "ekburg_permits_sync: year=%d done inserted=%d errors=%d",
        year,
        inserted,
        errors,
    )
    return {"inserted": inserted, "errors": errors}


@celery_app.task(name="tasks.ekburg_permits_sync.refresh_all", queue="celery")
def refresh_all() -> dict[str, dict[str, int]]:
    """Обновить все 5 лет (2022-2026). Планируется через Celery beat ежемесячно."""
    results: dict[str, dict[str, int]] = {}
    for year in sorted(EKBURG_PERMITS_URLS.keys()):
        logger.info("ekburg_permits_sync: starting year=%d", year)
        results[str(year)] = refresh_year(year)
    logger.info("ekburg_permits_sync: refresh_all done: %s", results)
    return results