"""Celery task: ingest ТЭП из PDF пояснительных записок ППТ/ПМТ (#1085). Fetch PDF → parse_ppt_tep → UPSERT в ekb_ppt_tep (ON CONFLICT по doc_ref). SAVEPOINT per-doc: ошибка одного документа не откатывает всю транзакцию. Beat НЕ регистрируется — обновление редкое, запуск ручной/через admin. """ from __future__ import annotations import json import logging import httpx from sqlalchemy import text from app.core.db import SessionLocal from app.services.scrapers.ekb_ppt_tep_parser import PptTep, parse_ppt_tep from app.workers.celery_app import celery_app logger = logging.getLogger(__name__) # ── Seed-документы ───────────────────────────────────────────────────────────── # Один образец: ППТ 22823 (2018), пояснительная записка. # URL — placeholder: VERIFY на екатеринбург.рф/документация/ппт/... # Реальный URL уточнить у пользователя перед прод-прогоном. _SEED_DOCS: list[dict] = [ { "doc_ref": "ppt2018_22823", "url": "# VERIFY URL: https://xn--80acgfbsl1azdqr.xn--p1ai/документация/ппт/22823", "project_type": "ppt", }, ] _UPSERT_SQL = text(""" INSERT INTO ekb_ppt_tep ( doc_ref, project_type, zone_balance, tep, phasing, raw_tables, source_pdf, fetched_at ) VALUES ( :doc_ref, :project_type, CAST(:zone_balance AS jsonb), CAST(:tep AS jsonb), CAST(:phasing AS jsonb), CAST(:raw_tables AS jsonb), :source_pdf, now() ) ON CONFLICT (doc_ref) DO UPDATE SET project_type = EXCLUDED.project_type, zone_balance = EXCLUDED.zone_balance, tep = EXCLUDED.tep, phasing = EXCLUDED.phasing, raw_tables = EXCLUDED.raw_tables, source_pdf = EXCLUDED.source_pdf, fetched_at = now() """) def _fetch_pdf(url: str) -> bytes: """Скачивает PDF по URL; verify=False из-за CA Минцифры (как ekburg_permits, #242).""" with httpx.Client(verify=False, timeout=60.0, follow_redirects=True) as client: resp = client.get(url, headers={"User-Agent": "GenDesign-SF/1.0 (+https://gendsgn.ru/)"}) resp.raise_for_status() return resp.content def _tep_to_params(doc_ref: str, project_type: str, source_pdf: str, tep: PptTep) -> dict: return { "doc_ref": doc_ref, "project_type": project_type, "zone_balance": json.dumps(tep.zone_balance, ensure_ascii=False), "tep": json.dumps(tep.tep, ensure_ascii=False), "phasing": json.dumps(tep.phasing, ensure_ascii=False), "raw_tables": json.dumps(tep.raw_tables, ensure_ascii=False), "source_pdf": source_pdf, } @celery_app.task(name="tasks.ekb_ppt_tep_sync.ingest_ppt_tep") def ingest_ppt_tep(docs: list[dict] | None = None) -> dict[str, int]: """Fetch PDF + parse ТЭП + UPSERT в ekb_ppt_tep. Args: docs: список [{doc_ref, url, project_type}] или None → используется _SEED_DOCS (1 seed-документ образца ppt2018_22823). Returns: {"docs": N} — число успешно обработанных документов. """ target_docs: list[dict] = docs if docs is not None else _SEED_DOCS logger.info("ekb_ppt_tep_sync: start, docs=%d", len(target_docs)) processed = 0 failed = 0 db = SessionLocal() try: for doc in target_docs: doc_ref = doc["doc_ref"] url = doc["url"] project_type = doc.get("project_type", "ppt") # Пропускаем placeholder-URL (seed без реального источника) if url.startswith("# VERIFY"): logger.warning( "ekb_ppt_tep_sync: doc_ref=%s — URL не задан (placeholder), пропускаем", doc_ref, ) continue try: logger.info("ekb_ppt_tep_sync: fetch %s (%s)", doc_ref, url) pdf_bytes = _fetch_pdf(url) tep = parse_ppt_tep(pdf_bytes) params = _tep_to_params(doc_ref, project_type, url, tep) with db.begin_nested(): db.execute(_UPSERT_SQL, params) processed += 1 logger.info( "ekb_ppt_tep_sync: upserted %s — zones=%d tep=%d phasing=%d", doc_ref, len(tep.zone_balance), len(tep.tep), len(tep.phasing), ) except Exception as exc: failed += 1 logger.warning( "ekb_ppt_tep_sync: failed for doc_ref=%s: %s", doc_ref, exc, ) db.commit() except Exception: db.rollback() raise finally: db.close() logger.info("ekb_ppt_tep_sync: done — processed=%d failed=%d", processed, failed) return {"docs": processed}