lekss361
ef15190690
Co-authored-by: lekss361 <lekss361@gendsgn.local> Co-committed-by: lekss361 <lekss361@gendsgn.local>
284 lines
11 KiB
Python
284 lines
11 KiB
Python
"""Celery task: monthly refresh ekburg permits xlsx (Issue #105).
|
||
|
||
Запускается через beat (1-е число каждого месяца в 05:00 МСК).
|
||
Добавить в beat_schedule через job_settings или hardcoded entry (Phase 2 followup).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.db import SessionLocal
|
||
from app.services.scrapers.ekburg_permits import (
|
||
EKBURG_PERMITS_URLS,
|
||
EkburgPermitsClient,
|
||
PermitRow,
|
||
msk66_to_wgs84,
|
||
)
|
||
from app.workers.celery_app import celery_app
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Размер батча для backfill_geom (SAVEPOINT per batch)
|
||
_BACKFILL_BATCH_SIZE = 200
|
||
|
||
|
||
def _upsert_permit(db: Session, row: PermitRow) -> None:
|
||
"""UPSERT одной строки разрешения в ekburg_construction_permits.
|
||
|
||
Если geocoded_lat/lon заполнены — записывает geom как ST_SetSRID(ST_MakePoint(lon,lat),4326).
|
||
"""
|
||
if row.geocoded_lon is not None and row.geocoded_lat is not None:
|
||
geom_expr = (
|
||
"ST_SetSRID(ST_MakePoint(CAST(:geocoded_lon AS double precision),"
|
||
" CAST(:geocoded_lat AS double precision)), 4326)"
|
||
)
|
||
else:
|
||
geom_expr = "NULL"
|
||
|
||
db.execute(
|
||
text(f"""
|
||
INSERT INTO ekburg_construction_permits (
|
||
permit_type, permit_number,
|
||
issue_date, expiry_date,
|
||
developer_inn, developer_name,
|
||
object_name, object_type,
|
||
construction_address, cadastral_number,
|
||
total_area_sqm, living_area_sqm, living_area_fact_sqm,
|
||
rve_number, rve_date,
|
||
raw_coord_x, raw_coord_y,
|
||
geocoded_lat, geocoded_lon, geom,
|
||
source_year, source_url, raw_row
|
||
)
|
||
VALUES (
|
||
:permit_type, :permit_number,
|
||
:issue_date, :expiry_date,
|
||
:developer_inn, :developer_name,
|
||
:object_name, :object_type,
|
||
:construction_address, :cadastral_number,
|
||
:total_area_sqm, :living_area_sqm, :living_area_fact_sqm,
|
||
:rve_number, :rve_date,
|
||
:raw_coord_x, :raw_coord_y,
|
||
:geocoded_lat, :geocoded_lon, {geom_expr},
|
||
:source_year, :source_url, CAST(:raw_row AS jsonb)
|
||
)
|
||
ON CONFLICT (permit_type, permit_number) DO UPDATE SET
|
||
issue_date = EXCLUDED.issue_date,
|
||
expiry_date = EXCLUDED.expiry_date,
|
||
developer_inn = EXCLUDED.developer_inn,
|
||
developer_name = EXCLUDED.developer_name,
|
||
object_name = EXCLUDED.object_name,
|
||
object_type = EXCLUDED.object_type,
|
||
construction_address = EXCLUDED.construction_address,
|
||
cadastral_number = EXCLUDED.cadastral_number,
|
||
total_area_sqm = EXCLUDED.total_area_sqm,
|
||
living_area_sqm = EXCLUDED.living_area_sqm,
|
||
living_area_fact_sqm = EXCLUDED.living_area_fact_sqm,
|
||
rve_number = EXCLUDED.rve_number,
|
||
rve_date = EXCLUDED.rve_date,
|
||
raw_coord_x = EXCLUDED.raw_coord_x,
|
||
raw_coord_y = EXCLUDED.raw_coord_y,
|
||
geocoded_lat = EXCLUDED.geocoded_lat,
|
||
geocoded_lon = EXCLUDED.geocoded_lon,
|
||
geom = EXCLUDED.geom,
|
||
raw_row = EXCLUDED.raw_row,
|
||
fetched_at = NOW()
|
||
"""),
|
||
{
|
||
"permit_type": row.permit_type,
|
||
"permit_number": row.permit_number,
|
||
"issue_date": row.issue_date,
|
||
"expiry_date": row.expiry_date,
|
||
"developer_inn": row.developer_inn,
|
||
"developer_name": row.developer_name,
|
||
"object_name": row.object_name,
|
||
"object_type": row.object_type,
|
||
"construction_address": row.construction_address,
|
||
"cadastral_number": row.cadastral_number,
|
||
"total_area_sqm": row.total_area_sqm,
|
||
"living_area_sqm": row.living_area_sqm,
|
||
"living_area_fact_sqm": row.living_area_fact_sqm,
|
||
"rve_number": row.rve_number,
|
||
"rve_date": row.rve_date,
|
||
"raw_coord_x": row.raw_coord_x,
|
||
"raw_coord_y": row.raw_coord_y,
|
||
"geocoded_lat": row.geocoded_lat,
|
||
"geocoded_lon": row.geocoded_lon,
|
||
"source_year": row.source_year,
|
||
"source_url": row.source_url,
|
||
"raw_row": json.dumps(row.raw_row, ensure_ascii=False),
|
||
},
|
||
)
|
||
|
||
|
||
def backfill_geom(db: Session | None = None) -> dict[str, int]:
|
||
"""Бэкфилл geom/geocoded_lat/lon для существующих строк с raw_coord_x/y но без geom.
|
||
|
||
Выполняется разово (admin-trigger или параметр refresh). Использует батч-UPDATE
|
||
с SAVEPOINT per batch для устойчивости.
|
||
|
||
Возвращает {"updated": N, "skipped": N, "errors": N}.
|
||
"""
|
||
own_session = db is None
|
||
if own_session:
|
||
db = SessionLocal()
|
||
|
||
updated = 0
|
||
skipped = 0
|
||
errors = 0
|
||
|
||
try:
|
||
# Выбираем строки с координатами, у которых geom пуст
|
||
rows = db.execute( # type: ignore[union-attr]
|
||
text("""
|
||
SELECT id, raw_coord_x, raw_coord_y
|
||
FROM ekburg_construction_permits
|
||
WHERE raw_coord_x IS NOT NULL
|
||
AND raw_coord_y IS NOT NULL
|
||
AND geom IS NULL
|
||
ORDER BY id
|
||
""")
|
||
).fetchall()
|
||
|
||
logger.info("ekburg_permits backfill_geom: %d строк для обработки", len(rows))
|
||
|
||
batch: list[dict] = []
|
||
|
||
def _flush_batch(b: list[dict]) -> tuple[int, int]:
|
||
u = e = 0
|
||
try:
|
||
with db.begin_nested(): # type: ignore[union-attr]
|
||
for params in b:
|
||
db.execute( # type: ignore[union-attr]
|
||
text("""
|
||
UPDATE ekburg_construction_permits
|
||
SET
|
||
geocoded_lat = CAST(:lat AS double precision),
|
||
geocoded_lon = CAST(:lon AS double precision),
|
||
geom = ST_SetSRID(
|
||
ST_MakePoint(
|
||
CAST(:lon AS double precision),
|
||
CAST(:lat AS double precision)
|
||
),
|
||
4326
|
||
),
|
||
fetched_at = NOW()
|
||
WHERE id = CAST(:id AS bigint)
|
||
"""),
|
||
params,
|
||
)
|
||
u += 1
|
||
except Exception as exc:
|
||
logger.warning("backfill_geom: batch failed: %s", exc)
|
||
e += len(b)
|
||
u = 0
|
||
return u, e
|
||
|
||
for row in rows:
|
||
coords = msk66_to_wgs84(row.raw_coord_x, row.raw_coord_y)
|
||
if coords is None:
|
||
skipped += 1
|
||
continue
|
||
|
||
lon, lat = coords
|
||
batch.append({"id": row.id, "lon": lon, "lat": lat})
|
||
|
||
if len(batch) >= _BACKFILL_BATCH_SIZE:
|
||
u, e = _flush_batch(batch)
|
||
updated += u
|
||
errors += e
|
||
batch = []
|
||
|
||
if batch:
|
||
u, e = _flush_batch(batch)
|
||
updated += u
|
||
errors += e
|
||
|
||
db.commit() # type: ignore[union-attr]
|
||
|
||
except Exception as exc:
|
||
logger.error("backfill_geom: unexpected error: %s", exc)
|
||
raise
|
||
finally:
|
||
if own_session:
|
||
db.close() # type: ignore[union-attr]
|
||
|
||
logger.info(
|
||
"ekburg_permits backfill_geom done: updated=%d skipped=%d errors=%d",
|
||
updated,
|
||
skipped,
|
||
errors,
|
||
)
|
||
return {"updated": updated, "skipped": skipped, "errors": errors}
|
||
|
||
|
||
@celery_app.task(name="tasks.ekburg_permits_sync.refresh_year", queue="celery")
|
||
def refresh_year(year: int) -> dict[str, int]:
|
||
"""Скачать + распарсить + upsert РНС/РВЭ за один год.
|
||
|
||
Возвращает {"inserted": N, "errors": N}.
|
||
"""
|
||
inserted = 0
|
||
errors = 0
|
||
|
||
url = EKBURG_PERMITS_URLS.get(year)
|
||
if not url:
|
||
logger.error("ekburg_permits_sync: no URL for year=%d", year)
|
||
return {"inserted": 0, "errors": 1}
|
||
|
||
with EkburgPermitsClient() as client:
|
||
try:
|
||
content = client.download_xlsx(year)
|
||
except Exception as exc:
|
||
logger.error("ekburg_permits_sync: download failed year=%d: %s", year, exc)
|
||
return {"inserted": 0, "errors": 1}
|
||
|
||
with SessionLocal() as db:
|
||
for row in client.parse_xlsx(content, year, url):
|
||
try:
|
||
with db.begin_nested():
|
||
_upsert_permit(db, row)
|
||
inserted += 1
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"ekburg_permits_sync: upsert failed %s/%s year=%d: %s",
|
||
row.permit_type,
|
||
row.permit_number,
|
||
year,
|
||
exc,
|
||
)
|
||
errors += 1
|
||
db.commit()
|
||
|
||
logger.info(
|
||
"ekburg_permits_sync: year=%d done inserted=%d errors=%d",
|
||
year,
|
||
inserted,
|
||
errors,
|
||
)
|
||
return {"inserted": inserted, "errors": errors}
|
||
|
||
|
||
@celery_app.task(name="tasks.ekburg_permits_sync.refresh_all", queue="celery")
|
||
def refresh_all() -> dict[str, dict[str, int]]:
|
||
"""Обновить все 5 лет (2022-2026). Планируется через Celery beat ежемесячно."""
|
||
results: dict[str, dict[str, int]] = {}
|
||
for year in sorted(EKBURG_PERMITS_URLS.keys()):
|
||
logger.info("ekburg_permits_sync: starting year=%d", year)
|
||
results[str(year)] = refresh_year(year)
|
||
logger.info("ekburg_permits_sync: refresh_all done: %s", results)
|
||
return results
|
||
|
||
|
||
@celery_app.task(name="tasks.ekburg_permits_sync.backfill_geom_task", queue="celery")
|
||
def backfill_geom_task() -> dict[str, int]:
|
||
"""Разовый бэкфилл geom для уже загруженных строк без geom.
|
||
|
||
Запускать вручную через admin или celery CLI:
|
||
celery -A app.workers.celery_app call tasks.ekburg_permits_sync.backfill_geom_task
|
||
"""
|
||
return backfill_geom()
|