gendesign/backend/app/services/objective_etl.py

"""ETL: SQLite Антона (/sf/) → наша PG (objective_* таблицы).

Reusable библиотека вокруг логики `data/sql/71_etl_anton_sqlite_to_pg.py`.
Используется Celery-task `tasks.objective_etl.import_anton_objective` и
admin-endpoint `POST /api/v1/admin/scrape/objective`.

Файл-источник по умолчанию: settings.objective_anton_sqlite_path
(на проде смонтирован bind-mount-ом docker-compose из /opt/gendesign/site-finder).

Стратегия:
  - lots:    INSERT … ON CONFLICT (objective_lot_id) DO UPDATE
  - crm:     INSERT … ON CONFLICT (report_month, group_name, project_name,
                                   corpus_name, room_bucket) DO UPDATE
  - mapping: дедупликация по (objective_complex_name, objective_group),
             потом INSERT … ON CONFLICT DO UPDATE

Heartbeat / прогресс (для UI) выставляется через callback `progress_cb`,
который Celery-task передаёт со UPDATE objective_scrape_runs.
"""

from __future__ import annotations

import logging
import re
import sqlite3
from collections.abc import Callable
from contextlib import closing
from datetime import date
from pathlib import Path
from typing import Any

import psycopg

logger = logging.getLogger(__name__)


def _bulk_upsert(cur: psycopg.Cursor, sql_template: str, rows: list[tuple]) -> None:
    """Эмулирует psycopg2.extras.execute_values для psycopg v3.

    `sql_template` должен содержать литеральное `VALUES %s` — заменим
    на `VALUES (%s, %s, ..., %s)` с числом placeholder'ов = len(rows[0])
    и пошлём через executemany. В psycopg3 executemany использует
    pipeline-mode, так что overhead на сетевые round-trips минимален.
    """
    if not rows:
        return
    n_cols = len(rows[0])
    values_placeholder = "(" + ",".join(["%s"] * n_cols) + ")"
    sql = sql_template.replace("VALUES %s", "VALUES " + values_placeholder)
    cur.executemany(sql, rows)


GROUP_NAME = "Екатеринбург"  # У Антона в SQLite нет колонки group_name


# ── helpers ─────────────────────────────────────────────────────────────────


def _parse_date(s) -> date | None:
    if s is None or s == "":
        return None
    try:
        return date.fromisoformat(str(s)[:10])
    except (ValueError, TypeError):
        return None


def _parse_month_to_first_day(s) -> date | None:
    if not s:
        return None
    try:
        y, m = str(s).split("-")
        return date(int(y), int(m), 1)
    except (ValueError, TypeError):
        return None


def _normalize_rooms(s) -> int | None:
    if not s:
        return None
    sl = str(s).strip().lower()
    if "студ" in sl:
        return 0
    m = re.match(r"^(\d+)", sl)
    if m:
        return min(int(m.group(1)), 5)
    return None


def _bool_from_yes(v) -> bool | None:
    if v is None or v == "":
        return None
    s = str(v).strip().lower()
    if s in ("да", "true", "1", "yes"):
        return True
    if s in ("нет", "false", "0", "no"):
        return False
    return None


def _site_id_to_obj_id(s) -> int | None:
    """'jk:49854' → 49854. 'parcel:...' → None."""
    if not s:
        return None
    s = str(s)
    if s.startswith("jk:"):
        try:
            return int(s[3:])
        except ValueError:
            return None
    return None


def _to_int(v) -> int | None:
    if v is None or v == "":
        return None
    try:
        return int(v)
    except (TypeError, ValueError):
        return None


def _to_num(v) -> float | None:
    if v is None or v == "":
        return None
    try:
        return float(v)
    except (TypeError, ValueError):
        return None


# ── row mappings ────────────────────────────────────────────────────────────


def _row_to_lot(r: sqlite3.Row, snapshot_date: date) -> tuple:
    return (
        r["lot_id"],
        r["project_id"],
        r["project"],
        r["developer"],
        r["city"],
        r["district"],
        r["corpus"],
        r["address"],
        r["obj_class"],
        r["section"],
        _to_int(r["floor"]),
        r["lot_num"],
        r["room_kind"],
        _parse_date(r["sales_start"]),
        _parse_date(r["plan_date"]),
        _parse_date(r["fact_date"]),
        _to_int(r["readiness_pct"]),
        r["construction_stage"],
        r["finish_type"],
        r["status"],
        _bool_from_yes(r["sold"]),
        r["rooms_dev"],
        r["rooms_pd"],
        r["rooms_obj"],
        _normalize_rooms(r["rooms_obj"]),
        _to_num(r["area_dev"]),
        _to_num(r["area_pd"]),
        _to_num(r["budget_rub"]),
        _to_num(r["price_per_m2"]),
        _to_num(r["offer_price"]),
        _to_num(r["delta_price_rub"]),
        _to_num(r["delta_price_pct"]),
        r["price_method"],
        _parse_date(r["price_set_date"]),
        _parse_date(r["price_actual_date"]),
        _parse_date(r["contract_date"]),
        _parse_date(r["register_date"]),
        r["deal_type"],
        r["buyer_type"],
        r["register_num"],
        r["encumbrance"],
        r["bank"],
        _parse_date(r["encumbrance_start"]),
        _parse_date(r["egrn_actual_date"]),
        snapshot_date,
    )


_LOT_INSERT = """
INSERT INTO objective_lots (
    objective_lot_id, objective_project_id, project_name, developer, city,
    district, corpus_name, address, class, section, floor, lot_number,
    premise_kind, sales_start_date, plan_completion_date, actual_completion_date,
    readiness_pct, construction_stage, finishing, status, is_sold,
    rooms_dev_site, rooms_pd, rooms_objective, rooms_int,
    area_dev_site, area_pd,
    price_calculated_total_rub, price_per_m2_rub, price_offer_total_rub,
    price_delta_rub, price_delta_pct, pricing_method, price_set_date,
    price_actual_date, contract_date, registration_date, contract_type,
    buyer_type, registration_number, encumbrance_type, bank_name,
    encumbrance_start_date, egrn_actual_date, snapshot_date
) VALUES %s
ON CONFLICT (objective_lot_id) DO UPDATE SET
    project_name = EXCLUDED.project_name,
    developer = EXCLUDED.developer, district = EXCLUDED.district,
    corpus_name = EXCLUDED.corpus_name, address = EXCLUDED.address,
    class = EXCLUDED.class, section = EXCLUDED.section, floor = EXCLUDED.floor,
    lot_number = EXCLUDED.lot_number, premise_kind = EXCLUDED.premise_kind,
    sales_start_date = EXCLUDED.sales_start_date,
    plan_completion_date = EXCLUDED.plan_completion_date,
    actual_completion_date = EXCLUDED.actual_completion_date,
    readiness_pct = EXCLUDED.readiness_pct,
    construction_stage = EXCLUDED.construction_stage,
    finishing = EXCLUDED.finishing,
    status = EXCLUDED.status, is_sold = EXCLUDED.is_sold,
    rooms_dev_site = EXCLUDED.rooms_dev_site,
    rooms_pd = EXCLUDED.rooms_pd,
    rooms_objective = EXCLUDED.rooms_objective,
    rooms_int = EXCLUDED.rooms_int,
    area_dev_site = EXCLUDED.area_dev_site, area_pd = EXCLUDED.area_pd,
    price_calculated_total_rub = EXCLUDED.price_calculated_total_rub,
    price_per_m2_rub = EXCLUDED.price_per_m2_rub,
    price_offer_total_rub = EXCLUDED.price_offer_total_rub,
    price_delta_rub = EXCLUDED.price_delta_rub,
    price_delta_pct = EXCLUDED.price_delta_pct,
    pricing_method = EXCLUDED.pricing_method,
    price_set_date = EXCLUDED.price_set_date,
    price_actual_date = EXCLUDED.price_actual_date,
    contract_date = EXCLUDED.contract_date,
    registration_date = EXCLUDED.registration_date,
    contract_type = EXCLUDED.contract_type, buyer_type = EXCLUDED.buyer_type,
    registration_number = EXCLUDED.registration_number,
    encumbrance_type = EXCLUDED.encumbrance_type,
    bank_name = EXCLUDED.bank_name,
    encumbrance_start_date = EXCLUDED.encumbrance_start_date,
    egrn_actual_date = EXCLUDED.egrn_actual_date,
    snapshot_date = EXCLUDED.snapshot_date,
    updated_at = NOW()
"""


def _row_to_crm(r: sqlite3.Row) -> tuple:
    return (
        _parse_month_to_first_day(r["month"]),
        GROUP_NAME,
        r["project"],
        r["developer"],
        r["district"],
        r["obj_class"],
        r["corpus"],
        r["rooms_bucket"],
        _normalize_rooms(r["rooms_bucket"]),
        _parse_date(r["sales_start"]),
        _parse_date(r["plan_date"]),
        _parse_date(r["fact_date"]),
        _to_int(r["months_in_sales"]),
        _to_int(r["lots_pd"]),
        _to_num(r["area_pd"]),
        # ДДУ — у Антона нет
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        # ДКП
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        # Всего
        _to_int(r["deals_total"]),
        _to_int(r["deals_priced"]),
        _to_num(r["sold_volume_m2"]),
        _to_num(r["sold_volume_m2"]),
        None,
        _to_num(r["avg_price_m2"]),
        _to_num(r["avg_area_m2"]),
        # Offer
        _to_int(r["stock_lots"]),
        _to_num(r["stock_m2"]),
        None,
        _to_num(r["stock_avg_price_m2"]),
    )


_CRM_INSERT = """
INSERT INTO objective_corpus_room_month (
    report_month, group_name, project_name, developer, district, class,
    corpus_name, room_bucket, rooms_int,
    sales_start_date, plan_completion_date, actual_completion_date,
    months_in_realization, lots_pd_count, lots_pd_area,
    deals_ddu_count, deals_ddu_count_priced, deals_ddu_vol_m2,
    deals_ddu_vol_m2_priced, deals_ddu_sum_mln_rub,
    deals_ddu_avg_price_thousand_rub_per_m2, deals_ddu_avg_area_m2,
    deals_dkp_count, deals_dkp_count_priced, deals_dkp_vol_m2,
    deals_dkp_vol_m2_priced, deals_dkp_sum_mln_rub,
    deals_dkp_avg_price_thousand_rub_per_m2, deals_dkp_avg_area_m2,
    deals_total_count, deals_total_count_priced, deals_total_vol_m2,
    deals_total_vol_m2_priced, deals_total_sum_mln_rub,
    deals_total_avg_price_thousand_rub_per_m2, deals_total_avg_area_m2,
    offer_count, offer_area_m2, offer_sum_mln_rub,
    offer_avg_price_thousand_rub_per_m2
) VALUES %s
ON CONFLICT (report_month, group_name, project_name, corpus_name, room_bucket)
DO UPDATE SET
    developer = EXCLUDED.developer, district = EXCLUDED.district,
    class = EXCLUDED.class, rooms_int = EXCLUDED.rooms_int,
    sales_start_date = EXCLUDED.sales_start_date,
    plan_completion_date = EXCLUDED.plan_completion_date,
    actual_completion_date = EXCLUDED.actual_completion_date,
    months_in_realization = EXCLUDED.months_in_realization,
    lots_pd_count = EXCLUDED.lots_pd_count,
    lots_pd_area = EXCLUDED.lots_pd_area,
    deals_total_count = EXCLUDED.deals_total_count,
    deals_total_count_priced = EXCLUDED.deals_total_count_priced,
    deals_total_vol_m2 = EXCLUDED.deals_total_vol_m2,
    deals_total_vol_m2_priced = EXCLUDED.deals_total_vol_m2_priced,
    deals_total_avg_price_thousand_rub_per_m2 = EXCLUDED.deals_total_avg_price_thousand_rub_per_m2,
    deals_total_avg_area_m2 = EXCLUDED.deals_total_avg_area_m2,
    offer_count = EXCLUDED.offer_count,
    offer_area_m2 = EXCLUDED.offer_area_m2,
    offer_avg_price_thousand_rub_per_m2 = EXCLUDED.offer_avg_price_thousand_rub_per_m2,
    fetched_at = NOW()
"""


def _row_to_mapping(r: sqlite3.Row) -> tuple | None:
    obj_id = _site_id_to_obj_id(r["site_id"])
    if obj_id is None:
        return None
    return (
        r["project"],
        None,
        GROUP_NAME,
        obj_id,
        False,
        r["method"],
        _to_num(r["score"]),
        "imported_from_anton_sf",
    )


_MAP_INSERT = """
INSERT INTO objective_complex_mapping (
    objective_complex_name, objective_project_id, objective_group,
    domrf_obj_id, is_reviewed, match_method, match_score, note
) VALUES %s
ON CONFLICT (objective_complex_name, objective_group) DO UPDATE SET
    domrf_obj_id = EXCLUDED.domrf_obj_id,
    match_method = EXCLUDED.match_method,
    match_score = EXCLUDED.match_score,
    note = EXCLUDED.note,
    updated_at = NOW()
"""


# ── orchestration ───────────────────────────────────────────────────────────


def _etl_lots(
    sl: sqlite3.Connection, pg, batch_size: int, progress_cb: Callable[[int], None] | None
) -> int:
    snap = date.today()
    cur = sl.execute("SELECT * FROM objective_lots")
    n = 0
    batch: list[tuple] = []
    pg_cur = pg.cursor()
    while True:
        rows = cur.fetchmany(batch_size)
        if not rows:
            break
        for r in rows:
            batch.append(_row_to_lot(r, snap))
            if len(batch) >= batch_size:
                _bulk_upsert(pg_cur, _LOT_INSERT, batch)
                pg.commit()
                n += len(batch)
                batch.clear()
                if progress_cb and n % (batch_size * 5) == 0:
                    progress_cb(n)
    if batch:
        _bulk_upsert(pg_cur, _LOT_INSERT, batch)
        pg.commit()
        n += len(batch)
    pg_cur.close()
    if progress_cb:
        progress_cb(n)
    return n


def _etl_crm(
    sl: sqlite3.Connection, pg, batch_size: int, progress_cb: Callable[[int], None] | None
) -> tuple[int, int]:
    cur = sl.execute("SELECT * FROM objective_corp_month")
    n = 0
    skipped = 0
    batch: list[tuple] = []
    pg_cur = pg.cursor()
    while True:
        rows = cur.fetchmany(batch_size)
        if not rows:
            break
        for r in rows:
            t = _row_to_crm(r)
            if not t[0] or not t[2] or not t[6] or not t[7]:
                skipped += 1
                continue
            batch.append(t)
            if len(batch) >= batch_size:
                _bulk_upsert(pg_cur, _CRM_INSERT, batch)
                pg.commit()
                n += len(batch)
                batch.clear()
    if batch:
        _bulk_upsert(pg_cur, _CRM_INSERT, batch)
        pg.commit()
        n += len(batch)
    pg_cur.close()
    if progress_cb:
        progress_cb(n)
    return n, skipped


def _etl_mapping(sl: sqlite3.Connection, pg) -> tuple[int, int]:
    cur = sl.execute("SELECT * FROM jk_objective_match ORDER BY score DESC NULLS LAST")
    rows = cur.fetchall()
    seen: set[tuple] = set()
    batch: list[tuple] = []
    skipped_dups = 0
    for r in rows:
        t = _row_to_mapping(r)
        if t is None:
            continue
        key = (t[0], t[2])  # (complex_name, group)
        if key in seen:
            skipped_dups += 1
            continue
        seen.add(key)
        batch.append(t)
    if not batch:
        return 0, skipped_dups
    pg_cur = pg.cursor()
    _bulk_upsert(pg_cur, _MAP_INSERT, batch)
    pg.commit()
    pg_cur.close()
    return len(batch), skipped_dups


# ── public API ──────────────────────────────────────────────────────────────


def get_sqlite_info(sqlite_path: str | Path) -> dict[str, Any]:
    """Метаданные SQLite-файла без полного запуска ETL.

    Используется в admin-coverage endpoint чтобы UI знал свежесть данных,
    размер файла и сколько потенциально rows будет залито.
    """
    p = Path(sqlite_path)
    info: dict[str, Any] = {
        "path": str(p),
        "exists": p.exists(),
    }
    if not p.exists():
        return info
    # stat() под защитой (#2464): между exists() и stat() файл может исчезнуть —
    # его переписывает выгрузка Объектива. Раньше try/except покрывал только
    # sqlite3.connect ниже, и OSError отсюда улетал наружу, превращая
    # диагностическую функцию в источник отказа. Отдаём то, что успели узнать.
    try:
        st = p.stat()
    except OSError as e:
        info["stat_error"] = f"{type(e).__name__}: {e}"
        return info
    info["size_bytes"] = st.st_size
    info["modified_at"] = st.st_mtime  # epoch seconds
    try:
        with closing(sqlite3.connect(p)) as c:
            info["lots"] = c.execute("SELECT COUNT(*) FROM objective_lots").fetchone()[0]
            info["corp_room_month"] = c.execute(
                "SELECT COUNT(*) FROM objective_corp_month"
            ).fetchone()[0]
            info["mappings"] = c.execute("SELECT COUNT(*) FROM jk_objective_match").fetchone()[0]
    except sqlite3.Error as e:
        info["error"] = str(e)
    return info


def run_etl(
    sqlite_path: str | Path,
    db_url: str,
    *,
    batch_size: int = 2000,
    progress_cb: Callable[[str, int], None] | None = None,
) -> dict[str, int]:
    """Полный ETL: SQLite → PG.

    Args:
        sqlite_path: путь к analysis.db (на проде — bind-mount /data/anton-sqlite/...)
        db_url: postgresql://… connection string
        batch_size: размер batch для execute_values (default 2000)
        progress_cb: optional callback(stage: str, n_rows: int) для UI прогресса

    Returns:
        dict с counts: {'lots', 'corp_room_month', 'mappings', 'crm_skipped',
                        'mapping_dups_skipped'}

    Raises:
        FileNotFoundError если SQLite не найден
        psycopg.Error при ошибках БД
    """
    p = Path(sqlite_path)
    if not p.exists():
        raise FileNotFoundError(f"SQLite не найден: {p}")

    logger.info("ETL Anton SQLite → PG: %s", p)
    sl = sqlite3.connect(p)
    sl.row_factory = sqlite3.Row
    pg = psycopg.connect(db_url)
    try:
        if progress_cb:
            progress_cb("mapping_start", 0)
        n_map, dups = _etl_mapping(sl, pg)
        logger.info("  mapping: %d upserted, %d dups skipped", n_map, dups)
        if progress_cb:
            progress_cb("mapping_done", n_map)

        if progress_cb:
            progress_cb("crm_start", 0)
        n_crm, crm_skipped = _etl_crm(
            sl,
            pg,
            batch_size,
            lambda n: progress_cb("crm_progress", n) if progress_cb else None,
        )
        logger.info("  crm: %d upserted, %d skipped", n_crm, crm_skipped)
        if progress_cb:
            progress_cb("crm_done", n_crm)

        if progress_cb:
            progress_cb("lots_start", 0)
        n_lots = _etl_lots(
            sl,
            pg,
            batch_size,
            lambda n: progress_cb("lots_progress", n) if progress_cb else None,
        )
        logger.info("  lots: %d upserted", n_lots)
        if progress_cb:
            progress_cb("lots_done", n_lots)
    finally:
        sl.close()
        pg.close()

    return {
        "lots": n_lots,
        "corp_room_month": n_crm,
        "mappings": n_map,
        "crm_skipped": crm_skipped,
        "mapping_dups_skipped": dups,
    }