gendesign/site-finder/05_objective_pull.py

"""Step 1.1: Pull Objective API and store in local DB.

We pull "Сводные/Корпуса" (corp_sum) for Ekb covering last 12 calendar months —
this is the primary source of:
  - per-corpus monthly: deals (DDU+DKP), volume sold (m²), avg price/m², stock left
  - готовность %, старт продаж, планируемая дата ввода
  - район (Objective's classification)

Data lands in two tables:
  objective_raw_reports — JSON payload archive (1 row per fetch)
  objective_corp_month — flattened rows (one per month × project × corpus × room-type)
"""
import sqlite3, pathlib, requests, json, time, datetime as dt

DB = pathlib.Path(__file__).parent / "analysis.db"
API = "https://api.objctv.ru"
KEY = "623f6a57-0179-434b-8202-259525bdc77c"

SCHEMA = """
CREATE TABLE IF NOT EXISTS objective_raw_reports (
    id            INTEGER PRIMARY KEY AUTOINCREMENT,
    fetched_at    TEXT DEFAULT CURRENT_TIMESTAMP,
    report_kind   TEXT,
    group_name    TEXT,
    start_date    TEXT,
    end_date      TEXT,
    n_rows        INTEGER,
    payload       TEXT
);

CREATE TABLE IF NOT EXISTS objective_corp_month (
    month         TEXT,                  -- 'YYYY-MM'
    project       TEXT,
    developer     TEXT,
    district      TEXT,
    obj_class     TEXT,
    corpus        TEXT,
    sales_start   TEXT,
    plan_date     TEXT,
    fact_date     TEXT,
    months_in_sales INTEGER,
    rooms_bucket  TEXT,                  -- '1', '2', '3', '4+', 'студия', etc.
    lots_pd       INTEGER,
    area_pd       REAL,
    deals_total   INTEGER,
    deals_priced  INTEGER,
    sold_volume_m2 REAL,
    avg_price_m2  REAL,
    avg_area_m2   REAL,
    stock_lots    INTEGER,
    stock_m2      REAL,
    stock_avg_price_m2 REAL,
    PRIMARY KEY (month, project, corpus, rooms_bucket)
);
CREATE INDEX IF NOT EXISTS oc_district ON objective_corp_month(district);
CREATE INDEX IF NOT EXISTS oc_project  ON objective_corp_month(project);
"""

def get_token():
    r = requests.get(f"{API}/Users/User/GetToken", params={"apiKey": KEY}, timeout=30)
    r.raise_for_status()
    return r.json()["token"]

def fetch_corp_sum(token, group, sd, ed):
    r = requests.get(f"{API}/v2/Report/GetReport",
        params={"Page":"Отчеты","ReportSection":"Объединенные данные",
                "ReportType":"Сводные","ReportName":"Корпуса",
                "GroupName":group,"StartDate":sd,"EndDate":ed,
                "UseDdu":"true","UseDkp":"true"},
        headers={"Authorization":f"Bearer {token}","Accept-Encoding":"br"},
        timeout=120)
    r.raise_for_status()
    return r.json().get("result", [])

# Map raw RU column names to our schema
COL_MAP = {
    "month":"Месяц","project":"Проект","developer":"Девелопер","district":"Район",
    "obj_class":"Класс","corpus":"Корпус","sales_start":"Старт продаж",
    "plan_date":"Планируемая дата ввода","fact_date":"Фактическая дата ввода",
    "months_in_sales":"Месяцев в реализации","rooms_bucket":"Количество комнат (Данные Объектива)",
    "lots_pd":"Лотов по ПД, шт.","area_pd":"Площадь по ПД, м2.",
    "deals_total":"Количество в сделках (всего), шт.",
    "deals_priced":"Количество лотов в сделках (с ценами), шт.",
    "sold_volume_m2":"Объем реализации (всего), м2.",
    "avg_price_m2":"Средняя цена м2 лота в сделках, тыс.Р/м2",
    "avg_area_m2":"Средняя площадь лота в сделках, м2",
    "stock_lots":"Объем предложения, шт.","stock_m2":"Объем предложения, м2.",
    "stock_avg_price_m2":"Средняя цена м2 лота в продаже, тыс.Р/м2",
}

RU_MONTHS = {"январь":1,"февраль":2,"март":3,"апрель":4,"май":5,"июнь":6,
             "июль":7,"август":8,"сентябрь":9,"октябрь":10,"ноябрь":11,"декабрь":12}

def normalize_month(s):
    # 'апрель-2026' → '2026-04'
    if not s: return None
    parts = str(s).lower().replace("—","-").split("-")
    if len(parts) != 2: return s
    m = RU_MONTHS.get(parts[0].strip())
    y = parts[1].strip()
    return f"{y}-{m:02d}" if m else s

def main():
    conn = sqlite3.connect(DB)
    conn.executescript(SCHEMA)

    today = dt.date.today()
    sd = (today.replace(day=1) - dt.timedelta(days=365)).strftime("%Y.%m.%d")
    ed = today.strftime("%Y.%m.%d")
    group = "Екатеринбург"

    print(f"Fetching corp_sum {group} {sd}..{ed}")
    token = get_token()
    rows = fetch_corp_sum(token, group, sd, ed)
    print(f"  {len(rows)} rows")

    conn.execute("INSERT INTO objective_raw_reports(report_kind,group_name,start_date,end_date,n_rows,payload) VALUES (?,?,?,?,?,?)",
                 ("corp_sum_v2", group, sd, ed, len(rows), json.dumps(rows, ensure_ascii=False)))

    conn.execute("DELETE FROM objective_corp_month")
    inserted = skipped = 0
    for r in rows:
        try:
            vals = {k: r.get(v) for k, v in COL_MAP.items()}
            vals["month"] = normalize_month(vals["month"])
            cols = list(vals.keys())
            placeholders = ",".join(["?"] * len(cols))
            conn.execute(f"INSERT OR REPLACE INTO objective_corp_month({','.join(cols)}) VALUES ({placeholders})",
                         [vals[c] for c in cols])
            inserted += 1
        except Exception as e:
            skipped += 1
    conn.commit()
    print(f"  inserted: {inserted}, skipped: {skipped}")

    # Sanity
    n = conn.execute("SELECT count(*) FROM objective_corp_month").fetchone()[0]
    n_proj = conn.execute("SELECT count(DISTINCT project) FROM objective_corp_month").fetchone()[0]
    n_dist = conn.execute("SELECT count(DISTINCT district) FROM objective_corp_month").fetchone()[0]
    print(f"\nLocal: {n} rows, {n_proj} проекта, {n_dist} районов")
    print("\nDistricts (Objective):")
    for r in conn.execute("""SELECT district, count(*) c, count(DISTINCT project) np
                             FROM objective_corp_month GROUP BY 1 ORDER BY 3 DESC""").fetchall():
        print(f"  {r[0]:<30} rows={r[1]:>5}  projects={r[2]}")
    conn.close()

if __name__ == "__main__":
    main()