"""Pull every available Objective report combination for Ekb. The API has 4 reports = ReportType × ReportName: Сводные / Корпуса — already pulled in 05_* Сводные / Лоты — probe Поквартирные / Корпуса — probe Поквартирные / Лоты — full per-flat data (most valuable) Earlier server graph said 2 of 4 returned HTTP 500. We probe again, then bulk-pull whatever returns 200. """ import sqlite3, pathlib, requests, time, json, datetime as dt, sys DB = pathlib.Path(__file__).parent / "analysis.db" RAW = pathlib.Path(__file__).parent / "cache" / "objective_raw" RAW.mkdir(parents=True, exist_ok=True) API = "https://api.objctv.ru" KEY = "623f6a57-0179-434b-8202-259525bdc77c" GROUP = "Екатеринбург" TODAY = dt.date.today() SD = (TODAY.replace(day=1) - dt.timedelta(days=365)).strftime("%Y.%m.%d") ED = TODAY.strftime("%Y.%m.%d") EXTRA_SCHEMA = """ -- per-flat snapshot (Поквартирные/Лоты) CREATE TABLE IF NOT EXISTS objective_lots ( lot_id INTEGER PRIMARY KEY, -- field "Id" project_id INTEGER, project TEXT, developer TEXT, city TEXT, district TEXT, corpus TEXT, address TEXT, obj_class TEXT, sales_start TEXT, plan_date TEXT, fact_date TEXT, readiness_pct REAL, construction_stage TEXT, finish_type TEXT, section TEXT, floor INTEGER, lot_num TEXT, room_kind TEXT, status TEXT, sold TEXT, rooms_dev TEXT, rooms_pd TEXT, rooms_obj TEXT, area_dev REAL, area_pd REAL, budget_rub REAL, price_per_m2 REAL, price_method TEXT, price_set_date TEXT, price_actual_date TEXT, offer_price REAL, delta_price_rub REAL, delta_price_pct REAL, contract_date TEXT, register_date TEXT, deal_type TEXT, buyer_type TEXT, register_num TEXT, encumbrance TEXT, bank TEXT, encumbrance_start TEXT, egrn_actual_date TEXT, fetched_at TEXT DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS lots_proj ON objective_lots(project); CREATE INDEX IF NOT EXISTS lots_dist ON objective_lots(district); CREATE INDEX IF NOT EXISTS lots_bank ON objective_lots(bank); CREATE INDEX IF NOT EXISTS lots_status ON objective_lots(status); """ # Mapping for per-flat (Поквартирные/Лоты) LOTS_MAP = { "lot_id":"Id","project_id":"Id проекта","project":"Проект","developer":"Девелопер", "city":"Город","district":"Район","corpus":"Корпус","address":"Адрес", "obj_class":"Класс","sales_start":"Старт продаж", "plan_date":"Планируемая дата ввода","fact_date":"Фактическая дата ввода", "readiness_pct":"Готовность","construction_stage":"Стадия строительства", "finish_type":"Отделка по корпусу","section":"Секция","floor":"Этаж", "lot_num":"Номер лота","room_kind":"Вид помещения","status":"Статус", "sold":"Продано","rooms_dev":"Количество комнат(Сайт девелопера)", "rooms_pd":"Количество комнат(ПД)","rooms_obj":"Количество комнат(Данные объектива)", "area_dev":"Площадь, м2(Сайт девелопера)","area_pd":"Площадь, м2(ПД)", "budget_rub":"Расчетный бюджет лота, Р","price_per_m2":"Цена за м2, Р", "price_method":"Способ определения цены","price_set_date":"Дата установки цены", "price_actual_date":"Дата актуальности цены","offer_price":"Цена предложения, Р", "delta_price_rub":"Дельта цена, Р","delta_price_pct":"Дельта цена, %", "contract_date":"Дата договора","register_date":"Дата регистрации", "deal_type":"Тип сделки","buyer_type":"Тип покупателя","register_num":"Номер регистрации", "encumbrance":"Тип обременения","bank":"Банк","encumbrance_start":"Дата начала обременения", "egrn_actual_date":"Дата актуальности данных из ЕГРН", } def get_token(): r = requests.get(f"{API}/Users/User/GetToken", params={"apiKey": KEY}, timeout=30) r.raise_for_status() return r.json()["token"] def fetch(token, params, attempts=5): """Fetch with backoff for 429/5xx.""" last = None for i in range(attempts): r = requests.get(f"{API}/v2/Report/GetReport", params=params, headers={"Authorization": f"Bearer {token}", "Accept-Encoding": "br"}, timeout=180) if r.ok: return r last = (r.status_code, r.text[:200]) # If 401 — refresh token if r.status_code == 401: token = get_token() continue wait = int(r.headers.get("Retry-After") or (10 * (2 ** i))) print(f" HTTP {r.status_code} · ждём {wait}с (попытка {i+1}/{attempts})") time.sleep(min(wait, 120)) raise RuntimeError(f"failed after {attempts}: {last}") def normalize_pct(s): if s is None: return None s = str(s).strip() if s.endswith("%"): s = s[:-1] try: return float(s) except: return None def parse_dec(v): if v in (None, ""): return None if isinstance(v, (int, float)): return v try: return float(str(v).replace(" ","").replace(",",".")) except: return None def main(): conn = sqlite3.connect(DB) conn.executescript(EXTRA_SCHEMA) token = get_token() print(f"Group={GROUP} window={SD}..{ED}") # -------- probe all four (without ComplexName, then with) -------- print("\n=== Probe all four reports ===") for rt in ("Сводные", "Поквартирные"): for rn in ("Корпуса", "Лоты"): base = {"Page":"Отчеты","ReportSection":"Объединенные данные", "ReportType":rt,"ReportName":rn,"GroupName":GROUP, "UseDdu":"true","UseDkp":"true"} # Сводные wants StartDate/EndDate if rt == "Сводные": base["StartDate"] = SD; base["EndDate"] = ED try: r = fetch(token, base, attempts=2) rows = r.json().get("result", []) rows_n = len(rows) if isinstance(rows, list) else "?" print(f" {rt}/{rn:<7} (whole Ekb): rows={rows_n}, bytes={len(r.content)}") ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S") fname = RAW / f"{rt}_{rn}_whole_{ts}.json" fname.write_text(json.dumps(rows, ensure_ascii=False)) if isinstance(rows, list) and rows: print(f" sample keys ({len(rows[0])}): {list(rows[0].keys())[:8]}") except Exception as e: print(f" {rt}/{rn} FAIL: {e}") # try with ComplexName fallback try: base2 = dict(base, ComplexName="Парк Культуры") r = fetch(token, base2, attempts=2) rows = r.json().get("result", []) rows_n = len(rows) if isinstance(rows, list) else "?" print(f" +ComplexName=Парк Культуры → rows={rows_n}") ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S") (RAW / f"{rt}_{rn}_pk_{ts}.json").write_text(json.dumps(rows, ensure_ascii=False)) except Exception as e2: print(f" +ComplexName FAIL: {e2}") time.sleep(15) # be nice — Objective rate-limits aggressively # -------- bulk-pull Поквартирные/Лоты (the most valuable per-flat data) -------- # If whole-Ekb works, prefer that. Otherwise iterate over distinct projects from corp_month. print("\n=== Bulk pull Поквартирные/Лоты ===") pf_params = {"Page":"Отчеты","ReportSection":"Объединенные данные", "ReportType":"Поквартирные","ReportName":"Лоты","GroupName":GROUP, "UseDdu":"true","UseDkp":"true"} rows = None try: r = fetch(token, pf_params, attempts=3) rows = r.json().get("result", []) print(f" whole Ekb: {len(rows)} flats, {len(r.content)/1024:.0f} KB") except Exception as e: print(f" whole-Ekb fail: {e}; falling back to per-project loop") if rows is None: projects = [p[0] for p in conn.execute( "SELECT DISTINCT project FROM objective_corp_month WHERE project IS NOT NULL ORDER BY project" ).fetchall()] print(f" iterating {len(projects)} projects") rows = [] for i, proj in enumerate(projects, 1): try: r = fetch(token, dict(pf_params, ComplexName=proj)) got = r.json().get("result", []) rows.extend(got) print(f" [{i}/{len(projects)}] {proj}: +{len(got)}") except Exception as e: print(f" [{i}/{len(projects)}] {proj}: FAIL {e}") time.sleep(10) if rows: ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S") (RAW / f"Поквартирные_Лоты_{ts}.json").write_text(json.dumps(rows, ensure_ascii=False)) # Insert conn.execute("DELETE FROM objective_lots") ins = 0; skip = 0 for r in rows: try: vals = {k: r.get(v) for k, v in LOTS_MAP.items()} vals["readiness_pct"] = normalize_pct(vals["readiness_pct"]) for f in ("area_dev","area_pd","budget_rub","price_per_m2","offer_price", "delta_price_rub","delta_price_pct","floor","lot_id","project_id"): vals[f] = parse_dec(vals[f]) cols = list(vals.keys()) conn.execute(f"INSERT OR REPLACE INTO objective_lots({','.join(cols)}) VALUES ({','.join(['?']*len(cols))})", [vals[c] for c in cols]) ins += 1 except Exception: skip += 1 conn.commit() print(f" inserted: {ins}, skipped: {skip}") # Sanity stats n = conn.execute("SELECT count(*) FROM objective_lots").fetchone()[0] print(f"\n Total flats stored: {n}") for line in conn.execute("""SELECT status, count(*) FROM objective_lots GROUP BY 1 ORDER BY 2 DESC""").fetchall(): print(f" status={line[0]:<25} {line[1]}") print("\n Top banks (mortgage encumbrance):") for line in conn.execute("""SELECT bank, count(*) FROM objective_lots WHERE bank IS NOT NULL AND bank!='' GROUP BY 1 ORDER BY 2 DESC LIMIT 10""").fetchall(): print(f" {line[0]:<35} {line[1]}") print("\n By district (priced lots):") for line in conn.execute("""SELECT district, count(*) total, SUM(CASE WHEN price_per_m2>0 THEN 1 ELSE 0 END) priced, ROUND(AVG(price_per_m2)/1000,1) avg_kp FROM objective_lots WHERE district IS NOT NULL GROUP BY 1 ORDER BY 2 DESC LIMIT 12""").fetchall(): print(f" {line[0]:<22} total={line[1]:>5} priced={line[2]:>5} avg={line[3]} тыс/м²") conn.close() print("\nDone.") if __name__ == "__main__": main()