Light1YT
97b19a0b85
Adds site-finder/ subfolder with:
- server.py — FastAPI scoring service v2 (35 endpoints, ~85KB)
- 01_load_sites.py … 12_more_pois.py — data ingest pipeline
- db_init.py — SQLite schema bootstrap
- static/ — Leaflet UI (index.html ~3500 lines + sw.js)
- cache/ — small persistent caches (admin districts, jk polygons,
geocode warm cache, parcel polygons drop-zone with README)
- reports/ — sample generated parcel report (HTML+JSON)
Excluded via .gitignore (regeneratable, too big for git):
- analysis.db (336MB SQLite — rebuild via 01_*..12_*.py)
- cache/objective_raw/ (1.2GB Объектив raw dumps)
- cache/overpass_raw.json, cache/osm_buildings_all.geojson
(regen from Overpass API)
Production deploy: /opt/gendesign/site-finder/ on gendsgn.ru
(container gendesign-site-finder-1, served at /sf/).
264 lines
12 KiB
Python
264 lines
12 KiB
Python
"""Pull every available Objective report combination for Ekb.
|
||
|
||
The API has 4 reports = ReportType × ReportName:
|
||
Сводные / Корпуса — already pulled in 05_*
|
||
Сводные / Лоты — probe
|
||
Поквартирные / Корпуса — probe
|
||
Поквартирные / Лоты — full per-flat data (most valuable)
|
||
|
||
Earlier server graph said 2 of 4 returned HTTP 500. We probe again,
|
||
then bulk-pull whatever returns 200.
|
||
"""
|
||
import sqlite3, pathlib, requests, time, json, datetime as dt, sys
|
||
|
||
DB = pathlib.Path(__file__).parent / "analysis.db"
|
||
RAW = pathlib.Path(__file__).parent / "cache" / "objective_raw"
|
||
RAW.mkdir(parents=True, exist_ok=True)
|
||
|
||
API = "https://api.objctv.ru"
|
||
KEY = "623f6a57-0179-434b-8202-259525bdc77c"
|
||
GROUP = "Екатеринбург"
|
||
TODAY = dt.date.today()
|
||
SD = (TODAY.replace(day=1) - dt.timedelta(days=365)).strftime("%Y.%m.%d")
|
||
ED = TODAY.strftime("%Y.%m.%d")
|
||
|
||
EXTRA_SCHEMA = """
|
||
-- per-flat snapshot (Поквартирные/Лоты)
|
||
CREATE TABLE IF NOT EXISTS objective_lots (
|
||
lot_id INTEGER PRIMARY KEY, -- field "Id"
|
||
project_id INTEGER,
|
||
project TEXT,
|
||
developer TEXT,
|
||
city TEXT,
|
||
district TEXT,
|
||
corpus TEXT,
|
||
address TEXT,
|
||
obj_class TEXT,
|
||
sales_start TEXT,
|
||
plan_date TEXT,
|
||
fact_date TEXT,
|
||
readiness_pct REAL,
|
||
construction_stage TEXT,
|
||
finish_type TEXT,
|
||
section TEXT,
|
||
floor INTEGER,
|
||
lot_num TEXT,
|
||
room_kind TEXT,
|
||
status TEXT,
|
||
sold TEXT,
|
||
rooms_dev TEXT,
|
||
rooms_pd TEXT,
|
||
rooms_obj TEXT,
|
||
area_dev REAL,
|
||
area_pd REAL,
|
||
budget_rub REAL,
|
||
price_per_m2 REAL,
|
||
price_method TEXT,
|
||
price_set_date TEXT,
|
||
price_actual_date TEXT,
|
||
offer_price REAL,
|
||
delta_price_rub REAL,
|
||
delta_price_pct REAL,
|
||
contract_date TEXT,
|
||
register_date TEXT,
|
||
deal_type TEXT,
|
||
buyer_type TEXT,
|
||
register_num TEXT,
|
||
encumbrance TEXT,
|
||
bank TEXT,
|
||
encumbrance_start TEXT,
|
||
egrn_actual_date TEXT,
|
||
fetched_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||
);
|
||
CREATE INDEX IF NOT EXISTS lots_proj ON objective_lots(project);
|
||
CREATE INDEX IF NOT EXISTS lots_dist ON objective_lots(district);
|
||
CREATE INDEX IF NOT EXISTS lots_bank ON objective_lots(bank);
|
||
CREATE INDEX IF NOT EXISTS lots_status ON objective_lots(status);
|
||
"""
|
||
|
||
# Mapping for per-flat (Поквартирные/Лоты)
|
||
LOTS_MAP = {
|
||
"lot_id":"Id","project_id":"Id проекта","project":"Проект","developer":"Девелопер",
|
||
"city":"Город","district":"Район","corpus":"Корпус","address":"Адрес",
|
||
"obj_class":"Класс","sales_start":"Старт продаж",
|
||
"plan_date":"Планируемая дата ввода","fact_date":"Фактическая дата ввода",
|
||
"readiness_pct":"Готовность","construction_stage":"Стадия строительства",
|
||
"finish_type":"Отделка по корпусу","section":"Секция","floor":"Этаж",
|
||
"lot_num":"Номер лота","room_kind":"Вид помещения","status":"Статус",
|
||
"sold":"Продано","rooms_dev":"Количество комнат(Сайт девелопера)",
|
||
"rooms_pd":"Количество комнат(ПД)","rooms_obj":"Количество комнат(Данные объектива)",
|
||
"area_dev":"Площадь, м2(Сайт девелопера)","area_pd":"Площадь, м2(ПД)",
|
||
"budget_rub":"Расчетный бюджет лота, Р","price_per_m2":"Цена за м2, Р",
|
||
"price_method":"Способ определения цены","price_set_date":"Дата установки цены",
|
||
"price_actual_date":"Дата актуальности цены","offer_price":"Цена предложения, Р",
|
||
"delta_price_rub":"Дельта цена, Р","delta_price_pct":"Дельта цена, %",
|
||
"contract_date":"Дата договора","register_date":"Дата регистрации",
|
||
"deal_type":"Тип сделки","buyer_type":"Тип покупателя","register_num":"Номер регистрации",
|
||
"encumbrance":"Тип обременения","bank":"Банк","encumbrance_start":"Дата начала обременения",
|
||
"egrn_actual_date":"Дата актуальности данных из ЕГРН",
|
||
}
|
||
|
||
|
||
def get_token():
|
||
r = requests.get(f"{API}/Users/User/GetToken", params={"apiKey": KEY}, timeout=30)
|
||
r.raise_for_status()
|
||
return r.json()["token"]
|
||
|
||
|
||
def fetch(token, params, attempts=5):
|
||
"""Fetch with backoff for 429/5xx."""
|
||
last = None
|
||
for i in range(attempts):
|
||
r = requests.get(f"{API}/v2/Report/GetReport",
|
||
params=params,
|
||
headers={"Authorization": f"Bearer {token}", "Accept-Encoding": "br"},
|
||
timeout=180)
|
||
if r.ok:
|
||
return r
|
||
last = (r.status_code, r.text[:200])
|
||
# If 401 — refresh token
|
||
if r.status_code == 401:
|
||
token = get_token()
|
||
continue
|
||
wait = int(r.headers.get("Retry-After") or (10 * (2 ** i)))
|
||
print(f" HTTP {r.status_code} · ждём {wait}с (попытка {i+1}/{attempts})")
|
||
time.sleep(min(wait, 120))
|
||
raise RuntimeError(f"failed after {attempts}: {last}")
|
||
|
||
|
||
def normalize_pct(s):
|
||
if s is None: return None
|
||
s = str(s).strip()
|
||
if s.endswith("%"): s = s[:-1]
|
||
try: return float(s)
|
||
except: return None
|
||
|
||
|
||
def parse_dec(v):
|
||
if v in (None, ""): return None
|
||
if isinstance(v, (int, float)): return v
|
||
try: return float(str(v).replace(" ","").replace(",","."))
|
||
except: return None
|
||
|
||
|
||
def main():
|
||
conn = sqlite3.connect(DB)
|
||
conn.executescript(EXTRA_SCHEMA)
|
||
|
||
token = get_token()
|
||
print(f"Group={GROUP} window={SD}..{ED}")
|
||
|
||
# -------- probe all four (without ComplexName, then with) --------
|
||
print("\n=== Probe all four reports ===")
|
||
for rt in ("Сводные", "Поквартирные"):
|
||
for rn in ("Корпуса", "Лоты"):
|
||
base = {"Page":"Отчеты","ReportSection":"Объединенные данные",
|
||
"ReportType":rt,"ReportName":rn,"GroupName":GROUP,
|
||
"UseDdu":"true","UseDkp":"true"}
|
||
# Сводные wants StartDate/EndDate
|
||
if rt == "Сводные":
|
||
base["StartDate"] = SD; base["EndDate"] = ED
|
||
try:
|
||
r = fetch(token, base, attempts=2)
|
||
rows = r.json().get("result", [])
|
||
rows_n = len(rows) if isinstance(rows, list) else "?"
|
||
print(f" {rt}/{rn:<7} (whole Ekb): rows={rows_n}, bytes={len(r.content)}")
|
||
ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S")
|
||
fname = RAW / f"{rt}_{rn}_whole_{ts}.json"
|
||
fname.write_text(json.dumps(rows, ensure_ascii=False))
|
||
if isinstance(rows, list) and rows:
|
||
print(f" sample keys ({len(rows[0])}): {list(rows[0].keys())[:8]}")
|
||
except Exception as e:
|
||
print(f" {rt}/{rn} FAIL: {e}")
|
||
# try with ComplexName fallback
|
||
try:
|
||
base2 = dict(base, ComplexName="Парк Культуры")
|
||
r = fetch(token, base2, attempts=2)
|
||
rows = r.json().get("result", [])
|
||
rows_n = len(rows) if isinstance(rows, list) else "?"
|
||
print(f" +ComplexName=Парк Культуры → rows={rows_n}")
|
||
ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S")
|
||
(RAW / f"{rt}_{rn}_pk_{ts}.json").write_text(json.dumps(rows, ensure_ascii=False))
|
||
except Exception as e2:
|
||
print(f" +ComplexName FAIL: {e2}")
|
||
time.sleep(15) # be nice — Objective rate-limits aggressively
|
||
|
||
# -------- bulk-pull Поквартирные/Лоты (the most valuable per-flat data) --------
|
||
# If whole-Ekb works, prefer that. Otherwise iterate over distinct projects from corp_month.
|
||
print("\n=== Bulk pull Поквартирные/Лоты ===")
|
||
pf_params = {"Page":"Отчеты","ReportSection":"Объединенные данные",
|
||
"ReportType":"Поквартирные","ReportName":"Лоты","GroupName":GROUP,
|
||
"UseDdu":"true","UseDkp":"true"}
|
||
rows = None
|
||
try:
|
||
r = fetch(token, pf_params, attempts=3)
|
||
rows = r.json().get("result", [])
|
||
print(f" whole Ekb: {len(rows)} flats, {len(r.content)/1024:.0f} KB")
|
||
except Exception as e:
|
||
print(f" whole-Ekb fail: {e}; falling back to per-project loop")
|
||
|
||
if rows is None:
|
||
projects = [p[0] for p in conn.execute(
|
||
"SELECT DISTINCT project FROM objective_corp_month WHERE project IS NOT NULL ORDER BY project"
|
||
).fetchall()]
|
||
print(f" iterating {len(projects)} projects")
|
||
rows = []
|
||
for i, proj in enumerate(projects, 1):
|
||
try:
|
||
r = fetch(token, dict(pf_params, ComplexName=proj))
|
||
got = r.json().get("result", [])
|
||
rows.extend(got)
|
||
print(f" [{i}/{len(projects)}] {proj}: +{len(got)}")
|
||
except Exception as e:
|
||
print(f" [{i}/{len(projects)}] {proj}: FAIL {e}")
|
||
time.sleep(10)
|
||
|
||
if rows:
|
||
ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S")
|
||
(RAW / f"Поквартирные_Лоты_{ts}.json").write_text(json.dumps(rows, ensure_ascii=False))
|
||
# Insert
|
||
conn.execute("DELETE FROM objective_lots")
|
||
ins = 0; skip = 0
|
||
for r in rows:
|
||
try:
|
||
vals = {k: r.get(v) for k, v in LOTS_MAP.items()}
|
||
vals["readiness_pct"] = normalize_pct(vals["readiness_pct"])
|
||
for f in ("area_dev","area_pd","budget_rub","price_per_m2","offer_price",
|
||
"delta_price_rub","delta_price_pct","floor","lot_id","project_id"):
|
||
vals[f] = parse_dec(vals[f])
|
||
cols = list(vals.keys())
|
||
conn.execute(f"INSERT OR REPLACE INTO objective_lots({','.join(cols)}) VALUES ({','.join(['?']*len(cols))})",
|
||
[vals[c] for c in cols])
|
||
ins += 1
|
||
except Exception:
|
||
skip += 1
|
||
conn.commit()
|
||
print(f" inserted: {ins}, skipped: {skip}")
|
||
|
||
# Sanity stats
|
||
n = conn.execute("SELECT count(*) FROM objective_lots").fetchone()[0]
|
||
print(f"\n Total flats stored: {n}")
|
||
for line in conn.execute("""SELECT status, count(*) FROM objective_lots
|
||
GROUP BY 1 ORDER BY 2 DESC""").fetchall():
|
||
print(f" status={line[0]:<25} {line[1]}")
|
||
|
||
print("\n Top banks (mortgage encumbrance):")
|
||
for line in conn.execute("""SELECT bank, count(*) FROM objective_lots
|
||
WHERE bank IS NOT NULL AND bank!=''
|
||
GROUP BY 1 ORDER BY 2 DESC LIMIT 10""").fetchall():
|
||
print(f" {line[0]:<35} {line[1]}")
|
||
|
||
print("\n By district (priced lots):")
|
||
for line in conn.execute("""SELECT district, count(*) total,
|
||
SUM(CASE WHEN price_per_m2>0 THEN 1 ELSE 0 END) priced,
|
||
ROUND(AVG(price_per_m2)/1000,1) avg_kp
|
||
FROM objective_lots
|
||
WHERE district IS NOT NULL
|
||
GROUP BY 1 ORDER BY 2 DESC LIMIT 12""").fetchall():
|
||
print(f" {line[0]:<22} total={line[1]:>5} priced={line[2]:>5} avg={line[3]} тыс/м²")
|
||
|
||
conn.close()
|
||
print("\nDone.")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|