Light1YT
86e9ea2937
Многоагентный аудит + имплементация: один воркер на файл, точечные правки. Верификация: py_compile (47/47 .py) + tsc --noEmit (0 ошибок). Unit-тесты не прогонялись (окружение не поднято: rollup native dep / нет pytest-venv). Полностью исправлено (169): #1336, #1337, #1339, #1340, #1341, #1342, #1343, #1345, #1346, #1348, #1349, #1350, #1351, #1354, #1356, #1358, #1359, #1360, #1362, #1364, #1365, #1366, #1367, #1368, #1369, #1370, #1371, #1372, #1373, #1374, #1375, #1376, #1377, #1378, #1379, #1380, #1381, #1382, #1384, #1385, #1386, #1387, #1388, #1389, #1390, #1391, #1392, #1394, #1395, #1396, #1397, #1399, #1400, #1401, #1402, #1403, #1404, #1408, #1409, #1410, #1411, #1412, #1413, #1414, #1415, #1416, #1417, #1418, #1420, #1423, #1425, #1426, #1427, #1428, #1429, #1430, #1431, #1432, #1433, #1434, #1435, #1437, #1438, #1439, #1440, #1441, #1442, #1443, #1444, #1445, #1446, #1447, #1448, #1449, #1450, #1451, #1452, #1453, #1454, #1455, #1456, #1457, #1458, #1459, #1460, #1461, #1462, #1463, #1464, #1465, #1466, #1467, #1468, #1469, #1471, #1472, #1473, #1474, #1476, #1478, #1479, #1481, #1482, #1483, #1484, #1485, #1487, #1488, #1489, #1490, #1491, #1492, #1493, #1494, #1495, #1496, #1497, #1499, #1500, #1501, #1502, #1504, #1505, #1506, #1507, #1510, #1514, #1515, #1516, #1517, #1518, #1519, #1521, #1522, #1523, #1524, #1525, #1526, #1527, #1528, #1529, #1531, #1532, #1533, #1534, #1535, #1536, #1537, #1538 Частично (9, in-file часть, остаток cross-file): #1361, #1419, #1422, #1424, #1470, #1475, #1477, #1480, #1498 Требуют cross-file (3, не тронуты): #1338, #1363, #1421 Пропущено (1): #1539 Не входило в партию: 22 needs-Leha issue (нужны решения владельца). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
944 lines
37 KiB
Python
944 lines
37 KiB
Python
"""Миграция knowledge graph (memory-gendesign.jsonl) → Obsidian vault через CouchDB.
|
||
|
||
Читает текущий KG и записывает каждую entity как `.md` файл прямо в CouchDB
|
||
в формате Self-hosted LiveSync (Vrtmrz/obsidian-livesync). Клиенты Obsidian
|
||
с LiveSync plugin автоматически подтянут изменения.
|
||
|
||
Формат LiveSync v0.x (classic chunks + children):
|
||
Chunk document:
|
||
{_id: "h:<base32-hash>", data: "<содержимое>", type: "leaf"}
|
||
|
||
Main document:
|
||
{_id: "entities/Foo.md", path: "entities/Foo.md",
|
||
children: ["h:..."], ctime, mtime, size, type:"plain", eden:{}}
|
||
|
||
eden — write-side staging area, при чтении не используется. Поэтому контент
|
||
ОБЯЗАН быть в отдельных chunk-docs (LiveSync читает через children).
|
||
PUT-ordering: chunks first, main docs second — иначе клиент видит пустой
|
||
body во время replication race.
|
||
|
||
Структура vault:
|
||
sessions/ — Session_End_* entities
|
||
entities/ — Module_*, Schema_*, прочие
|
||
decisions/ — *_Plan_*, *_Strategy_* (ADR-style)
|
||
runbooks/ — *_Scraper_*, *_Pipeline_* (how-to)
|
||
sources/ — DataSources_* (источники данных)
|
||
misc/ — всё остальное
|
||
|
||
Запуск:
|
||
cd backend
|
||
COUCHDB_URL=https://obsidian.gendsgn.ru \
|
||
COUCHDB_USER=obsidian \
|
||
COUCHDB_PASSWORD=<пароль> \
|
||
COUCHDB_DB=gendesign-vault \
|
||
uv run --no-project --with requests python -X utf8 \
|
||
../scripts/migrate_kg_to_obsidian.py [--dry-run]
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from urllib.parse import quote
|
||
|
||
import requests
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||
logger = logging.getLogger("kg_to_obsidian")
|
||
|
||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||
KG_PATH = REPO_ROOT / "memory" / "memory-gendesign.jsonl"
|
||
|
||
|
||
_B32 = "0123456789abcdefghijklmnopqrstuv"
|
||
|
||
|
||
def chunk_id_for(content: str) -> str:
|
||
"""Генерирует chunk_id похожий на LiveSync internal format:
|
||
`h:` + 13 base32-like chars. Уникальность достаточна — collision space 2^65.
|
||
|
||
LiveSync встроенный generator использует свой base32-кодированный hash.
|
||
Подражаем формату чтобы клиент не отбраковал по validation.
|
||
"""
|
||
digest = hashlib.sha256(content.encode("utf-8")).digest()
|
||
n = int.from_bytes(digest[:8], "big")
|
||
out = []
|
||
for _ in range(13):
|
||
out.append(_B32[n & 0x1F])
|
||
n >>= 5
|
||
return "h:" + "".join(out)
|
||
|
||
|
||
_MONTH_MAP = {
|
||
"jan": "01", "feb": "02", "mar": "03", "apr": "04", "may": "05", "jun": "06",
|
||
"jul": "07", "aug": "08", "sep": "09", "oct": "10", "nov": "11", "dec": "12",
|
||
}
|
||
_DATE_PATTERNS = [
|
||
# Apr27_2026, Apr30_2026, Apr29_2026_Late
|
||
# Ведущий lookbehind (?<![A-Za-z]) вместо \b: в именах entity месяц следует
|
||
# за '_' (Session_End_Apr30_2026), а '_' — это \w, поэтому \b между '_' и
|
||
# буквой не возникает. Завершающий (?![\dA-Za-z]) вместо \b — чтобы суффикс
|
||
# '_Late' (Apr29_2026_Late) не срывал совпадение.
|
||
re.compile(r"(?<![A-Za-z])(?P<m>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)(?P<d>\d{1,2})_(?P<y>20\d{2})(?![\dA-Za-z])", re.IGNORECASE),
|
||
# Apr26 (year implied 2026)
|
||
re.compile(r"(?<![A-Za-z])(?P<m>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)(?P<d>\d{1,2})(?![\dA-Za-z])(?!_20)", re.IGNORECASE),
|
||
# Nov2025
|
||
re.compile(r"(?<![A-Za-z])(?P<m>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)(?P<y>20\d{2})(?![\dA-Za-z])", re.IGNORECASE),
|
||
# Q1_2026, Q2_2026 — quarter→month
|
||
re.compile(r"(?<![A-Za-z])Q(?P<q>[1-4])_(?P<y>20\d{2})(?![\dA-Za-z])"),
|
||
]
|
||
|
||
|
||
def extract_date(name: str) -> str | None:
|
||
"""Из имени entity извлекает ISO-дату (YYYY-MM-DD или YYYY-MM).
|
||
|
||
Apr26 → 2026-04-26 (год дефолтный 2026 если не указан)
|
||
Nov2025 → 2025-11
|
||
Q1_2026 → 2026-Q1 (для квартала)
|
||
"""
|
||
for pat in _DATE_PATTERNS:
|
||
m = pat.search(name)
|
||
if not m:
|
||
continue
|
||
gd = m.groupdict()
|
||
if "q" in gd:
|
||
return f"{gd['y']}-Q{gd['q']}"
|
||
mon = _MONTH_MAP[gd["m"][:3].lower()]
|
||
if "d" in gd:
|
||
y = gd.get("y") or "2026"
|
||
return f"{y}-{mon}-{int(gd['d']):02d}"
|
||
return f"{gd['y']}-{mon}"
|
||
return None
|
||
|
||
|
||
# Категоризация: входим в первое совпавшее правило. Порядок важен —
|
||
# specific raньше generic.
|
||
def categorize_v2(name: str, entity_type: str) -> str:
|
||
"""Возвращает многоуровневый путь поддиректории (без имени файла, без расширения).
|
||
|
||
Стратегия — group по бизнес-домену, потом по технической роли.
|
||
"""
|
||
n = name
|
||
et = (entity_type or "").lower()
|
||
|
||
# 1. Top-level meta — флагманы проекта
|
||
if n in {"Project_GenDesign", "TechStack_v2.2", "NorthStar_DiscoveryMVP",
|
||
"Phase0_DiscoveryMVP", "Stages_MVP_v1", "Team_Setup",
|
||
"Onboarding_Colleague_KG", "WorkSplit_Generative_Stage1"}:
|
||
return "meta"
|
||
|
||
# 2. Sprints
|
||
if n.startswith("Sprint") and ("_" in n):
|
||
return "sprints"
|
||
|
||
# 3. Sessions — chronological logs
|
||
if n.startswith("Session") or et == "session":
|
||
return "sessions"
|
||
|
||
# 4. Bug fixes
|
||
if n.startswith("Bug_") or et == "bug":
|
||
return "fixes"
|
||
|
||
# 5. GitHub
|
||
if n.startswith("GitHub_"):
|
||
return "github"
|
||
|
||
# 6. Domain: DOM.РФ
|
||
if n.startswith("DomRF") or n == "DomRf_Kn_Scraper_Apr27":
|
||
if "_API_" in n or n.startswith("DomRF_API"):
|
||
return "domains/domrf/api"
|
||
if any(w in n for w in ("_Loaded_", "_Imported_", "_Wipe_")):
|
||
return "domains/domrf/events"
|
||
if "_Schema_" in n or n.endswith("_Schema"):
|
||
return "domains/domrf/schemas"
|
||
if "_Scraper" in n:
|
||
return "domains/domrf/scraper"
|
||
if "Enums" in n or "Architecture" in n or "Host" in n or "Integration" in n:
|
||
return "domains/domrf/api"
|
||
return "domains/domrf"
|
||
|
||
# 7. Domain: Cadastre (NSPD + Cad_*)
|
||
if n.startswith("NSPD") or n.startswith("Cad_") or n == "Cad_Quarter_Backfill_Apr30":
|
||
return "domains/cadastre"
|
||
|
||
# 8. Domain: Rosreestr
|
||
if (n.startswith("Rosreestr") or n.startswith("Sverdl_") or
|
||
(n.startswith("DataSource_Rosreestr") or n == "Backfill_District_Median_From_Kn_Flats")):
|
||
return "domains/rosreestr"
|
||
|
||
# 9. Domain: Analytics
|
||
if any(w in n for w in ("Recommend", "Velocity", "Affordability", "DemandIndex",
|
||
"AlternativeDeal", "Competitor_Benchmark", "Macro_Context",
|
||
"Mortgage", "PriceGap", "Tier2", "Tier_Alpha", "Quartirography",
|
||
"RegionGroups", "Regional_Migration", "RepeatedMortgage",
|
||
"QualityShift")):
|
||
if n.startswith("Component_") or n.startswith("Feature_"):
|
||
return "domains/analytics/ui"
|
||
if n.startswith("Endpoint_"):
|
||
return "domains/analytics/api"
|
||
if n.startswith("Hook_"):
|
||
return "domains/analytics/code"
|
||
if n.startswith("Bug_"):
|
||
return "fixes"
|
||
if "Trend" in n or "Index" in n or "Report" in n:
|
||
return "domains/analytics/research"
|
||
return "domains/analytics"
|
||
|
||
# 10. Domain: UI components/routes (общие, не analytics-specific)
|
||
if n.startswith("Component_") or n.startswith("Feature_") or n.startswith("UI_"):
|
||
return "domains/ui/components"
|
||
if n.startswith("Route_"):
|
||
return "domains/ui/routes"
|
||
|
||
# 11. Domain: API endpoints
|
||
if n.startswith("Endpoint_"):
|
||
return "domains/api/endpoints"
|
||
|
||
# 12. Domain: Geo
|
||
if (n.startswith("Geo_") or n == "Pattern_OSM_Overpass_Admin_Level"
|
||
or n == "Limit_District_Geometry_Closed" or n == "File_Ekb_Districts_Geojson_Cache"):
|
||
return "domains/geo"
|
||
|
||
# 13. Domain: Infra (deploy, docker, hosting, etc)
|
||
if any(w in n for w in ("Docker", "Deploy", "Sentry", "Hosting", "SSH",
|
||
"OpsBundle", "Domain_Setup", "HTTPS", "Redis",
|
||
"Caddy", "Celery", "TTL_Lock", "Stealth_Fingerprint",
|
||
"Runbook_VPS", "Production_Wipe")):
|
||
return "domains/infra"
|
||
|
||
# 14. Code: Modules / Schemas / Patterns / Hooks / Tasks
|
||
if n.startswith("Module_") or et == "code_module":
|
||
return "code/modules"
|
||
if n.startswith("Schema_") or et == "schema":
|
||
return "code/schemas"
|
||
if n.startswith("Pattern_"):
|
||
return "code/patterns"
|
||
if n.startswith("Hook_"):
|
||
return "code/hooks"
|
||
if n.startswith("Task_"):
|
||
return "code/tasks"
|
||
|
||
# 15. Sources (data source research, non-Rosreestr)
|
||
if n.startswith("DataSource_") or n.startswith("DataSources_"):
|
||
return "sources"
|
||
|
||
# 16. Limitations
|
||
if n.startswith("Limit_") or n == "Open_Limitations_Apr30" or et == "limitation":
|
||
return "limitations"
|
||
|
||
# 17. Decisions (plans, strategies, policies)
|
||
if n.startswith("PlanA_") or n.startswith("Plan_") or "_Strategy_" in n:
|
||
return "decisions"
|
||
if et in ("policy", "plan", "decision", "decision_pending", "strategy"):
|
||
return "decisions"
|
||
|
||
# 18. Events / milestones / objectives
|
||
if (n.startswith("Objective_") or "_Loaded_" in n or "_Imported_" in n
|
||
or n.startswith("Phase_") or et in ("event", "milestone")):
|
||
return "events"
|
||
|
||
# 19. Feedback
|
||
if n.startswith("Feedback_") or et == "feedback" or "MCP_AddFacts" in n:
|
||
return "feedback"
|
||
|
||
# 20. Research (generic)
|
||
if et == "research":
|
||
return "research"
|
||
|
||
# 21. Tests
|
||
if et == "test" or n.endswith("_Test_Apr25"):
|
||
return "research"
|
||
|
||
# 22. Reviews
|
||
if et == "review":
|
||
return "research"
|
||
|
||
# 23. Build (SiteFinder etc)
|
||
if et == "build" or n.startswith("SiteFinder"):
|
||
return "domains/sitefinder"
|
||
|
||
# 24. Deliverable — most likely analytics или infra
|
||
if et == "deliverable":
|
||
return "events"
|
||
|
||
# 25. Infrastructure type
|
||
if et in ("infrastructure", "infra"):
|
||
return "domains/infra"
|
||
|
||
return "unsorted"
|
||
|
||
|
||
_TAG_RE = re.compile(r"[^a-z0-9]+")
|
||
|
||
|
||
def tags_for_path(subdir: str, entity_type: str) -> list[str]:
|
||
"""Генерит теги из subpath + entity_type. Пример:
|
||
domains/domrf/api + api_spec → [domrf, api, domrf-api, api-spec]
|
||
"""
|
||
parts = [p for p in subdir.split("/") if p and p not in ("domains", "code")]
|
||
tags = list(dict.fromkeys(parts)) # uniq, preserve order
|
||
# composite tag — domain-subdomain
|
||
if len(parts) >= 2:
|
||
tags.append(f"{parts[0]}-{parts[1]}")
|
||
if entity_type:
|
||
et_norm = _TAG_RE.sub("-", entity_type.lower()).strip("-")
|
||
if et_norm and et_norm not in tags:
|
||
tags.append(et_norm)
|
||
return tags
|
||
|
||
|
||
# Старая функция для совместимости с CouchDB-режимом (он сейчас не используется,
|
||
# но всё ещё есть в коде на случай отката).
|
||
def categorize(name: str, entity_type: str) -> str:
|
||
subdir = categorize_v2(name, entity_type)
|
||
return subdir.split("/")[0]
|
||
|
||
|
||
_SLUG_RE = re.compile(r"[^a-zA-Zа-яА-Я0-9_\- ]+")
|
||
|
||
|
||
def safe_filename(name: str) -> str:
|
||
"""Превращает name в файл-безопасное имя (без / : * etc)."""
|
||
s = _SLUG_RE.sub("", name).strip().replace(" ", "_")
|
||
return s[:200] # Obsidian/Windows path limit
|
||
|
||
|
||
def md_for_entity(entity: dict, outgoing: list[dict], incoming: list[dict],
|
||
name_to_path: dict[str, str], subdir: str | None = None) -> str:
|
||
"""Генерит .md содержимое entity с frontmatter (теги + дата), observations,
|
||
relations через короткие [[wiki-links]] (Obsidian резолвит по basename).
|
||
"""
|
||
name = entity["name"]
|
||
etype = entity.get("entityType", "unknown")
|
||
observations = entity.get("observations") or []
|
||
date = extract_date(name)
|
||
tags = tags_for_path(subdir or "", etype) if subdir else []
|
||
|
||
lines: list[str] = []
|
||
|
||
# YAML frontmatter
|
||
lines.append("---")
|
||
lines.append(f"id: {name}")
|
||
lines.append(f"type: {etype}")
|
||
if date:
|
||
lines.append(f"date: {date}")
|
||
if tags:
|
||
lines.append(f"tags: [{', '.join(tags)}]")
|
||
lines.append("imported_from: knowledge_graph")
|
||
lines.append(f"imported_at: {time.strftime('%Y-%m-%d')}")
|
||
lines.append("---")
|
||
lines.append("")
|
||
lines.append(f"# {name}")
|
||
lines.append("")
|
||
meta_bits = [f"**Type:** `{etype}`"]
|
||
if date:
|
||
meta_bits.append(f"**Date:** {date}")
|
||
if subdir:
|
||
# breadcrumb: накопленный path → его MOC
|
||
crumbs = []
|
||
parts = subdir.split("/")
|
||
for i in range(len(parts)):
|
||
sub = "/".join(parts[: i + 1])
|
||
crumbs.append(f"[[{_moc_id(sub)}|{parts[i]}]]")
|
||
meta_bits.append("**Path:** " + " / ".join(crumbs))
|
||
lines.append(" · ".join(meta_bits))
|
||
lines.append("")
|
||
|
||
if observations:
|
||
lines.append("## Observations")
|
||
lines.append("")
|
||
for obs in observations:
|
||
txt = obs.replace("\n", " ").strip()
|
||
lines.append(f"- {txt}")
|
||
lines.append("")
|
||
|
||
if outgoing:
|
||
lines.append("## Relations (outgoing)")
|
||
lines.append("")
|
||
# группируем по relationType — нагляднее
|
||
by_rel: dict[str, list[str]] = defaultdict(list)
|
||
for rel in outgoing:
|
||
by_rel[rel["relationType"]].append(rel["to"])
|
||
for rtype, targets in sorted(by_rel.items()):
|
||
lines.append(f"### {rtype}")
|
||
for target in targets:
|
||
if target in name_to_path:
|
||
lines.append(f"- [[{target}]]")
|
||
else:
|
||
lines.append(f"- {target} *(no entity in KG)*")
|
||
lines.append("")
|
||
|
||
if incoming:
|
||
lines.append("## Relations (incoming)")
|
||
lines.append("")
|
||
by_rel2: dict[str, list[str]] = defaultdict(list)
|
||
for rel in incoming:
|
||
by_rel2[rel["relationType"]].append(rel["from"])
|
||
for rtype, sources in sorted(by_rel2.items()):
|
||
lines.append(f"### {rtype} ← from")
|
||
for src in sources:
|
||
if src in name_to_path:
|
||
lines.append(f"- [[{src}]]")
|
||
else:
|
||
lines.append(f"- {src} *(no entity)*")
|
||
lines.append("")
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
def put_doc(
|
||
base_url: str, db: str, auth: tuple[str, str], doc_id: str, doc: dict,
|
||
dry_run: bool = False,
|
||
) -> bool:
|
||
"""PUT/POST документ. При conflict — GET _rev и PUT с _rev."""
|
||
if dry_run:
|
||
return True
|
||
url = f"{base_url}/{db}/{quote(doc_id, safe='')}"
|
||
r = requests.put(url, json=doc, auth=auth, timeout=30)
|
||
if r.status_code in (201, 202):
|
||
return True
|
||
if r.status_code == 409:
|
||
# Conflict — try get existing and retry with _rev
|
||
rg = requests.get(url, auth=auth, timeout=30)
|
||
if rg.status_code == 200:
|
||
existing = rg.json()
|
||
doc["_rev"] = existing["_rev"]
|
||
r2 = requests.put(url, json=doc, auth=auth, timeout=30)
|
||
if r2.status_code in (201, 202):
|
||
return True
|
||
logger.warning("Retry-after-conflict PUT %s: HTTP %d %s",
|
||
doc_id, r2.status_code, r2.text[:200])
|
||
return False
|
||
logger.warning("PUT %s: HTTP %d %s", doc_id, r.status_code, r.text[:200])
|
||
return False
|
||
|
||
|
||
def delete_doc(base_url: str, db: str, auth: tuple[str, str], doc_id: str) -> bool:
|
||
"""DELETE документ (для перезаписи старой broken-версии)."""
|
||
url = f"{base_url}/{db}/{quote(doc_id, safe='')}"
|
||
rg = requests.get(url, auth=auth, timeout=30)
|
||
if rg.status_code != 200:
|
||
return True # уже нет — OK
|
||
rev = rg.json()["_rev"]
|
||
r = requests.delete(f"{url}?rev={rev}", auth=auth, timeout=30)
|
||
return r.status_code in (200, 202)
|
||
|
||
|
||
def _load_kg() -> tuple[dict[str, dict], list[dict]]:
|
||
"""Загружает KG из JSONL: returns (entities_by_name, relations_list)."""
|
||
entities: dict[str, dict] = {}
|
||
relations: list[dict] = []
|
||
with KG_PATH.open(encoding="utf-8") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
d = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if d.get("type") == "entity":
|
||
entities[d["name"]] = d
|
||
elif d.get("type") == "relation":
|
||
relations.append(d)
|
||
return entities, relations
|
||
|
||
|
||
_TOP_LEVEL_DESCRIPTIONS = {
|
||
"meta": "🧭 Проект, vision, стратегия, команда",
|
||
"sprints": "🏃 Sprint plans",
|
||
"sessions": "📅 Журналы сессий разработки",
|
||
"events": "📌 События — milestones, объективы, deliverables",
|
||
"decisions": "🎯 Решения, планы, политики (ADR-style)",
|
||
"domains": "🗺 Бизнес-домены — DomRF, Cadastre, Rosreestr, Analytics, UI, API, Geo, Infra",
|
||
"code": "🔧 Code modules, schemas, patterns, hooks, tasks",
|
||
"sources": "💾 Источники данных (research)",
|
||
"research": "🔬 Generic research, отчёты",
|
||
"fixes": "🐞 Bug fixes",
|
||
"limitations": "🚧 Ограничения, что не работает",
|
||
"github": "🐙 GitHub issues",
|
||
"feedback": "💬 Feedback / TODO заметки от пользователя",
|
||
"unsorted": "❓ Не сгруппировано",
|
||
}
|
||
|
||
|
||
def _moc_id(subdir: str) -> str:
|
||
"""Уникальный (per-vault) идентификатор MOC-файла для папки subdir.
|
||
Использует дефис-разделённый path: domains/domrf/api → domrf-api-MOC.
|
||
Префикс 'domains' опускается у вложенных, но сам domains→domains-MOC.
|
||
"""
|
||
parts = subdir.split("/")
|
||
if len(parts) > 1 and parts[0] == "domains":
|
||
parts = parts[1:]
|
||
return "-".join(parts) + "-MOC"
|
||
|
||
|
||
def _moc_for_subdir(subdir: str, entities_in_subdir: list[str],
|
||
entities_all: dict[str, dict],
|
||
subdirs_below: list[str]) -> str:
|
||
"""Генерит MOC.md (Map of Content) для папки subdir."""
|
||
parts = subdir.split("/")
|
||
top = parts[0]
|
||
title = " / ".join(parts) if len(parts) > 1 else parts[0]
|
||
desc = _TOP_LEVEL_DESCRIPTIONS.get(top, "")
|
||
moc_id = _moc_id(subdir)
|
||
|
||
lines = [
|
||
"---",
|
||
f"id: {moc_id}",
|
||
"type: moc",
|
||
f"tags: [moc, {top}]",
|
||
f"imported_at: {time.strftime('%Y-%m-%d')}",
|
||
"---",
|
||
"",
|
||
f"# 🗂 {title}",
|
||
"",
|
||
]
|
||
if desc and len(parts) == 1:
|
||
lines.append(f"> {desc}")
|
||
lines.append("")
|
||
|
||
if subdirs_below:
|
||
lines.append("## Подразделы")
|
||
lines.append("")
|
||
for sd in sorted(subdirs_below):
|
||
sub_moc = _moc_id(sd)
|
||
sd_label = sd.split("/")[-1]
|
||
lines.append(f"- [[{sub_moc}|{sd_label}/]]")
|
||
lines.append("")
|
||
|
||
if entities_in_subdir:
|
||
lines.append(f"## Entities ({len(entities_in_subdir)})")
|
||
lines.append("")
|
||
# сортируем по дате (новые сверху), затем по имени
|
||
def sort_key(name: str) -> tuple[str, str]:
|
||
d = extract_date(name) or "0000"
|
||
return (d, name)
|
||
for n in sorted(entities_in_subdir, key=sort_key, reverse=True):
|
||
ent = entities_all.get(n, {})
|
||
et = ent.get("entityType", "")
|
||
d = extract_date(n)
|
||
d_str = f" · `{d}`" if d else ""
|
||
et_str = f" · *{et}*" if et else ""
|
||
lines.append(f"- [[{n}]]{d_str}{et_str}")
|
||
lines.append("")
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _global_index_md(entities: dict[str, dict], relations_n: int,
|
||
by_subdir: dict[str, list[str]]) -> str:
|
||
"""Главный индекс (00_index.md) — entry point в vault."""
|
||
lines = [
|
||
"---",
|
||
"id: 00_index",
|
||
"type: index",
|
||
"tags: [index, hub]",
|
||
f"imported_at: {time.strftime('%Y-%m-%d')}",
|
||
"---",
|
||
"",
|
||
"# 🗂 Knowledge Graph — GenDesign",
|
||
"",
|
||
f"Vault содержит **{len(entities)} entities** и **{relations_n} relations**, "
|
||
"мигрированных из `memory/memory-gendesign.jsonl`.",
|
||
"",
|
||
"## Top-level разделы",
|
||
"",
|
||
]
|
||
# Группируем top-level папки
|
||
top_level: dict[str, int] = defaultdict(int)
|
||
for sd, names in by_subdir.items():
|
||
top_level[sd.split("/")[0]] += len(names)
|
||
for top in sorted(top_level):
|
||
desc = _TOP_LEVEL_DESCRIPTIONS.get(top, "")
|
||
lines.append(f"- [[{_moc_id(top)}|**{top}/**]] ({top_level[top]} entities) — {desc}")
|
||
lines.append("")
|
||
|
||
lines.append("## Граф")
|
||
lines.append("")
|
||
lines.append("Открой Graph View (Ctrl+G) — увидишь паутину связей entities по `[[wiki-links]]`.")
|
||
lines.append("")
|
||
lines.append("Фильтр по тегу: `tag:#domrf`, `tag:#analytics`, `tag:#cadastre` и т.д.")
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _clean_vault_subdirs(vault_path: Path, keep_files: set[str]) -> int:
|
||
"""Удаляет из vault'а всё ранее сгенерированное (по old structure).
|
||
Сохраняет файлы из keep_files (user content: welcome.md и пр).
|
||
Возвращает кол-во удалённых файлов.
|
||
"""
|
||
removed = 0
|
||
OLD_DIRS = ("sessions", "entities", "decisions", "runbooks", "sources",
|
||
"meta", "sprints", "events", "domains", "code", "research",
|
||
"fixes", "limitations", "github", "feedback", "unsorted")
|
||
for sub in OLD_DIRS:
|
||
d = vault_path / sub
|
||
if not d.exists():
|
||
continue
|
||
for f in d.rglob("*.md"):
|
||
try:
|
||
f.unlink()
|
||
removed += 1
|
||
except OSError as e:
|
||
logger.warning("Не удалить %s: %s", f, e)
|
||
# удалить пустые папки (rmdir каждый уровень снизу вверх)
|
||
for sub_d in sorted(d.rglob("*"), key=lambda p: -len(p.parts)):
|
||
if sub_d.is_dir():
|
||
try:
|
||
sub_d.rmdir()
|
||
except OSError:
|
||
pass
|
||
try:
|
||
d.rmdir()
|
||
except OSError:
|
||
pass
|
||
# Удалить 00_index.md из root если есть
|
||
idx = vault_path / "00_index.md"
|
||
if idx.exists() and "00_index.md" not in keep_files:
|
||
idx.unlink()
|
||
removed += 1
|
||
return removed
|
||
|
||
|
||
def _write_to_local_vault(vault_path: Path, limit: int | None) -> int:
|
||
"""Пишет .md файлы напрямую в локальную папку Obsidian vault с иерархической
|
||
структурой + MOC-страницы для навигации.
|
||
|
||
LiveSync plugin сам подхватит изменения через filesystem watcher и
|
||
загрузит их в CouchDB в V3-chunked формате.
|
||
"""
|
||
if not vault_path.exists() or not vault_path.is_dir():
|
||
logger.error("Vault path не существует: %s", vault_path)
|
||
return 2
|
||
|
||
entities, relations = _load_kg()
|
||
logger.info("Loaded %d entities, %d relations", len(entities), len(relations))
|
||
|
||
# Cleanup старой структуры (сохраняем user-files)
|
||
keep = {"welcome.md", "qqq.md", "untitled.md", "2026-05-11.md"}
|
||
removed = _clean_vault_subdirs(vault_path, keep)
|
||
logger.info("Cleaned %d old .md files from vault", removed)
|
||
|
||
# Relation indices
|
||
out_by: dict[str, list[dict]] = defaultdict(list)
|
||
in_by: dict[str, list[dict]] = defaultdict(list)
|
||
for r in relations:
|
||
out_by[r["from"]].append(r)
|
||
in_by[r["to"]].append(r)
|
||
|
||
# Path resolution
|
||
name_to_subdir: dict[str, str] = {}
|
||
name_to_path: dict[str, str] = {}
|
||
for name, ent in entities.items():
|
||
subdir = categorize_v2(name, ent.get("entityType", ""))
|
||
fname = safe_filename(name) + ".md"
|
||
name_to_subdir[name] = subdir
|
||
name_to_path[name] = f"{subdir}/{fname}"
|
||
|
||
names = list(entities.keys())
|
||
if limit:
|
||
names = names[:limit]
|
||
logger.info("Will write %d entities", len(names))
|
||
|
||
# PASS 1: запись всех entities
|
||
ok = 0
|
||
for i, name in enumerate(names, 1):
|
||
ent = entities[name]
|
||
rel_path = name_to_path[name]
|
||
subdir = name_to_subdir[name]
|
||
seen: set[tuple[str, str, str]] = set()
|
||
out_uniq: list[dict] = []
|
||
for r in out_by[name]:
|
||
key = (r["from"], r["to"], r["relationType"])
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
out_uniq.append(r)
|
||
in_uniq: list[dict] = []
|
||
for r in in_by[name]:
|
||
key = (r["from"], r["to"], r["relationType"])
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
in_uniq.append(r)
|
||
content = md_for_entity(ent, out_uniq, in_uniq, name_to_path, subdir)
|
||
|
||
target = vault_path / rel_path
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
target.write_text(content, encoding="utf-8")
|
||
ok += 1
|
||
|
||
logger.info("Entities written: %d", ok)
|
||
|
||
# PASS 2: MOC-страницы (для каждой подпапки)
|
||
by_subdir: dict[str, list[str]] = defaultdict(list)
|
||
for name in names:
|
||
by_subdir[name_to_subdir[name]].append(name)
|
||
|
||
# Найти все уровни подпапок и их subdirs
|
||
all_subdirs: set[str] = set()
|
||
for sd in by_subdir:
|
||
parts = sd.split("/")
|
||
for i in range(1, len(parts) + 1):
|
||
all_subdirs.add("/".join(parts[:i]))
|
||
|
||
moc_count = 0
|
||
for sd in sorted(all_subdirs):
|
||
entities_here = by_subdir.get(sd, [])
|
||
# подпапки на уровень ниже
|
||
subdirs_below = [s for s in all_subdirs
|
||
if s.startswith(sd + "/") and s.count("/") == sd.count("/") + 1]
|
||
moc_content = _moc_for_subdir(sd, entities_here, entities, subdirs_below)
|
||
moc_filename = _moc_id(sd) + ".md"
|
||
moc_path = vault_path / sd / moc_filename
|
||
moc_path.parent.mkdir(parents=True, exist_ok=True)
|
||
moc_path.write_text(moc_content, encoding="utf-8")
|
||
moc_count += 1
|
||
|
||
logger.info("MOC pages written: %d", moc_count)
|
||
|
||
# PASS 3: глобальный индекс
|
||
index_content = _global_index_md(entities, len(relations), by_subdir)
|
||
(vault_path / "00_index.md").write_text(index_content, encoding="utf-8")
|
||
|
||
logger.info("DONE: %d entities + %d MOC + 1 index в %s",
|
||
ok, moc_count, vault_path)
|
||
logger.info("Открой Obsidian — LiveSync plugin отсканирует и запушит в CouchDB.")
|
||
return 0
|
||
|
||
|
||
def _cleanup_orphan_chunks(base_url: str, db: str, auth: tuple[str, str],
|
||
dry_run: bool) -> int:
|
||
"""Удаляет h:kg* chunk-документы (legacy from old migration schema).
|
||
|
||
Использует _all_docs со startkey/endkey по префиксу 'h:kg'. Чанки уже
|
||
не нужны после перехода на eden-inline embedding.
|
||
"""
|
||
url = f"{base_url}/{db}/_all_docs"
|
||
params = {"startkey": '"h:kg"', "endkey": '"h:kg\\ufff0"', "limit": 10000}
|
||
r = requests.get(url, params=params, auth=auth, timeout=60)
|
||
if r.status_code != 200:
|
||
logger.error("_all_docs failed: HTTP %d %s", r.status_code, r.text[:200])
|
||
return 1
|
||
rows = r.json().get("rows", [])
|
||
logger.info("Found %d orphan h:kg* chunks", len(rows))
|
||
if dry_run:
|
||
return 0
|
||
# bulk_docs с _deleted:true — куда быстрее чем 549 индивидуальных DELETE.
|
||
docs = [{"_id": row["id"], "_rev": row["value"]["rev"], "_deleted": True}
|
||
for row in rows]
|
||
if not docs:
|
||
return 0
|
||
bulk_url = f"{base_url}/{db}/_bulk_docs"
|
||
rb = requests.post(bulk_url, json={"docs": docs}, auth=auth, timeout=120)
|
||
if rb.status_code not in (201, 200):
|
||
logger.error("_bulk_docs failed: HTTP %d %s", rb.status_code, rb.text[:200])
|
||
return 1
|
||
results = rb.json()
|
||
deleted = sum(1 for x in results if x.get("ok"))
|
||
logger.info("Deleted %d/%d orphan chunks", deleted, len(docs))
|
||
return 0
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--dry-run", action="store_true",
|
||
help="Только сгенерить .md (вывести count) без PUT в CouchDB")
|
||
ap.add_argument("--limit", type=int, default=None,
|
||
help="Загрузить только первые N entities (для smoke-теста)")
|
||
ap.add_argument("--cleanup-orphan-chunks", action="store_true",
|
||
help="Удалить осиротевшие h:kg* chunk-документы из предыдущей миграции")
|
||
ap.add_argument("--local-vault", type=str, default=None,
|
||
help="Путь к локальной папке Obsidian vault. В этом режиме скрипт "
|
||
"пишет .md файлы прямо в файловую систему, а LiveSync plugin "
|
||
"сам корректно загрузит их в CouchDB в правильном V3-chunked "
|
||
"формате. Не использует CouchDB API.")
|
||
args = ap.parse_args()
|
||
|
||
base_url = os.environ.get("COUCHDB_URL", "https://obsidian.gendsgn.ru")
|
||
db = os.environ.get("COUCHDB_DB", "gendesign-vault")
|
||
user = os.environ.get("COUCHDB_USER", "obsidian")
|
||
password = os.environ.get("COUCHDB_PASSWORD")
|
||
if not password and not args.dry_run and not args.local_vault:
|
||
logger.error("COUCHDB_PASSWORD env required (unless --dry-run or --local-vault)")
|
||
return 1
|
||
auth = (user, password or "")
|
||
|
||
if args.cleanup_orphan_chunks:
|
||
return _cleanup_orphan_chunks(base_url, db, auth, args.dry_run)
|
||
|
||
if not KG_PATH.exists():
|
||
logger.error("KG not found: %s", KG_PATH)
|
||
return 2
|
||
|
||
if args.local_vault:
|
||
return _write_to_local_vault(Path(args.local_vault), args.limit)
|
||
|
||
# 1. Load KG
|
||
entities: dict[str, dict] = {}
|
||
relations: list[dict] = []
|
||
with KG_PATH.open(encoding="utf-8") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
d = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if d.get("type") == "entity":
|
||
# Берём LATEST entity если name дублируется (auto-memory append)
|
||
entities[d["name"]] = d
|
||
elif d.get("type") == "relation":
|
||
relations.append(d)
|
||
logger.info("Loaded %d entities, %d relations", len(entities), len(relations))
|
||
|
||
# 2. Build relation indices
|
||
out_by: dict[str, list[dict]] = defaultdict(list)
|
||
in_by: dict[str, list[dict]] = defaultdict(list)
|
||
for r in relations:
|
||
# Дедуп: одна и та же relation может встречаться много раз в auto-memory log
|
||
out_by[r["from"]].append(r)
|
||
in_by[r["to"]].append(r)
|
||
|
||
# 3. Compute target paths for [[wiki-links]] resolution
|
||
name_to_path: dict[str, str] = {}
|
||
for name, ent in entities.items():
|
||
subdir = categorize(name, ent.get("entityType", ""))
|
||
fname = safe_filename(name) + ".md"
|
||
name_to_path[name] = f"{subdir}/{fname}"
|
||
|
||
# 4. Migrate
|
||
names = list(entities.keys())
|
||
if args.limit:
|
||
names = names[: args.limit]
|
||
logger.info("Will migrate %d entities (--limit=%s)", len(names), args.limit)
|
||
|
||
# PASS 1: pre-compute content + chunk_id for every entity, чтобы потом
|
||
# сделать bulk PUT chunks FIRST, потом — main docs.
|
||
prepared: list[tuple[str, str, str, int]] = [] # (path, chunk_id, content, size)
|
||
for name in names:
|
||
ent = entities[name]
|
||
path = name_to_path[name]
|
||
seen: set[tuple[str, str, str]] = set()
|
||
out_uniq: list[dict] = []
|
||
for r in out_by[name]:
|
||
key = (r["from"], r["to"], r["relationType"])
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
out_uniq.append(r)
|
||
in_uniq: list[dict] = []
|
||
for r in in_by[name]:
|
||
key = (r["from"], r["to"], r["relationType"])
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
in_uniq.append(r)
|
||
content = md_for_entity(ent, out_uniq, in_uniq, name_to_path)
|
||
size_bytes = len(content.encode("utf-8"))
|
||
chunk_id = chunk_id_for(content)
|
||
prepared.append((path, chunk_id, content, size_bytes))
|
||
|
||
# PASS 2: bulk PUT chunks (idempotent — если уже есть, CouchDB вернёт conflict,
|
||
# это OK).
|
||
unique_chunks: dict[str, str] = {} # chunk_id -> content
|
||
for _, cid, content, _ in prepared:
|
||
unique_chunks.setdefault(cid, content)
|
||
logger.info("Bulk PUT %d unique chunks…", len(unique_chunks))
|
||
chunk_docs = [{"_id": cid, "data": data, "type": "leaf"}
|
||
for cid, data in unique_chunks.items()]
|
||
if not args.dry_run:
|
||
rb = requests.post(f"{base_url}/{db}/_bulk_docs",
|
||
json={"docs": chunk_docs, "new_edits": True},
|
||
auth=auth, timeout=120)
|
||
if rb.status_code in (201, 200):
|
||
results = rb.json()
|
||
n_ok = sum(1 for x in results if x.get("ok"))
|
||
n_conflict = sum(1 for x in results if x.get("error") == "conflict")
|
||
logger.info(" chunks: %d created, %d already existed", n_ok, n_conflict)
|
||
else:
|
||
logger.error("Bulk chunks PUT failed: HTTP %d %s",
|
||
rb.status_code, rb.text[:200])
|
||
return 1
|
||
|
||
# PASS 3: PUT main docs sequentially (deleting old eden-only versions first
|
||
# to avoid _rev conflicts from прошлой неудачной миграции).
|
||
ok = 0
|
||
fail = 0
|
||
now_ms = int(time.time() * 1000)
|
||
for i, (path, chunk_id, _content, size_bytes) in enumerate(prepared, 1):
|
||
# delete старую версию (eden-only без children — broken)
|
||
delete_doc(base_url, db, auth, path)
|
||
main_doc = {
|
||
"_id": path,
|
||
"path": path,
|
||
"children": [chunk_id],
|
||
"ctime": now_ms,
|
||
"mtime": now_ms,
|
||
"size": size_bytes,
|
||
"type": "plain",
|
||
"eden": {},
|
||
}
|
||
if put_doc(base_url, db, auth, path, main_doc, args.dry_run):
|
||
ok += 1
|
||
else:
|
||
fail += 1
|
||
if i % 50 == 0:
|
||
logger.info(" main docs: %d/%d (ok=%d fail=%d)",
|
||
i, len(prepared), ok, fail)
|
||
|
||
# 5. Index page
|
||
index_md_lines = [
|
||
"---",
|
||
"id: 00_index",
|
||
"type: index",
|
||
f"imported_at: {time.strftime('%Y-%m-%d')}",
|
||
"---",
|
||
"",
|
||
"# 🗂 Knowledge graph index",
|
||
"",
|
||
f"Migrated from `memory/memory-gendesign.jsonl` ({len(entities)} entities,"
|
||
f" {len(relations)} relations).",
|
||
"",
|
||
]
|
||
by_subdir: dict[str, list[str]] = defaultdict(list)
|
||
for name, path in name_to_path.items():
|
||
by_subdir[path.split("/")[0]].append(name)
|
||
for subdir in sorted(by_subdir):
|
||
index_md_lines.append(f"## {subdir}/ ({len(by_subdir[subdir])})")
|
||
index_md_lines.append("")
|
||
for n in sorted(by_subdir[subdir]):
|
||
p = name_to_path[n][:-3]
|
||
index_md_lines.append(f"- [[{p}|{n}]]")
|
||
index_md_lines.append("")
|
||
index_content = "\n".join(index_md_lines)
|
||
index_chunk_id = chunk_id_for(index_content)
|
||
# chunk first
|
||
put_doc(base_url, db, auth, index_chunk_id,
|
||
{"_id": index_chunk_id, "data": index_content, "type": "leaf"},
|
||
args.dry_run)
|
||
delete_doc(base_url, db, auth, "00_index.md")
|
||
put_doc(base_url, db, auth, "00_index.md", {
|
||
"_id": "00_index.md", "path": "00_index.md",
|
||
"children": [index_chunk_id], "ctime": now_ms, "mtime": now_ms,
|
||
"size": len(index_content.encode("utf-8")),
|
||
"type": "plain", "eden": {},
|
||
}, args.dry_run)
|
||
|
||
logger.info("DONE: %d ok, %d failed", ok, fail)
|
||
return 0 if fail == 0 else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|