Light1YT
86e9ea2937
Многоагентный аудит + имплементация: один воркер на файл, точечные правки. Верификация: py_compile (47/47 .py) + tsc --noEmit (0 ошибок). Unit-тесты не прогонялись (окружение не поднято: rollup native dep / нет pytest-venv). Полностью исправлено (169): #1336, #1337, #1339, #1340, #1341, #1342, #1343, #1345, #1346, #1348, #1349, #1350, #1351, #1354, #1356, #1358, #1359, #1360, #1362, #1364, #1365, #1366, #1367, #1368, #1369, #1370, #1371, #1372, #1373, #1374, #1375, #1376, #1377, #1378, #1379, #1380, #1381, #1382, #1384, #1385, #1386, #1387, #1388, #1389, #1390, #1391, #1392, #1394, #1395, #1396, #1397, #1399, #1400, #1401, #1402, #1403, #1404, #1408, #1409, #1410, #1411, #1412, #1413, #1414, #1415, #1416, #1417, #1418, #1420, #1423, #1425, #1426, #1427, #1428, #1429, #1430, #1431, #1432, #1433, #1434, #1435, #1437, #1438, #1439, #1440, #1441, #1442, #1443, #1444, #1445, #1446, #1447, #1448, #1449, #1450, #1451, #1452, #1453, #1454, #1455, #1456, #1457, #1458, #1459, #1460, #1461, #1462, #1463, #1464, #1465, #1466, #1467, #1468, #1469, #1471, #1472, #1473, #1474, #1476, #1478, #1479, #1481, #1482, #1483, #1484, #1485, #1487, #1488, #1489, #1490, #1491, #1492, #1493, #1494, #1495, #1496, #1497, #1499, #1500, #1501, #1502, #1504, #1505, #1506, #1507, #1510, #1514, #1515, #1516, #1517, #1518, #1519, #1521, #1522, #1523, #1524, #1525, #1526, #1527, #1528, #1529, #1531, #1532, #1533, #1534, #1535, #1536, #1537, #1538 Частично (9, in-file часть, остаток cross-file): #1361, #1419, #1422, #1424, #1470, #1475, #1477, #1480, #1498 Требуют cross-file (3, не тронуты): #1338, #1363, #1421 Пропущено (1): #1539 Не входило в партию: 22 needs-Leha issue (нужны решения владельца). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
136 lines
5.7 KiB
Python
136 lines
5.7 KiB
Python
"""Удаляет ghost docs из decisions/, sessions/, sources/ в CouchDB —
|
||
те что есть в CouchDB но отсутствуют в локальном vault.
|
||
"""
|
||
import os
|
||
from pathlib import Path
|
||
from urllib.parse import quote
|
||
|
||
import requests
|
||
|
||
VAULT = Path(r"C:\Users\user\Documents\gendesign-memory")
|
||
BASE = "https://obsidian.gendsgn.ru/gendesign-vault"
|
||
AUTH = ("obsidian", os.environ["COUCHDB_PASSWORD"])
|
||
TARGETS = ("decisions/", "sessions/", "sources/")
|
||
|
||
|
||
def main() -> None:
|
||
# 1. Local files (case-insensitive set, forward-slash paths)
|
||
local: set[str] = set()
|
||
for top in ("decisions", "sessions", "sources"):
|
||
d = VAULT / top
|
||
if d.exists():
|
||
for f in d.rglob("*.md"):
|
||
rel = str(f.relative_to(VAULT)).replace(os.sep, "/").lower()
|
||
local.add(rel)
|
||
print(f"Local files in target dirs: {len(local)}")
|
||
|
||
# 2. Fetch ALL CouchDB docs (paginated — _all_docs усекается limit'ом,
|
||
# а неполный alive-набор приведёт к удалению живых chunks как orphan).
|
||
# Идём постранично через startkey_docid: last id страницы становится
|
||
# startkey следующей, дублирующая первая строка отбрасывается. Цикл
|
||
# завершается, когда страница вернула меньше page_size строк.
|
||
page_size = 10000
|
||
rows: list[dict] = []
|
||
startkey_docid: str | None = None
|
||
while True:
|
||
params: dict[str, object] = {"limit": page_size}
|
||
if startkey_docid is not None:
|
||
# startkey должен быть JSON-строкой
|
||
params["startkey"] = f'"{startkey_docid}"'
|
||
params["startkey_docid"] = startkey_docid
|
||
r = requests.get(f"{BASE}/_all_docs", auth=AUTH,
|
||
params=params, timeout=60)
|
||
page = r.json()["rows"]
|
||
if startkey_docid is not None and page and page[0]["id"] == startkey_docid:
|
||
# первая строка дублирует startkey предыдущей страницы — отбрасываем
|
||
page = page[1:]
|
||
rows.extend(page)
|
||
# Полная страница (page_size до отбрасывания дубля) => возможно есть ещё.
|
||
# Если строк меньше — это последняя страница.
|
||
if len(page) + (1 if startkey_docid is not None else 0) < page_size:
|
||
break
|
||
startkey_docid = rows[-1]["id"]
|
||
print(f"CouchDB docs fetched (paginated): {len(rows)}")
|
||
|
||
# 3. Identify ghosts
|
||
ghosts: list[tuple[str, str]] = []
|
||
keepers: list[str] = []
|
||
for row in rows:
|
||
did = row["id"]
|
||
if not did.endswith(".md"):
|
||
continue
|
||
if not did.startswith(TARGETS):
|
||
continue
|
||
if did.lower() in local:
|
||
keepers.append(did)
|
||
else:
|
||
ghosts.append((did, row["value"]["rev"]))
|
||
|
||
print(f"Keepers (matched local): {len(keepers)}")
|
||
print(f"Ghosts (no local match): {len(ghosts)}")
|
||
for did, _ in ghosts[:30]:
|
||
print(f" ghost: {did}")
|
||
if len(ghosts) > 30:
|
||
print(f" ... +{len(ghosts) - 30} more")
|
||
|
||
if not ghosts:
|
||
print("Nothing to do.")
|
||
return
|
||
|
||
# 4. Сначала найдём chunks которые ссылаются ghosts. Это потенциальные orphans.
|
||
ghost_chunk_refs: set[str] = set()
|
||
for did, _ in ghosts:
|
||
rd = requests.get(f"{BASE}/{quote(did, safe='')}", auth=AUTH, timeout=30)
|
||
if rd.status_code == 200:
|
||
d = rd.json()
|
||
for cid in d.get("children", []):
|
||
ghost_chunk_refs.add(cid)
|
||
for cid in d.get("eden", {}):
|
||
ghost_chunk_refs.add(cid)
|
||
print(f"Chunks referenced by ghosts: {len(ghost_chunk_refs)}")
|
||
|
||
# 5. Проверяем у скольких из них ещё есть alive parent
|
||
ghost_ids = {did for did, _ in ghosts}
|
||
alive_md = [r["id"] for r in rows
|
||
if r["id"].endswith(".md") and r["id"] not in ghost_ids]
|
||
print(f"Scanning {len(alive_md)} alive .md docs for chunk refs...")
|
||
alive_chunks: set[str] = set()
|
||
for i, did in enumerate(alive_md):
|
||
if i % 100 == 0 and i > 0:
|
||
print(f" {i}/{len(alive_md)}")
|
||
rd = requests.get(f"{BASE}/{quote(did, safe='')}", auth=AUTH, timeout=30)
|
||
if rd.status_code == 200:
|
||
d = rd.json()
|
||
for cid in d.get("children", []):
|
||
alive_chunks.add(cid)
|
||
for cid in d.get("eden", {}):
|
||
alive_chunks.add(cid)
|
||
|
||
orphans = ghost_chunk_refs - alive_chunks
|
||
print(f"Orphan chunks (only ghosts reference them): {len(orphans)}")
|
||
|
||
# 6. Получаем _rev для orphan chunks
|
||
chunk_revs: dict[str, str] = {}
|
||
chunk_id_to_rev = {r["id"]: r["value"]["rev"] for r in rows if r["id"].startswith("h:")}
|
||
for cid in orphans:
|
||
if cid in chunk_id_to_rev:
|
||
chunk_revs[cid] = chunk_id_to_rev[cid]
|
||
|
||
# 7. Bulk delete
|
||
del_docs = [{"_id": did, "_rev": rev, "_deleted": True} for did, rev in ghosts]
|
||
del_docs.extend({"_id": cid, "_rev": rev, "_deleted": True}
|
||
for cid, rev in chunk_revs.items())
|
||
|
||
print(f"Bulk delete {len(del_docs)} docs ({len(ghosts)} ghost main + "
|
||
f"{len(chunk_revs)} orphan chunks)...")
|
||
rb = requests.post(f"{BASE}/_bulk_docs", json={"docs": del_docs},
|
||
auth=AUTH, timeout=300)
|
||
if rb.status_code in (200, 201):
|
||
ok = sum(1 for x in rb.json() if x.get("ok"))
|
||
print(f"Deleted: {ok}/{len(del_docs)}")
|
||
else:
|
||
print(f"Bulk delete failed: HTTP {rb.status_code} {rb.text[:200]}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|