769b16df47
Worker крашился на старте с `ModuleNotFoundError: No module named 'psycopg2'` при импорте app/workers/tasks/objective_etl, потому что app/services/objective_etl писал на psycopg2 API, а в pyproject.toml есть только psycopg[binary]>=3.2.0. Изменения: - psycopg2 → psycopg (v3) - psycopg2.extras.execute_values → _bulk_upsert helper, эмулирующий тот же behavior через cur.executemany (в psycopg v3 это pipeline-mode, overhead на сетевые round-trips минимален) - psycopg.connect совместим с тем же connection string Тесты: AST parse ✓, ruff ✓, import ✓
113 lines
4.2 KiB
Python
113 lines
4.2 KiB
Python
"""Удаляет ghost docs из decisions/, sessions/, sources/ в CouchDB —
|
||
те что есть в CouchDB но отсутствуют в локальном vault.
|
||
"""
|
||
import os
|
||
from pathlib import Path
|
||
from urllib.parse import quote
|
||
|
||
import requests
|
||
|
||
VAULT = Path(r"C:\Users\user\Documents\gendesign-memory")
|
||
BASE = "https://obsidian.gendsgn.ru/gendesign-vault"
|
||
AUTH = ("obsidian", os.environ["COUCHDB_PASSWORD"])
|
||
TARGETS = ("decisions/", "sessions/", "sources/")
|
||
|
||
|
||
def main() -> None:
|
||
# 1. Local files (case-insensitive set, forward-slash paths)
|
||
local: set[str] = set()
|
||
for top in ("decisions", "sessions", "sources"):
|
||
d = VAULT / top
|
||
if d.exists():
|
||
for f in d.rglob("*.md"):
|
||
rel = str(f.relative_to(VAULT)).replace(os.sep, "/").lower()
|
||
local.add(rel)
|
||
print(f"Local files in target dirs: {len(local)}")
|
||
|
||
# 2. Fetch CouchDB docs
|
||
r = requests.get(f"{BASE}/_all_docs", auth=AUTH,
|
||
params={"limit": 10000}, timeout=60)
|
||
rows = r.json()["rows"]
|
||
|
||
# 3. Identify ghosts
|
||
ghosts: list[tuple[str, str]] = []
|
||
keepers: list[str] = []
|
||
for row in rows:
|
||
did = row["id"]
|
||
if not did.endswith(".md"):
|
||
continue
|
||
if not did.startswith(TARGETS):
|
||
continue
|
||
if did.lower() in local:
|
||
keepers.append(did)
|
||
else:
|
||
ghosts.append((did, row["value"]["rev"]))
|
||
|
||
print(f"Keepers (matched local): {len(keepers)}")
|
||
print(f"Ghosts (no local match): {len(ghosts)}")
|
||
for did, _ in ghosts[:30]:
|
||
print(f" ghost: {did}")
|
||
if len(ghosts) > 30:
|
||
print(f" ... +{len(ghosts) - 30} more")
|
||
|
||
if not ghosts:
|
||
print("Nothing to do.")
|
||
return
|
||
|
||
# 4. Сначала найдём chunks которые ссылаются ghosts. Это потенциальные orphans.
|
||
ghost_chunk_refs: set[str] = set()
|
||
for did, _ in ghosts:
|
||
rd = requests.get(f"{BASE}/{quote(did, safe='')}", auth=AUTH, timeout=30)
|
||
if rd.status_code == 200:
|
||
d = rd.json()
|
||
for cid in d.get("children", []):
|
||
ghost_chunk_refs.add(cid)
|
||
for cid in d.get("eden", {}):
|
||
ghost_chunk_refs.add(cid)
|
||
print(f"Chunks referenced by ghosts: {len(ghost_chunk_refs)}")
|
||
|
||
# 5. Проверяем у скольких из них ещё есть alive parent
|
||
ghost_ids = {did for did, _ in ghosts}
|
||
alive_md = [r["id"] for r in rows
|
||
if r["id"].endswith(".md") and r["id"] not in ghost_ids]
|
||
print(f"Scanning {len(alive_md)} alive .md docs for chunk refs...")
|
||
alive_chunks: set[str] = set()
|
||
for i, did in enumerate(alive_md):
|
||
if i % 100 == 0 and i > 0:
|
||
print(f" {i}/{len(alive_md)}")
|
||
rd = requests.get(f"{BASE}/{quote(did, safe='')}", auth=AUTH, timeout=30)
|
||
if rd.status_code == 200:
|
||
d = rd.json()
|
||
for cid in d.get("children", []):
|
||
alive_chunks.add(cid)
|
||
for cid in d.get("eden", {}):
|
||
alive_chunks.add(cid)
|
||
|
||
orphans = ghost_chunk_refs - alive_chunks
|
||
print(f"Orphan chunks (only ghosts reference them): {len(orphans)}")
|
||
|
||
# 6. Получаем _rev для orphan chunks
|
||
chunk_revs: dict[str, str] = {}
|
||
chunk_id_to_rev = {r["id"]: r["value"]["rev"] for r in rows if r["id"].startswith("h:")}
|
||
for cid in orphans:
|
||
if cid in chunk_id_to_rev:
|
||
chunk_revs[cid] = chunk_id_to_rev[cid]
|
||
|
||
# 7. Bulk delete
|
||
del_docs = [{"_id": did, "_rev": rev, "_deleted": True} for did, rev in ghosts]
|
||
del_docs.extend({"_id": cid, "_rev": rev, "_deleted": True}
|
||
for cid, rev in chunk_revs.items())
|
||
|
||
print(f"Bulk delete {len(del_docs)} docs ({len(ghosts)} ghost main + "
|
||
f"{len(chunk_revs)} orphan chunks)...")
|
||
rb = requests.post(f"{BASE}/_bulk_docs", json={"docs": del_docs},
|
||
auth=AUTH, timeout=300)
|
||
if rb.status_code in (200, 201):
|
||
ok = sum(1 for x in rb.json() if x.get("ok"))
|
||
print(f"Deleted: {ok}/{len(del_docs)}")
|
||
else:
|
||
print(f"Bulk delete failed: HTTP {rb.status_code} {rb.text[:200]}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|