"""data/atlas.json(파이프라인 결과)을 DB로 가져오기. .venv/bin/python -m server.seed # 사람이 고친 항목(edited=1)은 보존하며 갱신 .venv/bin/python -m server.seed --reset # DB 내용을 지우고 새로 가져옴 (피드백·기록은 유지) 자동 생성 행만 덮어쓰므로 파이프라인을 다시 돌려도 검수 결과가 사라지지 않는다. """ import argparse import json import os from pathlib import Path from . import db ATLAS_JSON = Path(os.environ.get("ATLAS_JSON", db.ROOT / "data" / "atlas.json")) def seed(con, data: dict, reset: bool = False) -> dict: ts = db.now() stats = {"terms_new": 0, "terms_updated": 0, "terms_kept": 0, "terms_removed": 0, "relations_new": 0, "relations_updated": 0, "relations_kept": 0, "relations_removed": 0} with con: if reset: for t in ("books", "galaxies", "toc", "terms", "term_sources", "relations"): con.execute(f"DELETE FROM {t}") for b in data["books"]: con.execute( "INSERT INTO books(book_id,title,short,edition,year,is_primary,filename) VALUES(?,?,?,?,?,?,?) " "ON CONFLICT(book_id) DO UPDATE SET title=excluded.title, short=excluded.short, edition=excluded.edition," " year=excluded.year, is_primary=excluded.is_primary, filename=excluded.filename", (b["book_id"], b["title"], b["short"], b["edition"], b["year"], int(b["primary"]), b["filename"]), ) for i, g in enumerate(data["galaxies"]): con.execute( "INSERT INTO galaxies(galaxy_id,name,sub,color,ord,chapter_no,updated_at) VALUES(?,?,?,?,?,?,?) " "ON CONFLICT(galaxy_id) DO UPDATE SET name=excluded.name, sub=excluded.sub, color=excluded.color," " ord=excluded.ord, chapter_no=excluded.chapter_no WHERE galaxies.edited=0", (g["id"], g["name"], g["sub"], g["color"], i, g["chapter_no"], ts), ) for n in data["toc"]: con.execute( "INSERT INTO toc(toc_id,book_id,kind,level,number,title,page,parent_id,galaxy_id) VALUES(?,?,?,?,?,?,?,?,?) " "ON CONFLICT(toc_id) DO UPDATE SET book_id=excluded.book_id, kind=excluded.kind, level=excluded.level," " number=excluded.number, title=excluded.title, page=excluded.page, parent_id=excluded.parent_id," " galaxy_id=excluded.galaxy_id WHERE toc.edited=0", (n["id"], n["book_id"], n["kind"], n["level"], n["number"], n["title"], n["page"], n["parent_id"], n["galaxy_id"]), ) toc_ids = {n["id"] for n in data["toc"]} for r in con.execute("SELECT toc_id FROM toc WHERE edited=0").fetchall(): if r["toc_id"] not in toc_ids: con.execute("DELETE FROM toc WHERE toc_id=?", (r["toc_id"],)) existing = {r["term_id"]: r for r in con.execute("SELECT term_id, edited, origin FROM terms")} seen = set() for t in data["terms"]: tid = t["term_id"] seen.add(tid) d = t.get("definition") or {} vals = dict( name=t["name"], kind=t["kind"], lang=t["lang"], aliases=db.dumps(t["aliases"]), search_terms=db.dumps(t.get("search_terms", [])), galaxy_id=t["galaxy_id"], section_id=t["section_id"], section_guess=int(t.get("section_guess", False)), item_id=t.get("item_id"), section_ids=db.dumps(t.get("section_ids", [])), definition_text=d.get("text"), definition_book=d.get("book_id"), definition_page=d.get("page"), definition_method=d.get("method"), easy=t.get("easy"), example=t.get("example"), easy_method=t.get("easy_method"), ) if tid not in existing: cols = ["term_id", *vals, "review_status", "origin", "created_at", "updated_at"] con.execute(f"INSERT INTO terms({','.join(cols)}) VALUES({','.join('?' * len(cols))})", (tid, *vals.values(), "draft", "auto", ts, ts)) stats["terms_new"] += 1 elif not existing[tid]["edited"]: sets = ",".join(f"{k}=?" for k in vals) con.execute(f"UPDATE terms SET {sets}, updated_at=? WHERE term_id=?", (*vals.values(), ts, tid)) stats["terms_updated"] += 1 else: stats["terms_kept"] += 1 con.execute("DELETE FROM term_sources WHERE term_id=? AND origin='auto'", (tid,)) manual = {(r["book_id"], r["page"]) for r in con.execute( "SELECT book_id, page FROM term_sources WHERE term_id=? AND origin='manual'", (tid,))} for s in t["sources"]: if (s["book_id"], s["page"]) in manual: # 사람이 고친 출처가 우선 continue con.execute( "INSERT INTO term_sources(term_id,book_id,page,pdf_page,toc_id,section_id,galaxy_id,path,snippet,origin)" " VALUES(?,?,?,?,?,?,?,?,?,'auto')", (tid, s["book_id"], s["page"], s.get("pdf_page"), s.get("toc_id"), s.get("section_id"), s.get("galaxy_id"), s.get("path"), s.get("snippet")), ) for tid, r in existing.items(): if tid not in seen and r["origin"] == "auto" and not r["edited"]: con.execute("DELETE FROM terms WHERE term_id=?", (tid,)) con.execute("DELETE FROM term_sources WHERE term_id=?", (tid,)) stats["terms_removed"] += 1 existing = {r["relation_id"]: r for r in con.execute("SELECT relation_id, edited, method FROM relations")} seen = set() for r in data["relations"]: rid = r["relation_id"] seen.add(rid) vals = dict(source=r["source"], target=r["target"], type=r["type"], label=r["label"], directed=int(r["directed"]), method=r["method"], evidence=db.dumps(r["evidence"])) if rid not in existing: cols = ["relation_id", *vals, "review_status", "created_at", "updated_at"] con.execute(f"INSERT INTO relations({','.join(cols)}) VALUES({','.join('?' * len(cols))})", (rid, *vals.values(), "draft", ts, ts)) stats["relations_new"] += 1 elif not existing[rid]["edited"]: sets = ",".join(f"{k}=?" for k in vals) con.execute(f"UPDATE relations SET {sets}, updated_at=? WHERE relation_id=?", (*vals.values(), ts, rid)) stats["relations_updated"] += 1 else: stats["relations_kept"] += 1 for rid, r in existing.items(): if rid not in seen and r["method"] != "manual" and not r["edited"]: con.execute("DELETE FROM relations WHERE relation_id=?", (rid,)) stats["relations_removed"] += 1 con.execute("INSERT INTO meta(key,value) VALUES('seeded_at',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (ts,)) con.execute("INSERT INTO meta(key,value) VALUES('generated_at',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (data.get("generated_at", ""),)) db.audit(con, "system", "seed", "atlas", None, None, stats) return stats def seed_if_empty(con) -> None: """DB가 비었거나 atlas.json이 새로 만들어졌으면 가져온다 (사람이 고친 항목은 보존).""" if not ATLAS_JSON.exists(): return n = con.execute("SELECT COUNT(*) FROM terms").fetchone()[0] data = json.loads(ATLAS_JSON.read_text()) cur = con.execute("SELECT value FROM meta WHERE key='generated_at'").fetchone() if n == 0: print(f"DB가 비어 있어 {ATLAS_JSON}을 가져옵니다…") elif not cur or cur[0] != data.get("generated_at"): print(f"새 데이터({data.get('generated_at')})를 가져옵니다…") else: return print(seed(con, data)) def main(): ap = argparse.ArgumentParser() ap.add_argument("--reset", action="store_true") ap.add_argument("--file", default=str(ATLAS_JSON)) a = ap.parse_args() con = db.connect() print(seed(con, json.loads(Path(a.file).read_text()), reset=a.reset)) if __name__ == "__main__": main()