jsahn
4e943700fc
- Introduced common.js for shared functions including API calls, error reporting, and UI helpers. - Implemented feedback.js for the feedback reporting page, including form handling and displaying feedback status. - Added layout.js for managing galaxy and section layouts in the visualization. - Created login.js for user authentication with session management. - Developed search.js for term searching based on user input and synonyms. - Implemented signup.js for user registration with validation and immediate login.
153 lines
8.2 KiB
Python
153 lines
8.2 KiB
Python
"""data/atlas.json(파이프라인 결과)을 DB로 가져오기.
|
|
|
|
.venv/bin/python -m server.seed # 사람이 고친 항목(edited=1)은 보존하며 갱신
|
|
.venv/bin/python -m server.seed --reset # DB 내용을 지우고 새로 가져옴 (피드백·기록은 유지)
|
|
|
|
자동 생성 행만 덮어쓰므로 파이프라인을 다시 돌려도 검수 결과가 사라지지 않는다.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from . import db
|
|
|
|
ATLAS_JSON = Path(os.environ.get("ATLAS_JSON", db.ROOT / "data" / "atlas.json"))
|
|
|
|
|
|
def seed(con, data: dict, reset: bool = False) -> dict:
|
|
ts = db.now()
|
|
stats = {"terms_new": 0, "terms_updated": 0, "terms_kept": 0, "terms_removed": 0,
|
|
"relations_new": 0, "relations_updated": 0, "relations_kept": 0, "relations_removed": 0}
|
|
with con:
|
|
if reset:
|
|
for t in ("books", "galaxies", "toc", "terms", "term_sources", "relations"):
|
|
con.execute(f"DELETE FROM {t}")
|
|
for b in data["books"]:
|
|
con.execute(
|
|
"INSERT INTO books(book_id,title,short,edition,year,is_primary,filename) VALUES(?,?,?,?,?,?,?) "
|
|
"ON CONFLICT(book_id) DO UPDATE SET title=excluded.title, short=excluded.short, edition=excluded.edition,"
|
|
" year=excluded.year, is_primary=excluded.is_primary, filename=excluded.filename",
|
|
(b["book_id"], b["title"], b["short"], b["edition"], b["year"], int(b["primary"]), b["filename"]),
|
|
)
|
|
for i, g in enumerate(data["galaxies"]):
|
|
con.execute(
|
|
"INSERT INTO galaxies(galaxy_id,name,sub,color,ord,chapter_no,updated_at) VALUES(?,?,?,?,?,?,?) "
|
|
"ON CONFLICT(galaxy_id) DO UPDATE SET name=excluded.name, sub=excluded.sub, color=excluded.color,"
|
|
" ord=excluded.ord, chapter_no=excluded.chapter_no WHERE galaxies.edited=0",
|
|
(g["id"], g["name"], g["sub"], g["color"], i, g["chapter_no"], ts),
|
|
)
|
|
for n in data["toc"]:
|
|
con.execute(
|
|
"INSERT INTO toc(toc_id,book_id,kind,level,number,title,page,parent_id,galaxy_id) VALUES(?,?,?,?,?,?,?,?,?) "
|
|
"ON CONFLICT(toc_id) DO UPDATE SET book_id=excluded.book_id, kind=excluded.kind, level=excluded.level,"
|
|
" number=excluded.number, title=excluded.title, page=excluded.page, parent_id=excluded.parent_id,"
|
|
" galaxy_id=excluded.galaxy_id WHERE toc.edited=0",
|
|
(n["id"], n["book_id"], n["kind"], n["level"], n["number"], n["title"], n["page"], n["parent_id"], n["galaxy_id"]),
|
|
)
|
|
|
|
toc_ids = {n["id"] for n in data["toc"]}
|
|
for r in con.execute("SELECT toc_id FROM toc WHERE edited=0").fetchall():
|
|
if r["toc_id"] not in toc_ids:
|
|
con.execute("DELETE FROM toc WHERE toc_id=?", (r["toc_id"],))
|
|
existing = {r["term_id"]: r for r in con.execute("SELECT term_id, edited, origin FROM terms")}
|
|
seen = set()
|
|
for t in data["terms"]:
|
|
tid = t["term_id"]
|
|
seen.add(tid)
|
|
d = t.get("definition") or {}
|
|
vals = dict(
|
|
name=t["name"], kind=t["kind"], lang=t["lang"], aliases=db.dumps(t["aliases"]),
|
|
search_terms=db.dumps(t.get("search_terms", [])), galaxy_id=t["galaxy_id"], section_id=t["section_id"],
|
|
section_guess=int(t.get("section_guess", False)), item_id=t.get("item_id"),
|
|
section_ids=db.dumps(t.get("section_ids", [])),
|
|
definition_text=d.get("text"), definition_book=d.get("book_id"), definition_page=d.get("page"),
|
|
definition_method=d.get("method"), easy=t.get("easy"), example=t.get("example"),
|
|
easy_method=t.get("easy_method"),
|
|
)
|
|
if tid not in existing:
|
|
cols = ["term_id", *vals, "review_status", "origin", "created_at", "updated_at"]
|
|
con.execute(f"INSERT INTO terms({','.join(cols)}) VALUES({','.join('?' * len(cols))})",
|
|
(tid, *vals.values(), "draft", "auto", ts, ts))
|
|
stats["terms_new"] += 1
|
|
elif not existing[tid]["edited"]:
|
|
sets = ",".join(f"{k}=?" for k in vals)
|
|
con.execute(f"UPDATE terms SET {sets}, updated_at=? WHERE term_id=?", (*vals.values(), ts, tid))
|
|
stats["terms_updated"] += 1
|
|
else:
|
|
stats["terms_kept"] += 1
|
|
con.execute("DELETE FROM term_sources WHERE term_id=? AND origin='auto'", (tid,))
|
|
manual = {(r["book_id"], r["page"]) for r in con.execute(
|
|
"SELECT book_id, page FROM term_sources WHERE term_id=? AND origin='manual'", (tid,))}
|
|
for s in t["sources"]:
|
|
if (s["book_id"], s["page"]) in manual: # 사람이 고친 출처가 우선
|
|
continue
|
|
con.execute(
|
|
"INSERT INTO term_sources(term_id,book_id,page,pdf_page,toc_id,section_id,galaxy_id,path,snippet,origin)"
|
|
" VALUES(?,?,?,?,?,?,?,?,?,'auto')",
|
|
(tid, s["book_id"], s["page"], s.get("pdf_page"), s.get("toc_id"), s.get("section_id"),
|
|
s.get("galaxy_id"), s.get("path"), s.get("snippet")),
|
|
)
|
|
for tid, r in existing.items():
|
|
if tid not in seen and r["origin"] == "auto" and not r["edited"]:
|
|
con.execute("DELETE FROM terms WHERE term_id=?", (tid,))
|
|
con.execute("DELETE FROM term_sources WHERE term_id=?", (tid,))
|
|
stats["terms_removed"] += 1
|
|
|
|
existing = {r["relation_id"]: r for r in con.execute("SELECT relation_id, edited, method FROM relations")}
|
|
seen = set()
|
|
for r in data["relations"]:
|
|
rid = r["relation_id"]
|
|
seen.add(rid)
|
|
vals = dict(source=r["source"], target=r["target"], type=r["type"], label=r["label"],
|
|
directed=int(r["directed"]), method=r["method"], evidence=db.dumps(r["evidence"]))
|
|
if rid not in existing:
|
|
cols = ["relation_id", *vals, "review_status", "created_at", "updated_at"]
|
|
con.execute(f"INSERT INTO relations({','.join(cols)}) VALUES({','.join('?' * len(cols))})",
|
|
(rid, *vals.values(), "draft", ts, ts))
|
|
stats["relations_new"] += 1
|
|
elif not existing[rid]["edited"]:
|
|
sets = ",".join(f"{k}=?" for k in vals)
|
|
con.execute(f"UPDATE relations SET {sets}, updated_at=? WHERE relation_id=?", (*vals.values(), ts, rid))
|
|
stats["relations_updated"] += 1
|
|
else:
|
|
stats["relations_kept"] += 1
|
|
for rid, r in existing.items():
|
|
if rid not in seen and r["method"] != "manual" and not r["edited"]:
|
|
con.execute("DELETE FROM relations WHERE relation_id=?", (rid,))
|
|
stats["relations_removed"] += 1
|
|
con.execute("INSERT INTO meta(key,value) VALUES('seeded_at',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (ts,))
|
|
con.execute("INSERT INTO meta(key,value) VALUES('generated_at',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
(data.get("generated_at", ""),))
|
|
db.audit(con, "system", "seed", "atlas", None, None, stats)
|
|
return stats
|
|
|
|
|
|
def seed_if_empty(con) -> None:
|
|
"""DB가 비었거나 atlas.json이 새로 만들어졌으면 가져온다 (사람이 고친 항목은 보존)."""
|
|
if not ATLAS_JSON.exists():
|
|
return
|
|
n = con.execute("SELECT COUNT(*) FROM terms").fetchone()[0]
|
|
data = json.loads(ATLAS_JSON.read_text())
|
|
cur = con.execute("SELECT value FROM meta WHERE key='generated_at'").fetchone()
|
|
if n == 0:
|
|
print(f"DB가 비어 있어 {ATLAS_JSON}을 가져옵니다…")
|
|
elif not cur or cur[0] != data.get("generated_at"):
|
|
print(f"새 데이터({data.get('generated_at')})를 가져옵니다…")
|
|
else:
|
|
return
|
|
print(seed(con, data))
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--reset", action="store_true")
|
|
ap.add_argument("--file", default=str(ATLAS_JSON))
|
|
a = ap.parse_args()
|
|
con = db.connect()
|
|
print(seed(con, json.loads(Path(a.file).read_text()), reset=a.reset))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|