first commit

embedding.py

@@ -0,0 +1,158 @@
+import json
+from openai import OpenAI
+from dotenv import load_dotenv
+import os
+import time
+import numpy as np
+
+load_dotenv()
+
+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
+
+INPUT_JSON = "spinnaker_issues_.json"
+OUTPUT_JSON = "spinnaker_embeddings_.json"
+MODEL = "text-embedding-3-small"
+
+CHUNK_CHAR_LIMIT = 3000   # 필요하면 조절
+
+
+def load_json(path):
+    if not os.path.exists(path):
+        return {}
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        print(f"[WARN] failed to load existing {path}: {e}")
+        return {}
+
+
+def atomic_save(path, data):
+    tmp = path + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False)
+    os.replace(tmp, path)
+
+
+def load_issues(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def build_issue_text(issue):
+    """title + body 묶어서 임베딩 입력 텍스트 생성"""
+    title = issue.get("title", "")
+    body = issue.get("body", "")
+    return f"{title}\n\n{body}".strip()
+
+
+def split_into_chunks(text: str, limit: int = CHUNK_CHAR_LIMIT):
+    """긴 텍스트를 limit 길이만큼 청크로 나눔."""
+    return [text[i:i + limit] for i in range(0, len(text), limit)]
+
+
+def embed_text_chunks(client, chunks):
+    """여러 chunk를 embedding 후 평균 벡터 반환"""
+    res = client.embeddings.create(
+        model=MODEL,
+        input=chunks
+    )
+
+    vectors = [np.array(item.embedding) for item in res.data]
+    avg_vec = np.mean(vectors, axis=0)
+    return avg_vec.tolist()
+
+
+def format_seconds(sec: float) -> str:
+    """초를 'HH:MM:SS' 문자열로 포맷"""
+    sec = int(sec)
+    h = sec // 3600
+    m = (sec % 3600) // 60
+    s = sec % 60
+    return f"{h:02d}:{m:02d}:{s:02d}"
+
+
+def main():
+    client = OpenAI()
+    issues = load_issues(INPUT_JSON)
+
+    # 🔥 기존 결과 로드 후 이어서 진행
+    embeddings = load_json(OUTPUT_JSON)
+    total_issues = len(issues)
+    already_embedded = len(embeddings)
+
+    print(f"[INFO] 전체 이슈 수: {total_issues}")
+    print(f"[INFO] 이미 임베딩된 이슈 수: {already_embedded}")
+    print(f"[INFO] 남은 이슈 수: {total_issues - already_embedded}")
+
+    start_time = time.time()
+    newly_processed = 0  # 이번 실행에서 새로 임베딩한 개수
+
+    for idx, issue in enumerate(issues, start=1):
+        issue_id = str(issue["id"])
+
+        # 🔥 이미 처리한 issue는 스킵
+        if issue_id in embeddings:
+            # 진행도 표시용: 스킵도 '이미 완료된 것'으로 취급
+            progress_count = already_embedded + newly_processed
+            progress_percent = (progress_count / total_issues) * 100 if total_issues > 0 else 0.0
+            print(f"[SKIP] issue {issue_id} already embedded "
+                  f"({progress_count}/{total_issues}, {progress_percent:.2f}%)")
+            continue
+
+        text = build_issue_text(issue)
+        if not text.strip():
+            print(f"[WARN] issue {issue_id} has empty text. skipping")
+            continue
+
+        progress_count = already_embedded + newly_processed
+        progress_percent = (progress_count / total_issues) * 100 if total_issues > 0 else 0.0
+
+        print(f"[INFO] embedding issue {issue_id}, index={idx}/{total_issues}, "
+              f"진행도={progress_percent:.2f}%, text length={len(text)}")
+
+        chunks = split_into_chunks(text)
+        print(f"[INFO] → split into {len(chunks)} chunks")
+
+        # 🔥 API 재시도 로직
+        for attempt in range(5):
+            try:
+                t0 = time.time()
+                vec = embed_text_chunks(client, chunks)
+                t1 = time.time()
+                elapsed_for_issue = t1 - t0
+                break
+            except Exception as e:
+                print(f"[ERROR] embedding issue {issue_id}, attempt {attempt+1}/5: {e}")
+                time.sleep(2)
+        else:
+            print(f"[FAIL] issue {issue_id} skipped after 5 retries")
+            continue
+
+        embeddings[issue_id] = vec
+        newly_processed += 1
+
+        # ✅ ETA 계산
+        total_elapsed = time.time() - start_time
+        if newly_processed > 0:
+            avg_per_issue = total_elapsed / newly_processed
+            remaining_issues = total_issues - (already_embedded + newly_processed)
+            eta_seconds = max(0, remaining_issues * avg_per_issue)
+            eta_str = format_seconds(eta_seconds)
+        else:
+            eta_str = "N/A"
+
+        print(f"[OK] Embedded issue {issue_id}, vector length={len(vec)}, "
+              f"소요시간(방금): {elapsed_for_issue:.2f}s, 예상 남은 시간: {eta_str}")
+
+        # 🔥 즉시 저장
+        atomic_save(OUTPUT_JSON, embeddings)
+        print(f"[SAVE] 현재까지 {already_embedded + newly_processed}/{total_issues}개 저장 완료 → {OUTPUT_JSON}")
+
+    print(f"[DONE] total embedded issues: {len(embeddings)}")
+    atomic_save(OUTPUT_JSON, embeddings)
+    print("[FINAL SAVE] completed")
+
+
+if __name__ == "__main__":
+    main()