commit c0b140d27f0e0494762c267aa32e7b623e00fc6b Author: jsahn Date: Wed Sep 2 14:57:29 2026 +0900 first commit diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..bf304be --- /dev/null +++ b/.env.example @@ -0,0 +1,27 @@ +# MiniMax (필수) — 글로벌 엔드포인트. sk-cp-… Coding/Token Plan 키 사용 가능 +MINIMAX_API_KEY= +MINIMAX_BASE_URL=https://api.minimax.io/v1 +MINIMAX_MODEL=MiniMax-M3 +MINIMAX_FALLBACK_MODELS=MiniMax-M2.7,MiniMax-M2.5 + +# OpenAlex (선택) — 이메일을 넣으면 polite pool로 요청이 더 빠르고 안정적 +SEBRIEF_OPENALEX_MAILTO= + +# 수집 정책 +# 수집 창을 주 시작보다 며칠 앞당길지 (늦게 색인된 논문 보완; DOI 중복은 자동 스킵) +SEBRIEF_BACKFILL_DAYS=7 +# 한 다이제스트에 담는 최대 논문 수 (초과분은 다음 주로 이월) +SEBRIEF_MAX_PAPERS=40 +# 보조 RSS 피드 CSV, 항목 형식 "이름|URL" (예: e-Informatica SEJ) +SEBRIEF_RSS_FEEDS= +# MDPI 기본 블랙리스트에 더해 차단할 ISSN CSV +SEBRIEF_EXTRA_BLACKLIST_ISSNS= + +# 실행 정책 — 기본: 월요일 07:00 KST, 직전 주(월~일) 다이제스트 +SEBRIEF_RUN_WEEKDAY=0 +SEBRIEF_RUN_TIME_KST=07:00 +SEBRIEF_CATCHUP_WINDOW_HOURS=72 + +# 요약 후 LLM 팩트체크 (초록과 모순 검사, major면 1회 자동 수정) +SEBRIEF_VERIFY_ENABLED=true +SEBRIEF_LOG_LEVEL=INFO diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8441ece --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.env +.venv/ +__pycache__/ +*.pyc +*.db +*.sqlite3 +data/ +reports/ +logs/ +.pytest_cache/ +.ruff_cache/ +node_modules/ +dashboard/dist/ +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..7051890 --- /dev/null +++ b/README.md @@ -0,0 +1,125 @@ +# SE-Brief — OA 소프트웨어공학 논문 주간 다이제스트 봇 + +매주 월요일 아침(07:00 KST), 지난주(월~일)에 발행된 오픈액세스(OA) 소프트웨어공학 논문을 +OpenAlex에서 수집하고 MiniMax LLM으로 한국어 다이제스트를 만들어 **내부 게시**(파일 아카이브 + +tailnet 대시보드)하는 상주 프로그램입니다. 설계 문서는 [SEBRIEF_PLAN.md](SEBRIEF_PLAN.md)를 참고하세요. +US2KR의 파이프라인 철학을 그대로 따르는 별도 프로젝트입니다. + +``` +수집(OpenAlex · RSS) → SQLite 저장(DOI 중복 방지) → MiniMax 요약(JSON 스키마) + → LLM 교차 검증(초록 대조 팩트체크, major 이슈면 1회 자동 수정) → Markdown 렌더 → 파일 아카이브 + 대시보드 +``` + +## 수집 소스 (sebrief/sources.py) + +| 소스 | 방식 | +|---|---| +| TOSEM (완전 OA) | ISSN 화이트리스트, 전량 | +| TSE · EMSE · IST · JSS (하이브리드) | ISSN 화이트리스트 + **논문 단위** `is_oa` 필터 | +| SoftwareX · JOSS | 전량 (도구/소프트웨어 논문) | +| IEEE Access · PeerJ CS (메가저널) | ISSN + OpenAlex Software 토픽(`topics.subfield.id:1712`) + **키워드 이중 필터** | +| e-Informatica SEJ 등 | `SEBRIEF_RSS_FEEDS`로 RSS 보조 (DOI 없는 항목은 스킵) | +| MDPI 전 저널 | **제외** — ISSN 블랙리스트 + 발행사 이름 검사로 이중 차단 | + +- 한 논문은 DOI 기준으로 한 번만 요약됩니다. 다이제스트에 실린 논문은 동결(frozen)되어 재수집돼도 + 스킵되고, 수집 창(`SEBRIEF_BACKFILL_DAYS`)에 늦게 들어온 논문·캡 초과 논문은 다음 주로 이월됩니다. +- 다이제스트 idempotency 키는 ISO 주차(`2026-W36`) — US2KR의 krx_date 역할. + +## 설치 + +```bash +# Python 3.12 + uv +uv venv --python 3.12 .venv +uv pip install --python .venv/bin/python -e ".[dev]" + +cp .env.example .env # MINIMAX_API_KEY 필수 +``` + +## 사용 + +```bash +.venv/bin/python -m sebrief.cli show-config # 설정 확인(시크릿 마스킹) +.venv/bin/python -m sebrief.cli next-run # 다음 다이제스트 주차/시각 +.venv/bin/python -m sebrief.cli run-once --dry-run # 지금 즉시 생성해서 터미널에 출력(발행 X) +.venv/bin/python -m sebrief.cli run-once --dry-run --no-llm # LLM 없이 논문 목록만(축소 모드) +.venv/bin/python -m sebrief.cli run-once --week 2026-W35 # 특정 주차 생성·발행(끝난 주만) +.venv/bin/python -m sebrief.cli run-once --force # 이미 발행된 주여도 다시(재발행) +.venv/bin/python -m sebrief.cli test-notify # 발행 채널 점검(errors.log 기록) +.venv/bin/python -m sebrief.cli daemon # 상주 실행(주간 스케줄러) +``` + +종료 코드: `0` 성공 · `1` 오류(발행 채널 실패 포함) · `2` 스킵(이미 발행) + +아직 끝나지 않은 주를 `--week`로 지정하면 `--dry-run`이 아닌 한 거부합니다(늦게 발행되는 논문이 +동결돼 누락되고, 데몬의 정규 실행이 "이미 발행"으로 건너뛰기 때문). 발행 테스트는 +`--week <지난 주> --force`로 하세요. + +## 상주 실행(배포) + +프로세스 생존은 OS에 맡기고, 프로그램은 "다음 실행 시각 계산 → 대기 → 실행"만 반복합니다. +재시작 후 실행 시각에서 `SEBRIEF_CATCHUP_WINDOW_HOURS`(기본 72시간) 이내면 놓친 다이제스트를 즉시 +따라잡고, SQLite 기록으로 중복 발행을 막습니다. 주 1회 작업이라 월~목 사이 언제 재기동해도 따라잡습니다. + +| 환경 | 방법 | +|---|---| +| macOS | `bash deploy/install-launchd.sh` (launchd, KeepAlive) | +| macOS (Apple container) | `bash deploy/install-apple-container.sh` — 데몬+대시보드 빌드·기동. `--boot`를 붙이면 로그인 시 자동 기동(LaunchAgent) 등록 | +| Linux | `deploy/sebrief.service` → `systemctl enable --now sebrief` | +| Docker | `docker compose -f deploy/compose.yaml up -d --build` | + +`data/` `reports/` `logs/`는 호스트 디렉터리를 그대로 마운트하므로 배포 방식과 무관하게 산출물 +위치가 같습니다. + +## 설정(.env) + +| 변수 | 기본값 | 설명 | +|---|---|---| +| `MINIMAX_API_KEY` | — | 필수. 글로벌 엔드포인트(`api.minimax.io`) 키 | +| `MINIMAX_MODEL` / `MINIMAX_FALLBACK_MODELS` | `MiniMax-M3` / `M2.7,M2.5` | 실패 시 순차 폴백 | +| `SEBRIEF_OPENALEX_MAILTO` | — | 이메일을 넣으면 OpenAlex polite pool (더 빠르고 안정적) | +| `SEBRIEF_BACKFILL_DAYS` | `7` | 수집 창을 주 시작보다 며칠 앞당길지 (늦은 색인 보완) | +| `SEBRIEF_MAX_PAPERS` | `40` | 다이제스트당 최대 논문 수 (초과분은 다음 주 이월) | +| `SEBRIEF_RSS_FEEDS` | — | 보조 RSS CSV, 항목 `이름\|URL` | +| `SEBRIEF_EXTRA_BLACKLIST_ISSNS` | — | 기본 MDPI 블랙리스트에 더해 차단할 ISSN CSV | +| `SEBRIEF_RUN_WEEKDAY` / `SEBRIEF_RUN_TIME_KST` | `0` / `07:00` | 실행 요일(0=월)·시각 | +| `SEBRIEF_CATCHUP_WINDOW_HOURS` | `72` | 재시작 시 따라잡기 허용 창 | +| `SEBRIEF_VERIFY_ENABLED` | `true` | 요약 후 LLM 팩트체크(초록과 대조). major 이슈 발견 시 1회 자동 수정 후 재검증. 검증 실패는 발행을 막지 않고 결과는 JSON·대시보드에 기록 | +| `SEBRIEF_LOG_LEVEL` | `INFO` | 로그 레벨 | + +## 산출물 + +- `reports/YYYY-Wnn.md`, `.json` — ISO 주차 기준 아카이브 +- `data/sebrief.sqlite3` — 논문(DOI 중복 방지·동결)·다이제스트·발행 기록 +- `logs/sebrief.log` — 일자별 회전, API 키 마스킹 + +## 대시보드 (dashboard/) + +쌓인 다이제스트를 보는 웹 대시보드(TypeScript, 런타임 의존성 없는 Node 24 서버 + 번들 클라이언트). +`reports/*.json`만 읽는 읽기 전용 뷰라 데몬의 SQLite(WAL)와 절대 경합하지 않습니다. + +- 선택 주차 요약 타일(논문·저널·하이라이트·주제 그룹·검증 상태), 주차별 논문 수 추이 차트(표 전환 가능) +- 다이제스트 목록(주차 갭 표시)/본문(Markdown 렌더, sanitize), 수집 결측·오류 로그 표시 +- tailnet 전용, Apple container 배포 시 함께 기동: `http://localhost:8091` (포트: `SEBRIEF_DASHBOARD_PORT`) + +```bash +cd dashboard +npm install +npm run build # tsc 타입체크 + esbuild 번들 +SEBRIEF_REPORTS_DIR=../reports npm start # http://localhost:8091 +``` + +## 개발 + +```bash +.venv/bin/python -m pytest -q +.venv/bin/ruff check sebrief tests +``` + +LLM은 `sebrief/llm/base.py`의 `LLMProvider` 인터페이스 뒤에 있으므로, 다른 모델로 바꾸려면 구현체만 +추가하면 됩니다. Blogger 공개 발행은 아직 붙이지 않았습니다 — US2KR의 `BloggerDeliverer`를 그대로 +이식해 `sebrief/deliver/__init__.py`의 `build_deliverers`에 추가하면 됩니다(Deliverer 프로토콜 동일). + +## 저작권 + +OA 논문의 한국어 요약 + 출처·DOI 링크 표기입니다. 초록 원문은 전재하지 않고 자기 언어의 요약으로 +재구성합니다(내부 게시라 리스크가 낮지만 공개 전환 대비 처음부터 지킵니다). diff --git a/SEBRIEF_PLAN.md b/SEBRIEF_PLAN.md new file mode 100644 index 0000000..8d037e2 --- /dev/null +++ b/SEBRIEF_PLAN.md @@ -0,0 +1,110 @@ +# SE-Brief (가칭) — OA 소프트웨어공학 논문 요약 봇 (설계 계획) + +작성일: 2026-09-01 +상태: 계획 확정 대기 (구현 미착수) · US2KR과 **별도 프로젝트/repo** + +## 1. 목표 + +오픈액세스(OA) 소프트웨어공학 논문을 주기적으로 수집하고, MiniMax LLM으로 한국어 요약 +다이제스트를 만들어 **내부 게시**(파일 아카이브 + tailnet 대시보드)하는 상주 프로그램. +US2KR의 파이프라인 철학을 그대로 따른다: + +``` +수집(OpenAlex · RSS) → SQLite 저장(DOI 중복 방지) → MiniMax 요약(JSON 스키마) + → LLM 교차 검증 → Markdown 렌더 → 파일 아카이브 + 대시보드 +``` + +## 2. 확정된 결정사항 + +| 항목 | 결정 | +|---|---| +| OA 기준 | **논문 단위 OA** (저널 단위 아님) — 하이브리드 저널의 OA 논문 포함 | +| MDPI | **전면 제외** (ISSN 블랙리스트로 명시적 차단) | +| IEEE Access | **포함하되 SE 토픽 필터 적용** (메가저널 노이즈 차단) | +| TOSEM | 완전 OA (2026-01 ACM Open 전환) — 전량 수집 | +| 게시 | **내부 전용**: reports/ 파일 + tailnet 대시보드. Blogger/공개는 나중 옵션 | +| 프로젝트 | US2KR과 별도 repo (US2KR 코드를 복제 후 도메인 교체로 시작) | + +## 3. 미결정 사항 + +- [ ] 주기 — **주간(월요일 아침) 추천**. 저널 볼륨(주 10~20편)상 일간은 얇아짐 +- [ ] 프로젝트/repo 이름 (후보: `sebrief`, `se-digest`, `seweekly`) +- [ ] 대시보드 포트 (US2KR이 :8090 사용 중 → :8091 제안) + +## 4. 소스 + +**백본: OpenAlex API** — 전 저널 통합 색인, 무료·API 키 불필요. +필터: `is_oa=true` + 발행일 범위 + (저널 ISSN 화이트리스트 ∪ SE 토픽). + +| 소스 | 수집 방식 | +|---|---| +| TOSEM (ACM, 완전 OA) | ISSN 화이트리스트, 전량 | +| TSE · EMSE · IST · JSS (하이브리드) | ISSN 화이트리스트 + 논문 단위 OA 필터 | +| IEEE Access | ISSN + **SE 토픽/키워드 이중 필터** | +| PeerJ Computer Science | 전량 (SE 토픽 필터) | +| SoftwareX, JOSS | 전량 (도구/소프트웨어 논문) | +| e-Informatica SEJ | RSS 보조 | +| MDPI 전 저널 | **제외** — 블랙리스트 | + +- 보조/검증용: Crossref API (메타데이터 보강), Unpaywall (OA PDF URL 확인) +- Phase 3 후보: ACM 학회 프로시딩(FSE·ISSTA·ASE — ACM Open으로 전부 OA), + arXiv cs.SE (일 20~40편, 일간 포맷 필요) + +## 5. 데이터 모델 (SQLite) + +``` +papers(doi PK, title, abstract, authors, venue, published_date, oa_url, pdf_url, + topics, collected_at) +digests(week PK, markdown, result_json, created_at) -- 주간 다이제스트 아카이브 +deliveries(digest_week, channel, receipt, sent_at) -- US2KR reports/deliveries 패턴 +``` + +- 중복 방지: DOI 기준 upsert. 한 논문은 한 번만 요약 (재수집돼도 스킵) +- 다이제스트 idempotency: 주(week) 기준 — US2KR의 krx_date 역할 + +## 6. LLM 요약 (US2KR llm 레이어 재사용) + +- 입력: 논문별 제목 + 초록 + 메타데이터 (Phase 1은 초록 기반 — 토큰 저렴·안정) +- 출력 JSON 스키마 (주간 다이제스트 단위): + - `overview` — 이번 주 흐름 총평 2~3문장 + - `highlights[3]` — 논문당 {문제, 방법, 결과, 실무 시사점} 4~5문장 + - `by_topic{}` — LLM4SE·테스팅·실증연구·보안·유지보수 등 주제별 그룹, 논문당 1~2문장 + 링크 + - `worth_reading[]` — "읽어볼 논문" 픽 (US2KR watchlist 격) +- **검증 패스 재사용**: 요약이 초록과 모순되는지 팩트체크 → major면 1회 수정 → 재검증 +- 프롬프트 규칙: 초록에 없는 결과 수치 창작 금지, 저자 주장 vs 검증된 결과 구분, + 실무 시사점은 "초록 근거 내"로 한정 + +## 7. 게시 (내부 전용) + +- `reports/YYYY-Wnn.md` + `.json` — 주차 기준 아카이브 (FileDeliverer 재사용) +- 대시보드: US2KR 대시보드 구조 복제 (TypeScript, reports/*.json 읽기 전용) + - 좌측: 주차 목록 / 본문: 다이제스트 렌더 / 상단: 주간 논문 수·소스별 통계 + - tailnet 전용 (:8091), 컨테이너 분리 +- Blogger 발행: 코드는 US2KR BloggerDeliverer 그대로 이식 가능 — + 공개 전환 결정 시 BLOG_ID만 발급 (OAuth 계정·클라이언트 재사용) + +## 8. 운영 + +- 맥미니 Apple container 배포 (US2KR install 스크립트 패턴 복제) +- 스케줄: 거래소 캘린더 불필요 → 단순 주간 스케줄 (예: 월 07:00 KST) + 재시작 따라잡기 +- 비용: 주 1회, 논문 10~20편 초록 요약 + 검증 = MiniMax 호출 2~4회 수준 (미미) + +## 9. 저작권 + +- OA 논문 요약 + 출처·DOI 링크 표기는 안전. CC-BY는 저자·출처 표기 준수 +- 초록 원문 전재 금지 — 자기 언어의 한국어 요약으로 재구성 (내부 게시라 리스크 낮지만 + 공개 전환 대비 처음부터 지킴) + +## 10. 단계 + +1. **Phase 1 (MVP)**: OpenAlex 수집 → 초록 기반 주간 다이제스트 → 파일 아카이브 +2. **Phase 2**: 검증 패스, 대시보드(:8091), 주제 분류 고도화, 맥미니 상주 배포 +3. **Phase 3 (옵션)**: 풀텍스트 딥다이브(주 1편), ACM 학회 특집, arXiv cs.SE 일간, + Blogger 공개 전환 + +## 11. US2KR에서 재사용하는 것 + +| 재사용 | 새로 작성 | +|---|---| +| `llm/`(minimax·검증 구조), `deliver/`(file·blogger), Store·run_safely 패턴 | 수집기(OpenAlex/RSS), 논문용 프롬프트 | +| 대시보드 골격(TS 서버+클라이언트), 배포 스크립트, 로깅 | 주간 스케줄러, 다이제스트 렌더러 | diff --git a/dashboard/Dockerfile b/dashboard/Dockerfile new file mode 100644 index 0000000..abcb73f --- /dev/null +++ b/dashboard/Dockerfile @@ -0,0 +1,16 @@ +FROM node:24-slim AS build +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci +COPY tsconfig.json ./ +COPY src ./src +COPY web ./web +RUN npm run build + +FROM node:24-slim +WORKDIR /app +ENV NODE_ENV=production SEBRIEF_REPORTS_DIR=/data/reports PORT=8091 +COPY --from=build /app/dist ./dist +EXPOSE 8091 +USER node +CMD ["node", "dist/server.js"] diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json new file mode 100644 index 0000000..8549781 --- /dev/null +++ b/dashboard/package-lock.json @@ -0,0 +1,565 @@ +{ + "name": "sebrief-dashboard", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "sebrief-dashboard", + "version": "0.1.0", + "devDependencies": { + "@types/node": "^24.3.0", + "dompurify": "^3.2.4", + "esbuild": "^0.25.0", + "marked": "^16.0.0", + "typescript": "^5.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/dompurify": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.14.tgz", + "integrity": "sha512-dVoH9z+MY+C9IilgGCk3YfFqjLi3fChm2OiKJMzh6axrJ5qwxqWaZamgmHrpv22CN/KdbZJuGEGgfQoL00LTdg==", + "dev": true, + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/marked": { + "version": "16.4.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz", + "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==", + "dev": true, + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/dashboard/package.json b/dashboard/package.json new file mode 100644 index 0000000..a46912b --- /dev/null +++ b/dashboard/package.json @@ -0,0 +1,21 @@ +{ + "name": "sebrief-dashboard", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "typecheck": "tsc --noEmit", + "build:server": "esbuild src/server.ts --bundle --platform=node --target=node24 --format=esm --outfile=dist/server.js", + "build:client": "esbuild src/client/main.ts --bundle --minify --target=es2022 --outfile=dist/public/app.js", + "build:static": "mkdir -p dist/public && cp web/index.html web/styles.css dist/public/", + "build": "npm run typecheck && npm run build:server && npm run build:client && npm run build:static", + "start": "node dist/server.js" + }, + "devDependencies": { + "@types/node": "^24.3.0", + "dompurify": "^3.2.4", + "esbuild": "^0.25.0", + "marked": "^16.0.0", + "typescript": "^5.9.0" + } +} diff --git a/dashboard/src/client/chart.ts b/dashboard/src/client/chart.ts new file mode 100644 index 0000000..cc2a22b --- /dev/null +++ b/dashboard/src/client/chart.ts @@ -0,0 +1,172 @@ +/** Hand-rolled single-series SVG line chart with crosshair + tooltip hover. */ + +const SVG_NS = "http://www.w3.org/2000/svg"; +const W = 920; +const H = 260; +const PAD = { top: 14, right: 16, bottom: 26, left: 58 }; + +interface Point { + label: string; + value: number; +} + +function svgEl( + tag: K, + attrs: Record = {}, +): SVGElementTagNameMap[K] { + const el = document.createElementNS(SVG_NS, tag); + for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v)); + return el; +} + +function niceTicks(min: number, max: number, count = 4): number[] { + if (min === max) { + const pad = Math.abs(min) * 0.01 || 1; + min -= pad; + max += pad; + } + const span = max - min; + const step = 10 ** Math.floor(Math.log10(span / count)); + const err = span / count / step; + const mult = err >= 7.5 ? 10 : err >= 3.5 ? 5 : err >= 1.5 ? 2 : 1; + const s = step * mult; + const first = Math.ceil(min / s) * s; + const ticks: number[] = []; + for (let v = first; v <= max + s * 1e-9; v += s) ticks.push(Number(v.toFixed(10))); + return ticks; +} + +export function formatValue(v: number): string { + const decimals = Number.isInteger(v) ? 0 : Math.abs(v) >= 10 ? 1 : 2; + return v.toLocaleString("ko-KR", { minimumFractionDigits: decimals, maximumFractionDigits: decimals }); +} + +/** Render the chart into `host` (cleared first). Null values break the line. + * `labels` are ISO weeks; the axis shows their short form ("W36"). */ +export function renderLineChart( + host: HTMLElement, + labels: string[], + values: (number | null)[], + seriesLabel: string, +): void { + host.textContent = ""; + const points: (Point | null)[] = labels.map((label, i) => { + const v = values[i]; + return v === null || v === undefined ? null : { label, value: v }; + }); + const present = points.filter((p): p is Point => p !== null); + if (present.length === 0) { + const empty = document.createElement("p"); + empty.className = "chart-empty"; + empty.textContent = "표시할 데이터가 없습니다."; + host.append(empty); + return; + } + + const lo = Math.min(0, ...present.map((p) => p.value)); + const hi = Math.max(...present.map((p) => p.value)); + const span = hi - lo || Math.abs(hi) * 0.02 || 1; + const yMin = lo; + const yMax = hi + span * 0.08; + + const innerW = W - PAD.left - PAD.right; + const innerH = H - PAD.top - PAD.bottom; + const x = (i: number) => PAD.left + (labels.length === 1 ? innerW / 2 : (i / (labels.length - 1)) * innerW); + const y = (v: number) => PAD.top + (1 - (v - yMin) / (yMax - yMin)) * innerH; + + const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, role: "img" }); + const title = svgEl("title"); + title.textContent = seriesLabel; + svg.append(title); + + for (const tick of niceTicks(yMin, yMax)) { + svg.append(svgEl("line", { class: "gridline", x1: PAD.left, x2: W - PAD.right, y1: y(tick), y2: y(tick) })); + const label = svgEl("text", { class: "axis-label", x: PAD.left - 8, y: y(tick) + 4, "text-anchor": "end" }); + label.textContent = formatValue(tick); + svg.append(label); + } + svg.append( + svgEl("line", { class: "baseline", x1: PAD.left, x2: W - PAD.right, y1: H - PAD.bottom, y2: H - PAD.bottom }), + ); + + const maxLabels = Math.max(2, Math.floor(innerW / 90)); + const stride = Math.max(1, Math.ceil(labels.length / maxLabels)); + labels.forEach((d, i) => { + if (i % stride !== 0 && i !== labels.length - 1) return; + const label = svgEl("text", { class: "axis-label", x: x(i), y: H - 8, "text-anchor": "middle" }); + label.textContent = d.slice(5); // "2026-W36" -> "W36" + svg.append(label); + }); + + // polyline per contiguous run so nulls read as gaps, not interpolation + let run: string[] = []; + const flush = () => { + if (run.length >= 2) svg.append(svgEl("polyline", { class: "series-line", points: run.join(" ") })); + else if (run.length === 1) { + const [px = "", py = ""] = (run[0] ?? "").split(","); + svg.append(svgEl("circle", { class: "hover-dot", cx: px, cy: py, r: 3 })); + } + run = []; + }; + points.forEach((pt, i) => { + if (pt === null) flush(); + else run.push(`${x(i)},${y(pt.value)}`); + }); + flush(); + + // hover layer: crosshair + dot + tooltip on the nearest present point + const crosshair = svgEl("line", { + class: "crosshair", + y1: PAD.top, + y2: H - PAD.bottom, + x1: -10, + x2: -10, + visibility: "hidden", + }); + const dot = svgEl("circle", { class: "hover-dot", r: 4.5, visibility: "hidden" }); + svg.append(crosshair, dot); + + const tooltip = document.createElement("div"); + tooltip.className = "chart-tooltip"; + tooltip.hidden = true; + host.append(svg, tooltip); + + const hide = () => { + crosshair.setAttribute("visibility", "hidden"); + dot.setAttribute("visibility", "hidden"); + tooltip.hidden = true; + }; + svg.addEventListener("pointerleave", hide); + svg.addEventListener("pointermove", (ev) => { + const rect = svg.getBoundingClientRect(); + const relX = ((ev.clientX - rect.left) / rect.width) * W; + let best: { i: number; pt: Point } | null = null; + points.forEach((pt, i) => { + if (pt === null) return; + if (best === null || Math.abs(x(i) - relX) < Math.abs(x(best.i) - relX)) best = { i, pt }; + }); + if (best === null) { + hide(); + return; + } + const { i, pt } = best as { i: number; pt: Point }; + const cx = x(i); + const cy = y(pt.value); + crosshair.setAttribute("x1", String(cx)); + crosshair.setAttribute("x2", String(cx)); + crosshair.setAttribute("visibility", "visible"); + dot.setAttribute("cx", String(cx)); + dot.setAttribute("cy", String(cy)); + dot.setAttribute("visibility", "visible"); + tooltip.innerHTML = ""; + const labelEl = document.createElement("div"); + labelEl.textContent = pt.label; + const valueEl = document.createElement("div"); + valueEl.className = "tt-value"; + valueEl.textContent = `${formatValue(pt.value)}편`; + tooltip.append(labelEl, valueEl); + tooltip.hidden = false; + tooltip.style.left = `${(cx / W) * rect.width}px`; + tooltip.style.top = `${(cy / H) * rect.height}px`; + }); +} diff --git a/dashboard/src/client/main.ts b/dashboard/src/client/main.ts new file mode 100644 index 0000000..98bc526 --- /dev/null +++ b/dashboard/src/client/main.ts @@ -0,0 +1,359 @@ +import DOMPurify from "dompurify"; +import { marked } from "marked"; + +import type { + DigestResult, + DigestSummary, + ErrorLine, + HealthResponse, + StatsResponse, +} from "../shared/types"; +import { formatValue, renderLineChart } from "./chart"; + +function el( + tag: K, + className?: string, + text?: string, +): HTMLElementTagNameMap[K] { + const node = document.createElement(tag); + if (className) node.className = className; + if (text !== undefined) node.textContent = text; + return node; +} + +function byId(id: string): T { + const node = document.getElementById(id); + if (!node) throw new Error(`missing #${id}`); + return node as T; +} + +async function getJson(url: string): Promise { + const res = await fetch(url); + if (!res.ok) throw new Error(`${url} -> ${res.status}`); + return (await res.json()) as T; +} + +// ------------------------------------------------------------------ ISO week helpers + +const WEEK_RE = /^(\d{4})-W(\d{2})$/; + +/** Monday (UTC midnight) of an ISO week id. */ +function weekToMonday(week: string): Date { + const m = WEEK_RE.exec(week); + if (!m) throw new Error(`bad week ${week}`); + const year = Number(m[1]); + const wk = Number(m[2]); + const jan4 = new Date(Date.UTC(year, 0, 4)); // Jan 4 is always in ISO week 1 + const week1Monday = new Date(jan4); + week1Monday.setUTCDate(jan4.getUTCDate() - ((jan4.getUTCDay() + 6) % 7)); + const monday = new Date(week1Monday); + monday.setUTCDate(week1Monday.getUTCDate() + (wk - 1) * 7); + return monday; +} + +/** ISO week id of a UTC date. */ +function dateToWeek(d: Date): string { + const thu = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())); + thu.setUTCDate(thu.getUTCDate() - ((thu.getUTCDay() + 6) % 7) + 3); // Thursday decides the ISO year + const isoYear = thu.getUTCFullYear(); + const jan4 = new Date(Date.UTC(isoYear, 0, 4)); + const week1Monday = new Date(jan4); + week1Monday.setUTCDate(jan4.getUTCDate() - ((jan4.getUTCDay() + 6) % 7)); + const wk = Math.floor((thu.getTime() - week1Monday.getTime()) / (7 * 864e5)) + 1; + return `${isoYear}-W${String(wk).padStart(2, "0")}`; +} + +function addWeeks(week: string, delta: number): string { + const monday = weekToMonday(week); + monday.setUTCDate(monday.getUTCDate() + delta * 7); + return dateToWeek(monday); +} + +/** Weeks are lexicographically ordered only within a year; compare via their Mondays. */ +function weekLt(a: string, b: string): boolean { + return weekToMonday(a).getTime() < weekToMonday(b).getTime(); +} + +/** The ISO week containing today (Seoul calendar date). */ +function kstCurrentWeek(): string { + const today = new Intl.DateTimeFormat("en-CA", { timeZone: "Asia/Seoul" }).format(new Date()); + return dateToWeek(new Date(`${today}T00:00:00Z`)); +} + +function fmtDateTime(iso: string): string { + return new Date(iso).toLocaleString("ko-KR", { dateStyle: "medium", timeStyle: "short" }); +} + +/** Live Seoul clock in the top bar. */ +function startClock(): void { + const host = byId("clocks"); + const seoul = el("span", "clock"); + host.append(seoul); + const tick = () => { + seoul.textContent = `서울 ${new Intl.DateTimeFormat("ko-KR", { + timeZone: "Asia/Seoul", + month: "2-digit", + day: "2-digit", + weekday: "short", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: false, + }).format(new Date())}`; + }; + tick(); + setInterval(tick, 1000); +} + +// ------------------------------------------------------------------ stat tiles + +function verificationBadge(result: DigestResult): { text: string; cls: string } { + const v = result.verification; + if (result.degraded) return { text: "요약 없음", cls: "badge-warn" }; + if (!v) return { text: "검증 안 함", cls: "" }; + if (v.ok) return { text: v.issues.length > 0 ? `통과 (minor ${v.issues.length})` : "통과", cls: "badge-ok" }; + return { text: `이슈 ${v.issues.length}`, cls: "badge-error" }; +} + +function renderTiles(result: DigestResult): void { + const host = byId("tiles"); + host.textContent = ""; + const venues = new Set(result.payload.papers.map((p) => p.venue || "(미상)")); + const tiles: Array<[string, string]> = [ + ["논문", `${result.payload.papers.length}편`], + ["저널", `${venues.size}곳`], + ["하이라이트", `${result.digest?.highlights.length ?? 0}편`], + ["주제 그룹", `${result.digest?.by_topic.length ?? 0}개`], + ]; + for (const [label, value] of tiles) { + const tile = el("div", "tile"); + tile.append(el("div", "tile-label", label)); + tile.append(el("div", "tile-value", value)); + host.append(tile); + } + const tile = el("div", "tile"); + tile.append(el("div", "tile-label", "검증")); + const { text, cls } = verificationBadge(result); + const value = el("div", "tile-value tile-value-small", text); + if (cls) value.classList.add(cls.replace("badge-", "tone-")); + tile.append(value); + host.append(tile); +} + +// ------------------------------------------------------------------ digest list + body + +function badgeFor(s: DigestSummary): HTMLElement { + if (s.error) return el("span", "badge badge-error", "오류"); + if (s.degraded) return el("span", "badge badge-warn", "축소"); + return el("span", "badge badge-ok", "정상"); +} + +function missingRow(week: string, currentWeek: string): HTMLElement { + const label = week === currentWeek ? "이번 주 · 다음 주 월요일 생성 예정" : "다이제스트 없음"; + const item = el("li", "week-missing"); + item.append(el("span", "digest-week", week), el("span", undefined, label)); + return item; +} + +function renderList( + summaries: DigestSummary[], + selected: string, + onSelect: (week: string) => void, +): void { + const list = byId("digest-list"); + list.textContent = ""; + const byWeek = new Map(summaries.map((s) => [s.week, s])); + const current = kstCurrentWeek(); + const newest = summaries[0]?.week ?? current; + const oldest = summaries[summaries.length - 1]?.week ?? current; + const start = weekLt(newest, current) ? current : newest; + // walk week by week so gaps (missed runs) are visible, not silent + for (let w = start; !weekLt(w, oldest); w = addWeeks(w, -1)) { + const s = byWeek.get(w); + if (!s) { + list.append(missingRow(w, current)); + continue; + } + const item = el("li"); + const button = el("button"); + button.type = "button"; + button.setAttribute("aria-current", String(s.week === selected)); + button.append(el("span", "digest-week", s.week)); + const line = el("span", "digest-line"); + line.append(`${s.week_start} ~ ${s.week_end}`, badgeFor(s)); + line.append(el("span", "badge", `${s.paper_count}편`)); + if (s.data_gaps.length > 0) line.append(el("span", "badge badge-warn", `결측 ${s.data_gaps.length}`)); + button.append(line); + if (s.overview) button.append(el("span", "digest-line digest-overview", s.overview)); + button.addEventListener("click", () => onSelect(s.week)); + item.append(button); + list.append(item); + } +} + +function metaChips(result: DigestResult): HTMLElement[] { + const p = result.payload; + const chips: HTMLElement[] = [ + el("span", undefined, `${p.week} (${p.week_start} ~ ${p.week_end})`), + el("span", undefined, `생성 ${fmtDateTime(p.generated_at)}`), + ]; + if (result.model_used) chips.push(el("span", undefined, `모델 ${result.model_used}`)); + if (result.degraded) chips.push(el("span", "badge badge-warn", "요약 없음")); + const { text, cls } = verificationBadge(result); + if (result.verification) chips.push(el("span", `badge ${cls}`.trim(), `검증 ${text}`)); + for (const gap of p.data_gaps) chips.push(el("span", "badge badge-warn", gap)); + if (result.error) chips.push(el("span", "badge badge-error", result.error)); + return chips; +} + +/** Compact "핵심 요약" card: overview + worth-reading picks, above the full markdown. */ +function renderSummaryCard(result: DigestResult): void { + const host = byId("digest-summary"); + host.textContent = ""; + const digest = result.digest; + if (!digest) return; + const card = el("section", "summary-card"); + card.setAttribute("aria-label", "핵심 요약"); + if (digest.overview) card.append(el("p", "summary-overview", digest.overview)); + if (digest.worth_reading.length > 0) { + const block = el("div", "summary-block"); + block.append(el("h3", undefined, "읽어볼 논문")); + const list = el("ul"); + const papers = new Map(result.payload.papers.map((p) => [p.doi, p])); + // 제목 우선 매칭: LLM이 DOI를 뒤섞어도 링크는 (독자가 보는) 제목을 따라간다 (렌더러와 동일 규칙) + const normTitle = (s: string) => s.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim(); + const byTitle = new Map(); + for (const p of result.payload.papers) { + const key = normTitle(p.title); + byTitle.set(key, byTitle.has(key) ? null : p); + } + for (const pick of digest.worth_reading) { + const item = el("li"); + const paper = byTitle.get(normTitle(pick.title)) ?? papers.get(pick.doi); + const url = paper?.oa_url ?? `https://doi.org/${pick.doi}`; + const link = el("a", undefined, pick.title || pick.doi); + link.setAttribute("href", url); + link.setAttribute("target", "_blank"); + link.setAttribute("rel", "noopener noreferrer"); + item.append(link); + if (pick.reason) item.append(` — ${pick.reason}`); + list.append(item); + } + block.append(list); + card.append(block); + } + host.append(card); +} + +async function showDigest(week: string): Promise { + const result = await getJson(`/api/digests/${week}`); + renderTiles(result); + const meta = byId("digest-meta"); + meta.textContent = ""; + meta.append(...metaChips(result)); + renderSummaryCard(result); + const body = byId("digest-body"); + // Paper titles/abstract prose come from external sources - sanitize the rendered HTML. + body.innerHTML = DOMPurify.sanitize(marked.parse(result.markdown, { async: false })); + for (const link of body.querySelectorAll("a")) { + link.setAttribute("target", "_blank"); + link.setAttribute("rel", "noopener noreferrer"); + } +} + +// ------------------------------------------------------------------ chart section + +function setupChart(stats: StatsResponse): void { + const chartHost = byId("chart"); + const tableHost = byId("chart-table"); + const toggle = byId("table-toggle"); + + if (stats.weeks.length === 0) { + chartHost.append(el("p", "chart-empty", "다이제스트가 쌓이면 주차별 논문 수가 표시됩니다.")); + toggle.disabled = true; + return; + } + + renderLineChart(chartHost, stats.weeks, stats.paper_counts, "주차별 수집 논문 수"); + + const table = el("table", "data-table"); + const head = el("tr"); + head.append(el("th", undefined, "주차"), el("th", undefined, "논문 수")); + table.append(head); + for (let i = stats.weeks.length - 1; i >= 0; i--) { + const row = el("tr"); + const v = stats.paper_counts[i]; + row.append( + el("td", undefined, stats.weeks[i] ?? ""), + el("td", undefined, v === null || v === undefined ? "—" : formatValue(v)), + ); + table.append(row); + } + tableHost.append(table); + + toggle.addEventListener("click", () => { + const showTable = tableHost.hidden; + tableHost.hidden = !showTable; + toggle.setAttribute("aria-pressed", String(showTable)); + toggle.textContent = showTable ? "표 숨기기" : "표로 보기"; + }); +} + +// ------------------------------------------------------------------ errors + health + +function renderErrors(errors: ErrorLine[]): void { + if (errors.length === 0) return; + const panel = byId("errors-panel"); + panel.hidden = false; + byId("errors-count").textContent = String(errors.length); + const list = byId("errors-list"); + for (const e of errors.slice(0, 30)) { + const item = el("li"); + item.append( + el("span", "err-ts", e.ts), + el("span", "err-subject", e.subject), + el("span", "err-detail", e.detail), + ); + list.append(item); + } +} + +// ------------------------------------------------------------------ boot + +async function main(): Promise { + startClock(); + const health = await getJson("/api/health"); + byId("health").textContent = + health.digest_count > 0 + ? `다이제스트 ${health.digest_count}건 · 최신 ${health.latest_week}` + : "아직 쌓인 다이제스트가 없습니다"; + + const [summaries, stats, errors] = await Promise.all([ + getJson("/api/digests"), + getJson("/api/stats"), + getJson("/api/errors"), + ]); + + renderErrors(errors); + setupChart(stats); + + if (summaries.length === 0) { + byId("digest-body").append( + el("p", "chart-empty", "다이제스트가 아직 없습니다. 다음 주 월요일 아침 첫 다이제스트가 생성됩니다."), + ); + return; + } + + let selected = summaries[0]!.week; + const select = async (week: string) => { + selected = week; + renderList(summaries, selected, (w) => void select(w)); + await showDigest(week); + }; + await select(selected); +} + +void main().catch((err) => { + console.error(err); + byId("health").textContent = "대시보드 로드 실패 — 서버 로그를 확인하세요"; +}); diff --git a/dashboard/src/server.ts b/dashboard/src/server.ts new file mode 100644 index 0000000..e7dece1 --- /dev/null +++ b/dashboard/src/server.ts @@ -0,0 +1,219 @@ +/** Read-only dashboard API + static file server. + * + * The single source of truth is the reports directory the sebrief daemon archives into + * (reports/YYYY-Wnn.{md,json} + errors.log). Nothing here touches the SQLite DB, so the + * dashboard container never contends with the daemon's WAL over virtiofs. + */ + +import { createServer, type ServerResponse } from "node:http"; +import { readFile, readdir, stat } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import type { + DigestResult, + DigestSummary, + ErrorLine, + HealthResponse, + StatsResponse, +} from "./shared/types"; + +const REPORTS_DIR = process.env.SEBRIEF_REPORTS_DIR ?? "/data/reports"; +const PORT = Number(process.env.PORT ?? 8091); +const PUBLIC_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "public"); + +const WEEK_RE = /^\d{4}-W\d{2}$/; +const JSON_FILE_RE = /^\d{4}-W\d{2}\.json$/; + +// ------------------------------------------------------------------ digest loading + +/** mtime+size keyed cache so repeated requests do not re-parse unchanged files. */ +const cache = new Map(); + +async function loadDigest(week: string): Promise { + const file = path.join(REPORTS_DIR, `${week}.json`); + let st; + try { + st = await stat(file); + } catch { + return null; + } + const cached = cache.get(file); + if (cached && cached.mtimeMs === st.mtimeMs && cached.size === st.size) return cached.data; + try { + const data = JSON.parse(await readFile(file, "utf8")) as DigestResult; + cache.set(file, { mtimeMs: st.mtimeMs, size: st.size, data }); + return data; + } catch (err) { + console.error(`unparseable digest ${file}:`, err); + return null; + } +} + +async function listWeeks(): Promise { + let names: string[]; + try { + names = await readdir(REPORTS_DIR); + } catch { + return []; // directory not mounted yet / no digest produced yet + } + return names + .filter((n) => JSON_FILE_RE.test(n)) + .map((n) => n.slice(0, 8)) + .sort() + .reverse(); +} + +function summarize(week: string, r: DigestResult): DigestSummary { + return { + week, + week_start: r.payload.week_start, + week_end: r.payload.week_end, + generated_at: r.payload.generated_at, + model_used: r.model_used, + degraded: r.degraded, + error: r.error, + overview: r.digest?.overview ?? null, + paper_count: r.payload.papers.length, + highlight_count: r.digest?.highlights.length ?? 0, + topic_count: r.digest?.by_topic.length ?? 0, + verification_ok: r.verification?.ok ?? null, + data_gaps: r.payload.data_gaps, + }; +} + +async function allDigests(): Promise> { + const out: Array<{ week: string; result: DigestResult }> = []; + for (const week of await listWeeks()) { + const result = await loadDigest(week); + if (result) out.push({ week, result }); + } + return out; // newest first (listWeeks is descending) +} + +async function buildStats(): Promise { + const digests = (await allDigests()).reverse(); // ascending for the time series + const weeks: string[] = []; + const paper_counts: (number | null)[] = []; + for (const { week, result } of digests) { + if (weeks.includes(week)) continue; + weeks.push(week); + paper_counts.push(result.payload.papers.length); + } + return { weeks, paper_counts }; +} + +async function readErrors(limit = 100): Promise { + let text: string; + try { + text = await readFile(path.join(REPORTS_DIR, "errors.log"), "utf8"); + } catch { + return []; + } + const lines = text.split("\n").filter((l) => l.trim().length > 0); + return lines + .slice(-limit) + .reverse() + .map((line) => { + const [ts = "", subject = "", ...rest] = line.split("\t"); + return { ts, subject, detail: rest.join("\t") }; + }); +} + +// ------------------------------------------------------------------ http plumbing + +function sendJson(res: ServerResponse, status: number, body: unknown): void { + const payload = JSON.stringify(body); + res.writeHead(status, { + "content-type": "application/json; charset=utf-8", + "cache-control": "no-cache", + }); + res.end(payload); +} + +const STATIC_TYPES: Record = { + ".html": "text/html; charset=utf-8", + ".js": "text/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".svg": "image/svg+xml", +}; + +async function sendStatic(res: ServerResponse, urlPath: string): Promise { + const rel = urlPath === "/" ? "index.html" : urlPath.replace(/^\/+/, ""); + const file = path.resolve(PUBLIC_DIR, rel); + if (!file.startsWith(PUBLIC_DIR + path.sep) && file !== path.join(PUBLIC_DIR, "index.html")) { + sendJson(res, 404, { error: "not found" }); + return; + } + try { + const body = await readFile(file); + res.writeHead(200, { + "content-type": STATIC_TYPES[path.extname(file)] ?? "application/octet-stream", + }); + res.end(body); + } catch { + sendJson(res, 404, { error: "not found" }); + } +} + +const server = createServer(async (req, res) => { + const url = new URL(req.url ?? "/", "http://localhost"); + const p = url.pathname; + try { + if (req.method !== "GET" && req.method !== "HEAD") { + sendJson(res, 405, { error: "method not allowed" }); + return; + } + if (p === "/api/health") { + const weeks = await listWeeks(); + const body: HealthResponse = { + ok: true, + digest_count: weeks.length, + reports_dir: REPORTS_DIR, + latest_week: weeks[0] ?? null, + }; + sendJson(res, 200, body); + return; + } + if (p === "/api/digests") { + const digests = await allDigests(); + sendJson(res, 200, digests.map(({ week, result }) => summarize(week, result))); + return; + } + const detail = p.match(/^\/api\/digests\/(\d{4}-W\d{2})$/); + if (detail) { + const week = detail[1]!; + if (!WEEK_RE.test(week)) { + sendJson(res, 400, { error: "bad week" }); + return; + } + const result = await loadDigest(week); + if (!result) { + sendJson(res, 404, { error: `no digest for ${week}` }); + return; + } + sendJson(res, 200, result); + return; + } + if (p === "/api/stats") { + sendJson(res, 200, await buildStats()); + return; + } + if (p === "/api/errors") { + sendJson(res, 200, await readErrors()); + return; + } + if (p.startsWith("/api/")) { + sendJson(res, 404, { error: "not found" }); + return; + } + await sendStatic(res, p); + } catch (err) { + console.error(`request ${p} failed:`, err); + sendJson(res, 500, { error: "internal error" }); + } +}); + +server.listen(PORT, "0.0.0.0", () => { + console.log(`sebrief dashboard listening on :${PORT} (reports: ${REPORTS_DIR})`); +}); diff --git a/dashboard/src/shared/types.ts b/dashboard/src/shared/types.ts new file mode 100644 index 0000000..1ecdfe2 --- /dev/null +++ b/dashboard/src/shared/types.ts @@ -0,0 +1,128 @@ +/** Mirrors sebrief/models.py (pydantic model_dump_json → snake_case). */ + +export interface PaperSummary { + tldr: string; + problem: string; + approach: string; + findings: string; + implication: string; +} + +export interface Paper { + doi: string; + title: string; + abstract: string; + authors: string[]; + venue: string; + published_date: string | null; + oa_url: string | null; + pdf_url: string | null; + topics: string[]; + source: string; + /** Absent/null until the per-paper summarisation step has run for this DOI. */ + summary?: PaperSummary | null; +} + +export interface WeekPayload { + week: string; + week_start: string; + week_end: string; + generated_at: string; + papers: Paper[]; + data_gaps: string[]; +} + +export interface Highlight { + doi: string; + title: string; + problem: string; + approach: string; + findings: string; + implication: string; +} + +export interface TopicPaper { + doi: string; + title: string; + note: string; +} + +export interface TopicGroup { + topic: string; + papers: TopicPaper[]; +} + +export interface Pick { + doi: string; + title: string; + reason: string; +} + +export interface Digest { + overview: string; + highlights: Highlight[]; + by_topic: TopicGroup[]; + worth_reading: Pick[]; + data_gaps: string[]; +} + +export interface VerificationIssue { + claim: string; + problem: string; + severity: "minor" | "major"; +} + +export interface Verification { + ok: boolean; + issues: VerificationIssue[]; +} + +export interface DigestResult { + payload: WeekPayload; + digest: Digest | null; + markdown: string; + model_used: string | null; + degraded: boolean; + error: string | null; + /** Absent/null: verification not run (disabled, degraded, or the verifier failed). */ + verification?: Verification | null; +} + +// ---------------------------------------------------------------- API shapes + +export interface DigestSummary { + week: string; + week_start: string; + week_end: string; + generated_at: string; + model_used: string | null; + degraded: boolean; + error: string | null; + overview: string | null; + paper_count: number; + highlight_count: number; + topic_count: number; + /** null: verification not run. */ + verification_ok: boolean | null; + data_gaps: string[]; +} + +export interface StatsResponse { + /** ISO weeks, ascending. */ + weeks: string[]; + /** papers per week, aligned with `weeks`. */ + paper_counts: (number | null)[]; +} + +export interface ErrorLine { + ts: string; + subject: string; + detail: string; +} + +export interface HealthResponse { + ok: boolean; + digest_count: number; + reports_dir: string; + latest_week: string | null; +} diff --git a/dashboard/tsconfig.json b/dashboard/tsconfig.json new file mode 100644 index 0000000..cb9ca01 --- /dev/null +++ b/dashboard/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noFallthroughCasesInSwitch": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src"] +} diff --git a/dashboard/web/index.html b/dashboard/web/index.html new file mode 100644 index 0000000..bc027b4 --- /dev/null +++ b/dashboard/web/index.html @@ -0,0 +1,53 @@ + + + + + + SE-Brief 대시보드 + + + +
+

SE-Brief OA 소프트웨어공학 논문 주간 다이제스트

+
+
+
+
+
+ +
+
+ +
+
+

주차별 수집 논문 수

+
+ +
+
+
+ +
+ +
+ + +
+
+
+
+
+
+ + +
+ + + + diff --git a/dashboard/web/styles.css b/dashboard/web/styles.css new file mode 100644 index 0000000..54006bc --- /dev/null +++ b/dashboard/web/styles.css @@ -0,0 +1,241 @@ +/* Design tokens - light is the base, dark overrides via prefers-color-scheme. */ +:root { + color-scheme: light; + --page: #f9f9f7; + --surface: #fcfcfb; + --ink: #0b0b0b; + --ink-2: #52514e; + --muted: #898781; + --grid: #e1e0d9; + --baseline: #c3c2b7; + --border: rgba(11, 11, 11, 0.1); + --series-1: #4a7c59; + --badge-warn-bg: #fdf3d7; + --badge-warn-ink: #7a5600; + --badge-err-bg: #fbe3e3; + --badge-err-ink: #8f2020; + --badge-ok-bg: #e2f2e2; + --badge-ok-ink: #106310; +} +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --ink: #ffffff; + --ink-2: #c3c2b7; + --muted: #898781; + --grid: #2c2c2a; + --baseline: #383835; + --border: rgba(255, 255, 255, 0.1); + --series-1: #6faf82; + --badge-warn-bg: #3a2f10; + --badge-warn-ink: #fab219; + --badge-err-bg: #3c1a1a; + --badge-err-ink: #e66767; + --badge-ok-bg: #143214; + --badge-ok-ink: #4fc24f; + } +} + +* { box-sizing: border-box; } +body { + margin: 0; + background: var(--page); + color: var(--ink); + font: 14px/1.55 system-ui, -apple-system, "Segoe UI", sans-serif; +} +h1, h2 { margin: 0; font-weight: 650; } +h1 { font-size: 17px; } +h2 { font-size: 14px; color: var(--ink-2); } + +.topbar { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 12px; + padding: 14px 20px; + border-bottom: 1px solid var(--border); + background: var(--surface); +} +.topbar-sub { font-size: 13px; font-weight: 400; color: var(--muted); margin-left: 6px; } +.topbar-right { display: flex; flex-direction: column; align-items: flex-end; gap: 2px; } +.clocks { display: flex; gap: 14px; font-size: 12.5px; color: var(--ink-2); font-variant-numeric: tabular-nums; } +.clock::before { content: "●"; font-size: 7px; vertical-align: 2px; margin-right: 5px; color: var(--series-1); } +.health { font-size: 12px; color: var(--muted); } + +.layout { + max-width: 1180px; + margin: 0 auto; + padding: 16px 20px 48px; + display: flex; + flex-direction: column; + gap: 16px; +} + +.panel { + background: var(--surface); + border: 1px solid var(--border); + border-radius: 10px; + padding: 14px 16px; +} +.panel-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + flex-wrap: wrap; + margin-bottom: 6px; +} +.chart-controls { display: flex; gap: 8px; align-items: center; } +select, button { + font: inherit; + color: var(--ink); + background: var(--surface); + border: 1px solid var(--baseline); + border-radius: 7px; + padding: 5px 9px; +} +button { cursor: pointer; } +button[aria-pressed="true"] { background: var(--grid); } + +/* ------------------------------------------------------------------ stat tiles */ +.tiles { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); + gap: 10px; +} +.tile { + background: var(--surface); + border: 1px solid var(--border); + border-radius: 10px; + padding: 10px 13px; +} +.tile-label { font-size: 12px; color: var(--ink-2); } +.tile-value { font-size: 21px; font-weight: 650; margin-top: 2px; } +.tile-value-small { font-size: 15px; } +.tile-value.tone-ok { color: var(--badge-ok-ink); } +.tile-value.tone-warn { color: var(--badge-warn-ink); } +.tile-value.tone-error { color: var(--badge-err-ink); } + +/* ------------------------------------------------------------------ chart */ +.chart { position: relative; } +.chart svg { display: block; width: 100%; height: auto; } +.chart .axis-label { font-size: 11px; fill: var(--muted); } +.chart .gridline { stroke: var(--grid); stroke-width: 1; } +.chart .baseline { stroke: var(--baseline); stroke-width: 1; } +.chart .series-line { stroke: var(--series-1); stroke-width: 2; fill: none; } +.chart .hover-dot { fill: var(--series-1); stroke: var(--surface); stroke-width: 2; } +.chart .crosshair { stroke: var(--baseline); stroke-width: 1; stroke-dasharray: 3 3; } +.chart-tooltip { + position: absolute; + pointer-events: none; + background: var(--surface); + border: 1px solid var(--baseline); + border-radius: 7px; + padding: 5px 9px; + font-size: 12px; + color: var(--ink); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.12); + white-space: nowrap; + transform: translate(-50%, -110%); +} +.chart-tooltip .tt-value { font-weight: 650; } +.chart-empty { color: var(--muted); padding: 24px 4px; } + +.chart-table { overflow-x: auto; margin-top: 8px; } + +/* ------------------------------------------------------------------ columns */ +.columns { + display: grid; + grid-template-columns: 270px 1fr; + gap: 16px; + align-items: start; +} +@media (max-width: 820px) { + .columns { grid-template-columns: 1fr; } +} + +.digest-list { list-style: none; margin: 10px 0 0; padding: 0; display: flex; flex-direction: column; gap: 6px; } +.digest-list button { + width: 100%; + text-align: left; + display: flex; + flex-direction: column; + gap: 2px; + border-color: var(--border); +} +.digest-list button[aria-current="true"] { border-color: var(--series-1); background: var(--grid); } +.digest-week { font-weight: 650; } +.week-missing { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + align-items: baseline; + column-gap: 8px; + padding: 4px 10px; + font-size: 12px; + color: var(--muted); + border: 1px dashed var(--grid); + border-radius: 7px; +} +.week-missing .digest-week { font-weight: 500; white-space: nowrap; } +.digest-line { font-size: 12px; color: var(--ink-2); display: flex; gap: 6px; align-items: center; flex-wrap: wrap; } +.digest-overview { + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + +.badge { + display: inline-block; + font-size: 11px; + border-radius: 20px; + padding: 1px 8px; + font-weight: 600; + background: var(--grid); + color: var(--ink-2); +} +.badge-ok { background: var(--badge-ok-bg); color: var(--badge-ok-ink); } +.badge-warn { background: var(--badge-warn-bg); color: var(--badge-warn-ink); } +.badge-error { background: var(--badge-err-bg); color: var(--badge-err-ink); } + +/* ------------------------------------------------------------------ summary card */ +.summary-card { + border: 1px solid var(--border); + border-left: 3px solid var(--series-1); + border-radius: 10px; + background: var(--page); + padding: 12px 16px; + margin: 10px 0 14px; +} +.summary-overview { margin: 0 0 10px; font-size: 15px; } +.summary-block h3 { font-size: 12.5px; margin: 0 0 4px; color: var(--muted); font-weight: 650; } +.summary-block ul { margin: 0; padding-left: 18px; } +.summary-block li { margin: 2px 0; } + +/* ------------------------------------------------------------------ digest body */ +.digest-meta { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; font-size: 12.5px; color: var(--ink-2); margin-bottom: 4px; } +.markdown { max-width: 80ch; } +.markdown h1 { font-size: 19px; margin: 10px 0 6px; } +.markdown h2 { font-size: 15px; color: var(--ink); margin: 18px 0 6px; } +.markdown h3 { font-size: 14px; margin: 14px 0 4px; } +.markdown p, .markdown li { color: var(--ink); } +.markdown a { color: var(--series-1); } +.markdown em { color: var(--muted); font-style: normal; font-size: 12.5px; } +.markdown table { border-collapse: collapse; font-size: 13px; margin: 8px 0; display: block; overflow-x: auto; } +.markdown th, .markdown td { border: 1px solid var(--grid); padding: 4px 9px; text-align: left; } +.markdown th { background: var(--page); color: var(--ink-2); font-weight: 600; } +.markdown blockquote { margin: 8px 0; padding: 2px 12px; border-left: 3px solid var(--baseline); color: var(--ink-2); } +.markdown code { background: var(--page); border: 1px solid var(--grid); border-radius: 4px; padding: 0 4px; font-size: 12.5px; } + +table.data-table { border-collapse: collapse; font-size: 13px; } +table.data-table th, table.data-table td { border: 1px solid var(--grid); padding: 4px 10px; text-align: right; font-variant-numeric: tabular-nums; } +table.data-table th:first-child, table.data-table td:first-child { text-align: left; } +table.data-table th { background: var(--page); color: var(--ink-2); font-weight: 600; } + +.errors-list { list-style: none; margin: 8px 0 0; padding: 0; font-size: 12.5px; display: flex; flex-direction: column; gap: 6px; } +.errors-list .err-ts { color: var(--muted); margin-right: 8px; } +.errors-list .err-subject { font-weight: 650; color: var(--badge-err-ink); margin-right: 8px; } +.errors-list .err-detail { color: var(--ink-2); overflow-wrap: anywhere; } diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000..4ff9785 --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY sebrief ./sebrief +RUN pip install --no-cache-dir . \ + && mkdir -p /app/data /app/reports /app/logs + +VOLUME ["/app/data", "/app/reports", "/app/logs"] +CMD ["python", "-m", "sebrief.cli", "daemon"] diff --git a/deploy/com.sebrief.container.plist b/deploy/com.sebrief.container.plist new file mode 100644 index 0000000..46c6bdc --- /dev/null +++ b/deploy/com.sebrief.container.plist @@ -0,0 +1,18 @@ + + + + + + Labelcom.sebrief.container + ProgramArguments + + /bin/bash + -c + __CONTAINER__ system start && __CONTAINER__ start sebrief; __CONTAINER__ start sebrief-dashboard + + RunAtLoad + StandardOutPath__ROOT__/logs/container-boot.log + StandardErrorPath__ROOT__/logs/container-boot.log + + diff --git a/deploy/com.sebrief.daemon.plist b/deploy/com.sebrief.daemon.plist new file mode 100644 index 0000000..9818e49 --- /dev/null +++ b/deploy/com.sebrief.daemon.plist @@ -0,0 +1,40 @@ + + + + + Label + com.sebrief.daemon + + ProgramArguments + + /Users/jsahn/Personal_works/sebrief/.venv/bin/python + -m + sebrief.cli + daemon + + + WorkingDirectory + /Users/jsahn/Personal_works/sebrief + + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + PYTHONUNBUFFERED + 1 + + + + KeepAlive + + RunAtLoad + + ThrottleInterval + 30 + + StandardOutPath + /Users/jsahn/Personal_works/sebrief/logs/launchd.out.log + StandardErrorPath + /Users/jsahn/Personal_works/sebrief/logs/launchd.err.log + + diff --git a/deploy/compose.yaml b/deploy/compose.yaml new file mode 100644 index 0000000..112ef7f --- /dev/null +++ b/deploy/compose.yaml @@ -0,0 +1,28 @@ +# docker compose -f deploy/compose.yaml up -d --build +services: + sebrief: + build: + context: .. + dockerfile: deploy/Dockerfile + container_name: sebrief + env_file: + - ../.env + environment: + SEBRIEF_DATA_DIR: /app/data + SEBRIEF_REPORTS_DIR: /app/reports + SEBRIEF_LOG_DIR: /app/logs + volumes: + - ../data:/app/data + - ../reports:/app/reports + - ../logs:/app/logs + restart: unless-stopped + + dashboard: + build: + context: ../dashboard + container_name: sebrief-dashboard + volumes: + - ../reports:/data/reports:ro + ports: + - "8091:8091" + restart: unless-stopped diff --git a/deploy/install-apple-container.sh b/deploy/install-apple-container.sh new file mode 100755 index 0000000..63dda45 --- /dev/null +++ b/deploy/install-apple-container.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# macOS: Apple container(https://github.com/apple/container)로 빌드·상주 실행. +# 실행: bash deploy/install-apple-container.sh # 이미지 빌드 + 컨테이너 (재)기동 +# bash deploy/install-apple-container.sh --boot # + 로그인 시 자동 기동(LaunchAgent) 등록 +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +NAME=sebrief + +CONTAINER_BIN="$(command -v container || true)" +if [ -z "$CONTAINER_BIN" ]; then + echo "Apple container CLI가 없습니다: brew install container (macOS 15+, Apple Silicon)" >&2 + exit 1 +fi +if [ ! -f "$ROOT/.env" ]; then + echo "$ROOT/.env 가 없습니다: cp .env.example .env 후 MINIMAX_API_KEY를 넣으세요" >&2 + exit 1 +fi + +DASHBOARD=sebrief-dashboard +DASHBOARD_PORT="${SEBRIEF_DASHBOARD_PORT:-8091}" + +mkdir -p "$ROOT/data" "$ROOT/reports" "$ROOT/logs" +container system start +container build -t "$NAME" -f "$ROOT/deploy/Dockerfile" "$ROOT" +container build -t "$DASHBOARD" -f "$ROOT/dashboard/Dockerfile" "$ROOT/dashboard" + +# 기존 컨테이너는 갈아끼운다 (이미지 갱신 반영). container에는 restart 정책이 없어 +# 자동 기동은 --boot 의 LaunchAgent가 담당한다. +container stop "$NAME" "$DASHBOARD" >/dev/null 2>&1 || true +container delete "$NAME" "$DASHBOARD" >/dev/null 2>&1 || true +container run --detach --name "$NAME" \ + --env-file "$ROOT/.env" \ + --env SEBRIEF_DATA_DIR=/app/data \ + --env SEBRIEF_REPORTS_DIR=/app/reports \ + --env SEBRIEF_LOG_DIR=/app/logs \ + --volume "$ROOT/data:/app/data" \ + --volume "$ROOT/reports:/app/reports" \ + --volume "$ROOT/logs:/app/logs" \ + "$NAME" +container run --detach --name "$DASHBOARD" \ + --volume "$ROOT/reports:/data/reports:ro" \ + --publish "0.0.0.0:$DASHBOARD_PORT:8091" \ + "$DASHBOARD" + +if [ "${1:-}" = "--boot" ]; then + PLIST_SRC="$ROOT/deploy/com.sebrief.container.plist" + PLIST_DST="$HOME/Library/LaunchAgents/com.sebrief.container.plist" + mkdir -p "$HOME/Library/LaunchAgents" + sed -e "s#__CONTAINER__#$CONTAINER_BIN#g" -e "s#__ROOT__#$ROOT#g" "$PLIST_SRC" > "$PLIST_DST" + launchctl bootout "gui/$(id -u)/com.sebrief.container" 2>/dev/null || true + launchctl bootstrap "gui/$(id -u)" "$PLIST_DST" + echo "boot agent: $PLIST_DST (로그인 시 container system start + container start $NAME $DASHBOARD)" +fi + +echo "dashboard: http://localhost:$DASHBOARD_PORT" +echo "status: container list --all" +echo "logs: container logs --follow $NAME (파일: tail -f $ROOT/logs/sebrief.log)" +echo "stop: container stop $NAME $DASHBOARD" +echo "remove: container stop $NAME $DASHBOARD && container delete $NAME $DASHBOARD" diff --git a/deploy/install-launchd.sh b/deploy/install-launchd.sh new file mode 100755 index 0000000..c4555da --- /dev/null +++ b/deploy/install-launchd.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# macOS: launchd 사용자 에이전트로 등록/갱신. 실행: bash deploy/install-launchd.sh +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PLIST_SRC="$ROOT/deploy/com.sebrief.daemon.plist" +PLIST_DST="$HOME/Library/LaunchAgents/com.sebrief.daemon.plist" +mkdir -p "$ROOT/logs" "$HOME/Library/LaunchAgents" +# 프로젝트 경로가 다르면 plist 안의 경로를 현재 위치로 치환 +sed "s#/Users/jsahn/Personal_works/sebrief#$ROOT#g" "$PLIST_SRC" > "$PLIST_DST" +launchctl bootout "gui/$(id -u)/com.sebrief.daemon" 2>/dev/null || true +launchctl bootstrap "gui/$(id -u)" "$PLIST_DST" +launchctl kickstart -k "gui/$(id -u)/com.sebrief.daemon" +echo "installed: $PLIST_DST" +echo "status: launchctl print gui/$(id -u)/com.sebrief.daemon | head -20" +echo "logs: tail -f $ROOT/logs/sebrief.log" +echo "remove: launchctl bootout gui/$(id -u)/com.sebrief.daemon && rm $PLIST_DST" diff --git a/deploy/sebrief.service b/deploy/sebrief.service new file mode 100644 index 0000000..35ee785 --- /dev/null +++ b/deploy/sebrief.service @@ -0,0 +1,26 @@ +# Linux systemd unit. 설치: +# sudo cp deploy/sebrief.service /etc/systemd/system/sebrief.service +# sudo systemctl daemon-reload && sudo systemctl enable --now sebrief +# journalctl -u sebrief -f +[Unit] +Description=SE-Brief - weekly OA software engineering papers digest daemon +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=sebrief +WorkingDirectory=/opt/sebrief +EnvironmentFile=/opt/sebrief/.env +Environment=PYTHONUNBUFFERED=1 +ExecStart=/opt/sebrief/.venv/bin/python -m sebrief.cli daemon +Restart=always +RestartSec=30 +# 하드닝 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ReadWritePaths=/opt/sebrief/data /opt/sebrief/reports /opt/sebrief/logs + +[Install] +WantedBy=multi-user.target diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ffe4fd4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[project] +name = "sebrief" +version = "0.1.0" +description = "Weekly Korean digest of open-access software engineering papers" +requires-python = ">=3.12,<3.13" +dependencies = [ + "openai>=1.50", + "feedparser>=6.0", + "apscheduler>=3.10,<4", + "pydantic>=2.7", + "pydantic-settings>=2.3", + "python-dotenv>=1.0", + "requests>=2.31", +] + +[project.optional-dependencies] +dev = ["pytest>=8", "pytest-mock>=3.14", "ruff>=0.6", "freezegun>=1.5"] + +[project.scripts] +sebrief = "sebrief.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["sebrief"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" + +[tool.ruff] +line-length = 110 +target-version = "py312" diff --git a/sebrief/__init__.py b/sebrief/__init__.py new file mode 100644 index 0000000..e73d0b8 --- /dev/null +++ b/sebrief/__init__.py @@ -0,0 +1,3 @@ +"""SE-Brief - weekly Korean digest of open-access software engineering papers.""" + +__version__ = "0.1.0" diff --git a/sebrief/cli.py b/sebrief/cli.py new file mode 100644 index 0000000..19a949f --- /dev/null +++ b/sebrief/cli.py @@ -0,0 +1,172 @@ +"""Command-line entry point: ``sebrief run-once | daemon | next-run | test-notify | show-config``.""" + +from __future__ import annotations + +import argparse +import logging +import re +import sys +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +from sebrief import __version__ +from sebrief.config import KST, Settings, get_settings +from sebrief.daemon import run_daemon +from sebrief.deliver import build_deliverers +from sebrief.logging_setup import setup_logging +from sebrief.pipeline import SKIP_ALREADY_SENT, delivery_failed, run_digest +from sebrief.week import WeeklyClock, week_range + +log = logging.getLogger(__name__) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +EXIT_OK = 0 +EXIT_ERROR = 1 +EXIT_SKIPPED = 2 +SECRET_FIELD_RE = re.compile(r"(api_key|_key|_token|secret|password)$") +_PATH_FIELDS = ("data_dir", "reports_dir", "log_dir") + + +def _resolve_paths(settings: Settings, root: Path = PROJECT_ROOT) -> Settings: + """Make relative path settings absolute against the project root (launchd starts with cwd=/).""" + update = {} + for name in _PATH_FIELDS: + value = Path(getattr(settings, name)) + if not value.is_absolute(): + update[name] = root / value + return settings.model_copy(update=update) if update else settings + + +def _load_settings() -> Settings: + """Load settings; fall back to the project-root .env when none exists in the current directory.""" + root_env = PROJECT_ROOT / ".env" + if not Path(".env").is_file() and root_env.is_file(): + settings = Settings(_env_file=root_env) # type: ignore[call-arg] + else: + settings = get_settings() + return _resolve_paths(settings, PROJECT_ROOT) + + +def mask_secret(value: object) -> str: + if value is None or value == "": + return "(unset)" + text = str(value) + return f"{text[:4]}***" if len(text) > 8 else "***" + + +def _parse_week(text: str) -> str: + try: + week_range(text) + except ValueError as exc: + raise argparse.ArgumentTypeError(str(exc)) from exc + return text + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="sebrief", description="Weekly OA software engineering papers digest bot" + ) + parser.add_argument("--version", action="version", version=f"sebrief {__version__}") + sub = parser.add_subparsers(dest="command", required=True) + + once = sub.add_parser("run-once", help="build (and deliver) one weekly digest") + once.add_argument( + "--week", + type=_parse_week, + default=None, + help="ISO week YYYY-Wnn (must have ended already unless --dry-run)", + ) + once.add_argument("--dry-run", action="store_true", help="print markdown, do not save or deliver") + once.add_argument("--force", action="store_true", help="re-run (resend) even if already sent") + once.add_argument("--no-llm", action="store_true", help="skip the LLM (paper list only)") + + sub.add_parser("daemon", help="run the resident weekly scheduler") + sub.add_parser("next-run", help="show the next scheduled digest") + + notify = sub.add_parser("test-notify", help="send a test message through every delivery channel") + notify.add_argument("--message", default="SE-Brief test notification", help="text to send") + + sub.add_parser("show-config", help="print effective settings (secrets masked)") + return parser + + +# ---------------------------------------------------------------------------- commands +def cmd_run_once(args: argparse.Namespace, settings: Settings) -> int: + result = run_digest(args.week, settings, dry_run=args.dry_run, force=args.force, no_llm=args.no_llm) + if result.error == SKIP_ALREADY_SENT: + print(f"skipped: {result.error} (week {result.payload.week})", file=sys.stderr) + return EXIT_SKIPPED + if result.degraded: + log.warning("digest produced in degraded mode: %s", result.error or "no LLM") + if delivery_failed(result): + print(f"error: {result.error} (week {result.payload.week})", file=sys.stderr) + return EXIT_ERROR + return EXIT_OK + + +def cmd_daemon(_args: argparse.Namespace, settings: Settings) -> int: + run_daemon(settings) + return EXIT_OK + + +def cmd_next_run(_args: argparse.Namespace, settings: Settings) -> int: + clock = WeeklyClock(settings.run_weekday, settings.run_time_kst) + now = datetime.now(ZoneInfo(KST)) + week, run_at = clock.next_run(now) + start, end = week_range(week) + print(f"week : {week} ({start} ~ {end})") + print(f"digest at KST : {run_at.isoformat(timespec='minutes')}") + return EXIT_OK + + +def cmd_test_notify(args: argparse.Namespace, settings: Settings) -> int: + deliverers = build_deliverers(settings) + if not deliverers: + print("no delivery channels configured", file=sys.stderr) + return EXIT_ERROR + for deliverer in deliverers: + deliverer.notify_error("SE-Brief 테스트 알림", args.message) + print(f"sent test notification via {deliverer.name}") + return EXIT_OK + + +def cmd_show_config(_args: argparse.Namespace, settings: Settings) -> int: + for name, value in settings.model_dump().items(): + shown = mask_secret(value) if SECRET_FIELD_RE.search(name) else value + print(f"{name} = {shown}") + print(f"db_path = {settings.db_path}") + print(f"project_root = {PROJECT_ROOT}") + return EXIT_OK + + +COMMANDS = { + "run-once": cmd_run_once, + "daemon": cmd_daemon, + "next-run": cmd_next_run, + "test-notify": cmd_test_notify, + "show-config": cmd_show_config, +} + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + settings = _load_settings() + except Exception as exc: # noqa: BLE001 - bad .env must produce a readable message + print(f"error: could not load settings: {exc}", file=sys.stderr) + return EXIT_ERROR + setup_logging(settings) + try: + return COMMANDS[args.command](args, settings) + except KeyboardInterrupt: + return EXIT_ERROR + except Exception as exc: # top-level guard: log and exit non-zero + log.exception("command %s failed", args.command) + print(f"error: {exc}", file=sys.stderr) + return EXIT_ERROR + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sebrief/collectors/__init__.py b/sebrief/collectors/__init__.py new file mode 100644 index 0000000..b30b0c8 --- /dev/null +++ b/sebrief/collectors/__init__.py @@ -0,0 +1 @@ +"""Collectors: OpenAlex (backbone) + supplementary RSS feeds.""" diff --git a/sebrief/collectors/base.py b/sebrief/collectors/base.py new file mode 100644 index 0000000..082c97d --- /dev/null +++ b/sebrief/collectors/base.py @@ -0,0 +1,29 @@ +"""Helpers shared by collectors: failure isolation and timing.""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable + +from sebrief.logging_setup import mask_secrets + +log = logging.getLogger(__name__) + + +def run_safely[T](name: str, fn: Callable[[], T], default: T, gaps: list[str]) -> T: + """Run a collector; on any exception log it, append a gap note and return `default`. + + The pipeline must never die because one data source is down. + """ + t0 = time.monotonic() + try: + result = fn() + log.info("collector %s ok in %.1fs", name, time.monotonic() - t0) + return result + except Exception as exc: # deliberate: isolate every failure + log.exception("collector %s failed", name) + # Gap notes end up in the digest / LLM prompt: never let a URL with a key through. + detail = mask_secrets(str(exc)).split(" for url: ", 1)[0][:120] + gaps.append(f"{name}: {type(exc).__name__}: {detail}") + return default diff --git a/sebrief/collectors/openalex.py b/sebrief/collectors/openalex.py new file mode 100644 index 0000000..56f7c89 --- /dev/null +++ b/sebrief/collectors/openalex.py @@ -0,0 +1,204 @@ +"""OpenAlex collector: paper-level OA works from the journal whitelist, cursor-paginated. + +Two queries per run (§4의 수집 전략): +1. 일반 화이트리스트 저널(TOSEM·TSE·EMSE·IST·JSS·SoftwareX·JOSS): ISSN + is_oa + 발행일 범위 +2. 메가저널(IEEE Access·PeerJ CS): 위 조건 + SE 서브필드 필터, 결과에 키워드 이중 필터 + +Only if *both* queries fail does `fetch_openalex_papers` raise; a single failed query becomes a +data-gap note via the caller's `run_safely`. +""" + +from __future__ import annotations + +import logging +import re +from datetime import date +from typing import Any + +import requests + +from sebrief.config import Settings +from sebrief.models import Paper +from sebrief.sources import is_blacklisted, issns_for, looks_like_se + +log = logging.getLogger(__name__) + +USER_AGENT = "sebrief/0.1 (OA software engineering papers weekly digest)" +PER_PAGE = 200 +MAX_PAGES = 10 # 안전판: 주간 볼륨(10~20편)에서 절대 도달하지 않는다 + +SELECT_FIELDS = ( + "doi,title,display_name,abstract_inverted_index,authorships,primary_location," + "best_oa_location,publication_date,topics,primary_topic,open_access,type" +) + +_DOI_PREFIX_RE = re.compile(r"^(?:https?://(?:dx\.)?doi\.org/|doi:)\s*", re.IGNORECASE) + + +def normalize_doi(raw: str | None) -> str | None: + """Bare lowercase DOI ("10.xxxx/...") from a DOI URL / doi: prefix / plain DOI; None if invalid.""" + if not raw: + return None + doi = _DOI_PREFIX_RE.sub("", raw.strip()).strip().lower() + return doi if doi.startswith("10.") and "/" in doi else None + + +def reconstruct_abstract(inverted: dict[str, list[int]] | None) -> str: + """OpenAlex stores abstracts as an inverted index (word -> positions); rebuild the plain text.""" + if not inverted: + return "" + positions: list[tuple[int, str]] = [] + for word, idxs in inverted.items(): + positions.extend((i, word) for i in idxs) + positions.sort() + return " ".join(word for _, word in positions) + + +def _work_issns(work: dict[str, Any]) -> list[str]: + source = (work.get("primary_location") or {}).get("source") or {} + issns: list[str] = [] + if source.get("issn_l"): + issns.append(source["issn_l"]) + for issn in source.get("issn") or []: + if issn not in issns: + issns.append(issn) + return issns + + +def _work_publisher(work: dict[str, Any]) -> str | None: + source = (work.get("primary_location") or {}).get("source") or {} + return source.get("host_organization_name") + + +def paper_from_work(work: dict[str, Any]) -> Paper | None: + """Map one OpenAlex work JSON to a Paper; None when it has no usable DOI or title.""" + doi = normalize_doi(work.get("doi")) + title = (work.get("title") or work.get("display_name") or "").strip() + if not doi or not title: + return None + authors = [ + name + for a in work.get("authorships") or [] + if (name := ((a.get("author") or {}).get("display_name") or "").strip()) + ] + source = (work.get("primary_location") or {}).get("source") or {} + venue = (source.get("display_name") or "").strip() + published = None + if work.get("publication_date"): + try: + published = date.fromisoformat(work["publication_date"]) + except ValueError: + log.warning("unparseable publication_date %r for %s", work["publication_date"], doi) + oa = work.get("open_access") or {} + best = work.get("best_oa_location") or {} + topics = [ + name for t in (work.get("topics") or [])[:5] if (name := (t.get("display_name") or "").strip()) + ] + return Paper( + doi=doi, + title=title, + abstract=reconstruct_abstract(work.get("abstract_inverted_index")), + authors=authors, + venue=venue, + published_date=published, + oa_url=oa.get("oa_url") or best.get("landing_page_url"), + pdf_url=best.get("pdf_url"), + topics=topics, + source="openalex", + ) + + +def _base_filter(start: date, end: date) -> list[str]: + return [ + "is_oa:true", + "type:article", + f"from_publication_date:{start.isoformat()}", + f"to_publication_date:{end.isoformat()}", + ] + + +def _fetch_works(filter_str: str, settings: Settings) -> list[dict[str, Any]]: + """All works matching `filter_str`, via cursor pagination. Raises on HTTP / shape errors.""" + works: list[dict[str, Any]] = [] + cursor = "*" + url = settings.openalex_base_url.rstrip("/") + "/works" + for _ in range(MAX_PAGES): + params: dict[str, str] = { + "filter": filter_str, + "per-page": str(PER_PAGE), + "cursor": cursor, + "select": SELECT_FIELDS, + } + if settings.openalex_mailto: + params["mailto"] = settings.openalex_mailto + resp = requests.get( + url, + params=params, + timeout=settings.openalex_timeout_seconds, + headers={"User-Agent": USER_AGENT}, + ) + resp.raise_for_status() + body = resp.json() + page = body.get("results") or [] + works.extend(page) + cursor = (body.get("meta") or {}).get("next_cursor") + if not cursor or not page: + return works + log.warning("openalex pagination hit MAX_PAGES=%d for filter %s", MAX_PAGES, filter_str) + return works + + +def fetch_openalex_papers(start: date, end: date, settings: Settings) -> list[Paper]: + """Collect OA papers published in [start, end] from the whitelist, deduplicated by DOI. + + Raises RuntimeError only when every query failed. + """ + plain_issns = issns_for(se_filter=False) + mega_issns = issns_for(se_filter=True) + queries: list[tuple[str, str, bool]] = [ + ("whitelist", ",".join([f"primary_location.source.issn:{'|'.join(plain_issns)}", *_base_filter(start, end)]), False), + ( + "mega", + # topics.subfield.id (any topic position): primary_topic alone misses most SE papers - + # even TSE/TOSEM works carry "Software" as a secondary topic more often than primary. + ",".join( + [ + f"primary_location.source.issn:{'|'.join(mega_issns)}", + f"topics.subfield.id:{settings.se_subfield_id}", + *_base_filter(start, end), + ] + ), + True, + ), + ] + + papers: list[Paper] = [] + seen: set[str] = set() + failures: list[str] = [] + for name, filter_str, needs_se_check in queries: + try: + works = _fetch_works(filter_str, settings) + except Exception as exc: # noqa: BLE001 - per-query isolation + log.warning("openalex query %s failed: %s: %s", name, type(exc).__name__, str(exc)[:200]) + failures.append(f"{name}: {type(exc).__name__}") + continue + kept = 0 + for work in works: + paper = paper_from_work(work) + if paper is None or paper.doi in seen: + continue + if is_blacklisted(_work_issns(work), _work_publisher(work), settings.extra_blacklist_issns): + log.info("blacklisted paper skipped: %s (%s)", paper.doi, paper.venue) + continue + if needs_se_check and not looks_like_se(paper.title, paper.abstract, paper.topics): + log.debug("mega-journal paper failed SE keyword filter: %s", paper.doi) + continue + seen.add(paper.doi) + papers.append(paper) + kept += 1 + log.info("openalex query %s: %d works -> %d papers kept", name, len(works), kept) + + if failures and len(failures) == len(queries): + raise RuntimeError("all openalex queries failed: " + "; ".join(failures)) + log.info("openalex: %d papers for %s..%s (queries failed: %d/%d)", len(papers), start, end, len(failures), len(queries)) + return papers diff --git a/sebrief/collectors/rss.py b/sebrief/collectors/rss.py new file mode 100644 index 0000000..da44921 --- /dev/null +++ b/sebrief/collectors/rss.py @@ -0,0 +1,188 @@ +"""Supplementary RSS collector (e-Informatica SEJ 등 OpenAlex 색인이 느린 저널 보조). + +Feeds are configured as ``SEBRIEF_RSS_FEEDS`` CSV of ``이름|URL``. Entries must carry a DOI (in the +link, id or summary) - papers are keyed by DOI in the store, so DOI-less entries are skipped with a +log line. Each feed is fetched independently; only if *every* feed fails does `fetch_rss_papers` +raise. +""" + +from __future__ import annotations + +import calendar +import html +import logging +import re +from datetime import UTC, date, datetime +from html.parser import HTMLParser + +import feedparser +import requests + +from sebrief.collectors.openalex import normalize_doi +from sebrief.models import Paper +from sebrief.sources import is_blacklisted + +log = logging.getLogger(__name__) + +USER_AGENT = "sebrief/0.1 (OA software engineering papers weekly digest) feed-reader" +FEED_TIMEOUT_SECONDS = 15.0 +ABSTRACT_MAX_CHARS = 2000 + +# Bare DOI anywhere in a link / id / summary (trailing punctuation trimmed afterwards). +_DOI_RE = re.compile(r"10\.\d{4,9}/[^\s\"'<>)\]]+", re.IGNORECASE) +_WS_RE = re.compile(r"\s+") + + +def parse_feed_specs(specs: list[str]) -> tuple[list[tuple[str, str]], list[str]]: + """CSV items "이름|URL" -> ([(name, url)], [invalid items]).""" + feeds: list[tuple[str, str]] = [] + invalid: list[str] = [] + for spec in specs: + name, sep, url = spec.partition("|") + if not sep or not name.strip() or not url.strip().startswith(("http://", "https://")): + invalid.append(spec) + continue + feeds.append((name.strip(), url.strip())) + return feeds, invalid + + +class _TextExtractor(HTMLParser): + """Minimal HTML -> text: keeps text nodes, drops tags/scripts/styles.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self._parts: list[str] = [] + self._skip = 0 + + def handle_starttag(self, tag: str, attrs) -> None: + if tag in ("script", "style"): + self._skip += 1 + elif tag in ("br", "p", "div", "li"): + self._parts.append(" ") + + def handle_endtag(self, tag: str) -> None: + if tag in ("script", "style") and self._skip: + self._skip -= 1 + elif tag in ("p", "div", "li"): + self._parts.append(" ") + + def handle_data(self, data: str) -> None: + if not self._skip: + self._parts.append(data) + + def text(self) -> str: + return "".join(self._parts) + + +def strip_html(raw: str | None, max_chars: int = ABSTRACT_MAX_CHARS) -> str: + """Remove tags/entities, collapse whitespace and truncate to `max_chars`.""" + if not raw: + return "" + parser = _TextExtractor() + try: + parser.feed(raw) + parser.close() + text = parser.text() + except Exception: # noqa: BLE001 - malformed HTML must not kill the collector + text = re.sub(r"<[^>]+>", " ", raw) + text = _WS_RE.sub(" ", html.unescape(text)).strip() + if len(text) > max_chars: + text = text[: max_chars - 1].rstrip() + "…" + return text + + +def extract_doi(entry) -> str | None: + """DOI from an entry's link / id / dc:identifier / summary, normalised; None when absent.""" + candidates = [ + entry.get("link") or "", + entry.get("id") or "", + entry.get("prism_doi") or "", + entry.get("dc_identifier") or "", + entry.get("summary") or "", + ] + for text in candidates: + m = _DOI_RE.search(text) + if m: + doi = normalize_doi(m.group(0).rstrip(".,;")) + if doi: + return doi + return None + + +def _entry_date(entry) -> date | None: + for key in ("published_parsed", "updated_parsed", "created_parsed"): + tm = entry.get(key) + if tm: + try: + return datetime.fromtimestamp(calendar.timegm(tm), tz=UTC).date() + except (OverflowError, ValueError, TypeError): + continue + return None + + +def _parse_feed(name: str, content: bytes, start: date, end: date, extra_blacklist: list[str]) -> list[Paper]: + parsed = feedparser.parse(content) + if parsed.get("bozo") and not parsed.entries: + raise ValueError(f"unparseable feed: {parsed.get('bozo_exception')}") + papers: list[Paper] = [] + for entry in parsed.entries: + title = _WS_RE.sub(" ", html.unescape(entry.get("title") or "")).strip() + if not title: + continue + published = _entry_date(entry) + if published is None or not (start <= published <= end): + continue + doi = extract_doi(entry) + if doi is None: + log.info("rss %s: entry without DOI skipped: %s", name, title[:80]) + continue + if is_blacklisted((), None, extra_blacklist): # publisher unknown in RSS; ISSN-only check + continue + papers.append( + Paper( + doi=doi, + title=title, + abstract=strip_html(entry.get("summary") or entry.get("description") or ""), + authors=[a.get("name", "").strip() for a in entry.get("authors") or [] if a.get("name")], + venue=name, + published_date=published, + oa_url=(entry.get("link") or "").strip() or None, + topics=[], + source=f"rss:{name}", + ) + ) + return papers + + +def fetch_rss_papers( + feeds: list[tuple[str, str]], start: date, end: date, extra_blacklist: list[str] | None = None +) -> list[Paper]: + """Collect papers published in [start, end] from the configured feeds, deduplicated by DOI. + + Raises RuntimeError only when every feed failed (and at least one is configured). + """ + collected: list[Paper] = [] + failures: list[str] = [] + for name, url in feeds: + try: + resp = requests.get(url, timeout=FEED_TIMEOUT_SECONDS, headers={"User-Agent": USER_AGENT}) + resp.raise_for_status() + items = _parse_feed(name, resp.content, start, end, extra_blacklist or []) + except Exception as exc: # noqa: BLE001 - per-feed isolation + log.warning("rss feed %s failed: %s: %s", name, type(exc).__name__, str(exc)[:200]) + failures.append(f"{name}: {type(exc).__name__}") + continue + log.info("rss feed %s: %d papers in %s..%s", name, len(items), start, end) + collected.extend(items) + + if feeds and failures and len(failures) == len(feeds): + raise RuntimeError("all rss feeds failed: " + "; ".join(failures)) + + unique: list[Paper] = [] + seen: set[str] = set() + for paper in collected: + if paper.doi in seen: + continue + seen.add(paper.doi) + unique.append(paper) + return unique diff --git a/sebrief/config.py b/sebrief/config.py new file mode 100644 index 0000000..4cef55c --- /dev/null +++ b/sebrief/config.py @@ -0,0 +1,95 @@ +"""Runtime settings (env / .env) - single source of truth for configuration. + +Journal sources are NOT configured here; see `sebrief.sources`. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated + +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict + +KST = "Asia/Seoul" + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", env_file_encoding="utf-8", extra="ignore", populate_by_name=True + ) + + # --- MiniMax (OpenAI-compatible endpoint) --- + minimax_api_key: str = Field(default="", alias="MINIMAX_API_KEY") + minimax_base_url: str = Field(default="https://api.minimax.io/v1", alias="MINIMAX_BASE_URL") + minimax_model: str = Field(default="MiniMax-M3", alias="MINIMAX_MODEL") + minimax_fallback_models: Annotated[list[str], NoDecode] = Field( + default_factory=lambda: ["MiniMax-M2.7", "MiniMax-M2.5"], alias="MINIMAX_FALLBACK_MODELS" + ) + # Digest + adaptive-thinking output for a 40-paper week runs past 16k tokens (observed on the + # revise pass); 32k keeps M3 from truncating and falling back. + minimax_max_completion_tokens: int = Field(default=32768, alias="MINIMAX_MAX_COMPLETION_TOKENS") + minimax_timeout_seconds: float = Field(default=240.0, alias="MINIMAX_TIMEOUT_SECONDS") + + # --- OpenAlex (no API key needed; mailto joins the polite pool -> faster, more reliable) --- + openalex_base_url: str = Field(default="https://api.openalex.org", alias="SEBRIEF_OPENALEX_BASE_URL") + openalex_mailto: str = Field(default="", alias="SEBRIEF_OPENALEX_MAILTO") + openalex_timeout_seconds: float = Field(default=30.0, alias="SEBRIEF_OPENALEX_TIMEOUT_SECONDS") + + # --- Collection policy --- + # The collection window starts this many days before the target week so papers indexed late by + # OpenAlex are still picked up; DOI dedup in the store keeps them from being summarised twice. + backfill_days: int = Field(default=7, alias="SEBRIEF_BACKFILL_DAYS") + # Papers beyond this cap are left undigested and roll over to the next week's digest. + max_papers: int = Field(default=40, alias="SEBRIEF_MAX_PAPERS") + # Supplementary RSS feeds, CSV of "이름|URL" (e.g. e-Informatica SEJ). Entries without a DOI are skipped. + rss_feeds: Annotated[list[str], NoDecode] = Field(default_factory=list, alias="SEBRIEF_RSS_FEEDS") + # Extra ISSNs to block on top of sebrief.sources.BLACKLIST_ISSNS (MDPI journals etc.). + extra_blacklist_issns: Annotated[list[str], NoDecode] = Field( + default_factory=list, alias="SEBRIEF_EXTRA_BLACKLIST_ISSNS" + ) + # OpenAlex subfield id used to keep mega-journals (IEEE Access, PeerJ CS) on-topic. + # 1712 = "Software" (Scopus ASJC code reused by OpenAlex). + se_subfield_id: str = Field(default="1712", alias="SEBRIEF_SE_SUBFIELD_ID") + + # --- Paths --- + data_dir: Path = Field(default=Path("data"), alias="SEBRIEF_DATA_DIR") + reports_dir: Path = Field(default=Path("reports"), alias="SEBRIEF_REPORTS_DIR") + log_dir: Path = Field(default=Path("logs"), alias="SEBRIEF_LOG_DIR") + + # --- Scheduling policy (weekly; default Monday 07:00 KST covering the week that just ended) --- + run_weekday: int = Field(default=0, alias="SEBRIEF_RUN_WEEKDAY") # 0=Mon .. 6=Sun + run_time_kst: str = Field(default="07:00", alias="SEBRIEF_RUN_TIME_KST") + catchup_window_hours: int = Field(default=72, alias="SEBRIEF_CATCHUP_WINDOW_HOURS") + + # --- LLM verification (fact-check pass over the generated digest) --- + verify_enabled: bool = Field(default=True, alias="SEBRIEF_VERIFY_ENABLED") + + # --- Per-paper summaries (논문 한 편씩 구조화 요약; DOI당 한 번 생성, papers 테이블에 저장) --- + paper_summary_enabled: bool = Field(default=True, alias="SEBRIEF_PAPER_SUMMARY_ENABLED") + paper_summary_batch_size: int = Field(default=8, alias="SEBRIEF_PAPER_SUMMARY_BATCH_SIZE") + + log_level: str = Field(default="INFO", alias="SEBRIEF_LOG_LEVEL") + + @field_validator("minimax_fallback_models", "rss_feeds", "extra_blacklist_issns", mode="before") + @classmethod + def _split_csv(cls, v): + if isinstance(v, str): + return [s.strip() for s in v.split(",") if s.strip()] + return v + + @field_validator("run_weekday") + @classmethod + def _valid_weekday(cls, v: int) -> int: + if not (0 <= v <= 6): + raise ValueError("SEBRIEF_RUN_WEEKDAY must be 0 (Mon) .. 6 (Sun)") + return v + + @property + def db_path(self) -> Path: + return self.data_dir / "sebrief.sqlite3" + + +def get_settings() -> Settings: + """Load settings from environment / .env (called once per process by the CLI).""" + return Settings() diff --git a/sebrief/daemon.py b/sebrief/daemon.py new file mode 100644 index 0000000..30e0ca0 --- /dev/null +++ b/sebrief/daemon.py @@ -0,0 +1,294 @@ +"""Resident scheduler: catch-up on start, then one-shot weekly digest jobs from the WeeklyClock.""" + +from __future__ import annotations + +import logging +import signal +import threading +from collections.abc import Callable +from datetime import datetime, timedelta +from typing import Any, Protocol +from zoneinfo import ZoneInfo + +from apscheduler.events import EVENT_JOB_MISSED, JobExecutionEvent +from apscheduler.schedulers import SchedulerNotRunningError +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.triggers.date import DateTrigger +from apscheduler.triggers.interval import IntervalTrigger + +from sebrief.config import KST, Settings +from sebrief.deliver import build_deliverers +from sebrief.pipeline import run_digest +from sebrief.store import Store +from sebrief.week import WeeklyClock + +log = logging.getLogger(__name__) + +DIGEST_JOB_ID = "digest" +RECOVERY_JOB_ID = "digest-recover" +HEARTBEAT_JOB_ID = "heartbeat" + +# None => APScheduler runs the one-shot digest job whenever it becomes due, even hours late (machine +# sleep, SIGSTOP, clock jump). With a finite grace APScheduler *drops* a late job instead of running +# it, and a dropped one-shot job never reaches _digest_job, whose `finally: schedule_next()` is what +# keeps the chain of digests alive. _digest_job enforces the catch-up window itself. +DIGEST_MISFIRE_GRACE: int | None = None +# A digest whose run time is older than this and which is still recorded as "next" was either lost by +# the scheduler or is running right now; the heartbeat re-arms the schedule in both cases (idempotent). +HEARTBEAT_STALE_AFTER = timedelta(minutes=5) + + +class SchedulerLike(Protocol): + """The subset of APScheduler's BaseScheduler the daemon relies on (fakeable in tests).""" + + def add_job(self, func: Callable[..., Any], trigger: Any = None, **kwargs: Any) -> Any: ... + + def add_listener(self, callback: Callable[[Any], None], mask: int = ...) -> None: ... + + def shutdown(self, wait: bool = True) -> None: ... + + +class Daemon: + """Wires clock, store and pipeline into a scheduler. Every collaborator is injectable.""" + + def __init__( + self, + settings: Settings, + *, + scheduler: SchedulerLike, + clock: WeeklyClock, + store: Store, + run_fn: Callable[[str], Any], + now_fn: Callable[[], datetime] | None = None, + deliverers_fn: Callable[[Settings], list[Any]] | None = None, + ) -> None: + self.settings = settings + self.scheduler = scheduler + self.clock = clock + self.store = store + self.run_fn = run_fn + self.now_fn = now_fn or (lambda: datetime.now(ZoneInfo(KST))) + self.deliverers_fn = deliverers_fn or build_deliverers + self.next_run: tuple[str, datetime] | None = None + self.stop_requested = False + # Serialises pipeline runs (scheduled job / catch-up / heartbeat recovery) across executor threads. + self._run_lock = threading.Lock() + + # ----------------------------------------------------------------- lifecycle + def start(self) -> None: + """Catch up a missed digest, then schedule the next one plus the heartbeat.""" + self.scheduler.add_listener(self._on_job_missed, EVENT_JOB_MISSED) + self.run_catchup() + if self.stop_requested: # signal arrived during catch-up, before the scheduler was running + log.info("shutdown requested during catch-up - not scheduling jobs") + return + self.schedule_next() + self.schedule_heartbeat() + + def install_signal_handlers(self) -> None: + def _handler(signum: int, _frame: Any) -> None: + log.info("signal %s received - shutting down", signal.Signals(signum).name) + if not self.shutdown(): + # The scheduler is not running yet (start-up catch-up in progress): the only way to stop + # the synchronous digest is to unwind the main thread. run_catchup and the pipeline + # catch Exception only, so KeyboardInterrupt propagates up to run_daemon. + raise KeyboardInterrupt + + try: + signal.signal(signal.SIGTERM, _handler) + signal.signal(signal.SIGINT, _handler) + except ValueError: # not in the main thread + log.warning("signal handlers not installed (not main thread)") + + def shutdown(self) -> bool: + """Request a stop. Returns True when a running scheduler was told to shut down, False otherwise.""" + self.stop_requested = True + try: + self.scheduler.shutdown(wait=False) + except SchedulerNotRunningError: + log.debug("scheduler not running - nothing to shut down") + return False + except Exception: # shutting down twice must not raise + log.debug("scheduler shutdown raised", exc_info=True) + return False + return True + + # ------------------------------------------------------------------ catch-up + def run_catchup(self) -> bool: + """Run a digest immediately if one was missed within the catch-up window. Returns True if run.""" + now = self.now_fn() + candidate = self.clock.catchup_candidate(now, self.settings.catchup_window_hours) + if candidate is None: + log.info("no catch-up candidate at %s", now.isoformat(timespec="minutes")) + return False + if self.store.has_digest(candidate): + log.info("catch-up: digest for %s already sent", candidate) + return False + if not self._run_lock.acquire(blocking=False): + log.info("catch-up: a digest is already running - skipping") + return False + try: + log.info("catch-up: running missed digest for week %s", candidate) + self.run_fn(candidate) + except Exception: # daemon must keep going + log.exception("catch-up digest for %s failed", candidate) + self._notify_error("SE-Brief catch-up 다이제스트 실패", f"week={candidate}") + finally: + self._run_lock.release() + return True + + # ---------------------------------------------------------------- scheduling + def schedule_next(self) -> tuple[str, datetime] | None: + """Compute the next digest time from the clock and (re)schedule the one-shot job.""" + now = self.now_fn() + try: + week, run_at = self.clock.next_run(now) + except Exception: # clock failure: retry via heartbeat rather than die + log.exception("could not compute next digest time") + self.next_run = None + return None + self.scheduler.add_job( + self._digest_job, + DateTrigger(run_date=run_at), + args=[week, run_at], + id=DIGEST_JOB_ID, + name=f"digest {week}", + replace_existing=True, + misfire_grace_time=DIGEST_MISFIRE_GRACE, + ) + self.next_run = (week, run_at) + log.info("next digest: week %s at %s", week, run_at.isoformat(timespec="minutes")) + return self.next_run + + def schedule_heartbeat(self) -> None: + self.scheduler.add_job( + self._heartbeat_job, + IntervalTrigger(hours=1), + id=HEARTBEAT_JOB_ID, + name="heartbeat", + replace_existing=True, + ) + + # --------------------------------------------------------------------- jobs + def _digest_job(self, week: str, run_at: datetime | None = None) -> None: + """Scheduled digest. Late fires (sleep/wake) honour the catch-up window; ALWAYS reschedules.""" + try: + if run_at is not None: + late = self.now_fn() - run_at + if late >= timedelta(hours=self.settings.catchup_window_hours): + log.warning( + "digest for %s scheduled at %s fired %s late - outside catch-up window, skipping", + week, + run_at.isoformat(timespec="minutes"), + late, + ) + return + if late > HEARTBEAT_STALE_AFTER: + log.warning("digest for %s fired %s late - running (inside window)", week, late) + log.info("scheduled digest starting for week %s", week) + with self._run_lock: + self.run_fn(week) + except Exception as exc: # never let a job kill the scheduler + log.exception("scheduled digest for %s failed", week) + self._notify_error("SE-Brief 다이제스트 실패", f"week={week}: {type(exc).__name__}: {exc}"[:500]) + finally: + self.schedule_next() + + def _heartbeat_job(self) -> None: + """Hourly liveness log; also the backstop that re-arms a missing or stale digest schedule.""" + now = self.now_fn() + if self.next_run is None: + log.warning("heartbeat: no digest scheduled - retrying schedule") + self.schedule_next() + return + week, run_at = self.next_run + if now - run_at > HEARTBEAT_STALE_AFTER: + # The one-shot job should have fired by now. Either it is running (run_catchup then finds + # the lock held and does nothing) or the scheduler lost it: catch up inside the window and + # re-arm. schedule_next() is idempotent, so re-arming during a run is harmless. + log.warning( + "heartbeat: digest for %s at %s is stale - recovering schedule", + week, + run_at.isoformat(timespec="minutes"), + ) + self._recover_schedule() + return + log.info("heartbeat: next digest for %s at %s", week, run_at.isoformat(timespec="minutes")) + + def _on_job_missed(self, event: JobExecutionEvent) -> None: + """EVENT_JOB_MISSED listener: a dropped one-shot digest job must be replaced. + + Cannot happen while DIGEST_MISFIRE_GRACE is None, but kept as defence in depth. APScheduler + removes the exhausted one-shot job *after* dispatching this event (possibly on this very + thread, inside its job-processing loop), so re-adding a job with the same id here would be + deleted immediately. Defer the recovery to a separately named one-shot job instead. + """ + if getattr(event, "job_id", None) != DIGEST_JOB_ID: + return + log.warning( + "digest job scheduled at %s was missed - queuing schedule recovery", + getattr(event, "scheduled_run_time", "?"), + ) + try: + self.scheduler.add_job( + self._recover_schedule, + DateTrigger(run_date=self.now_fn()), + id=RECOVERY_JOB_ID, + name="digest schedule recovery", + replace_existing=True, + misfire_grace_time=None, + ) + except Exception: # the heartbeat is the last backstop + log.exception("could not queue schedule recovery") + + def _recover_schedule(self) -> None: + """Catch-up policy (window + store idempotency), then reschedule. Never raises.""" + try: + self.run_catchup() + except Exception: # clock/store failure must not kill the scheduler thread + log.exception("catch-up during schedule recovery failed") + finally: + self.schedule_next() + + # ------------------------------------------------------------------ helpers + def _notify_error(self, subject: str, detail: str) -> None: + try: + for deliverer in self.deliverers_fn(self.settings): + try: + deliverer.notify_error(subject, detail) + except Exception: + log.exception("notify_error via %s failed", getattr(deliverer, "name", deliverer)) + except Exception: # building deliverers itself may fail + log.exception("could not build deliverers for error notification") + + +def run_daemon(settings: Settings) -> None: + """Build the real daemon and block until SIGTERM/SIGINT.""" + scheduler = BlockingScheduler(timezone=ZoneInfo(KST)) + clock = WeeklyClock(settings.run_weekday, settings.run_time_kst) + store = Store(settings.db_path) + daemon = Daemon( + settings, + scheduler=scheduler, + clock=clock, + store=store, + run_fn=lambda week: run_digest(week, settings), + ) + daemon.install_signal_handlers() + log.info( + "sebrief daemon starting (weekday=%d %s KST, catch-up window=%dh)", + settings.run_weekday, + settings.run_time_kst, + settings.catchup_window_hours, + ) + try: + daemon.start() + if daemon.stop_requested: + log.info("stop requested during start-up - not starting scheduler") + else: + scheduler.start() + except (KeyboardInterrupt, SystemExit): + daemon.shutdown() + finally: + store.close() + log.info("sebrief daemon stopped") diff --git a/sebrief/deliver/__init__.py b/sebrief/deliver/__init__.py new file mode 100644 index 0000000..25b5b20 --- /dev/null +++ b/sebrief/deliver/__init__.py @@ -0,0 +1,29 @@ +"""Delivery channels: local file archive (내부 게시 전용). + +Blogger 공개 발행은 계획(§7)대로 나중 옵션 - US2KR의 BloggerDeliverer를 그대로 이식해 +`build_deliverers`에 추가하면 된다 (Deliverer 프로토콜 동일). +""" + +from __future__ import annotations + +import logging + +from sebrief.config import Settings +from sebrief.deliver.base import Deliverer, DeliveryError +from sebrief.deliver.file import FileDeliverer + +log = logging.getLogger(__name__) + +__all__ = [ + "Deliverer", + "DeliveryError", + "FileDeliverer", + "build_deliverers", +] + + +def build_deliverers(settings: Settings) -> list[Deliverer]: + """FileDeliverer always (the dashboard reads its output).""" + deliverers: list[Deliverer] = [FileDeliverer(settings.reports_dir)] + log.info("deliverers: %s", ", ".join(d.name for d in deliverers)) + return deliverers diff --git a/sebrief/deliver/base.py b/sebrief/deliver/base.py new file mode 100644 index 0000000..ed72f5b --- /dev/null +++ b/sebrief/deliver/base.py @@ -0,0 +1,23 @@ +"""Delivery channel interface.""" + +from __future__ import annotations + +from typing import Protocol + +from sebrief.models import DigestResult + + +class DeliveryError(RuntimeError): + pass + + +class Deliverer(Protocol): + name: str + + def deliver(self, result: DigestResult) -> str: + """Publish the digest. Return a short receipt string (path / post id). Raise DeliveryError.""" + ... + + def notify_error(self, subject: str, detail: str) -> None: + """Operator alert for pipeline failures. Must never raise.""" + ... diff --git a/sebrief/deliver/file.py b/sebrief/deliver/file.py new file mode 100644 index 0000000..0813fe9 --- /dev/null +++ b/sebrief/deliver/file.py @@ -0,0 +1,52 @@ +"""Local archive deliverer: writes reports/YYYY-Wnn.md (+ .json) keyed by payload.week.""" + +from __future__ import annotations + +import logging +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +from sebrief.config import KST +from sebrief.deliver.base import DeliveryError +from sebrief.models import DigestResult + +log = logging.getLogger(__name__) + + +class FileDeliverer: + """Archive each digest as Markdown and as the full DigestResult JSON.""" + + name = "file" + + def __init__(self, reports_dir: Path) -> None: + self.reports_dir = Path(reports_dir) + + def _ensure_dir(self) -> None: + self.reports_dir.mkdir(parents=True, exist_ok=True) + + def deliver(self, result: DigestResult) -> str: + """Write {week}.md and {week}.json; return the Markdown path.""" + stem = result.payload.week + md_path = self.reports_dir / f"{stem}.md" + json_path = self.reports_dir / f"{stem}.json" + try: + self._ensure_dir() + md_path.write_text(result.markdown, encoding="utf-8") + json_path.write_text(result.model_dump_json(indent=2), encoding="utf-8") + except OSError as exc: + raise DeliveryError(f"file delivery failed for {stem}: {exc}") from exc + log.info("digest archived: %s (+ .json)", md_path) + return str(md_path) + + def notify_error(self, subject: str, detail: str) -> None: + """Append a timestamped line to {reports_dir}/errors.log. Never raises.""" + try: + self._ensure_dir() + ts = datetime.now(ZoneInfo(KST)).isoformat(timespec="seconds") + flat_detail = " | ".join(part.strip() for part in detail.splitlines() if part.strip()) + line = f"{ts}\t{subject}\t{flat_detail}\n" + with (self.reports_dir / "errors.log").open("a", encoding="utf-8") as fh: + fh.write(line) + except Exception: + log.exception("could not append to errors.log") diff --git a/sebrief/llm/__init__.py b/sebrief/llm/__init__.py new file mode 100644 index 0000000..df43646 --- /dev/null +++ b/sebrief/llm/__init__.py @@ -0,0 +1 @@ +"""LLM layer: provider interface, prompts, MiniMax implementation.""" diff --git a/sebrief/llm/base.py b/sebrief/llm/base.py new file mode 100644 index 0000000..866f1e3 --- /dev/null +++ b/sebrief/llm/base.py @@ -0,0 +1,31 @@ +"""LLM provider interface. Implementations must be side-effect free apart from the HTTP call.""" + +from __future__ import annotations + +from typing import Protocol + +from sebrief.models import Digest, Paper, PaperSummaryItem, Verification, VerificationIssue, WeekPayload + + +class LLMError(RuntimeError): + """Raised when every configured model failed to produce a valid Digest.""" + + +class LLMProvider(Protocol): + def summarize(self, payload: WeekPayload) -> tuple[Digest, str]: + """Return (validated Digest, model id that produced it). Raise LLMError on total failure.""" + ... + + def summarize_papers(self, papers: list[Paper]) -> tuple[list[PaperSummaryItem], str]: + """Per-paper structured summaries for one batch. Raise LLMError on total failure.""" + ... + + def verify(self, payload: WeekPayload, digest: Digest) -> tuple[Verification, str]: + """Fact-check `digest` against the payload. Raise LLMError on total failure.""" + ... + + def revise( + self, payload: WeekPayload, digest: Digest, issues: list[VerificationIssue] + ) -> tuple[Digest, str]: + """Return a corrected Digest addressing `issues`. Raise LLMError on total failure.""" + ... diff --git a/sebrief/llm/minimax.py b/sebrief/llm/minimax.py new file mode 100644 index 0000000..23e6528 --- /dev/null +++ b/sebrief/llm/minimax.py @@ -0,0 +1,386 @@ +"""MiniMax provider: OpenAI-compatible chat completions with JSON-schema output and model fallback.""" + +from __future__ import annotations + +import copy +import json +import logging +import re +from typing import Any, get_origin + +import openai +from pydantic import BaseModel, ValidationError + +from sebrief.config import Settings +from sebrief.llm.base import LLMError +from sebrief.llm.prompts import ( + PAPER_SUMMARY_SYSTEM_PROMPT, + REVISE_INSTRUCTION, + SYSTEM_PROMPT, + VERIFY_SYSTEM_PROMPT, + build_papers_message, + build_user_message, + build_verify_message, + format_issues, +) +from sebrief.models import ( + Digest, + Paper, + PaperSummaryBatch, + PaperSummaryItem, + Verification, + VerificationIssue, + WeekPayload, +) + +log = logging.getLogger(__name__) + +# Errors from the openai SDK that mean "this model/attempt failed; try the next model". +OPENAI_ERRORS: tuple[type[Exception], ...] = ( + openai.APIStatusError, + openai.APIConnectionError, + openai.APITimeoutError, + openai.RateLimitError, +) + +_THINK_BLOCK = re.compile(r".*?", re.DOTALL) +_CODE_FENCE = re.compile(r"```(?:json|JSON)?\s*(.*?)```", re.DOTALL) + +REPAIR_INSTRUCTION = ( + "위 응답은 JSON 스키마 검증에 실패했습니다. 아래 오류를 참고하여, 설명 없이 스키마에 맞는 " + "완전한 JSON 객체 하나만 다시 출력하세요.\n\n오류:\n" +) + + +def _strip_titles(node: Any) -> None: + """Recursively remove JSON-schema "title" keys (noise for the model, not needed for validation).""" + if isinstance(node, dict): + node.pop("title", None) + for value in node.values(): + _strip_titles(value) + elif isinstance(node, list): + for value in node: + _strip_titles(value) + + +def model_json_schema(model_cls: type[BaseModel]) -> dict[str, Any]: + """JSON schema for a pydantic model suitable for `response_format` (no titles, closed object).""" + schema = copy.deepcopy(model_cls.model_json_schema()) + _strip_titles(schema) + schema["additionalProperties"] = False + return schema + + +def model_response_format(model_cls: type[BaseModel], name: str) -> dict[str, Any]: + return {"type": "json_schema", "json_schema": {"name": name, "schema": model_json_schema(model_cls)}} + + +def digest_json_schema() -> dict[str, Any]: + return model_json_schema(Digest) + + +def digest_response_format() -> dict[str, Any]: + return model_response_format(Digest, "digest") + + +def _first_json_object(text: str) -> str | None: + """Return the first balanced `{...}` block in `text` (string-aware), or None.""" + start = text.find("{") + if start < 0: + return None + depth = 0 + in_str = False + escape = False + for i in range(start, len(text)): + ch = text[i] + if in_str: + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return text[start : i + 1] + return None + + +def extract_json_text(raw: str) -> str: + """Strip blocks and code fences from a model reply, returning the JSON candidate text. + + Handles: complete ... blocks anywhere, an unterminated (take text after the + last , or the first {...} object), and ```json fences. Raises ValueError if nothing remains. + """ + text = _THINK_BLOCK.sub("", raw) + if "" in text: + if "" in raw: + text = raw.rsplit("", 1)[1] + else: + text = _first_json_object(raw) or "" + fence = _CODE_FENCE.search(text) + if fence: + text = fence.group(1) + text = _first_json_object(text.strip()) or "" + if not text: + raise ValueError("no JSON object found in model reply") + return text + + +def _list_fields(model_cls: type[BaseModel]) -> frozenset[str]: + """Names of the model's plain string-list fields (targets for the string->list coercion).""" + return frozenset( + name + for name, info in model_cls.model_fields.items() + if get_origin(info.annotation) is list and info.annotation == list[str] + ) + + +_BULLET_SPLIT = re.compile(r"(?:^|\n)\s*(?:[-*•]|\d+[.)])\s+") +_SENTENCE_SPLIT = re.compile(r"(?<=[.!?。])\s+") + + +def _split_prose(text: str) -> list[str]: + """Split a prose string into list items: bullets/newlines first, then sentence boundaries.""" + parts = [p.strip() for p in _BULLET_SPLIT.split(text) if p.strip()] + if len(parts) <= 1: + parts = [p.strip() for p in re.split(r"\n+", text) if p.strip()] + if len(parts) <= 1: + parts = [p.strip() for p in _SENTENCE_SPLIT.split(text) if p.strip()] + return parts or [text.strip()] + + +def coerce_model_data(data: Any, model_cls: type[BaseModel]) -> tuple[Any, list[str]]: + """Fix the common MiniMax schema slip of emitting string-list fields as a single string. + + Returns (data, names of coerced fields). Anything that is not a dict is returned untouched so that + pydantic can report the real error. + """ + if not isinstance(data, dict): + return data, [] + coerced: list[str] = [] + for name in _list_fields(model_cls): + value = data.get(name) + if isinstance(value, str): + data[name] = _split_prose(value) + coerced.append(name) + return data, coerced + + +def parse_model[T: BaseModel](raw: str, model_cls: type[T]) -> T: + """Turn raw model content into a validated instance (raises ValueError / ValidationError).""" + text = extract_json_text(raw) + try: + data = json.loads(text) + except json.JSONDecodeError: + return model_cls.model_validate_json(text) # let pydantic produce the canonical error + data, coerced = coerce_model_data(data, model_cls) + if coerced: + log.warning("coerced string -> list for %s fields: %s", model_cls.__name__, ", ".join(sorted(coerced))) + return model_cls.model_validate(data) + + +def parse_digest(raw: str) -> Digest: + return parse_model(raw, Digest) + + +class MiniMaxProvider: + """LLMProvider backed by MiniMax's OpenAI-compatible endpoint with a model fallback chain.""" + + def __init__(self, settings: Settings, client: Any | None = None) -> None: + self._settings = settings + models = [settings.minimax_model, *settings.minimax_fallback_models] + self._models = list(dict.fromkeys(m for m in models if m)) + if not self._models: + raise LLMError("no MiniMax model configured") + if client is None: + if not settings.minimax_api_key: + raise LLMError("MINIMAX_API_KEY is not set") + client = openai.OpenAI( + api_key=settings.minimax_api_key, + base_url=settings.minimax_base_url, + timeout=settings.minimax_timeout_seconds, + max_retries=2, + ) + self._client = client + + @property + def models(self) -> list[str]: + return list(self._models) + + # ------------------------------------------------------------------ public + def summarize(self, payload: WeekPayload) -> tuple[Digest, str]: + messages: list[dict[str, Any]] = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": build_user_message(payload)}, + ] + return self._run(messages, Digest, "digest", task="summarize") + + def summarize_papers(self, papers: list[Paper]) -> tuple[list[PaperSummaryItem], str]: + messages: list[dict[str, Any]] = [ + {"role": "system", "content": PAPER_SUMMARY_SYSTEM_PROMPT}, + {"role": "user", "content": build_papers_message(papers)}, + ] + batch, model = self._run(messages, PaperSummaryBatch, "paper_summaries", task="summarize_papers") + return batch.summaries, model + + def verify(self, payload: WeekPayload, digest: Digest) -> tuple[Verification, str]: + """Fact-check `digest` against the payload. Raise LLMError on total failure.""" + messages: list[dict[str, Any]] = [ + {"role": "system", "content": VERIFY_SYSTEM_PROMPT}, + {"role": "user", "content": build_verify_message(payload, digest)}, + ] + return self._run(messages, Verification, "verification", task="verify") + + def revise( + self, payload: WeekPayload, digest: Digest, issues: list[VerificationIssue] + ) -> tuple[Digest, str]: + """Ask for a corrected Digest given verification issues. Raise LLMError on total failure.""" + messages: list[dict[str, Any]] = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": build_user_message(payload)}, + {"role": "assistant", "content": digest.model_dump_json()}, + {"role": "user", "content": REVISE_INSTRUCTION + format_issues(issues)}, + ] + return self._run(messages, Digest, "digest", task="revise") + + # ----------------------------------------------------------------- helpers + def _run[T: BaseModel]( + self, messages: list[dict[str, Any]], model_cls: type[T], schema_name: str, task: str + ) -> tuple[T, str]: + """Try every configured model in order; return (parsed instance, model id).""" + response_format = model_response_format(model_cls, schema_name) + attempts: list[str] = [] + for model in self._models: + result = self._attempt_model(model, messages, attempts, model_cls, response_format) + if result is not None: + return result, model + raise LLMError(f"all MiniMax models failed ({task}): " + " | ".join(attempts)) + + def _attempt_model[T: BaseModel]( + self, + model: str, + messages: list[dict[str, Any]], + attempts: list[str], + model_cls: type[T], + response_format: dict[str, Any], + ) -> T | None: + """One model: initial call plus at most one repair retry. Returns None when the model failed.""" + content = self._call(model, messages, attempts, response_format) + if content is None: + return None + try: + return parse_model(content, model_cls) + except (ValidationError, ValueError) as exc: + error_text = _short(str(exc), 800) + attempts.append(f"{model}: invalid digest ({_short(str(exc), 160)})") + log.warning( + "model %s returned invalid digest, retrying once with repair hint: %s", + model, + _short(str(exc), 300), + ) + + repair_messages = [ + *messages, + {"role": "assistant", "content": _short(content, 12000)}, + {"role": "user", "content": REPAIR_INSTRUCTION + error_text}, + ] + content = self._call(model, repair_messages, attempts, response_format, label="repair") + if content is None: + return None + try: + return parse_model(content, model_cls) + except (ValidationError, ValueError) as exc: + attempts.append(f"{model} (repair): invalid digest ({_short(str(exc), 160)})") + log.warning("model %s repair retry still invalid: %s", model, _short(str(exc), 300)) + return None + + def _call( + self, + model: str, + messages: list[dict[str, Any]], + attempts: list[str], + response_format: dict[str, Any], + label: str = "initial", + ) -> str | None: + """Perform one chat completion. Returns content text, or None (attempt recorded) on failure.""" + tag = model if label == "initial" else f"{model} ({label})" + extra_body: dict[str, Any] = {"reasoning_split": True} + if model.startswith("MiniMax-M3"): + extra_body["thinking"] = {"type": "adaptive"} + try: + completion = self._client.chat.completions.create( + model=model, + messages=messages, + max_completion_tokens=self._settings.minimax_max_completion_tokens, + response_format=response_format, + extra_body=extra_body, + ) + except OPENAI_ERRORS as exc: + status = getattr(exc, "status_code", None) + detail = f"{type(exc).__name__}" + (f" {status}" if status else "") + f": {_short(str(exc), 160)}" + attempts.append(f"{tag}: {detail}") + log.warning("MiniMax call failed for %s: %s", tag, detail) + return None + + choices = getattr(completion, "choices", None) or [] + if not choices: + attempts.append(f"{tag}: empty choices") + log.warning("MiniMax %s returned no choices", tag) + return None + choice = choices[0] + finish_reason = getattr(choice, "finish_reason", None) + self._log_usage(model, completion, finish_reason) + if finish_reason == "length": + attempts.append(f"{tag}: truncated (finish_reason=length)") + log.warning("MiniMax %s output truncated (finish_reason=length)", tag) + return None + content = getattr(choice.message, "content", None) + if not content: + attempts.append(f"{tag}: empty content") + log.warning("MiniMax %s returned empty content", tag) + return None + return content + + @staticmethod + def _log_usage(model: str, completion: Any, finish_reason: str | None) -> None: + usage = getattr(completion, "usage", None) + prompt_tokens = getattr(usage, "prompt_tokens", None) + completion_tokens = getattr(usage, "completion_tokens", None) + details = getattr(usage, "prompt_tokens_details", None) + cached = getattr(details, "cached_tokens", None) if details is not None else None + log.info( + "MiniMax response model=%s served_as=%s finish=%s prompt_tokens=%s completion_tokens=%s " + "cached_tokens=%s", + model, + getattr(completion, "model", None), + finish_reason, + prompt_tokens, + completion_tokens, + cached, + ) + + +def _short(text: str, limit: int) -> str: + text = " ".join(text.split()) + return text if len(text) <= limit else text[: limit - 1] + "…" + + +__all__ = [ + "OPENAI_ERRORS", + "MiniMaxProvider", + "coerce_model_data", + "digest_json_schema", + "digest_response_format", + "extract_json_text", + "model_json_schema", + "model_response_format", + "parse_digest", + "parse_model", +] diff --git a/sebrief/llm/prompts.py b/sebrief/llm/prompts.py new file mode 100644 index 0000000..55ccb31 --- /dev/null +++ b/sebrief/llm/prompts.py @@ -0,0 +1,193 @@ +"""Prompt construction for the weekly digest LLM. + +`SYSTEM_PROMPT` is static (no dates, no timestamps) so that the provider's prompt cache can hit on +every run. All per-week data goes into the user message built by `build_user_message`, which is a +deterministic, human-readable serialisation of a `WeekPayload`. +""" + +from __future__ import annotations + +from sebrief.models import Digest, Paper, VerificationIssue, WeekPayload + +# Maximum characters of an abstract carried into the prompt (token budget control). +ABSTRACT_MAX_CHARS = 1800 +# Authors listed per paper before "외 N명". +MAX_AUTHORS = 6 + +# 주제 그룹의 권장 이름 - 모델이 그대로 쓰거나 필요하면 새 이름을 만든다. +SUGGESTED_TOPICS = "LLM4SE · 테스팅·분석 · 실증 연구 · 보안·신뢰성 · 유지보수·진화 · 프로세스·인간 요인 · 도구·데이터셋" + +_SKELETON = """{ + "overview": "이번 주 흐름 총평 2~3문장", + "highlights": [ + {"doi": "10.xxxx/yyyy", "title": "원제 그대로", "problem": "문장", "approach": "문장", + "findings": "문장", "implication": "문장"} + ], + "by_topic": [ + {"topic": "주제명", "papers": [{"doi": "10.xxxx/yyyy", "title": "원제 그대로", "note": "1~2문장"}]} + ], + "worth_reading": [{"doi": "10.xxxx/yyyy", "title": "원제 그대로", "reason": "1~2문장"}], + "data_gaps": ["수집 누락 항목"] +}""" + +SYSTEM_PROMPT: str = f"""당신은 소프트웨어공학(SE) 연구자와 실무 개발자를 위한 주간 논문 다이제스트 작성자입니다. +한 주 동안 발행된 오픈액세스 SE 논문들의 제목·초록·메타데이터를 받아 한국어 다이제스트를 만듭니다. + +## 작성 규칙 +1. 판단 근거는 사용자 메시지에 제공된 제목·초록·메타데이터뿐입니다. 초록에 없는 결과 수치·실험 조건· + 결론을 새로 만들거나 추정하지 않습니다. 배경지식으로 내용을 보태지 않습니다. +2. 저자 주장과 검증된 사실을 구분합니다. 초록의 성과 서술은 "저자들은 ~라고 보고합니다" 같은 + 주장 표기 톤으로 씁니다. 초록만으로 확인되지 않는 인과관계를 단정하지 않습니다. +3. 실무 시사점(implication)은 초록 근거 내로 한정하고 과장하지 않습니다. +4. highlights: 가장 주목할 논문 최대 3편. 논문당 문제(problem)·방법(approach)·결과(findings)· + 실무 시사점(implication)을 각 1~2문장으로 씁니다. +5. by_topic: 제공된 모든 논문을 주제 그룹으로 나눕니다. 한 논문은 정확히 한 그룹에만 넣고, + 하이라이트로 뽑은 논문도 해당 주제 그룹에 포함합니다. 논문당 note 1~2문장. + 권장 주제명: {SUGGESTED_TOPICS}. 맞는 것이 없으면 간결한 새 주제명을 만듭니다. +6. worth_reading: 시간을 들여 읽을 가치가 있는 논문 1~3편과 이유. +7. doi 필드는 논문 목록의 DOI를 한 글자도 바꾸지 않고 그대로 복사합니다. 목록에 없는 DOI나 논문을 + 만들지 않습니다. title은 논문 원제 그대로(영문 유지) 씁니다. +8. 그 밖의 모든 서술은 간결한 한국어 경어체(~습니다)로 작성합니다. 논문 제목·전문용어·고유명사는 + 영문을 유지해도 됩니다. 중국어·일본어 문장을 섞지 않습니다. +9. 초록이 없는 논문은 제목·저널·주제만으로 by_topic에 배치하고, note에 "초록 미제공"을 밝힙니다. + 그런 논문은 highlights와 worth_reading으로 뽑지 않습니다. +10. 수집 실패로 제공되지 않은 항목은 data_gaps에 그대로 적습니다. +11. 출력은 지정된 JSON 스키마만 반환합니다. 스키마 밖의 텍스트, 마크다운 코드 펜스, 설명을 추가하지 + 않습니다. + +출력 형태 예시(값은 자리표시자): +{_SKELETON} +""" + +_PAPER_SUMMARY_SKELETON = """{ + "summaries": [ + {"doi": "10.xxxx/yyyy", "tldr": "한 문장", "problem": "문장", "approach": "문장", + "findings": "문장", "implication": "문장"} + ] +}""" + +PAPER_SUMMARY_SYSTEM_PROMPT: str = f"""당신은 소프트웨어공학(SE) 논문 요약자입니다. 논문들의 제목·초록·메타데이터를 받아 +논문마다 하나씩 구조화된 한국어 요약을 만듭니다. + +## 작성 규칙 +1. 판단 근거는 각 논문의 제목·초록·메타데이터뿐입니다. 초록에 없는 결과 수치·실험 조건·결론을 + 새로 만들거나 추정하지 않습니다. 배경지식으로 내용을 보태지 않습니다. +2. 저자 주장과 검증된 사실을 구분합니다. 초록의 성과 서술은 "저자들은 ~라고 보고합니다" 같은 + 주장 표기 톤으로 씁니다. +3. 필드: tldr(한 문장 핵심), problem(다루는 문제 1~2문장), approach(방법 1~2문장), + findings(결과 1~2문장 - 초록에 있는 내용만), implication(실무 시사점 1문장 - 초록 근거 내). +4. doi 필드는 논문 목록의 DOI를 한 글자도 바꾸지 않고 그대로 복사합니다. 목록의 모든 논문을 + 빠짐없이 하나씩 요약하고, 목록에 없는 논문을 만들지 않습니다. +5. 간결한 한국어 경어체(~습니다)로 작성합니다. 논문 제목·전문용어·고유명사는 영문을 유지해도 + 됩니다. +6. 출력은 지정된 JSON 스키마만 반환합니다. 스키마 밖의 텍스트, 마크다운 코드 펜스, 설명을 + 추가하지 않습니다. + +출력 형태 예시(값은 자리표시자): +{_PAPER_SUMMARY_SKELETON} +""" + +VERIFY_SYSTEM_PROMPT: str = """당신은 소프트웨어공학 주간 다이제스트의 팩트체커입니다. 다이제스트 초안이 함께 제공된 +논문 목록(제목·초록·메타데이터)과 모순되지 않는지만 검사합니다. 문체·구성·의견은 평가하지 않습니다. + +## 검사 규칙 +1. 판단 근거는 사용자 메시지에 제공된 논문 목록뿐입니다. 당신의 배경지식이나 최신 연구 동향과 달라 + 보여도, 제공된 초록과 일치하면 문제가 아닙니다. +2. major: 다이제스트의 서술이 초록과 모순되거나, 초록에 없는 수치·실험 결과·결론을 사실처럼 + 서술하거나, 목록에 없는 DOI·논문을 언급하거나, DOI와 제목이 서로 다른 논문을 가리키는 경우. +3. minor: 초록만으로 확인되지 않는 인과관계를 단정, 저자 주장을 검증된 사실처럼 표현, 모호하지만 + 틀렸다고 단정할 수 없는 서술. +4. 각 이슈의 claim에는 다이제스트 원문을 그대로 인용하고, problem에는 초록의 실제 내용과 함께 한 + 문장으로 설명합니다. +5. major 이슈가 하나도 없으면 ok=true 입니다. 사소한 트집을 만들지 말고, 문제가 없으면 빈 issues로 + 반환합니다. +6. 출력은 지정된 JSON 스키마만 반환합니다. 스키마 밖 텍스트·설명·코드 펜스를 추가하지 않습니다. +""" + +REVISE_INSTRUCTION: str = ( + "위 다이제스트 초안에서 아래 검증 이슈가 발견되었습니다. 제공된 논문 목록에 맞게 해당 서술만 고치고," + " 문제없는 부분은 그대로 유지하여 동일한 JSON 스키마로 전체 다이제스트를 다시 출력하세요." + " 설명 없이 JSON 객체 하나만 출력합니다.\n\n검증 이슈:\n" +) + + +def _fmt_authors(authors: list[str]) -> str: + if not authors: + return "(저자 정보 없음)" + shown = ", ".join(authors[:MAX_AUTHORS]) + rest = len(authors) - MAX_AUTHORS + return f"{shown} 외 {rest}명" if rest > 0 else shown + + +def _fmt_abstract(abstract: str) -> str: + text = " ".join(abstract.split()) + if not text: + return "(초록 없음)" + if len(text) > ABSTRACT_MAX_CHARS: + text = text[: ABSTRACT_MAX_CHARS - 1].rstrip() + "…" + return text + + +def _fmt_paper(idx: int, p: Paper) -> list[str]: + published = p.published_date.isoformat() if p.published_date else "발행일 미상" + return [ + f"### [{idx}] DOI: {p.doi}", + f"- 제목: {p.title}", + f"- 저널: {p.venue or '(저널 미상)'} | 발행일: {published}", + f"- 저자: {_fmt_authors(p.authors)}", + f"- 주제: {', '.join(p.topics) if p.topics else '(주제 정보 없음)'}", + f"- 초록: {_fmt_abstract(p.abstract)}", + ] + + +def _bullets(items: list[str]) -> list[str]: + return [f"- {item}" for item in items] or ["(없음)"] + + +def build_papers_message(papers: list[Paper]) -> str: + """Serialise one batch of papers for the per-paper summarisation call.""" + out: list[str] = ["## 논문 목록", f"- 논문 수: {len(papers)}편", ""] + for idx, paper in enumerate(papers, 1): + out.extend(_fmt_paper(idx, paper)) + out.append("") + return "\n".join(out) + + +def build_user_message(payload: WeekPayload) -> str: + """Serialise the payload into a deterministic Korean text block for the user message.""" + out: list[str] = [] + + out.append("## 대상 주차") + out.append(f"- 주차: {payload.week} ({payload.week_start.isoformat()} ~ {payload.week_end.isoformat()})") + out.append(f"- 논문 수: {len(payload.papers)}편") + out.append("") + + out.append("## 논문 목록") + if not payload.papers: + out.append("(수집된 논문 없음)") + for idx, paper in enumerate(payload.papers, 1): + out.extend(_fmt_paper(idx, paper)) + out.append("") + + out.append("## 수집 누락") + out.extend(_bullets(payload.data_gaps)) + + return "\n".join(out) + "\n" + + +def build_verify_message(payload: WeekPayload, digest: Digest) -> str: + """User message for the fact-check pass: the payload serialisation plus the draft digest.""" + return ( + build_user_message(payload) + + "\n## 검증 대상 다이제스트 초안 (JSON)\n" + + digest.model_dump_json(indent=2) + + "\n" + ) + + +def format_issues(issues: list[VerificationIssue]) -> str: + """Issues -> numbered plain-text block for the revise instruction.""" + lines = [] + for i, issue in enumerate(issues, 1): + lines.append(f"{i}. [{issue.severity}] 서술: {issue.claim}\n 문제: {issue.problem}") + return "\n".join(lines) or "(없음)" diff --git a/sebrief/logging_setup.py b/sebrief/logging_setup.py new file mode 100644 index 0000000..3b209d6 --- /dev/null +++ b/sebrief/logging_setup.py @@ -0,0 +1,129 @@ +"""Logging configuration: console + daily-rotating file, with API-key masking.""" + +from __future__ import annotations + +import logging +import re +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path + +from sebrief.config import Settings + +LOG_FILE_NAME = "sebrief.log" +LOG_FORMAT = "%(asctime)s %(levelname)-7s %(name)s: %(message)s" +_HANDLER_TAG = "_sebrief_handler" + +# MiniMax keys (sk-...) and secrets passed as URL query parameters (requests/urllib3 embed the full +# URL in HTTPError/ConnectionError messages, which then land in tracebacks via log.exception). +# The parameter name is kept so the log stays readable. +SECRET_RE = re.compile( + r"(sk-[A-Za-z0-9_-]{6,})" + r"|(?i:([?&](?:api_key|apikey|api-key|key|token|access_token)=))[^&\s\"'<>]+" +) +MASK = "***" + +# Exact secret values registered by setup_logging(); replaced verbatim whatever their shape. +_SECRET_VALUES: tuple[str, ...] = () +_MIN_SECRET_LEN = 8 + +NOISY_LOGGERS = ( + "httpx", + "httpx2", # transport used by openai>=3 (logs request URLs at INFO) + "httpcore", + "apscheduler.executors", + "apscheduler.executors.default", + "urllib3", +) + + +def register_secret_values(*values: str | None) -> None: + """Register configured secret values so they are masked even when they match no known shape. + + Calling with no arguments clears the registry. Longest values first so a value that contains + another registered value leaves no residue. + """ + global _SECRET_VALUES + _SECRET_VALUES = tuple( + sorted({v for v in values if v and len(v) >= _MIN_SECRET_LEN}, key=len, reverse=True) + ) + + +def _sub(m: re.Match[str]) -> str: + prefix = m.group(2) + return f"{prefix}{MASK}" if prefix else MASK + + +def mask_secrets(text: str) -> str: + """Replace anything that looks like an API key / URL query secret with ``***``.""" + for value in _SECRET_VALUES: + text = text.replace(value, MASK) + return SECRET_RE.sub(_sub, text) + + +def _mask_value(value: object) -> object: + return mask_secrets(value) if isinstance(value, str) else value + + +class SecretMaskingFilter(logging.Filter): + """Masks secrets in ``record.msg`` and ``record.args`` before formatting.""" + + def filter(self, record: logging.LogRecord) -> bool: + if isinstance(record.msg, str): + record.msg = mask_secrets(record.msg) + args = record.args + if isinstance(args, tuple): + record.args = tuple(_mask_value(a) for a in args) + elif isinstance(args, dict): + record.args = {k: _mask_value(v) for k, v in args.items()} + return True + + +class SecretMaskingFormatter(logging.Formatter): + """Second line of defence: masks the fully formatted line (covers tracebacks too).""" + + def format(self, record: logging.LogRecord) -> str: + return mask_secrets(super().format(record)) + + +def _remove_our_handlers(root: logging.Logger) -> None: + for handler in list(root.handlers): + if getattr(handler, _HANDLER_TAG, False): + root.removeHandler(handler) + handler.close() + + +def setup_logging(settings: Settings) -> None: + """Configure the root logger. Safe to call more than once (handlers are replaced, not duplicated).""" + level_name = (settings.log_level or "INFO").upper() + level = logging.getLevelNamesMapping().get(level_name, logging.INFO) + + root = logging.getLogger() + _remove_our_handlers(root) + root.setLevel(level) + + register_secret_values(settings.minimax_api_key) + formatter = SecretMaskingFormatter(LOG_FORMAT) + mask_filter = SecretMaskingFilter() + + console = logging.StreamHandler() + handlers: list[logging.Handler] = [console] + + log_dir = Path(settings.log_dir) + try: + log_dir.mkdir(parents=True, exist_ok=True) + file_handler = TimedRotatingFileHandler( + log_dir / LOG_FILE_NAME, when="midnight", backupCount=30, encoding="utf-8" + ) + handlers.append(file_handler) + except OSError as exc: # unwritable log dir must not prevent start-up + logging.getLogger(__name__).warning("file logging disabled (%s): %s", log_dir, exc) + + for handler in handlers: + handler.setLevel(level) + handler.setFormatter(formatter) + handler.addFilter(mask_filter) + setattr(handler, _HANDLER_TAG, True) + root.addHandler(handler) + + for name in NOISY_LOGGERS: + logging.getLogger(name).setLevel(logging.WARNING) diff --git a/sebrief/models.py b/sebrief/models.py new file mode 100644 index 0000000..e1a863d --- /dev/null +++ b/sebrief/models.py @@ -0,0 +1,128 @@ +"""Shared data models. Every module talks to the others through these types only.""" + +from __future__ import annotations + +from datetime import date, datetime +from typing import Literal + +from pydantic import BaseModel, Field + + +class PaperSummary(BaseModel): + """Per-paper Korean summary derived from the abstract (generated once per DOI).""" + + tldr: str = Field(description="한 줄 요약 (한 문장)") + problem: str = Field(description="다루는 문제 1~2문장") + approach: str = Field(description="방법·접근 1~2문장") + findings: str = Field(description="결과 1~2문장 - 초록에 있는 내용만, 저자 주장은 주장으로 표기") + implication: str = Field(description="실무 시사점 1문장 - 초록 근거 내로 한정") + + +class PaperSummaryItem(PaperSummary): + """One entry of a batch summarisation reply, keyed back to its paper by DOI.""" + + doi: str = Field(description="논문 목록의 DOI를 그대로 복사") + + +class PaperSummaryBatch(BaseModel): + """LLM output schema for one batch of per-paper summaries.""" + + summaries: list[PaperSummaryItem] = Field(description="목록의 모든 논문에 대해 하나씩") + + +class Paper(BaseModel): + """One collected OA paper (abstract-level metadata only in Phase 1).""" + + doi: str # normalised bare DOI, lowercase (e.g. "10.1145/1234567") + title: str + abstract: str = "" + authors: list[str] = Field(default_factory=list) + venue: str = "" + published_date: date | None = None + oa_url: str | None = None + pdf_url: str | None = None + topics: list[str] = Field(default_factory=list) + source: str = "openalex" # collector key ("openalex" | "rss:<이름>") + summary: PaperSummary | None = None # filled by the per-paper summarisation step + + +class WeekPayload(BaseModel): + """Everything collected for one weekly digest, before LLM summarisation.""" + + week: str # ISO week id, e.g. "2026-W36" (the digest idempotency key) + week_start: date # Monday + week_end: date # Sunday + generated_at: datetime # tz-aware (Asia/Seoul) + papers: list[Paper] = Field(default_factory=list) + data_gaps: list[str] = Field(default_factory=list) # collectors that failed / partial + + def paper_by_doi(self) -> dict[str, Paper]: + return {p.doi: p for p in self.papers} + + +class Highlight(BaseModel): + """One featured paper: problem / approach / findings / practical implication.""" + + doi: str = Field(description="논문 목록의 DOI를 그대로 복사") + title: str = Field(description="논문 원제 그대로 (영문 유지)") + problem: str = Field(description="다루는 문제 1~2문장") + approach: str = Field(description="방법·접근 1~2문장") + findings: str = Field(description="결과 1~2문장 - 초록에 있는 내용만, 저자 주장은 주장으로 표기") + implication: str = Field(description="실무 시사점 1문장 - 초록 근거 내로 한정") + + +class TopicPaper(BaseModel): + doi: str = Field(description="논문 목록의 DOI를 그대로 복사") + title: str = Field(description="논문 원제 그대로") + note: str = Field(description="논문 요약 1~2문장") + + +class TopicGroup(BaseModel): + topic: str = Field(description="주제명 (예: LLM4SE, 테스팅·분석, 실증 연구)") + papers: list[TopicPaper] = Field(description="이 주제의 논문들") + + +class Pick(BaseModel): + doi: str = Field(description="논문 목록의 DOI를 그대로 복사") + title: str = Field(description="논문 원제 그대로") + reason: str = Field(description="읽어볼 이유 1~2문장") + + +class Digest(BaseModel): + """LLM output schema. Used verbatim as the JSON Schema for structured output.""" + + overview: str = Field(description="이번 주 흐름 총평 2~3문장") + highlights: list[Highlight] = Field(description="가장 주목할 논문 최대 3편") + by_topic: list[TopicGroup] = Field(description="주제별 그룹 - 모든 논문을 정확히 한 그룹에 배치") + worth_reading: list[Pick] = Field(description="읽어볼 논문 픽 1~3편") + data_gaps: list[str] = Field(default_factory=list, description="수집 실패로 반영하지 못한 항목") + + +class VerificationIssue(BaseModel): + claim: str = Field(description="문제가 된 다이제스트 속 서술 (원문 그대로 인용)") + problem: str = Field(description="제공된 초록·메타데이터와 어떻게 다른지 한 문장") + severity: Literal["minor", "major"] = Field( + default="minor", + description="major: 초록과 모순되거나 초록에 없는 수치·결과를 사실처럼 서술 / minor: 근거 불충분·표현 문제", + ) + + +class Verification(BaseModel): + """LLM fact-check verdict for a generated Digest, judged only against the payload.""" + + ok: bool = Field(description="major 이슈가 하나도 없으면 true") + issues: list[VerificationIssue] = Field(default_factory=list, description="발견된 문제 목록 (없으면 빈 배열)") + + @property + def has_major(self) -> bool: + return any(i.severity == "major" for i in self.issues) + + +class DigestResult(BaseModel): + payload: WeekPayload + digest: Digest | None + markdown: str + model_used: str | None = None + degraded: bool = False # True when the LLM failed (or --no-llm) and only the paper list was produced + error: str | None = None + verification: Verification | None = None # None: not run (disabled / degraded / verifier failed) diff --git a/sebrief/pipeline.py b/sebrief/pipeline.py new file mode 100644 index 0000000..3df3840 --- /dev/null +++ b/sebrief/pipeline.py @@ -0,0 +1,277 @@ +"""Digest pipeline: collect -> store -> summarise -> render -> deliver. + +All collaborators are imported at module level and referenced as module attributes so that tests +can monkeypatch e.g. ``sebrief.pipeline.fetch_openalex_papers``. +""" + +from __future__ import annotations + +import logging +from collections.abc import Sequence +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +from sebrief.collectors.base import run_safely +from sebrief.collectors.openalex import fetch_openalex_papers +from sebrief.collectors.rss import fetch_rss_papers, parse_feed_specs +from sebrief.config import KST, Settings +from sebrief.deliver import build_deliverers +from sebrief.llm.base import LLMError +from sebrief.models import Digest, DigestResult, Paper, Verification, WeekPayload +from sebrief.render import render_markdown +from sebrief.store import Store +from sebrief.summarize import build_provider, summarize_papers, summarize_payload, verify_and_revise +from sebrief.week import WeeklyClock, week_range + +log = logging.getLogger(__name__) + +SKIP_ALREADY_SENT = "already sent" +DEGRADED_SUBJECT = "LLM 요약 실패 - 축소 모드 발송" +DELIVERY_FAILED_SUBJECT = "다이제스트 발송 실패" +DELIVERY_FAILED_PREFIX = "delivery failed" + + +def _now() -> datetime: + return datetime.now(ZoneInfo(KST)) + + +def _dedup_by_doi(papers: Sequence[Paper]) -> list[Paper]: + """First occurrence wins (OpenAlex before RSS, so richer metadata is kept).""" + seen: set[str] = set() + unique: list[Paper] = [] + for p in papers: + if p.doi in seen: + continue + seen.add(p.doi) + unique.append(p) + return unique + + +def build_payload(week: str, settings: Settings, store: Store) -> WeekPayload: + """Run every collector with failure isolation and assemble the WeekPayload.""" + now = _now() + week_start, week_end = week_range(week) + gaps: list[str] = [] + + # Collection window reaches back so late-indexed papers are still caught; DOI dedup + the + # digest_week stamp keep them from being summarised twice. + collect_from = week_start - timedelta(days=settings.backfill_days) + papers = run_safely( + "openalex", lambda: fetch_openalex_papers(collect_from, week_end, settings), [], gaps + ) + + feeds, invalid = parse_feed_specs(settings.rss_feeds) + for spec in invalid: + gaps.append(f"rss: invalid feed spec {spec!r} (expected 이름|URL)") + if feeds: + rss_papers = run_safely( + "rss", + lambda: fetch_rss_papers(feeds, collect_from, week_end, settings.extra_blacklist_issns), + [], + gaps, + ) + if rss_papers: + papers = _dedup_by_doi([*papers, *rss_papers]) + + if papers: + run_safely("store", lambda: store.upsert_papers(papers), None, gaps) + pending = run_safely("store", lambda: store.pending_papers(week), [], gaps) + + if len(pending) > settings.max_papers: + gaps.append( + f"papers: {len(pending)}편 수집, {settings.max_papers}편만 요약 (초과분은 다음 주로 이월)" + ) + pending = pending[: settings.max_papers] + + payload = WeekPayload( + week=week, + week_start=week_start, + week_end=week_end, + generated_at=now, + papers=pending, + data_gaps=gaps, + ) + log.info( + "payload built for %s: %d collected, %d in digest, %d gaps", + week, + len(papers), + len(pending), + len(gaps), + ) + return payload + + +def _minimal_payload(week: str, now: datetime) -> WeekPayload: + week_start, week_end = week_range(week) + return WeekPayload(week=week, week_start=week_start, week_end=week_end, generated_at=now) + + +def _skip_result(payload: WeekPayload, reason: str) -> DigestResult: + return DigestResult(payload=payload, digest=None, markdown="", error=reason) + + +def _summarize(payload: WeekPayload, settings: Settings) -> tuple[Digest | None, str | None, str | None]: + """Return (digest, model_used, error). Any failure degrades instead of aborting the digest.""" + try: + provider = build_provider(settings) + digest, model_used = summarize_payload(payload, provider) + return digest, model_used, None + except LLMError as exc: + log.error("LLM summarisation failed, switching to degraded mode: %s", exc) + return None, None, str(exc) + except Exception as exc: # provider construction / unexpected SDK errors + log.exception("unexpected LLM failure, switching to degraded mode") + return None, None, f"{type(exc).__name__}: {exc}" + + +def _summarize_papers(payload: WeekPayload, settings: Settings, store: Store) -> None: + """Fill per-paper summaries in place (persisted per DOI). Any failure only leaves gaps.""" + try: + provider = build_provider(settings) + summaries, notes = summarize_papers( + payload.papers, provider, settings.paper_summary_batch_size + ) + except Exception as exc: # noqa: BLE001 - per-paper summaries must never block the digest + log.warning("paper summarisation failed entirely: %s: %s", type(exc).__name__, exc) + payload.data_gaps.append(f"paper-summaries: {type(exc).__name__} (다음 실행에서 재시도)") + return + payload.data_gaps.extend(notes) + if not summaries: + return + for paper in payload.papers: + if paper.doi in summaries: + paper.summary = summaries[paper.doi] + run_safely("store", lambda: store.save_paper_summaries(summaries), None, payload.data_gaps) + + +def _verify(payload: WeekPayload, digest: Digest, settings: Settings) -> tuple[Digest, Verification | None]: + """Fact-check (and possibly revise) the digest. Any failure keeps the digest and returns None.""" + try: + provider = build_provider(settings) + return verify_and_revise(payload, digest, provider) + except Exception as exc: # noqa: BLE001 - verification must never block the digest + log.warning("verification failed, delivering unverified digest: %s: %s", type(exc).__name__, exc) + return digest, None + + +def _notify_all(deliverers: Sequence[object], subject: str, detail: str) -> None: + for deliverer in deliverers: + try: + deliverer.notify_error(subject, detail) # type: ignore[attr-defined] + except Exception: # notify_error must never propagate + log.exception("notify_error via %s failed", getattr(deliverer, "name", deliverer)) + + +def _deliver(result: DigestResult, settings: Settings, store: Store) -> list[str]: + """Deliver on every channel. Returns the names of channels that failed (empty on success). + + Failures are isolated per channel but never silent: ``result.error`` is annotated (see + ``delivery_failed``) and an operator alert goes through every channel - FileDeliverer always + writes errors.log. + """ + deliverers = build_deliverers(settings) + week = result.payload.week + failures: list[tuple[str, str]] = [] + for deliverer in deliverers: + name = getattr(deliverer, "name", str(deliverer)) + try: + receipt = deliverer.deliver(result) + store.mark_sent(week, name, receipt) + log.info("delivered via %s: %s", name, receipt) + except Exception as exc: # one channel failing must not block the others + log.exception("delivery via %s failed", name) + failures.append((name, f"{type(exc).__name__}: {exc}"[:300])) + + if result.degraded and result.error: + _notify_all(deliverers, DEGRADED_SUBJECT, result.error) + + if failures: + detail = "; ".join(f"{name}: {err}" for name, err in failures) + note = f"{DELIVERY_FAILED_PREFIX}: {detail}" + result.error = f"{result.error} | {note}" if result.error else note + log.error("%s for week %s", note, week) + _notify_all( + deliverers, + DELIVERY_FAILED_SUBJECT, + f"week {week} - {detail} - 재발송: sebrief run-once --force --week {week}", + ) + return [name for name, _ in failures] + + +def delivery_failed(result: DigestResult) -> bool: + """True when at least one delivery channel failed for this result.""" + return bool(result.error) and DELIVERY_FAILED_PREFIX in (result.error or "") + + +def run_digest( + week: str | None, + settings: Settings, + *, + dry_run: bool = False, + force: bool = False, + no_llm: bool = False, +) -> DigestResult: + """Produce and deliver one weekly digest for ``week`` (default: latest fully elapsed ISO week).""" + clock = WeeklyClock(settings.run_weekday, settings.run_time_kst) + store = Store(settings.db_path) + try: + now = _now() + if week is None: + week = clock.latest_completed_week(now) + week_start, week_end = week_range(week) # validates the week id + log.info("digest for week %s (%s ~ %s)", week, week_start, week_end) + + if week_end >= now.date() and not dry_run: + # An unfinished week must never reach save_digest: papers published later in the week + # would be frozen out (digest_week already stamped) and the daemon's scheduled run would + # skip as "already sent". Preview with --dry-run instead. + raise ValueError( + f"week {week} has not ended yet (ends {week_end}); refusing to save/deliver a partial " + "digest. Use --dry-run for a preview, or --week --force to test delivery." + ) + if not dry_run and not force and store.has_digest(week): + log.info("digest for %s already sent - skipping (use --force to resend)", week) + return _skip_result(_minimal_payload(week, now), SKIP_ALREADY_SENT) + + payload = build_payload(week, settings, store) + + digest: Digest | None = None + model_used: str | None = None + error: str | None = None + verification: Verification | None = None + degraded = False + if no_llm: + degraded = True + log.info("--no-llm: skipping summarisation") + elif not payload.papers: + log.info("no papers collected for %s - producing an empty digest without the LLM", week) + else: + if settings.paper_summary_enabled: + _summarize_papers(payload, settings, store) + digest, model_used, error = _summarize(payload, settings) + degraded = digest is None + if digest is not None and settings.verify_enabled: + digest, verification = _verify(payload, digest, settings) + + markdown = render_markdown(payload, digest, model_used) + result = DigestResult( + payload=payload, + digest=digest, + markdown=markdown, + model_used=model_used, + degraded=degraded, + error=error, + verification=verification, + ) + if dry_run: + print(markdown) + return result + + # Save before delivering: a crash mid-delivery must not cause a duplicate on catch-up. + store.save_digest(result) + store.mark_digested([p.doi for p in payload.papers], week) + log.info("digest saved for %s (degraded=%s, %d papers)", week, degraded, len(payload.papers)) + _deliver(result, settings, store) + return result + finally: + store.close() diff --git a/sebrief/render.py b/sebrief/render.py new file mode 100644 index 0000000..242117d --- /dev/null +++ b/sebrief/render.py @@ -0,0 +1,227 @@ +"""Rendering: WeekPayload (+ optional LLM Digest) -> Korean Markdown. + +Every paper link in the output resolves through the payload (DOI -> oa_url / doi.org); Digest prose +is used verbatim but never trusted for URLs. +""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime +from urllib.parse import quote +from zoneinfo import ZoneInfo + +from sebrief.models import Digest, Paper, WeekPayload + +KST = ZoneInfo("Asia/Seoul") +DASH = "—" + + +# ------------------------------------------------------------------ formatting helpers + + +def _kst(dt: datetime) -> datetime: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(KST) + + +def _cell(text: str) -> str: + """Escape a Markdown table cell (pipes, newlines).""" + return text.replace("|", "\\|").replace("\n", " ").strip() + + +def _text(s: object) -> str: + """Normalise a prose string from the Digest: strip, collapse whitespace/newlines.""" + return re.sub(r"\s*\n\s*", " ", str(s or "")).strip() + + +def _link_text(s: str) -> str: + return _text(s).replace("[", "(").replace("]", ")") + + +# Normal URL punctuation and existing %XX escapes stay as-is; '(' ')' and whitespace become %28 %29 %20 so +# they cannot terminate the Markdown ``(url)`` destination early. +_URL_SAFE = ":/?#[]@!$&'*+,;=%" + + +def _link_url(url: str) -> str: + """Percent-encode characters that would break a Markdown link destination.""" + return quote(url.strip(), safe=_URL_SAFE) + + +def _paper_url(paper: Paper) -> str: + return paper.oa_url or f"https://doi.org/{paper.doi}" + + +def _norm_title(s: str) -> str: + return re.sub(r"\W+", " ", str(s or "").casefold()).strip() + + +class _PaperIndex: + """Resolves digest references to payload papers, title first. + + LLM은 제목보다 DOI를 뒤섞는 실수가 훨씬 흔하고 독자가 보는 것은 제목이므로, 제목이 payload의 + 논문과 일치하면 그 논문으로 링크한다 (동명 논문이 있으면 제목 매칭은 포기하고 DOI로만). + """ + + def __init__(self, papers: list[Paper]) -> None: + self.by_doi: dict[str, Paper] = {p.doi: p for p in papers} + by_title: dict[str, Paper | None] = {} + for p in papers: + key = _norm_title(p.title) + by_title[key] = None if key in by_title else p # collision -> ambiguous + self.by_title: dict[str, Paper] = {k: v for k, v in by_title.items() if v is not None} + + def resolve(self, title: str, doi: str) -> Paper | None: + return self.by_title.get(_norm_title(title)) or self.by_doi.get(doi.strip().lower()) + + +def _linked_title(title: str, doi: str, index: _PaperIndex) -> str: + """Markdown link for a digest-referenced paper; plain text when it matches no payload paper.""" + paper = index.resolve(title, doi) + if paper is None: + return _link_text(title) + return f"[{_link_text(title or paper.title)}]({_link_url(_paper_url(paper))})" + + +# ------------------------------------------------------------------ Markdown sections + + +def _header(payload: WeekPayload, digest: Digest | None) -> list[str]: + lines = [ + ( + f"# 📚 SE 주간 브리핑 — {payload.week} " + f"({payload.week_start.isoformat()} ~ {payload.week_end.isoformat()})" + ) + ] + gen_kst = _kst(payload.generated_at).strftime("%Y-%m-%d %H:%M") + lines.append(f"수집 논문 {len(payload.papers)}편 · 생성 {gen_kst} KST") + if not payload.papers: + lines.append("이번 주 수집된 OA 논문이 없습니다.") + elif digest is None: + lines.append("🧩 축소 모드(LLM 요약 없음) — 논문 목록만 제공합니다.") + return lines + + +def _bullets(items: list[str]) -> list[str]: + return [f"- {_text(it)}" for it in items if _text(it)] + + +def _section(title: str, body: list[str]) -> list[str]: + if not body: + return [] + return [f"## {title}", *body, ""] + + +def _highlight_lines(digest: Digest, index: _PaperIndex) -> list[str]: + out: list[str] = [] + for h in digest.highlights: + out.append(f"### {_linked_title(h.title, h.doi, index)}") + paper = index.resolve(h.title, h.doi) + if paper is not None: + out.append(_paper_meta_line(paper)) + for label, value in ( + ("문제", h.problem), + ("방법", h.approach), + ("결과", h.findings), + ("시사점", h.implication), + ): + if _text(value): + out.append(f"- **{label}**: {_text(value)}") + out.append("") + while out and out[-1] == "": + out.pop() + return out + + +def _topic_lines(digest: Digest, index: _PaperIndex) -> list[str]: + out: list[str] = [] + for group in digest.by_topic: + if not group.papers: + continue + out.append(f"### {_text(group.topic)}") + for tp in group.papers: + note = _text(tp.note) + line = f"- {_linked_title(tp.title, tp.doi, index)}" + if note: + line += f" — {note}" + out.append(line) + out.append("") + while out and out[-1] == "": + out.pop() + return out + + +def _pick_lines(digest: Digest, index: _PaperIndex) -> list[str]: + out: list[str] = [] + for pick in digest.worth_reading: + reason = _text(pick.reason) + line = f"- {_linked_title(pick.title, pick.doi, index)}" + if reason: + line += f" — {reason}" + out.append(line) + return out + + +def _digest_sections(digest: Digest, index: _PaperIndex) -> list[str]: + out: list[str] = [] + if _text(digest.overview): + out += _section("이번 주 흐름", [_text(digest.overview)]) + out += _section("하이라이트", _highlight_lines(digest, index)) + out += _section("주제별 훑어보기", _topic_lines(digest, index)) + out += _section("읽어볼 논문", _pick_lines(digest, index)) + return out + + +def _paper_meta_line(paper: Paper) -> str: + published = paper.published_date.isoformat() if paper.published_date else DASH + return f"*{_text(paper.venue) or DASH} · {published}*" + + +def _paper_summary_lines(papers: list[Paper]) -> list[str]: + """One card per paper with a stored summary (papers without one only appear in the table).""" + out: list[str] = [] + for p in papers: + s = p.summary + if s is None: + continue + out.append(f"### [{_link_text(p.title)}]({_link_url(_paper_url(p))})") + out.append(_paper_meta_line(p)) + if _text(s.tldr): + out.append(f"**{_text(s.tldr)}**") + for label, value in ( + ("문제", s.problem), + ("방법", s.approach), + ("결과", s.findings), + ("시사점", s.implication), + ): + if _text(value): + out.append(f"- **{label}**: {_text(value)}") + out.append("") + while out and out[-1] == "": + out.pop() + return out + + +def _paper_table(papers: list[Paper]) -> list[str]: + if not papers: + return [] + lines = ["| 제목 | 저널 | 발행일 |", "|---|---|---|"] + for p in papers: + title = f"[{_cell(_link_text(p.title))}]({_link_url(_paper_url(p))})" + published = p.published_date.isoformat() if p.published_date else DASH + lines.append(f"| {title} | {_cell(p.venue) or DASH} | {published} |") + return lines + + +def render_markdown(payload: WeekPayload, digest: Digest | None, model_used: str | None) -> str: + """Render the Korean Markdown digest. Works with ``digest=None`` (degraded: paper list only).""" + lines: list[str] = _header(payload, digest) + lines.append("") + if digest is not None: + lines += _digest_sections(digest, _PaperIndex(payload.papers)) + lines += _section("논문별 요약", _paper_summary_lines(payload.papers)) + lines += _section("이번 주 수집 논문", _paper_table(payload.papers)) + lines += _section("수집 누락", _bullets(payload.data_gaps)) + return "\n".join(lines).rstrip() + "\n" diff --git a/sebrief/sources.py b/sebrief/sources.py new file mode 100644 index 0000000..4aead56 --- /dev/null +++ b/sebrief/sources.py @@ -0,0 +1,144 @@ +"""Journal sources: OA whitelist (ISSN), MDPI blacklist, and the SE topic/keyword filter. + +계획(SEBRIEF_PLAN.md §2, §4)의 결정사항을 코드로 옮긴 곳: +- 논문 단위 OA 기준이므로 하이브리드 저널(TSE·EMSE·IST·JSS)도 화이트리스트에 두고 + 수집 쿼리에서 `is_oa:true`로 거른다. +- 메가저널(IEEE Access, PeerJ CS)은 `se_filter=True` - OpenAlex 쿼리의 SE 서브필드 필터에 더해 + `looks_like_se`로 키워드 이중 필터를 적용한다. +- MDPI는 전면 제외. 화이트리스트 수집이라 원래 들어오지 않지만, RSS 등 다른 경로로 흘러들어도 + 걸러지도록 명시적 ISSN 블랙리스트 + 발행사 이름 검사로 이중 차단한다. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Source: + key: str + name: str + issns: tuple[str, ...] + se_filter: bool = False # True: mega-journal - needs the SE topic/keyword double filter + + +SOURCES: tuple[Source, ...] = ( + # 완전 OA (2026-01 ACM Open 전환) - 전량 수집 + Source("tosem", "ACM Transactions on Software Engineering and Methodology", ("1049-331X", "1557-7392")), + # 하이브리드 - 논문 단위 OA 필터로 수집 + Source("tse", "IEEE Transactions on Software Engineering", ("0098-5589", "1939-3520", "2326-3881")), + Source("emse", "Empirical Software Engineering", ("1382-3256", "1573-7616")), + Source("ist", "Information and Software Technology", ("0950-5849",)), + Source("jss", "Journal of Systems and Software", ("0164-1212", "1873-1228")), + # 완전 OA 도구/소프트웨어 논문 + Source("softwarex", "SoftwareX", ("2352-7110",)), + Source("joss", "Journal of Open Source Software", ("2475-9066",)), + # 메가저널 - SE 토픽/키워드 이중 필터 + Source("ieee_access", "IEEE Access", ("2169-3536",), se_filter=True), + Source("peerj_cs", "PeerJ Computer Science", ("2376-5992",), se_filter=True), +) + +# MDPI 주요 CS 저널 ISSN (방어용 - 발행사 이름 검사가 본선). 운영 중 발견되면 +# SEBRIEF_EXTRA_BLACKLIST_ISSNS로 추가한다. +BLACKLIST_ISSNS: frozenset[str] = frozenset( + { + "2079-9292", # Electronics + "2076-3417", # Applied Sciences + "1424-8220", # Sensors + "2227-7390", # Mathematics + "2078-2489", # Information + "2079-8954", # Systems + "2073-431X", # Computers + "1999-4893", # Algorithms + "1999-5903", # Future Internet + "2674-113X", # Software (MDPI) + } +) + +BLACKLIST_PUBLISHER_KEYWORDS: tuple[str, ...] = ("mdpi",) + +# 메가저널 이중 필터용 SE 키워드 - 제목·주제·초록(소문자)에 부분 문자열 매칭. +SE_KEYWORDS: tuple[str, ...] = ( + "software engineering", + "software testing", + "software test", + "software maintenance", + "software evolution", + "software develop", + "software quality", + "software architect", + "software project", + "software repositor", + "software vulnerabilit", + "code review", + "code generation", + "code completion", + "code summarization", + "code smell", + "code clone", + "source code", + "program repair", + "program analysis", + "program comprehension", + "static analysis", + "bug report", + "bug localization", + "bug detection", + "bug fix", + "defect prediction", + "fault localization", + "test generation", + "test case", + "unit test", + "regression test", + "mutation testing", + "fuzzing", + "requirements engineering", + "technical debt", + "refactoring", + "continuous integration", + "continuous delivery", + "devops", + "microservice", + "api usage", + "large language models for code", + "llm-based agent", + "developer productivity", + "empirical study of software", +) + + +def issns_for(se_filter: bool) -> tuple[str, ...]: + """All whitelist ISSNs of sources with the given `se_filter` flag, order preserved.""" + out: list[str] = [] + for src in SOURCES: + if src.se_filter == se_filter: + out.extend(src.issns) + return tuple(out) + + +def is_blacklisted( + issns: Iterable[str], publisher: str | None, extra_issns: Iterable[str] = () +) -> bool: + """True when any ISSN is blacklisted or the publisher name matches a blocked keyword (MDPI).""" + blocked = BLACKLIST_ISSNS | {i.strip() for i in extra_issns if i and i.strip()} + if any(issn in blocked for issn in issns): + return True + if publisher: + lowered = publisher.lower() + return any(kw in lowered for kw in BLACKLIST_PUBLISHER_KEYWORDS) + return False + + +def looks_like_se(title: str, abstract: str, topics: Iterable[str]) -> bool: + """Keyword double filter for mega-journal papers (IEEE Access, PeerJ CS). + + OpenAlex의 서브필드 필터가 1차이고 이 함수가 2차다: 제목·주제·초록 어디든 SE 키워드가 + 하나라도 나타나면 통과. 주제명에 "Software"가 들어가는 경우(예: "Software Engineering + Research", "Software Reliability")도 통과시킨다. + """ + haystack = " ".join([title, abstract, *topics]).lower() + if any(kw in haystack for kw in SE_KEYWORDS): + return True + return any("software" in t.lower() for t in topics) diff --git a/sebrief/store.py b/sebrief/store.py new file mode 100644 index 0000000..9558a24 --- /dev/null +++ b/sebrief/store.py @@ -0,0 +1,247 @@ +"""SQLite persistence: collected papers (DOI dedup), digest idempotency, delivery receipts. + +Single-threaded use only (one connection, WAL mode). Dates are stored as ISO ``YYYY-MM-DD`` text and +weeks as ``YYYY-Wnn`` so lexicographic ordering equals chronological ordering. + +Paper lifecycle: `upsert_papers` inserts new DOIs and refreshes metadata of papers that are not yet +in a digest; once `mark_digested` stamps a paper with its digest week it is frozen (재수집돼도 스킵) +and never summarised again. +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from datetime import UTC, date, datetime +from pathlib import Path +from typing import Self + +from sebrief.models import DigestResult, Paper, PaperSummary + +log = logging.getLogger(__name__) + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS papers ( + doi TEXT PRIMARY KEY, + title TEXT NOT NULL, + abstract TEXT NOT NULL DEFAULT '', + authors TEXT NOT NULL DEFAULT '[]', + venue TEXT NOT NULL DEFAULT '', + published_date TEXT, + oa_url TEXT, + pdf_url TEXT, + topics TEXT NOT NULL DEFAULT '[]', + source TEXT NOT NULL DEFAULT '', + collected_at TEXT, + digest_week TEXT, + summary TEXT +); +CREATE INDEX IF NOT EXISTS idx_papers_digest_week ON papers (digest_week); +CREATE TABLE IF NOT EXISTS digests ( + week TEXT PRIMARY KEY, + week_start TEXT NOT NULL, + week_end TEXT NOT NULL, + model_used TEXT, + degraded INTEGER NOT NULL DEFAULT 0, + markdown TEXT, + result_json TEXT, + created_at TEXT +); +CREATE TABLE IF NOT EXISTS deliveries ( + id INTEGER PRIMARY KEY, + week TEXT NOT NULL, + channel TEXT NOT NULL, + receipt TEXT, + sent_at TEXT +); +""" + + +def _now_iso() -> str: + return datetime.now(UTC).isoformat(timespec="seconds") + + +def _row_to_paper(row: sqlite3.Row) -> Paper: + return Paper( + doi=row["doi"], + title=row["title"], + abstract=row["abstract"], + authors=json.loads(row["authors"]), + venue=row["venue"], + published_date=date.fromisoformat(row["published_date"]) if row["published_date"] else None, + oa_url=row["oa_url"], + pdf_url=row["pdf_url"], + topics=json.loads(row["topics"]), + source=row["source"], + summary=PaperSummary.model_validate_json(row["summary"]) if row["summary"] else None, + ) + + +class Store: + """Thin sqlite3 wrapper. Creates parent directories and tables on construction.""" + + def __init__(self, db_path: Path) -> None: + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._conn = sqlite3.connect(str(self.db_path), timeout=30, isolation_level=None) + self._conn.row_factory = sqlite3.Row + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA synchronous=NORMAL") + self._conn.executescript(_SCHEMA) + self._migrate() + log.debug("store opened at %s", self.db_path) + + def _migrate(self) -> None: + """Add columns introduced after the first release to pre-existing databases.""" + columns = {row["name"] for row in self._conn.execute("PRAGMA table_info(papers)")} + if "summary" not in columns: + with self._conn: + self._conn.execute("ALTER TABLE papers ADD COLUMN summary TEXT") + log.info("migrated papers table: added summary column") + + # ------------------------------------------------------------------ papers + def upsert_papers(self, papers: list[Paper]) -> int: + """Insert new papers and refresh metadata of undigested ones; return the number of new DOIs. + + Papers already stamped with a digest week are frozen: re-collection never changes them. + """ + new = 0 + now = _now_iso() + with self._conn: + for p in papers: + row = self._conn.execute( + "SELECT digest_week FROM papers WHERE doi = ?", (p.doi,) + ).fetchone() + if row is not None and row["digest_week"] is not None: + continue # already digested - frozen + values = ( + p.title, + p.abstract, + json.dumps(p.authors, ensure_ascii=False), + p.venue, + p.published_date.isoformat() if p.published_date else None, + p.oa_url, + p.pdf_url, + json.dumps(p.topics, ensure_ascii=False), + p.source, + now, + ) + if row is None: + self._conn.execute( + """ + INSERT INTO papers (title, abstract, authors, venue, published_date, oa_url, + pdf_url, topics, source, collected_at, doi) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (*values, p.doi), + ) + new += 1 + else: + self._conn.execute( + """ + UPDATE papers SET title = ?, abstract = ?, authors = ?, venue = ?, + published_date = ?, oa_url = ?, pdf_url = ?, topics = ?, + source = ?, collected_at = ? + WHERE doi = ? + """, + (*values, p.doi), + ) + log.info("upserted %d papers (%d new)", len(papers), new) + return new + + def save_paper_summaries(self, summaries: dict[str, PaperSummary]) -> None: + """Attach per-paper summaries by DOI. Allowed on digested (frozen) papers - the summary is + derived data, not collector metadata - and never cleared by later metadata refreshes.""" + with self._conn: + self._conn.executemany( + "UPDATE papers SET summary = ? WHERE doi = ?", + [(summary.model_dump_json(), doi) for doi, summary in summaries.items()], + ) + log.info("saved %d paper summaries", len(summaries)) + + def pending_papers(self, week: str) -> list[Paper]: + """Papers to include in the digest for `week`: never digested, or digested in this very week. + + Including `digest_week = week` makes --force re-runs reproduce the same paper set; papers + collected earlier but never digested (late arrivals, cap overflow) roll in automatically. + """ + cur = self._conn.execute( + """ + SELECT * FROM papers WHERE digest_week IS NULL OR digest_week = ? + ORDER BY COALESCE(published_date, '9999-12-31'), venue, doi + """, + (week,), + ) + return [_row_to_paper(row) for row in cur.fetchall()] + + def mark_digested(self, dois: list[str], week: str) -> None: + with self._conn: + self._conn.executemany( + "UPDATE papers SET digest_week = ? WHERE doi = ?", [(week, doi) for doi in dois] + ) + log.info("marked %d papers as digested in %s", len(dois), week) + + def paper_count(self) -> int: + row = self._conn.execute("SELECT COUNT(*) AS n FROM papers").fetchone() + return int(row["n"]) + + # ----------------------------------------------------------------- digests + def has_digest(self, week: str) -> bool: + cur = self._conn.execute("SELECT 1 FROM digests WHERE week = ? LIMIT 1", (week,)) + return cur.fetchone() is not None + + def save_digest(self, result: DigestResult) -> None: + """Upsert the digest keyed by its week.""" + payload = result.payload + with self._conn: + self._conn.execute( + """ + INSERT INTO digests (week, week_start, week_end, model_used, degraded, markdown, + result_json, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(week) DO UPDATE SET + week_start = excluded.week_start, + week_end = excluded.week_end, + model_used = excluded.model_used, + degraded = excluded.degraded, + markdown = excluded.markdown, + result_json = excluded.result_json, + created_at = excluded.created_at + """, + ( + payload.week, + payload.week_start.isoformat(), + payload.week_end.isoformat(), + result.model_used, + int(result.degraded), + result.markdown, + result.model_dump_json(), + _now_iso(), + ), + ) + log.info("saved digest for %s (degraded=%s)", payload.week, result.degraded) + + def mark_sent(self, week: str, channel: str, receipt: str) -> None: + with self._conn: + self._conn.execute( + "INSERT INTO deliveries (week, channel, receipt, sent_at) VALUES (?, ?, ?, ?)", + (week, channel, receipt, _now_iso()), + ) + + def deliveries_for(self, week: str) -> list[sqlite3.Row]: + cur = self._conn.execute("SELECT * FROM deliveries WHERE week = ? ORDER BY id", (week,)) + return list(cur.fetchall()) + + # ------------------------------------------------------------------- misc + def close(self) -> None: + try: + self._conn.close() + except sqlite3.Error: # pragma: no cover - defensive + log.debug("store close failed", exc_info=True) + + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc: object) -> None: + self.close() diff --git a/sebrief/summarize.py b/sebrief/summarize.py new file mode 100644 index 0000000..e192bae --- /dev/null +++ b/sebrief/summarize.py @@ -0,0 +1,111 @@ +"""Glue between the pipeline and the LLM layer.""" + +from __future__ import annotations + +import logging + +from sebrief.config import Settings +from sebrief.llm.base import LLMError, LLMProvider +from sebrief.llm.minimax import MiniMaxProvider +from sebrief.models import Digest, Paper, PaperSummary, Verification, WeekPayload + +log = logging.getLogger(__name__) + + +def build_provider(settings: Settings) -> LLMProvider: + """Construct the configured LLM provider (MiniMax only for now).""" + return MiniMaxProvider(settings) + + +def summarize_payload(payload: WeekPayload, provider: LLMProvider) -> tuple[Digest, str]: + """Summarise via the provider and guarantee `digest.data_gaps` includes every payload gap.""" + digest, model_used = provider.summarize(payload) + merged: list[str] = [] + for gap in [*payload.data_gaps, *digest.data_gaps]: + if gap and gap not in merged: + merged.append(gap) + if merged != list(digest.data_gaps): + digest = digest.model_copy(update={"data_gaps": merged}) + log.info("digest produced by %s (%d data gaps)", model_used, len(merged)) + return digest, model_used + + +def summarize_papers( + papers: list[Paper], provider: LLMProvider, batch_size: int = 8 +) -> tuple[dict[str, PaperSummary], list[str]]: + """Per-paper summaries for papers that still lack one, in batches of `batch_size`. + + Returns (doi -> summary, gap notes). A failed batch is skipped with a note - partial results are + fine because summaries are persisted per DOI and retried on the next run. Papers without an + abstract are never summarised (제목만으로 요약하면 창작이 됨). + """ + todo = [p for p in papers if p.summary is None and p.abstract.strip()] + skipped = sum(1 for p in papers if p.summary is None and not p.abstract.strip()) + if skipped: + log.info("paper summaries: %d papers without an abstract skipped", skipped) + results: dict[str, PaperSummary] = {} + notes: list[str] = [] + batches = [todo[i : i + batch_size] for i in range(0, len(todo), batch_size)] + for n, batch in enumerate(batches, 1): + try: + items, model = provider.summarize_papers(batch) + except LLMError as exc: + log.warning("paper summary batch %d/%d failed: %s", n, len(batches), exc) + notes.append(f"paper-summaries: batch {n}/{len(batches)} 실패 ({len(batch)}편, 다음 실행에서 재시도)") + continue + wanted = {p.doi for p in batch} + matched = 0 + for item in items: + doi = item.doi.strip().lower() + if doi not in wanted: + log.warning("paper summary for unknown DOI %s discarded", item.doi) + continue + results[doi] = PaperSummary(**item.model_dump(exclude={"doi"})) + matched += 1 + missing = wanted - set(results) + if missing: + log.warning("paper summary batch %d/%d missing %d DOIs: %s", n, len(batches), len(missing), sorted(missing)) + log.info("paper summary batch %d/%d by %s: %d/%d papers", n, len(batches), model, matched, len(batch)) + return results, notes + + +def _consistent(verification: Verification) -> Verification: + """Force `ok` to mean "no major issues" - verifier models are inconsistent about minor-only cases.""" + ok = not verification.has_major + return verification if verification.ok == ok else verification.model_copy(update={"ok": ok}) + + +def verify_and_revise( + payload: WeekPayload, digest: Digest, provider: LLMProvider +) -> tuple[Digest, Verification]: + """Fact-check the digest; on major issues ask for one revision and re-check it. + + Returns (possibly revised digest, verification of the returned digest). Raises LLMError only when + the *initial* verification cannot be produced; a failed revision keeps the original digest. + """ + raw_verification, model = provider.verify(payload, digest) + verification = _consistent(raw_verification) + log.info( + "verification by %s: ok=%s issues=%d (major=%s)", + model, + verification.ok, + len(verification.issues), + verification.has_major, + ) + if not verification.has_major: + return digest, verification + + for issue in verification.issues: + log.warning("verification issue [%s] %s -> %s", issue.severity, issue.claim, issue.problem) + try: + revised, model = provider.revise(payload, digest, verification.issues) + log.info("digest revised by %s after verification issues", model) + re_verification, model = provider.verify(payload, revised) + re_verification = _consistent(re_verification) + log.info( + "re-verification by %s: ok=%s issues=%d", model, re_verification.ok, len(re_verification.issues) + ) + return revised, re_verification + except LLMError as exc: + log.warning("revision failed; keeping the original digest and verdict: %s", exc) + return digest, verification diff --git a/sebrief/week.py b/sebrief/week.py new file mode 100644 index 0000000..90edcf3 --- /dev/null +++ b/sebrief/week.py @@ -0,0 +1,105 @@ +"""ISO-week helpers and the weekly digest schedule clock. + +The digest idempotency key is the ISO week id ("2026-W36", Monday..Sunday) - the role krx_date plays +in US2KR. A digest for week W is scheduled at the first run-weekday after W ends (default: Monday +07:00 KST, i.e. the morning right after the covered week's Sunday). +""" + +from __future__ import annotations + +import re +from datetime import date, datetime, time, timedelta +from zoneinfo import ZoneInfo + +from sebrief.config import KST + +KST_TZ = ZoneInfo(KST) + +WEEK_RE = re.compile(r"^(\d{4})-W(\d{2})$") + + +def week_id(d: date) -> str: + """ISO week id for the week containing `d` (ISO year, not calendar year).""" + iso = d.isocalendar() + return f"{iso.year}-W{iso.week:02d}" + + +def week_range(week: str) -> tuple[date, date]: + """(Monday, Sunday) of an ISO week id. Raises ValueError for malformed / non-existent weeks.""" + m = WEEK_RE.match(week) + if not m: + raise ValueError(f"invalid week id {week!r}, expected YYYY-Wnn (e.g. 2026-W36)") + start = date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1) + return start, start + timedelta(days=6) + + +def parse_hhmm(value: str) -> tuple[int, int]: + try: + hh, mm = value.strip().split(":") + hour, minute = int(hh), int(mm) + except ValueError as exc: + raise ValueError(f"invalid HH:MM time: {value!r}") from exc + if not (0 <= hour <= 23 and 0 <= minute <= 59): + raise ValueError(f"invalid HH:MM time: {value!r}") + return hour, minute + + +def _require_aware(now: datetime) -> datetime: + if now.tzinfo is None or now.utcoffset() is None: + raise ValueError("`now` must be a tz-aware datetime") + return now + + +class WeeklyClock: + """Digest schedule calculator: plain weekday arithmetic in Asia/Seoul, no exchange calendars.""" + + def __init__(self, run_weekday: int = 0, run_time: str = "07:00") -> None: + if not (0 <= run_weekday <= 6): + raise ValueError("run_weekday must be 0 (Mon) .. 6 (Sun)") + self.run_weekday = run_weekday + hour, minute = parse_hhmm(run_time) + self.run_time = time(hour, minute) + + # ------------------------------------------------------------------ mapping + def _run_at(self, d: date) -> datetime: + return datetime.combine(d, self.run_time, tzinfo=KST_TZ) + + def covered_week(self, run_date: date) -> str: + """Week a run on `run_date` covers: the ISO week before the one containing `run_date`.""" + return week_id(run_date - timedelta(days=7)) + + def run_at_for(self, week: str) -> datetime: + """Scheduled run time for the digest covering `week`: first run-weekday after the week ends.""" + _, end = week_range(week) + d = end + timedelta(days=1) + while d.weekday() != self.run_weekday: + d += timedelta(days=1) + return self._run_at(d) + + # ------------------------------------------------------------------ schedule + def next_run(self, now: datetime) -> tuple[str, datetime]: + """(week, run_at KST) for the earliest digest run strictly after `now`.""" + now = _require_aware(now) + d = now.astimezone(KST_TZ).date() + while d.weekday() != self.run_weekday: + d += timedelta(days=1) + if self._run_at(d) <= now: + d += timedelta(days=7) + return self.covered_week(d), self._run_at(d) + + def last_run(self, now: datetime) -> tuple[str, datetime]: + """(week, run_at KST) of the most recent scheduled run at or before `now`.""" + _, next_at = self.next_run(now) + last_at = next_at - timedelta(days=7) + return self.covered_week(last_at.date()), last_at + + def latest_completed_week(self, now: datetime) -> str: + """Most recent fully elapsed ISO week (the default run-once target).""" + now = _require_aware(now) + return week_id(now.astimezone(KST_TZ).date() - timedelta(days=7)) + + def catchup_candidate(self, now: datetime, window_hours: int) -> str | None: + """Week whose run time lies in (now - window, now]; None when nothing was missed.""" + now = _require_aware(now) + week, run_at = self.last_run(now) + return week if now - run_at <= timedelta(hours=window_hours) else None diff --git a/tests/_fakes.py b/tests/_fakes.py new file mode 100644 index 0000000..caf13ed --- /dev/null +++ b/tests/_fakes.py @@ -0,0 +1,244 @@ +"""Shared fakes and payload builders for the orchestration tests.""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from zoneinfo import ZoneInfo + +from sebrief.models import ( + Digest, + Highlight, + Paper, + PaperSummary, + PaperSummaryItem, + Pick, + TopicGroup, + TopicPaper, + WeekPayload, +) + +KST = ZoneInfo("Asia/Seoul") + +# Fixed test timeline: 2026-W36 = 2026-08-31 (Mon) .. 2026-09-06 (Sun); the covering run fires +# Monday 2026-09-07 07:00 KST. NOW is that Monday at noon. +WEEK = "2026-W36" +WEEK_START = date(2026, 8, 31) +WEEK_END = date(2026, 9, 6) +RUN_AT = datetime(2026, 9, 7, 7, 0, tzinfo=KST) +NOW = datetime(2026, 9, 7, 12, 0, tzinfo=KST) + + +def make_paper(doi: str = "10.1145/1234567", **overrides) -> Paper: + values = { + "doi": doi, + "title": "Test-Driven Digest Generation", + "abstract": "We study how weekly digests are generated. Results show 42% faster reading.", + "authors": ["Alice Kim", "Bob Lee"], + "venue": "ACM Transactions on Software Engineering and Methodology", + "published_date": date(2026, 9, 2), + "oa_url": "https://dl.acm.org/doi/10.1145/1234567", + "pdf_url": None, + "topics": ["Software Engineering Research"], + "source": "openalex", + } + values.update(overrides) + return Paper(**values) + + +def make_summary(**overrides) -> PaperSummary: + values = { + "tldr": "테스트 주도 다이제스트 생성 기법을 제안합니다.", + "problem": "주간 다이제스트 생성이 느립니다.", + "approach": "저자들은 테스트 주도 생성 기법을 제안합니다.", + "findings": "저자들은 읽기 속도가 42% 빨라졌다고 보고합니다.", + "implication": "다이제스트 파이프라인에 적용해볼 만합니다.", + } + values.update(overrides) + return PaperSummary(**values) + + +def make_payload(papers: list[Paper] | None = None, gaps: list[str] | None = None) -> WeekPayload: + return WeekPayload( + week=WEEK, + week_start=WEEK_START, + week_end=WEEK_END, + generated_at=NOW, + papers=papers if papers is not None else [make_paper()], + data_gaps=gaps or [], + ) + + +def make_digest(doi: str = "10.1145/1234567", **overrides) -> Digest: + values = { + "overview": "이번 주는 다이제스트 생성 연구가 눈에 띕니다.", + "highlights": [ + Highlight( + doi=doi, + title="Test-Driven Digest Generation", + problem="주간 다이제스트 생성이 느립니다.", + approach="저자들은 테스트 주도 생성 기법을 제안합니다.", + findings="저자들은 읽기 속도가 42% 빨라졌다고 보고합니다.", + implication="다이제스트 파이프라인에 적용해볼 만합니다.", + ) + ], + "by_topic": [ + TopicGroup( + topic="도구·데이터셋", + papers=[TopicPaper(doi=doi, title="Test-Driven Digest Generation", note="다이제스트 생성 도구 연구입니다.")], + ) + ], + "worth_reading": [Pick(doi=doi, title="Test-Driven Digest Generation", reason="방법론이 참신합니다.")], + "data_gaps": [], + } + values.update(overrides) + return Digest(**values) + + +class FakeClock: + """Deterministic stand-in for sebrief.week.WeeklyClock.""" + + def __init__( + self, + *, + catchup: str | None = None, + next_run: tuple[str, datetime] | None = None, + ) -> None: + self.catchup = catchup + self._next_run = next_run + self.calls: list[tuple[str, object]] = [] + + def next_run(self, now: datetime) -> tuple[str, datetime]: + self.calls.append(("next_run", now)) + if self._next_run is not None: + return self._next_run + # default: the Monday 07:00 after `now`, covering the week before it + d = now.astimezone(KST).date() + while d.weekday() != 0: + d += timedelta(days=1) + run_at = datetime(d.year, d.month, d.day, 7, 0, tzinfo=KST) + if run_at <= now: + run_at += timedelta(days=7) + from sebrief.week import week_id + + return week_id(run_at.date() - timedelta(days=7)), run_at + + def latest_completed_week(self, now: datetime) -> str: + from sebrief.week import week_id + + return week_id(now.astimezone(KST).date() - timedelta(days=7)) + + def catchup_candidate(self, now: datetime, window_hours: int) -> str | None: + self.calls.append(("catchup_candidate", (now, window_hours))) + return self.catchup + + +class FakeStore: + """In-memory stand-in for sebrief.store.Store.""" + + def __init__(self, *, pending: list[Paper] | None = None) -> None: + self.upserted: list[list[Paper]] = [] + self.saved_summaries: list[dict] = [] + self.pending = pending if pending is not None else [] + self.digests: dict[str, object] = {} + self.digested: list[tuple[list[str], str]] = [] + self.sent: list[tuple[str, str, str]] = [] + self.closed = False + + def upsert_papers(self, papers: list[Paper]) -> int: + self.upserted.append(list(papers)) + return len(papers) + + def save_paper_summaries(self, summaries: dict) -> None: + self.saved_summaries.append(dict(summaries)) + + def pending_papers(self, week: str) -> list[Paper]: + return list(self.pending) + + def mark_digested(self, dois: list[str], week: str) -> None: + self.digested.append((list(dois), week)) + + def has_digest(self, week: str) -> bool: + return week in self.digests + + def save_digest(self, result) -> None: + self.digests[result.payload.week] = result + + def mark_sent(self, week: str, channel: str, receipt: str) -> None: + self.sent.append((week, channel, receipt)) + + def close(self) -> None: + self.closed = True + + +class FakeDeliverer: + def __init__(self, name: str = "fake", *, fail: bool = False) -> None: + self.name = name + self.fail = fail + self.delivered: list = [] + self.errors: list[tuple[str, str]] = [] + + def deliver(self, result) -> str: + if self.fail: + raise RuntimeError(f"{self.name} is down") + self.delivered.append(result) + return f"{self.name}-receipt-{len(self.delivered)}" + + def notify_error(self, subject: str, detail: str) -> None: + self.errors.append((subject, detail)) + + +class FakeProvider: + """LLMProvider stand-in: returns canned digest / verification objects.""" + + def __init__( + self, + digest: Digest | None = None, + verifications: list | None = None, + revised: Digest | None = None, + fail_summarize: bool = False, + fail_revise: bool = False, + fail_paper_summaries: bool = False, + ) -> None: + self.digest = digest or make_digest() + self.verifications = list(verifications or []) + self.revised = revised + self.fail_summarize = fail_summarize + self.fail_revise = fail_revise + self.fail_paper_summaries = fail_paper_summaries + self.calls: list[str] = [] + + def summarize(self, payload): + self.calls.append("summarize") + if self.fail_summarize: + from sebrief.llm.base import LLMError + + raise LLMError("all models failed") + return self.digest, "fake-model" + + def summarize_papers(self, papers): + self.calls.append(f"summarize_papers:{len(papers)}") + if self.fail_paper_summaries: + from sebrief.llm.base import LLMError + + raise LLMError("paper batch failed") + items = [ + PaperSummaryItem(doi=p.doi, **make_summary(tldr=f"{p.doi} 한 줄 요약입니다.").model_dump()) + for p in papers + ] + return items, "fake-model" + + def verify(self, payload, digest): + self.calls.append("verify") + if not self.verifications: + from sebrief.models import Verification + + return Verification(ok=True, issues=[]), "fake-model" + return self.verifications.pop(0), "fake-model" + + def revise(self, payload, digest, issues): + self.calls.append("revise") + if self.fail_revise: + from sebrief.llm.base import LLMError + + raise LLMError("revise failed") + return self.revised or self.digest, "fake-model" diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..c458a4e --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,138 @@ +"""CLI: argument parsing, exit codes, config masking, path resolution.""" + +from __future__ import annotations + +import pytest +from _fakes import make_payload + +from sebrief import cli +from sebrief.config import Settings +from sebrief.models import DigestResult +from sebrief.pipeline import DELIVERY_FAILED_PREFIX, SKIP_ALREADY_SENT + + +def make_settings(tmp_path) -> Settings: + return Settings( + _env_file=None, + SEBRIEF_DATA_DIR=str(tmp_path / "data"), + SEBRIEF_REPORTS_DIR=str(tmp_path / "reports"), + SEBRIEF_LOG_DIR=str(tmp_path / "logs"), + ) + + +def make_result(error: str | None = None, degraded: bool = False) -> DigestResult: + return DigestResult(payload=make_payload(), digest=None, markdown="# md", degraded=degraded, error=error) + + +@pytest.fixture +def run(monkeypatch, tmp_path): + """Invoke the CLI with settings pinned to tmp_path and run_digest captured.""" + calls: dict = {} + + def fake_run_digest(week, settings, *, dry_run=False, force=False, no_llm=False): + calls.update(week=week, dry_run=dry_run, force=force, no_llm=no_llm) + return calls.get("result") or make_result() + + monkeypatch.setattr(cli, "_load_settings", lambda: make_settings(tmp_path)) + monkeypatch.setattr(cli, "run_digest", fake_run_digest) + + def invoke(*argv: str, result: DigestResult | None = None) -> int: + if result is not None: + calls["result"] = result + return cli.main(list(argv)) + + invoke.calls = calls # type: ignore[attr-defined] + return invoke + + +# ---------------------------------------------------------------------------------- parsing +def test_parser_requires_command(): + with pytest.raises(SystemExit): + cli.build_parser().parse_args([]) + + +def test_parser_rejects_bad_week(): + with pytest.raises(SystemExit): + cli.build_parser().parse_args(["run-once", "--week", "2026-09-01"]) + + +def test_run_once_flags_forwarded(run): + assert run("run-once", "--week", "2026-W30", "--dry-run", "--force", "--no-llm") == cli.EXIT_OK + assert run.calls["week"] == "2026-W30" + assert run.calls["dry_run"] and run.calls["force"] and run.calls["no_llm"] + + +# ------------------------------------------------------------------------------- exit codes +def test_run_once_ok(run): + assert run("run-once") == cli.EXIT_OK + + +def test_run_once_skip_exits_2(run, capsys): + assert run("run-once", result=make_result(error=SKIP_ALREADY_SENT)) == cli.EXIT_SKIPPED + assert "skipped" in capsys.readouterr().err + + +def test_run_once_delivery_failure_exits_1(run, capsys): + result = make_result(error=f"{DELIVERY_FAILED_PREFIX}: file: boom") + assert run("run-once", result=result) == cli.EXIT_ERROR + assert "delivery failed" in capsys.readouterr().err + + +def test_run_once_degraded_still_ok(run): + assert run("run-once", result=make_result(degraded=True, error="llm down")) == cli.EXIT_OK + + +def test_unexpected_exception_exits_1(run, monkeypatch, capsys): + def boom(*a, **k): + raise RuntimeError("kaboom") + + monkeypatch.setattr(cli, "run_digest", boom) + assert run("run-once") == cli.EXIT_ERROR + assert "kaboom" in capsys.readouterr().err + + +# --------------------------------------------------------------------------------- commands +def test_next_run_prints_week_and_time(run, capsys): + assert run("next-run") == cli.EXIT_OK + out = capsys.readouterr().out + assert "week" in out and "digest at KST" in out + + +def test_show_config_masks_secrets(monkeypatch, tmp_path, capsys): + settings = make_settings(tmp_path).model_copy(update={"minimax_api_key": "sk-super-secret-key"}) + monkeypatch.setattr(cli, "_load_settings", lambda: settings) + assert cli.main(["show-config"]) == cli.EXIT_OK + out = capsys.readouterr().out + assert "sk-super-secret-key" not in out + assert "minimax_api_key = sk-s***" in out + assert "db_path" in out + + +def test_test_notify_writes_errors_log(monkeypatch, tmp_path, capsys): + monkeypatch.setattr(cli, "_load_settings", lambda: make_settings(tmp_path)) + assert cli.main(["test-notify", "--message", "hello"]) == cli.EXIT_OK + assert "sent test notification via file" in capsys.readouterr().out + log_text = (tmp_path / "reports" / "errors.log").read_text(encoding="utf-8") + assert "hello" in log_text + + +# ------------------------------------------------------------------------------------ paths +def test_resolve_paths_makes_relative_paths_absolute(tmp_path): + settings = Settings(_env_file=None) # default relative data/reports/logs + resolved = cli._resolve_paths(settings, root=tmp_path) + assert resolved.data_dir == tmp_path / "data" + assert resolved.reports_dir == tmp_path / "reports" + assert resolved.log_dir == tmp_path / "logs" + + +def test_resolve_paths_keeps_absolute(tmp_path): + settings = Settings(_env_file=None, SEBRIEF_DATA_DIR=str(tmp_path / "abs")) + resolved = cli._resolve_paths(settings, root=tmp_path / "elsewhere") + assert resolved.data_dir == tmp_path / "abs" + + +def test_mask_secret(): + assert cli.mask_secret(None) == "(unset)" + assert cli.mask_secret("") == "(unset)" + assert cli.mask_secret("short") == "***" + assert cli.mask_secret("sk-1234567890") == "sk-1***" diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..89f8e90 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,52 @@ +"""Settings loading: CSV splitting, validation, derived properties.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from sebrief.config import Settings + + +def make_settings(**overrides) -> Settings: + return Settings(_env_file=None, **overrides) + + +def test_defaults(): + s = make_settings() + assert s.minimax_model == "MiniMax-M3" + assert s.minimax_fallback_models == ["MiniMax-M2.7", "MiniMax-M2.5"] + assert s.run_weekday == 0 + assert s.run_time_kst == "07:00" + assert s.catchup_window_hours == 72 + assert s.max_papers == 40 + assert s.backfill_days == 7 + assert s.verify_enabled is True + assert s.rss_feeds == [] + assert s.db_path == Path("data") / "sebrief.sqlite3" + + +def test_csv_fields_split(): + s = make_settings( + MINIMAX_FALLBACK_MODELS=" a , b ,", + SEBRIEF_RSS_FEEDS="e-Informatica|https://example.org/rss, X|https://x.example/feed", + SEBRIEF_EXTRA_BLACKLIST_ISSNS="1234-5678, 2345-6789", + ) + assert s.minimax_fallback_models == ["a", "b"] + assert s.rss_feeds == ["e-Informatica|https://example.org/rss", "X|https://x.example/feed"] + assert s.extra_blacklist_issns == ["1234-5678", "2345-6789"] + + +def test_run_weekday_validated(): + with pytest.raises(ValidationError): + make_settings(SEBRIEF_RUN_WEEKDAY="7") + assert make_settings(SEBRIEF_RUN_WEEKDAY="6").run_weekday == 6 + + +def test_env_aliases(tmp_path): + s = make_settings(SEBRIEF_DATA_DIR=str(tmp_path / "d"), SEBRIEF_MAX_PAPERS="10") + assert s.data_dir == tmp_path / "d" + assert s.max_papers == 10 + assert s.db_path == tmp_path / "d" / "sebrief.sqlite3" diff --git a/tests/test_daemon.py b/tests/test_daemon.py new file mode 100644 index 0000000..0656bef --- /dev/null +++ b/tests/test_daemon.py @@ -0,0 +1,354 @@ +"""Daemon tests with a fake scheduler (captures add_job) and a fake weekly clock.""" + +from __future__ import annotations + +import signal +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +import pytest +from _fakes import NOW, WEEK, FakeClock, FakeDeliverer, FakeStore +from apscheduler.events import EVENT_JOB_MISSED, JobExecutionEvent +from apscheduler.schedulers import SchedulerNotRunningError +from apscheduler.triggers.date import DateTrigger +from apscheduler.triggers.interval import IntervalTrigger + +from sebrief import daemon as daemon_mod +from sebrief.config import Settings +from sebrief.daemon import DIGEST_JOB_ID, HEARTBEAT_JOB_ID, RECOVERY_JOB_ID, Daemon + +KST = ZoneInfo("Asia/Seoul") +NEXT_RUN = ("2026-W37", datetime(2026, 9, 14, 7, 0, tzinfo=KST)) + + +class FakeScheduler: + def __init__(self): + self.jobs: dict[str, dict] = {} + self.add_calls: list[dict] = [] + self.shutdown_calls: list[bool] = [] + self.listeners: list[tuple[object, int | None]] = [] + + def add_listener(self, callback, mask=None): + self.listeners.append((callback, mask)) + + def add_job(self, func, trigger=None, **kwargs): + entry = {"func": func, "trigger": trigger, **kwargs} + self.add_calls.append(entry) + self.jobs[kwargs.get("id", func.__name__)] = entry + return entry + + def shutdown(self, wait=True): + self.shutdown_calls.append(wait) + + +class RunRecorder: + def __init__(self, fail: bool = False): + self.weeks: list[str] = [] + self.fail = fail + + def __call__(self, week: str): + self.weeks.append(week) + if self.fail: + raise RuntimeError("pipeline exploded") + + +def make_settings(tmp_path, **overrides) -> Settings: + values = {"SEBRIEF_DATA_DIR": str(tmp_path / "data"), "SEBRIEF_LOG_DIR": str(tmp_path / "logs")} + values.update(overrides) + return Settings(_env_file=None, **values) + + +def make_daemon(tmp_path, *, clock=None, store=None, run_fn=None, settings=None, deliverer=None): + scheduler = FakeScheduler() + clock = clock or FakeClock(next_run=NEXT_RUN) + store = store or FakeStore() + run_fn = run_fn or RunRecorder() + deliverer = deliverer or FakeDeliverer("file") + d = Daemon( + settings or make_settings(tmp_path), + scheduler=scheduler, + clock=clock, + store=store, + run_fn=run_fn, + now_fn=lambda: NOW, + deliverers_fn=lambda s: [deliverer], + ) + return d, scheduler, clock, store, run_fn, deliverer + + +# -------------------------------------------------------------------------------- scheduling +def test_schedule_next_schedules_one_shot_job(tmp_path): + d, scheduler, *_ = make_daemon(tmp_path) + assert d.schedule_next() == NEXT_RUN + job = scheduler.jobs[DIGEST_JOB_ID] + assert isinstance(job["trigger"], DateTrigger) + assert job["trigger"].run_date == NEXT_RUN[1] + assert job["args"] == [NEXT_RUN[0], NEXT_RUN[1]] + assert job["replace_existing"] is True + # None: a late (sleep/wake) job must still FIRE so it can reschedule itself + assert job["misfire_grace_time"] is None + + +def test_schedule_next_survives_clock_error(tmp_path): + class BrokenClock(FakeClock): + def next_run(self, now): + raise RuntimeError("clock broken") + + d, scheduler, *_ = make_daemon(tmp_path, clock=BrokenClock()) + assert d.schedule_next() is None + assert DIGEST_JOB_ID not in scheduler.jobs + d.clock = FakeClock(next_run=NEXT_RUN) + d._heartbeat_job() # heartbeat retries scheduling + assert DIGEST_JOB_ID in scheduler.jobs + + +def test_start_schedules_digest_and_heartbeat(tmp_path): + d, scheduler, *_ = make_daemon(tmp_path) + d.start() + assert set(scheduler.jobs) == {DIGEST_JOB_ID, HEARTBEAT_JOB_ID} + hb = scheduler.jobs[HEARTBEAT_JOB_ID] + assert isinstance(hb["trigger"], IntervalTrigger) + assert hb["trigger"].interval.total_seconds() == 3600 + assert (d._on_job_missed, EVENT_JOB_MISSED) in scheduler.listeners + + +# ---------------------------------------------------------------------------------- catch-up +def test_catchup_runs_when_candidate_and_not_yet_digested(tmp_path): + clock = FakeClock(catchup=WEEK, next_run=NEXT_RUN) + d, _scheduler, clock, _store, run_fn, _ = make_daemon(tmp_path, clock=clock) + assert d.run_catchup() is True + assert run_fn.weeks == [WEEK] + assert clock.calls[0] == ("catchup_candidate", (NOW, 72)) + + +def test_catchup_skipped_when_digest_exists(tmp_path): + store = FakeStore() + store.digests[WEEK] = object() + d, _s, _c, _st, run_fn, _ = make_daemon(tmp_path, clock=FakeClock(catchup=WEEK), store=store) + assert d.run_catchup() is False + assert run_fn.weeks == [] + + +def test_catchup_skipped_without_candidate(tmp_path): + d, _s, _c, _st, run_fn, _ = make_daemon(tmp_path) + assert d.run_catchup() is False + assert run_fn.weeks == [] + + +def test_catchup_failure_is_logged_and_start_continues(tmp_path): + d, scheduler, _c, _st, run_fn, deliverer = make_daemon( + tmp_path, clock=FakeClock(catchup=WEEK, next_run=NEXT_RUN), run_fn=RunRecorder(fail=True) + ) + d.start() # must not raise + assert run_fn.weeks == [WEEK] + assert DIGEST_JOB_ID in scheduler.jobs + assert deliverer.errors and "catch-up" in deliverer.errors[0][0] + + +def test_catchup_window_comes_from_settings(tmp_path): + settings = make_settings(tmp_path, SEBRIEF_CATCHUP_WINDOW_HOURS="24") + d, _s, clock, *_ = make_daemon(tmp_path, settings=settings) + d.run_catchup() + assert clock.calls[0] == ("catchup_candidate", (NOW, 24)) + + +# -------------------------------------------------------------------------------------- job +def test_job_runs_pipeline_and_reschedules(tmp_path): + d, scheduler, clock, _st, run_fn, deliverer = make_daemon(tmp_path) + d.schedule_next() + job = scheduler.jobs[DIGEST_JOB_ID] + clock._next_run = ("2026-W38", datetime(2026, 9, 21, 7, 0, tzinfo=KST)) + + job["func"](*job["args"]) + + assert run_fn.weeks == [NEXT_RUN[0]] + assert scheduler.jobs[DIGEST_JOB_ID]["args"] == ["2026-W38", clock._next_run[1]] + assert d.next_run == clock._next_run + assert deliverer.errors == [] + + +def test_job_exception_notifies_and_still_reschedules(tmp_path): + d, scheduler, _clock, _st, run_fn, deliverer = make_daemon(tmp_path, run_fn=RunRecorder(fail=True)) + d.schedule_next() + job = scheduler.jobs[DIGEST_JOB_ID] + job["func"](*job["args"]) # must not raise + assert run_fn.weeks == [NEXT_RUN[0]] + assert scheduler.jobs[DIGEST_JOB_ID]["trigger"].run_date == NEXT_RUN[1] + subject, detail = deliverer.errors[0] + assert "다이제스트 실패" in subject and "pipeline exploded" in detail + + +@pytest.mark.parametrize(("lateness_h", "expect_run"), [(0, True), (48, True), (72, False), (100, False)]) +def test_late_job_honours_catchup_window(tmp_path, lateness_h, expect_run): + d, scheduler, _c, _st, run_fn, deliverer = make_daemon(tmp_path) + run_at = NOW - timedelta(hours=lateness_h) + + d._digest_job(WEEK, run_at) + + assert (run_fn.weeks == [WEEK]) is expect_run + assert scheduler.jobs[DIGEST_JOB_ID]["trigger"].run_date == NEXT_RUN[1] # always rescheduled + assert deliverer.errors == [] + + +# --------------------------------------------------------------------------- heartbeat / missed +def test_heartbeat_recovers_dropped_job_inside_window(tmp_path): + now = {"t": NOW} + d, scheduler, clock, _st, run_fn, _ = make_daemon(tmp_path) + d.now_fn = lambda: now["t"] + d.schedule_next() + del scheduler.jobs[DIGEST_JOB_ID] # what APScheduler does when it drops a misfired job + now["t"] = NEXT_RUN[1] + timedelta(hours=2) + clock.catchup = NEXT_RUN[0] + + d._heartbeat_job() + + assert run_fn.weeks == [NEXT_RUN[0]] + assert DIGEST_JOB_ID in scheduler.jobs + + +def test_heartbeat_leaves_schedule_alone_when_run_is_imminent(tmp_path): + now = {"t": NOW} + d, scheduler, _c, _st, run_fn, _ = make_daemon(tmp_path) + d.now_fn = lambda: now["t"] + d.schedule_next() + for delta in (timedelta(minutes=-30), timedelta(0), timedelta(minutes=2)): + now["t"] = NEXT_RUN[1] + delta + d._heartbeat_job() + assert len(scheduler.add_calls) == 1 and run_fn.weeks == [] + + +def test_heartbeat_does_not_run_while_digest_in_progress(tmp_path): + now = {"t": NOW} + d, scheduler, clock, _st, run_fn, _ = make_daemon(tmp_path) + d.now_fn = lambda: now["t"] + d.schedule_next() + now["t"] = NEXT_RUN[1] + timedelta(hours=1) + clock.catchup = NEXT_RUN[0] + with d._run_lock: # a digest is running on another executor thread + d._heartbeat_job() + assert run_fn.weeks == [] # no concurrent second run + assert DIGEST_JOB_ID in scheduler.jobs # but re-armed + + +def test_on_job_missed_queues_recovery_under_a_different_id(tmp_path): + d, scheduler, clock, _st, run_fn, _ = make_daemon(tmp_path) + d.start() + del scheduler.jobs[DIGEST_JOB_ID] + clock.catchup = NEXT_RUN[0] + + d._on_job_missed(JobExecutionEvent(EVENT_JOB_MISSED, DIGEST_JOB_ID, "default", NEXT_RUN[1])) + + assert scheduler.add_calls[-1]["id"] == RECOVERY_JOB_ID + recovery = scheduler.jobs[RECOVERY_JOB_ID] + assert isinstance(recovery["trigger"], DateTrigger) and recovery["misfire_grace_time"] is None + recovery["func"]() + assert run_fn.weeks == [NEXT_RUN[0]] + assert DIGEST_JOB_ID in scheduler.jobs + + n_calls = len(scheduler.add_calls) + d._on_job_missed(JobExecutionEvent(EVENT_JOB_MISSED, HEARTBEAT_JOB_ID, "default", NEXT_RUN[1])) + assert len(scheduler.add_calls) == n_calls # other jobs are ignored + + +# --------------------------------------------------------------------------------- shutdown +def test_signal_handler_shuts_scheduler_down(tmp_path): + d, scheduler, *_ = make_daemon(tmp_path) + old_term, old_int = signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT) + try: + d.install_signal_handlers() + handler = signal.getsignal(signal.SIGTERM) + assert callable(handler) and handler is not old_term + handler(signal.SIGTERM, None) + finally: + signal.signal(signal.SIGTERM, old_term) + signal.signal(signal.SIGINT, old_int) + assert scheduler.shutdown_calls == [False] + + +def test_signal_during_catchup_unwinds_when_scheduler_not_running(tmp_path): + class NotStarted(FakeScheduler): + def shutdown(self, wait=True): + super().shutdown(wait) + raise SchedulerNotRunningError + + d, *_ = make_daemon(tmp_path) + d.scheduler = NotStarted() + old_term, old_int = signal.getsignal(signal.SIGTERM), signal.getsignal(signal.SIGINT) + try: + d.install_signal_handlers() + handler = signal.getsignal(signal.SIGTERM) + with pytest.raises(KeyboardInterrupt): + handler(signal.SIGTERM, None) + finally: + signal.signal(signal.SIGTERM, old_term) + signal.signal(signal.SIGINT, old_int) + assert d.stop_requested is True + + +def test_shutdown_during_catchup_prevents_scheduling(tmp_path): + class NotStarted(FakeScheduler): + def shutdown(self, wait=True): + super().shutdown(wait) + raise SchedulerNotRunningError + + holder: dict = {} + + def run_fn(week): + holder["d"].shutdown() + + d, *_ = make_daemon(tmp_path, clock=FakeClock(catchup=WEEK, next_run=NEXT_RUN), run_fn=run_fn) + d.scheduler = NotStarted() + holder["d"] = d + d.start() + assert d.stop_requested is True + assert d.scheduler.jobs == {} # nothing scheduled; run_daemon skips scheduler.start() + + +# ------------------------------------------------------------------------------- run_daemon +def test_run_daemon_wires_real_parts_and_blocks_on_scheduler(monkeypatch, tmp_path): + started = {} + fake_store = FakeStore() + + class FakeBlocking(FakeScheduler): + def __init__(self, timezone=None): + super().__init__() + started["timezone"] = timezone + + def start(self, paused=False): + started["started"] = True + raise KeyboardInterrupt # simulate Ctrl-C ending the blocking loop + + monkeypatch.setattr(daemon_mod, "BlockingScheduler", FakeBlocking) + monkeypatch.setattr(daemon_mod, "WeeklyClock", lambda wd, t: FakeClock(next_run=NEXT_RUN)) + monkeypatch.setattr(daemon_mod, "Store", lambda path: fake_store) + monkeypatch.setattr(daemon_mod.signal, "signal", lambda *a, **k: None) + + daemon_mod.run_daemon(make_settings(tmp_path)) + + assert started["started"] is True + assert str(started["timezone"]) == "Asia/Seoul" + assert fake_store.closed is True + + +def test_run_lock_serialises_catchup_and_job(tmp_path): + seen = {} + d, *_ = make_daemon(tmp_path) + + def run_fn(week): + seen["locked"] = d._run_lock.locked() + + d.run_fn = run_fn + d._digest_job(WEEK, NOW) + assert seen["locked"] is True + + +def test_notify_error_failures_are_swallowed(tmp_path): + class Exploding(FakeDeliverer): + def notify_error(self, subject, detail): + raise RuntimeError("channel down") + + d, *_ = make_daemon(tmp_path, run_fn=RunRecorder(fail=True), deliverer=Exploding("x")) + d._digest_job(WEEK) # must not raise + + d.deliverers_fn = lambda s: (_ for _ in ()).throw(RuntimeError("cannot build")) + d._digest_job(WEEK) # must not raise either diff --git a/tests/test_deliver.py b/tests/test_deliver.py new file mode 100644 index 0000000..fe1bb23 --- /dev/null +++ b/tests/test_deliver.py @@ -0,0 +1,62 @@ +"""File deliverer: archive writes, receipts, errors.log alerting.""" + +from __future__ import annotations + +import json + +import pytest +from _fakes import make_payload + +from sebrief.config import Settings +from sebrief.deliver import build_deliverers +from sebrief.deliver.base import DeliveryError +from sebrief.deliver.file import FileDeliverer +from sebrief.models import DigestResult + + +def make_result() -> DigestResult: + return DigestResult(payload=make_payload(), digest=None, markdown="# 다이제스트\n", model_used="m") + + +def test_deliver_writes_md_and_json_keyed_by_week(tmp_path): + deliverer = FileDeliverer(tmp_path / "reports") + receipt = deliverer.deliver(make_result()) + + md = tmp_path / "reports" / "2026-W36.md" + js = tmp_path / "reports" / "2026-W36.json" + assert receipt == str(md) + assert md.read_text(encoding="utf-8") == "# 다이제스트\n" + data = json.loads(js.read_text(encoding="utf-8")) + assert data["payload"]["week"] == "2026-W36" + assert data["markdown"].startswith("# 다이제스트") + + +def test_deliver_raises_delivery_error_on_os_error(tmp_path): + blocker = tmp_path / "reports" + blocker.write_text("file, not a dir") + deliverer = FileDeliverer(blocker) + with pytest.raises(DeliveryError): + deliverer.deliver(make_result()) + + +def test_notify_error_appends_flattened_line(tmp_path): + deliverer = FileDeliverer(tmp_path / "reports") + deliverer.notify_error("제목", "줄1\n줄2\n") + deliverer.notify_error("두번째", "detail") + lines = (tmp_path / "reports" / "errors.log").read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + ts, subject, detail = lines[0].split("\t") + assert subject == "제목" and detail == "줄1 | 줄2" and ts + assert lines[1].split("\t")[1] == "두번째" + + +def test_notify_error_never_raises(tmp_path): + blocker = tmp_path / "reports" + blocker.write_text("file, not a dir") + FileDeliverer(blocker).notify_error("s", "d") # must not raise + + +def test_build_deliverers_is_file_only(tmp_path): + settings = Settings(_env_file=None, SEBRIEF_REPORTS_DIR=str(tmp_path / "reports")) + deliverers = build_deliverers(settings) + assert [d.name for d in deliverers] == ["file"] diff --git a/tests/test_llm.py b/tests/test_llm.py new file mode 100644 index 0000000..6bf3d91 --- /dev/null +++ b/tests/test_llm.py @@ -0,0 +1,201 @@ +"""LLM layer: JSON extraction/coercion, prompt building, provider fallback and repair retry.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest +from _fakes import make_digest, make_payload + +from sebrief.config import Settings +from sebrief.llm.base import LLMError +from sebrief.llm.minimax import ( + MiniMaxProvider, + coerce_model_data, + digest_json_schema, + extract_json_text, + parse_digest, +) +from sebrief.llm.prompts import build_user_message, build_verify_message, format_issues +from sebrief.models import Digest, VerificationIssue + +VALID_DIGEST_JSON = make_digest().model_dump_json() + + +# ------------------------------------------------------------------------ extraction/parsing +def test_extract_json_text_plain_and_fenced(): + assert extract_json_text('{"a": 1}') == '{"a": 1}' + assert extract_json_text('```json\n{"a": 1}\n```') == '{"a": 1}' + + +def test_extract_json_text_strips_think_blocks(): + raw = f"reasoning {{nested}} {VALID_DIGEST_JSON}" + assert json.loads(extract_json_text(raw)) == json.loads(VALID_DIGEST_JSON) + + +def test_extract_json_text_unterminated_think_falls_back_to_first_object(): + raw = f"never closed... {VALID_DIGEST_JSON}" + assert json.loads(extract_json_text(raw))["overview"].startswith("이번 주") + + +def test_extract_json_text_raises_without_object(): + with pytest.raises(ValueError): + extract_json_text("no json here") + + +def test_parse_digest_valid(): + digest = parse_digest(VALID_DIGEST_JSON) + assert isinstance(digest, Digest) + assert digest.highlights[0].doi == "10.1145/1234567" + + +def test_coerce_string_list_fields(): + data = json.loads(VALID_DIGEST_JSON) + data["data_gaps"] = "누락 1. 누락 2." + coerced, fields = coerce_model_data(data, Digest) + assert fields == ["data_gaps"] + assert coerced["data_gaps"] == ["누락 1.", "누락 2."] + # nested object lists are left for pydantic to validate + digest = Digest.model_validate(coerced) + assert len(digest.data_gaps) == 2 + + +def test_digest_json_schema_is_closed_and_untitled(): + schema = digest_json_schema() + assert schema["additionalProperties"] is False + assert "title" not in schema + assert "overview" in schema["properties"] + + +# ------------------------------------------------------------------------------- prompts +def test_build_user_message_contains_papers_and_gaps(): + payload = make_payload(gaps=["rss: down"]) + msg = build_user_message(payload) + assert "## 대상 주차" in msg and "2026-W36" in msg + assert "DOI: 10.1145/1234567" in msg + assert "Test-Driven Digest Generation" in msg + assert "- rss: down" in msg + + +def test_build_user_message_truncates_long_abstracts(): + payload = make_payload(papers=[make_payload().papers[0].model_copy(update={"abstract": "x" * 5000})]) + msg = build_user_message(payload) + assert "x" * 1800 not in msg and "…" in msg + + +def test_build_verify_message_appends_draft(): + payload = make_payload() + msg = build_verify_message(payload, make_digest()) + assert msg.startswith(build_user_message(payload)) + assert "## 검증 대상 다이제스트 초안 (JSON)" in msg + + +def test_format_issues(): + issues = [VerificationIssue(claim="c", problem="p", severity="major")] + text = format_issues(issues) + assert "[major]" in text and "서술: c" in text and "문제: p" in text + assert format_issues([]) == "(없음)" + + +# ------------------------------------------------------------------------------- provider +class FakeClient: + """chat.completions.create stub answering from a queue of contents (or exceptions).""" + + def __init__(self, replies: list): + self.replies = list(replies) + self.calls: list[dict] = [] + self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create)) + + def _create(self, **kwargs): + self.calls.append(kwargs) + reply = self.replies.pop(0) + if isinstance(reply, Exception): + raise reply + finish = "length" if reply == "" else "stop" + message = SimpleNamespace(content=None if reply == "" else reply) + choice = SimpleNamespace(message=message, finish_reason=finish) + return SimpleNamespace(choices=[choice], model=kwargs["model"], usage=None) + + +def make_provider(replies: list, models: str = "m1", fallbacks: str = "m2") -> tuple[MiniMaxProvider, FakeClient]: + settings = Settings( + _env_file=None, + MINIMAX_API_KEY="sk-test-key", + MINIMAX_MODEL=models, + MINIMAX_FALLBACK_MODELS=fallbacks, + ) + client = FakeClient(replies) + return MiniMaxProvider(settings, client=client), client + + +def test_summarize_happy_path(): + provider, client = make_provider([VALID_DIGEST_JSON]) + digest, model = provider.summarize(make_payload()) + assert model == "m1" and digest.overview + assert client.calls[0]["response_format"]["json_schema"]["name"] == "digest" + + +def test_invalid_reply_triggers_repair_then_fallback_model(): + # m1: garbage twice (initial + repair) -> m2 succeeds + provider, client = make_provider(["not json", "{\"still\": \"wrong\"}", VALID_DIGEST_JSON]) + _digest, model = provider.summarize(make_payload()) + assert model == "m2" + assert len(client.calls) == 3 + repair_call = client.calls[1] + assert any("JSON 스키마 검증에 실패" in m["content"] for m in repair_call["messages"] if m["role"] == "user") + + +def test_truncated_and_empty_replies_move_to_next_model(): + provider, _ = make_provider(["", VALID_DIGEST_JSON]) + _, model = provider.summarize(make_payload()) + assert model == "m2" + + provider, _ = make_provider(["", VALID_DIGEST_JSON]) + _, model = provider.summarize(make_payload()) + assert model == "m2" + + +def test_all_models_failed_raises_llm_error(): + provider, _ = make_provider(["nope", "nope", "nope", "nope"]) # 2 models x (initial + repair) + with pytest.raises(LLMError, match="all MiniMax models failed"): + provider.summarize(make_payload()) + + +def test_summarize_papers_uses_batch_schema_and_prompt(): + reply = ( + '{"summaries": [{"doi": "10.1145/1234567", "tldr": "한 줄.", "problem": "p", ' + '"approach": "a", "findings": "f", "implication": "i"}]}' + ) + provider, client = make_provider([reply]) + items, model = provider.summarize_papers(make_payload().papers) + assert model == "m1" and len(items) == 1 and items[0].doi == "10.1145/1234567" + call = client.calls[0] + assert call["response_format"]["json_schema"]["name"] == "paper_summaries" + assert call["messages"][0]["content"].startswith("당신은 소프트웨어공학(SE) 논문 요약자") + assert "DOI: 10.1145/1234567" in call["messages"][1]["content"] + + +def test_verify_and_revise_use_expected_message_shapes(): + verification_json = '{"ok": false, "issues": [{"claim": "c", "problem": "p", "severity": "major"}]}' + provider, client = make_provider([verification_json]) + verification, _ = provider.verify(make_payload(), make_digest()) + assert verification.has_major + assert client.calls[0]["messages"][0]["content"].startswith("당신은 소프트웨어공학 주간 다이제스트의 팩트체커") + + provider, client = make_provider([VALID_DIGEST_JSON]) + issues = [VerificationIssue(claim="c", problem="p", severity="major")] + provider.revise(make_payload(), make_digest(), issues) + roles = [m["role"] for m in client.calls[0]["messages"]] + assert roles == ["system", "user", "assistant", "user"] + assert "검증 이슈" in client.calls[0]["messages"][-1]["content"] + + +def test_provider_requires_api_key_and_models(): + with pytest.raises(LLMError, match="MINIMAX_API_KEY"): + MiniMaxProvider(Settings(_env_file=None, MINIMAX_API_KEY="")) + with pytest.raises(LLMError, match="no MiniMax model"): + MiniMaxProvider( + Settings(_env_file=None, MINIMAX_API_KEY="sk-x", MINIMAX_MODEL="", MINIMAX_FALLBACK_MODELS=""), + client=object(), + ) diff --git a/tests/test_logging_setup.py b/tests/test_logging_setup.py new file mode 100644 index 0000000..e2e5c34 --- /dev/null +++ b/tests/test_logging_setup.py @@ -0,0 +1,72 @@ +"""Secret masking and logging handler setup.""" + +from __future__ import annotations + +import logging + +from sebrief.config import Settings +from sebrief.logging_setup import ( + LOG_FILE_NAME, + mask_secrets, + register_secret_values, + setup_logging, +) + + +def teardown_function(): + register_secret_values() # clear the registry between tests + + +def test_mask_sk_keys_and_url_params(): + assert mask_secrets("key sk-abcdef123456 leaked") == "key *** leaked" + assert mask_secrets("GET https://x/api?api_key=deadbeef&x=1") == "GET https://x/api?api_key=***&x=1" + assert mask_secrets("token=verysecretvalue123") == "token=verysecretvalue123" # no ?/& prefix + + +def test_registered_values_masked_verbatim(): + register_secret_values("my-odd-shaped-secret") + assert mask_secrets("value my-odd-shaped-secret here") == "value *** here" + register_secret_values() # cleared + assert "my-odd-shaped-secret" in mask_secrets("value my-odd-shaped-secret here") + + +def test_short_values_not_registered(): + register_secret_values("short") + assert mask_secrets("short") == "short" + + +def test_setup_logging_masks_in_file_and_is_idempotent(tmp_path): + settings = Settings( + _env_file=None, + MINIMAX_API_KEY="sk-verysecretkey12345", + SEBRIEF_LOG_DIR=str(tmp_path / "logs"), + ) + setup_logging(settings) + setup_logging(settings) # handlers replaced, not duplicated + root = logging.getLogger() + ours = [h for h in root.handlers if getattr(h, "_sebrief_handler", False)] + assert len(ours) == 2 # console + file + + logging.getLogger("sebrief.test").info("key=%s", "sk-verysecretkey12345") + for handler in ours: + handler.flush() + text = (tmp_path / "logs" / LOG_FILE_NAME).read_text(encoding="utf-8") + assert "sk-verysecretkey12345" not in text + assert "***" in text + + for handler in ours: # clean up for other tests + root.removeHandler(handler) + handler.close() + + +def test_setup_logging_survives_unwritable_log_dir(tmp_path): + blocker = tmp_path / "logs" + blocker.write_text("file, not a dir") + settings = Settings(_env_file=None, SEBRIEF_LOG_DIR=str(blocker)) + setup_logging(settings) # must not raise + root = logging.getLogger() + ours = [h for h in root.handlers if getattr(h, "_sebrief_handler", False)] + assert len(ours) == 1 # console only + for handler in ours: + root.removeHandler(handler) + handler.close() diff --git a/tests/test_openalex.py b/tests/test_openalex.py new file mode 100644 index 0000000..046dc85 --- /dev/null +++ b/tests/test_openalex.py @@ -0,0 +1,205 @@ +"""OpenAlex collector: abstract reconstruction, work mapping, filters, pagination, failure modes.""" + +from __future__ import annotations + +from datetime import date + +import pytest + +from sebrief.collectors import openalex +from sebrief.collectors.openalex import ( + fetch_openalex_papers, + normalize_doi, + paper_from_work, + reconstruct_abstract, +) +from sebrief.config import Settings + +SETTINGS = Settings(_env_file=None) +START, END = date(2026, 8, 24), date(2026, 9, 6) + + +# ------------------------------------------------------------------------------- pure parts +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("https://doi.org/10.1145/1234567", "10.1145/1234567"), + ("http://dx.doi.org/10.1109/TSE.2026.1", "10.1109/tse.2026.1"), + ("doi:10.1000/ABC", "10.1000/abc"), + ("10.1000/abc", "10.1000/abc"), + ("https://example.org/paper", None), + ("", None), + (None, None), + ], +) +def test_normalize_doi(raw, expected): + assert normalize_doi(raw) == expected + + +def test_reconstruct_abstract(): + inv = {"study": [2], "We": [0], "digests.": [3], "weekly": [1]} + assert reconstruct_abstract(inv) == "We weekly study digests." + assert reconstruct_abstract(None) == "" + assert reconstruct_abstract({}) == "" + + +def make_work(**overrides) -> dict: + work = { + "doi": "https://doi.org/10.1145/1234567", + "title": "Test-Driven Digest Generation", + "abstract_inverted_index": {"We": [0], "test.": [1]}, + "publication_date": "2026-09-02", + "authorships": [ + {"author": {"display_name": "Alice Kim"}}, + {"author": {"display_name": "Bob Lee"}}, + ], + "primary_location": { + "source": { + "display_name": "ACM Transactions on Software Engineering and Methodology", + "issn_l": "1049-331X", + "issn": ["1049-331X", "1557-7392"], + "host_organization_name": "Association for Computing Machinery", + } + }, + "best_oa_location": { + "landing_page_url": "https://dl.acm.org/doi/10.1145/1234567", + "pdf_url": "https://dl.acm.org/doi/pdf/10.1145/1234567", + }, + "open_access": {"oa_url": "https://dl.acm.org/doi/10.1145/1234567"}, + "topics": [{"display_name": "Software Engineering Research"}], + } + work.update(overrides) + return work + + +def test_paper_from_work_maps_all_fields(): + paper = paper_from_work(make_work()) + assert paper is not None + assert paper.doi == "10.1145/1234567" + assert paper.title == "Test-Driven Digest Generation" + assert paper.abstract == "We test." + assert paper.authors == ["Alice Kim", "Bob Lee"] + assert paper.venue.startswith("ACM Transactions") + assert paper.published_date == date(2026, 9, 2) + assert paper.oa_url == "https://dl.acm.org/doi/10.1145/1234567" + assert paper.pdf_url == "https://dl.acm.org/doi/pdf/10.1145/1234567" + assert paper.topics == ["Software Engineering Research"] + + +def test_paper_from_work_rejects_missing_doi_or_title(): + assert paper_from_work(make_work(doi=None)) is None + assert paper_from_work(make_work(doi="https://example.org/x")) is None + assert paper_from_work(make_work(title="", display_name="")) is None + + +def test_paper_from_work_tolerates_sparse_work(): + paper = paper_from_work({"doi": "https://doi.org/10.1/x", "title": "T"}) + assert paper is not None + assert paper.abstract == "" and paper.authors == [] and paper.published_date is None + + +# -------------------------------------------------------------------------------- fetching +class FakeResponse: + def __init__(self, body: dict, status: int = 200): + self._body = body + self.status_code = status + + def raise_for_status(self) -> None: + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + def json(self) -> dict: + return self._body + + +def install_fetch(monkeypatch, pages_by_query: dict[str, list[dict]]): + """Route requests.get by whether the filter string contains the subfield filter (mega query).""" + calls: list[dict] = [] + + def fake_get(url, params=None, timeout=None, headers=None): + calls.append({"url": url, "params": params}) + key = "mega" if "topics.subfield.id" in params["filter"] else "whitelist" + pages = pages_by_query.get(key, []) + idx = sum(1 for c in calls[:-1] if ("topics.subfield.id" in c["params"]["filter"]) == (key == "mega")) + if isinstance(pages, Exception): + raise pages + body = pages[idx] if idx < len(pages) else {"results": [], "meta": {"next_cursor": None}} + return FakeResponse(body) + + monkeypatch.setattr(openalex.requests, "get", fake_get) + return calls + + +def test_fetch_merges_queries_applies_filters_and_dedups(monkeypatch): + mdpi = make_work( + doi="https://doi.org/10.3390/xyz", + primary_location={"source": {"display_name": "Electronics", "issn_l": "2079-9292", "issn": [], "host_organization_name": "MDPI AG"}}, + ) + off_topic = make_work( + doi="https://doi.org/10.1109/access.1", + title="Optimal Beamforming for 6G Networks", + abstract_inverted_index={"MIMO": [0]}, + topics=[{"display_name": "Wireless Communication"}], + ) + on_topic = make_work( + doi="https://doi.org/10.1109/access.2", + title="LLM-based Code Review at Scale", + topics=[{"display_name": "Software Engineering Research"}], + ) + dup = make_work() # same DOI as the whitelist hit + pages = { + "whitelist": [{"results": [make_work(), mdpi], "meta": {"next_cursor": None}}], + "mega": [{"results": [off_topic, on_topic, dup], "meta": {"next_cursor": None}}], + } + calls = install_fetch(monkeypatch, pages) + + papers = fetch_openalex_papers(START, END, SETTINGS) + + assert [p.doi for p in papers] == ["10.1145/1234567", "10.1109/access.2"] + filters = [c["params"]["filter"] for c in calls] + assert any("primary_location.source.issn:" in f and "is_oa:true" in f for f in filters) + assert any(f"topics.subfield.id:{SETTINGS.se_subfield_id}" in f for f in filters) + assert all(f"from_publication_date:{START}" in f and f"to_publication_date:{END}" in f for f in filters) + + +def test_fetch_follows_cursor_pagination(monkeypatch): + page1 = {"results": [make_work()], "meta": {"next_cursor": "c2"}} + page2 = {"results": [make_work(doi="https://doi.org/10.1/second")], "meta": {"next_cursor": None}} + calls = install_fetch(monkeypatch, {"whitelist": [page1, page2], "mega": []}) + + papers = fetch_openalex_papers(START, END, SETTINGS) + + assert {p.doi for p in papers} == {"10.1145/1234567", "10.1/second"} + whitelist_calls = [c for c in calls if "subfield" not in c["params"]["filter"]] + assert [c["params"]["cursor"] for c in whitelist_calls] == ["*", "c2"] + + +def test_one_failed_query_keeps_the_other(monkeypatch): + def fake_get(url, params=None, timeout=None, headers=None): + if "topics.subfield.id" in params["filter"]: + raise RuntimeError("mega query down") + return FakeResponse({"results": [make_work()], "meta": {"next_cursor": None}}) + + monkeypatch.setattr(openalex.requests, "get", fake_get) + papers = fetch_openalex_papers(START, END, SETTINGS) + assert [p.doi for p in papers] == ["10.1145/1234567"] + + +def test_all_queries_failed_raises(monkeypatch): + def fake_get(*a, **k): + raise RuntimeError("openalex down") + + monkeypatch.setattr(openalex.requests, "get", fake_get) + with pytest.raises(RuntimeError, match="all openalex queries failed"): + fetch_openalex_papers(START, END, SETTINGS) + + +def test_mailto_param_only_when_configured(monkeypatch): + calls = install_fetch(monkeypatch, {"whitelist": [], "mega": []}) + fetch_openalex_papers(START, END, SETTINGS) + assert all("mailto" not in c["params"] for c in calls) + + calls.clear() + settings = Settings(_env_file=None, SEBRIEF_OPENALEX_MAILTO="ops@example.org") + fetch_openalex_papers(START, END, settings) + assert all(c["params"]["mailto"] == "ops@example.org" for c in calls) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..bec0f94 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,212 @@ +"""Pipeline orchestration with fakes: happy path, idempotency, degrade and delivery failures.""" + +from __future__ import annotations + +import pytest +from _fakes import NOW, WEEK, FakeDeliverer, FakeProvider, FakeStore, make_digest, make_paper + +from sebrief import pipeline +from sebrief.config import Settings +from sebrief.models import Verification, VerificationIssue +from sebrief.pipeline import SKIP_ALREADY_SENT, build_payload, delivery_failed, run_digest + + +@pytest.fixture +def env(monkeypatch, tmp_path): + """Patch every pipeline collaborator; return the mutable test context.""" + ctx = { + "store": FakeStore(pending=[make_paper()]), + "provider": FakeProvider(), + "deliverers": [FakeDeliverer("file")], + "openalex": [make_paper()], + "rss": [], + "settings": Settings(_env_file=None, SEBRIEF_DATA_DIR=str(tmp_path / "data")), + } + monkeypatch.setattr(pipeline, "_now", lambda: NOW) + monkeypatch.setattr(pipeline, "Store", lambda path: ctx["store"]) + monkeypatch.setattr(pipeline, "fetch_openalex_papers", lambda start, end, s: ctx["openalex"]) + monkeypatch.setattr(pipeline, "fetch_rss_papers", lambda feeds, start, end, bl: ctx["rss"]) + monkeypatch.setattr(pipeline, "build_provider", lambda s: ctx["provider"]) + monkeypatch.setattr(pipeline, "build_deliverers", lambda s: ctx["deliverers"]) + return ctx + + +# ------------------------------------------------------------------------------ build_payload +def test_build_payload_collects_upserts_and_reads_pending(env): + payload = build_payload(WEEK, env["settings"], env["store"]) + assert payload.week == WEEK + assert env["store"].upserted == [[make_paper()]] + assert [p.doi for p in payload.papers] == ["10.1145/1234567"] + assert payload.data_gaps == [] + # collection window reaches back by backfill_days + assert payload.week_start.isoformat() == "2026-08-31" + + +def test_build_payload_collector_failure_becomes_gap(env, monkeypatch): + def boom(start, end, s): + raise RuntimeError("openalex down") + + monkeypatch.setattr(pipeline, "fetch_openalex_papers", boom) + payload = build_payload(WEEK, env["settings"], env["store"]) + assert payload.data_gaps == ["openalex: RuntimeError: openalex down"] + assert env["store"].upserted == [] # nothing collected -> nothing upserted + assert [p.doi for p in payload.papers] == ["10.1145/1234567"] # pending leftovers still flow in + + +def test_build_payload_merges_rss_dedup_by_doi(env): + env["settings"] = env["settings"].model_copy( + update={"rss_feeds": ["e-Inf|https://example.org/rss"]} + ) + dup = make_paper(source="rss:e-Inf", venue="e-Inf") # same DOI as openalex + fresh = make_paper("10.5555/rss-only", source="rss:e-Inf") + env["rss"] = [dup, fresh] + build_payload(WEEK, env["settings"], env["store"]) + [upserted] = env["store"].upserted + assert [p.doi for p in upserted] == ["10.1145/1234567", "10.5555/rss-only"] + assert upserted[0].source == "openalex" # openalex metadata wins on the duplicate + + +def test_build_payload_invalid_feed_spec_is_a_gap(env): + env["settings"] = env["settings"].model_copy(update={"rss_feeds": ["broken-spec"]}) + payload = build_payload(WEEK, env["settings"], env["store"]) + assert any("invalid feed spec" in g for g in payload.data_gaps) + + +def test_build_payload_caps_papers_and_notes_rollover(env): + env["store"].pending = [make_paper(f"10.1/p{i}") for i in range(5)] + env["settings"] = env["settings"].model_copy(update={"max_papers": 3}) + payload = build_payload(WEEK, env["settings"], env["store"]) + assert len(payload.papers) == 3 + assert any("이월" in g for g in payload.data_gaps) + + +# -------------------------------------------------------------------------------- run_digest +def test_run_digest_happy_path_saves_marks_and_delivers(env): + result = run_digest(WEEK, env["settings"]) + assert result.error is None and not result.degraded + assert result.model_used == "fake-model" + assert result.verification is not None and result.verification.ok + assert WEEK in env["store"].digests + assert env["store"].digested == [(["10.1145/1234567"], WEEK)] + assert env["store"].sent == [(WEEK, "file", "file-receipt-1")] + assert env["store"].closed is True + assert "SE 주간 브리핑" in result.markdown + # per-paper summaries were generated, attached, persisted and rendered + assert result.payload.papers[0].summary is not None + assert env["store"].saved_summaries == [{"10.1145/1234567": result.payload.papers[0].summary}] + assert "## 논문별 요약" in result.markdown + + +def test_run_digest_paper_summaries_disabled(env): + env["settings"] = env["settings"].model_copy(update={"paper_summary_enabled": False}) + result = run_digest(WEEK, env["settings"]) + assert result.payload.papers[0].summary is None + assert env["store"].saved_summaries == [] + assert not any(c.startswith("summarize_papers") for c in env["provider"].calls) + + +def test_run_digest_paper_summary_failure_is_a_gap_not_a_blocker(env): + env["provider"] = FakeProvider(fail_paper_summaries=True) + result = run_digest(WEEK, env["settings"]) + assert result.error is None and not result.degraded # digest itself still produced + assert any("paper-summaries" in g for g in result.payload.data_gaps) + assert env["store"].saved_summaries == [] + + +def test_run_digest_defaults_to_latest_completed_week(env): + result = run_digest(None, env["settings"]) + assert result.payload.week == WEEK # NOW is Monday 2026-09-07 -> last completed week is W36 + + +def test_run_digest_skips_when_already_sent(env): + env["store"].digests[WEEK] = object() + result = run_digest(WEEK, env["settings"]) + assert result.error == SKIP_ALREADY_SENT + assert env["store"].sent == [] + + +def test_run_digest_force_resends(env): + env["store"].digests[WEEK] = object() + result = run_digest(WEEK, env["settings"], force=True) + assert result.error is None + assert env["store"].sent == [(WEEK, "file", "file-receipt-1")] + + +def test_run_digest_refuses_unfinished_week(env): + with pytest.raises(ValueError, match="has not ended yet"): + run_digest("2026-W37", env["settings"]) # NOW (Mon 09-07) is inside W37 + + +def test_run_digest_dry_run_previews_without_side_effects(env, capsys): + result = run_digest("2026-W37", env["settings"], dry_run=True) # even an unfinished week + assert "SE 주간 브리핑" in capsys.readouterr().out + assert env["store"].digests == {} and env["store"].sent == [] and env["store"].digested == [] + assert result.markdown + + +def test_run_digest_llm_failure_degrades_but_still_delivers(env): + env["provider"] = FakeProvider(fail_summarize=True) + result = run_digest(WEEK, env["settings"]) + assert result.degraded and result.digest is None + assert "all models failed" in (result.error or "") + assert env["store"].sent == [(WEEK, "file", "file-receipt-1")] + # degraded delivery raises an operator alert + assert any("축소 모드" in s for s, _ in env["deliverers"][0].errors) + + +def test_run_digest_no_llm_flag(env): + result = run_digest(WEEK, env["settings"], no_llm=True) + assert result.degraded and result.digest is None and result.error is None + assert env["provider"].calls == [] + + +def test_run_digest_empty_week_is_not_degraded(env): + env["openalex"] = [] + env["store"].pending = [] + result = run_digest(WEEK, env["settings"]) + assert not result.degraded and result.digest is None and result.error is None + assert "이번 주 수집된 OA 논문이 없습니다." in result.markdown + assert env["provider"].calls == [] + assert env["store"].digested == [([], WEEK)] + + +def test_run_digest_verification_disabled(env): + env["settings"] = env["settings"].model_copy(update={"verify_enabled": False}) + result = run_digest(WEEK, env["settings"]) + assert result.verification is None + assert env["provider"].calls == ["summarize_papers:1", "summarize"] + + +def test_run_digest_verification_revises_major_issue(env): + major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")]) + clean = Verification(ok=True, issues=[]) + env["provider"] = FakeProvider( + verifications=[major, clean], revised=make_digest(overview="수정본입니다.") + ) + result = run_digest(WEEK, env["settings"]) + assert result.digest is not None and result.digest.overview == "수정본입니다." + assert result.verification is not None and result.verification.ok + + +def test_run_digest_delivery_failure_is_annotated_and_alerted(env): + failing = FakeDeliverer("file", fail=True) + env["deliverers"] = [failing] + result = run_digest(WEEK, env["settings"]) + assert delivery_failed(result) + assert "delivery failed" in (result.error or "") + assert WEEK in env["store"].digests # saved before delivering + assert any("발송 실패" in s for s, _ in failing.errors) + assert any(f"--week {WEEK}" in d for _, d in failing.errors) + + +def test_run_digest_one_channel_failing_does_not_block_the_other(env): + ok = FakeDeliverer("ok") + env["deliverers"] = [FakeDeliverer("bad", fail=True), ok] + result = run_digest(WEEK, env["settings"]) + assert ok.delivered and env["store"].sent == [(WEEK, "ok", "ok-receipt-1")] + assert delivery_failed(result) + + +def test_run_digest_rejects_malformed_week(env): + with pytest.raises(ValueError): + run_digest("2026-36", env["settings"]) diff --git a/tests/test_render.py b/tests/test_render.py new file mode 100644 index 0000000..72a77cb --- /dev/null +++ b/tests/test_render.py @@ -0,0 +1,108 @@ +"""Markdown rendering: sections, DOI link resolution, degraded and empty-week modes.""" + +from __future__ import annotations + +from _fakes import make_digest, make_paper, make_payload, make_summary + +from sebrief.render import render_markdown + + +def test_full_digest_renders_all_sections_with_links(): + payload = make_payload() + md = render_markdown(payload, make_digest(), "fake-model") + assert md.startswith("# 📚 SE 주간 브리핑 — 2026-W36 (2026-08-31 ~ 2026-09-06)") + assert "수집 논문 1편" in md + for section in ("## 이번 주 흐름", "## 하이라이트", "## 주제별 훑어보기", "## 읽어볼 논문", "## 이번 주 수집 논문"): + assert section in md + # DOI resolves through the payload's oa_url, not a bare doi.org link + assert "[Test-Driven Digest Generation](https://dl.acm.org/doi/10.1145/1234567)" in md + assert "- **문제**:" in md and "- **시사점**:" in md + assert "축소 모드" not in md + + +def test_doi_mixup_is_rescued_by_title_match(): + # The LLM shuffled DOIs but kept the title: the link must follow the (visible) title. + other = make_paper("10.1/other", title="A Different Paper", oa_url="https://x.example/other") + payload = make_payload(papers=[make_paper(), other]) + digest = make_digest(doi="10.1/other") # wrong DOI, correct title "Test-Driven Digest Generation" + md = render_markdown(payload, digest, None) + assert "[Test-Driven Digest Generation](https://dl.acm.org/doi/10.1145/1234567)" in md + assert "[Test-Driven Digest Generation](https://x.example/other)" not in md + + +def test_unknown_doi_and_title_render_plain_text(): + digest = make_digest(doi="10.9999/unknown") + for h in digest.highlights: + h.title = "Invented Paper Title" + md = render_markdown(make_payload(), digest, None) + assert "### Invented Paper Title" in md # highlight title without a link + assert "[Invented Paper Title]" not in md + + +def test_title_collision_falls_back_to_doi(): + twin_a = make_paper("10.1/a", title="Same Title", oa_url="https://x.example/a") + twin_b = make_paper("10.1/b", title="Same Title", oa_url="https://x.example/b") + payload = make_payload(papers=[twin_a, twin_b]) + digest = make_digest(doi="10.1/b") + for h in digest.highlights: + h.title = "Same Title" + md = render_markdown(payload, digest, None) + assert "[Same Title](https://x.example/b)" in md # ambiguous title -> trust the DOI + + +def test_doi_without_oa_url_falls_back_to_doi_org(): + paper = make_paper(oa_url=None) + md = render_markdown(make_payload(papers=[paper]), make_digest(), None) + assert "(https://doi.org/10.1145/1234567)" in md + + +def test_degraded_mode_paper_table_only(): + payload = make_payload(gaps=["openalex: down"]) + md = render_markdown(payload, None, None) + assert "🧩 축소 모드(LLM 요약 없음)" in md + assert "## 이번 주 수집 논문" in md + assert "| 제목 | 저널 | 발행일 |" in md + assert "## 하이라이트" not in md + assert "- openalex: down" in md + + +def test_empty_week(): + md = render_markdown(make_payload(papers=[]), None, None) + assert "이번 주 수집된 OA 논문이 없습니다." in md + assert "축소 모드" not in md + assert "## 이번 주 수집 논문" not in md + + +def test_paper_summary_section_renders_cards(): + with_summary = make_paper(summary=make_summary(tldr="한 줄 요약입니다.")) + without = make_paper("10.1/nosummary", title="No Summary Yet") + md = render_markdown(make_payload(papers=[with_summary, without]), make_digest(), None) + assert "## 논문별 요약" in md + assert "**한 줄 요약입니다.**" in md + assert "- **문제**: 주간 다이제스트 생성이 느립니다." in md + # summary-less papers appear only in the table, not as a summary card + section = md.split("## 논문별 요약")[1].split("## 이번 주 수집 논문")[0] + assert "No Summary Yet" not in section + + +def test_paper_summaries_render_even_without_digest(): + md = render_markdown(make_payload(papers=[make_paper(summary=make_summary())]), None, None) + assert "## 논문별 요약" in md and "축소 모드" in md + + +def test_no_summary_section_when_no_paper_has_one(): + md = render_markdown(make_payload(), make_digest(), None) + assert "## 논문별 요약" not in md + + +def test_table_cells_and_link_text_are_escaped(): + paper = make_paper(title="Pipes | and [brackets]", venue="Ven|ue") + md = render_markdown(make_payload(papers=[paper]), None, None) + assert "Pipes \\| and (brackets)" in md + assert "Ven\\|ue" in md + + +def test_url_with_parentheses_is_encoded(): + paper = make_paper(oa_url="https://example.org/paper(1)") + md = render_markdown(make_payload(papers=[paper]), None, None) + assert "https://example.org/paper%281%29" in md diff --git a/tests/test_rss.py b/tests/test_rss.py new file mode 100644 index 0000000..04e62a6 --- /dev/null +++ b/tests/test_rss.py @@ -0,0 +1,106 @@ +"""RSS collector: feed spec parsing, DOI extraction, date filtering, failure isolation.""" + +from __future__ import annotations + +from datetime import date + +import pytest + +from sebrief.collectors import rss +from sebrief.collectors.rss import extract_doi, fetch_rss_papers, parse_feed_specs, strip_html + +START, END = date(2026, 8, 31), date(2026, 9, 6) + +FEED_XML = """ + + e-Informatica SEJ + + Mutation Testing & Coverage + https://doi.org/10.37190/e-inf/2026.1 + <p>We study <b>mutation</b> testing.</p> + Wed, 02 Sep 2026 08:00:00 GMT + + + Too Old Paper + https://doi.org/10.37190/e-inf/2025.9 + Mon, 03 Aug 2026 08:00:00 GMT + + + No DOI Entry + https://example.org/announcement + Thu, 03 Sep 2026 08:00:00 GMT + + +""" + + +class FakeResponse: + def __init__(self, content: bytes): + self.content = content + + def raise_for_status(self) -> None: + pass + + +def test_parse_feed_specs(): + feeds, invalid = parse_feed_specs( + ["e-Inf|https://example.org/rss", "broken", "|https://x.example", "Name|ftp://bad"] + ) + assert feeds == [("e-Inf", "https://example.org/rss")] + assert invalid == ["broken", "|https://x.example", "Name|ftp://bad"] + + +def test_strip_html(): + assert strip_html("

We study mutation testing.

") == "We study mutation testing." + assert strip_html(None) == "" + assert strip_html("x" * 3000, max_chars=10).endswith("…") + + +@pytest.mark.parametrize( + ("entry", "expected"), + [ + ({"link": "https://doi.org/10.37190/e-inf/2026.1"}, "10.37190/e-inf/2026.1"), + ({"link": "https://x.org/a", "summary": "see doi 10.1000/xyz."}, "10.1000/xyz"), + ({"id": "urn:doi:10.5555/abc"}, "10.5555/abc"), + ({"link": "https://example.org/none"}, None), + ], +) +def test_extract_doi(entry, expected): + assert extract_doi(entry) == expected + + +def test_fetch_maps_entries_and_filters_by_date(monkeypatch): + monkeypatch.setattr(rss.requests, "get", lambda *a, **k: FakeResponse(FEED_XML.encode())) + + papers = fetch_rss_papers([("e-Inf", "https://example.org/rss")], START, END) + + assert [p.doi for p in papers] == ["10.37190/e-inf/2026.1"] + paper = papers[0] + assert paper.title == "Mutation Testing & Coverage" + assert paper.abstract == "We study mutation testing." + assert paper.venue == "e-Inf" and paper.source == "rss:e-Inf" + assert paper.published_date == date(2026, 9, 2) + + +def test_one_feed_down_keeps_the_other(monkeypatch): + def fake_get(url, timeout=None, headers=None): + if "down" in url: + raise RuntimeError("boom") + return FakeResponse(FEED_XML.encode()) + + monkeypatch.setattr(rss.requests, "get", fake_get) + papers = fetch_rss_papers([("A", "https://down.example/rss"), ("B", "https://ok.example/rss")], START, END) + assert len(papers) == 1 + + +def test_all_feeds_down_raises(monkeypatch): + def fake_get(*a, **k): + raise RuntimeError("boom") + + monkeypatch.setattr(rss.requests, "get", fake_get) + with pytest.raises(RuntimeError, match="all rss feeds failed"): + fetch_rss_papers([("A", "https://a.example/rss")], START, END) + + +def test_no_feeds_is_a_noop(): + assert fetch_rss_papers([], START, END) == [] diff --git a/tests/test_sources.py b/tests/test_sources.py new file mode 100644 index 0000000..437724e --- /dev/null +++ b/tests/test_sources.py @@ -0,0 +1,37 @@ +"""Whitelist/blacklist and the SE keyword double filter.""" + +from __future__ import annotations + +from sebrief.sources import SOURCES, is_blacklisted, issns_for, looks_like_se + + +def test_issns_for_splits_plain_and_mega(): + plain = issns_for(se_filter=False) + mega = issns_for(se_filter=True) + assert "1049-331X" in plain # TOSEM + assert "2169-3536" in mega # IEEE Access + assert set(plain).isdisjoint(mega) + assert len(plain) + len(mega) == sum(len(s.issns) for s in SOURCES) + + +def test_mdpi_blocked_by_issn_and_publisher(): + assert is_blacklisted(["2079-9292"], None) is True # Electronics (MDPI) + assert is_blacklisted([], "MDPI AG") is True + assert is_blacklisted([], "MDPI") is True + assert is_blacklisted(["1049-331X"], "Association for Computing Machinery") is False + + +def test_extra_blacklist_issns(): + assert is_blacklisted(["9999-0001"], None) is False + assert is_blacklisted(["9999-0001"], None, extra_issns=["9999-0001"]) is True + + +def test_looks_like_se_by_keyword_title_and_abstract(): + assert looks_like_se("A Study on Automated Program Repair", "", []) is True + assert looks_like_se("Deep Models", "we improve defect prediction accuracy", []) is True + assert looks_like_se("Optimal Beamforming for 6G Networks", "channel estimation and MIMO", []) is False + + +def test_looks_like_se_by_topic_name(): + assert looks_like_se("Untitled", "", ["Software Reliability and Analysis"]) is True + assert looks_like_se("Untitled", "", ["Wireless Communication"]) is False diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 0000000..c0f81b1 --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,139 @@ +"""SQLite store: paper lifecycle (upsert/freeze/rollover), digest idempotency, deliveries.""" + +from __future__ import annotations + +from datetime import date + +from _fakes import WEEK, make_paper, make_payload, make_summary + +from sebrief.models import DigestResult +from sebrief.store import Store + + +def make_store(tmp_path) -> Store: + return Store(tmp_path / "data" / "test.sqlite3") + + +def make_result(papers=None, degraded=False) -> DigestResult: + payload = make_payload(papers=papers) + return DigestResult( + payload=payload, digest=None, markdown="# md", model_used="m", degraded=degraded + ) + + +# ----------------------------------------------------------------------------------- papers +def test_upsert_inserts_and_counts_new(tmp_path): + with make_store(tmp_path) as store: + p1, p2 = make_paper("10.1/a"), make_paper("10.1/b") + assert store.upsert_papers([p1, p2]) == 2 + assert store.upsert_papers([p1]) == 0 # re-collection is not new + assert store.paper_count() == 2 + + +def test_upsert_refreshes_undigested_metadata(tmp_path): + with make_store(tmp_path) as store: + store.upsert_papers([make_paper("10.1/a", abstract="")]) + store.upsert_papers([make_paper("10.1/a", abstract="now with abstract")]) + [paper] = store.pending_papers(WEEK) + assert paper.abstract == "now with abstract" + + +def test_digested_papers_are_frozen_and_skipped(tmp_path): + with make_store(tmp_path) as store: + store.upsert_papers([make_paper("10.1/a", title="original")]) + store.mark_digested(["10.1/a"], WEEK) + store.upsert_papers([make_paper("10.1/a", title="rewritten")]) + # not pending for a later week, and metadata unchanged + assert store.pending_papers("2026-W37") == [] + [paper] = store.pending_papers(WEEK) # --force re-run of the same week still sees it + assert paper.title == "original" + + +def test_pending_includes_undigested_leftovers_and_orders_by_date(tmp_path): + with make_store(tmp_path) as store: + old = make_paper("10.1/old", published_date=date(2026, 8, 20)) # last week's late arrival + new = make_paper("10.1/new", published_date=date(2026, 9, 2)) + undated = make_paper("10.1/undated", published_date=None) + store.upsert_papers([new, undated, old]) + dois = [p.doi for p in store.pending_papers(WEEK)] + assert dois == ["10.1/old", "10.1/new", "10.1/undated"] # dated first (ascending), undated last + + +def test_paper_roundtrip_preserves_fields(tmp_path): + with make_store(tmp_path) as store: + original = make_paper("10.1/a", authors=["김철수", "Bob"], topics=["Testing", "LLM"]) + store.upsert_papers([original]) + [loaded] = store.pending_papers(WEEK) + assert loaded == original + + +# ---------------------------------------------------------------------------- paper summaries +def test_paper_summaries_roundtrip_and_survive_metadata_refresh(tmp_path): + with make_store(tmp_path) as store: + store.upsert_papers([make_paper("10.1/a")]) + summary = make_summary(tldr="한 줄.") + store.save_paper_summaries({"10.1/a": summary}) + [paper] = store.pending_papers(WEEK) + assert paper.summary == summary + # a later re-collection refreshes metadata but must not clear the summary + store.upsert_papers([make_paper("10.1/a", abstract="updated")]) + [paper] = store.pending_papers(WEEK) + assert paper.abstract == "updated" and paper.summary == summary + + +def test_paper_summaries_allowed_on_frozen_papers(tmp_path): + with make_store(tmp_path) as store: + store.upsert_papers([make_paper("10.1/a")]) + store.mark_digested(["10.1/a"], WEEK) + store.save_paper_summaries({"10.1/a": make_summary()}) + [paper] = store.pending_papers(WEEK) # digest_week = WEEK: visible on --force re-run + assert paper.summary is not None + + +def test_migration_adds_summary_column_to_old_db(tmp_path): + import sqlite3 + + db = tmp_path / "data" / "old.sqlite3" + db.parent.mkdir(parents=True) + conn = sqlite3.connect(db) + conn.executescript( + """ + CREATE TABLE papers ( + doi TEXT PRIMARY KEY, title TEXT NOT NULL, abstract TEXT NOT NULL DEFAULT '', + authors TEXT NOT NULL DEFAULT '[]', venue TEXT NOT NULL DEFAULT '', + published_date TEXT, oa_url TEXT, pdf_url TEXT, topics TEXT NOT NULL DEFAULT '[]', + source TEXT NOT NULL DEFAULT '', collected_at TEXT, digest_week TEXT + ); + INSERT INTO papers (doi, title) VALUES ('10.1/legacy', 'Legacy Paper'); + """ + ) + conn.commit() + conn.close() + + with Store(db) as store: + [paper] = store.pending_papers(WEEK) + assert paper.doi == "10.1/legacy" and paper.summary is None + store.save_paper_summaries({"10.1/legacy": make_summary()}) + [paper] = store.pending_papers(WEEK) + assert paper.summary is not None + + +# ---------------------------------------------------------------------------------- digests +def test_digest_idempotency_and_upsert(tmp_path): + with make_store(tmp_path) as store: + assert store.has_digest(WEEK) is False + store.save_digest(make_result()) + assert store.has_digest(WEEK) is True + store.save_digest(make_result(degraded=True)) # --force resend overwrites + row = store._conn.execute("SELECT degraded FROM digests WHERE week = ?", (WEEK,)).fetchone() + assert row["degraded"] == 1 + + +def test_deliveries(tmp_path): + with make_store(tmp_path) as store: + store.save_digest(make_result()) + store.mark_sent(WEEK, "file", "/tmp/2026-W36.md") + store.mark_sent(WEEK, "file", "/tmp/2026-W36.md (resend)") + rows = store.deliveries_for(WEEK) + assert [r["channel"] for r in rows] == ["file", "file"] + assert rows[1]["receipt"].endswith("(resend)") diff --git a/tests/test_summarize.py b/tests/test_summarize.py new file mode 100644 index 0000000..13ab86e --- /dev/null +++ b/tests/test_summarize.py @@ -0,0 +1,105 @@ +"""Summarise glue: gap merging, per-paper batching, verification consistency, revise-once policy.""" + +from __future__ import annotations + +from _fakes import FakeProvider, make_digest, make_paper, make_payload, make_summary + +from sebrief.llm.base import LLMError +from sebrief.models import PaperSummaryItem, Verification, VerificationIssue +from sebrief.summarize import summarize_papers, summarize_payload, verify_and_revise + + +def test_summarize_merges_payload_gaps_into_digest(): + payload = make_payload(gaps=["openalex: down"]) + provider = FakeProvider(digest=make_digest(data_gaps=["LLM이 적은 누락"])) + digest, model = summarize_payload(payload, provider) + assert model == "fake-model" + assert digest.data_gaps == ["openalex: down", "LLM이 적은 누락"] + + +def test_summarize_dedups_gaps(): + payload = make_payload(gaps=["g1"]) + provider = FakeProvider(digest=make_digest(data_gaps=["g1", "g2"])) + digest, _ = summarize_payload(payload, provider) + assert digest.data_gaps == ["g1", "g2"] + + +# -------------------------------------------------------------------- per-paper summaries +def test_summarize_papers_batches_and_maps_by_doi(): + papers = [make_paper(f"10.1/p{i}") for i in range(5)] + provider = FakeProvider() + summaries, notes = summarize_papers(papers, provider, batch_size=2) + assert notes == [] + assert set(summaries) == {p.doi for p in papers} + assert provider.calls == ["summarize_papers:2", "summarize_papers:2", "summarize_papers:1"] + assert summaries["10.1/p0"].tldr == "10.1/p0 한 줄 요약입니다." + + +def test_summarize_papers_skips_already_summarized_and_abstractless(): + done = make_paper("10.1/done", summary=make_summary()) + no_abstract = make_paper("10.1/empty", abstract=" ") + todo = make_paper("10.1/todo") + provider = FakeProvider() + summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8) + assert set(summaries) == {"10.1/todo"} + assert provider.calls == ["summarize_papers:1"] + + +def test_summarize_papers_failed_batch_is_noted_and_others_survive(): + class FlakyProvider(FakeProvider): + def summarize_papers(self, papers): + if any(p.doi == "10.1/p0" for p in papers): + raise LLMError("boom") + return super().summarize_papers(papers) + + papers = [make_paper(f"10.1/p{i}") for i in range(4)] + summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2) + assert set(summaries) == {"10.1/p2", "10.1/p3"} + assert len(notes) == 1 and "batch 1/2" in notes[0] and "재시도" in notes[0] + + +def test_summarize_papers_discards_unknown_dois(): + class InventingProvider(FakeProvider): + def summarize_papers(self, papers): + items, model = super().summarize_papers(papers) + items.append(PaperSummaryItem(doi="10.9/invented", **make_summary().model_dump())) + return items, model + + summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8) + assert set(summaries) == {"10.1/a"} and notes == [] + + +def test_verify_ok_returns_original(): + provider = FakeProvider(verifications=[Verification(ok=True, issues=[])]) + digest = make_digest() + out, verification = verify_and_revise(make_payload(), digest, provider) + assert out is digest and verification.ok + assert provider.calls == ["verify"] + + +def test_minor_only_issues_force_ok_true_and_skip_revision(): + minor = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="minor")]) + provider = FakeProvider(verifications=[minor]) + _out, verification = verify_and_revise(make_payload(), make_digest(), provider) + assert verification.ok is True and len(verification.issues) == 1 + assert provider.calls == ["verify"] + + +def test_major_issue_revises_once_and_reverifies(): + major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")]) + clean = Verification(ok=True, issues=[]) + revised = make_digest(overview="수정된 총평입니다.") + provider = FakeProvider(verifications=[major, clean], revised=revised) + out, verification = verify_and_revise(make_payload(), make_digest(), provider) + assert out.overview == "수정된 총평입니다." + assert verification.ok is True + assert provider.calls == ["verify", "revise", "verify"] + + +def test_failed_revision_keeps_original_and_verdict(): + major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")]) + provider = FakeProvider(verifications=[major], fail_revise=True) + digest = make_digest() + out, verification = verify_and_revise(make_payload(), digest, provider) + assert out is digest + assert verification.has_major and verification.ok is False diff --git a/tests/test_week.py b/tests/test_week.py new file mode 100644 index 0000000..3b414a5 --- /dev/null +++ b/tests/test_week.py @@ -0,0 +1,116 @@ +"""Week helpers and WeeklyClock schedule arithmetic.""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from zoneinfo import ZoneInfo + +import pytest + +from sebrief.week import WeeklyClock, parse_hhmm, week_id, week_range + +KST = ZoneInfo("Asia/Seoul") + + +# ------------------------------------------------------------------------- week id / range +def test_week_id_and_range_roundtrip(): + assert week_id(date(2026, 8, 31)) == "2026-W36" + assert week_id(date(2026, 9, 6)) == "2026-W36" + assert week_range("2026-W36") == (date(2026, 8, 31), date(2026, 9, 6)) + + +def test_week_id_uses_iso_year_at_boundaries(): + # 2026-01-01 is a Thursday -> belongs to ISO 2026-W01 starting Mon 2025-12-29 + assert week_id(date(2025, 12, 29)) == "2026-W01" + assert week_range("2026-W01")[0] == date(2025, 12, 29) + + +@pytest.mark.parametrize("bad", ["2026-36", "2026-W99", "2026W36", "abcd-W01", "2026-W1"]) +def test_week_range_rejects_invalid(bad): + with pytest.raises(ValueError): + week_range(bad) + + +@pytest.mark.parametrize("bad", ["8:", "25:00", "08:60", "abc", "0820"]) +def test_parse_hhmm_rejects_invalid(bad): + with pytest.raises(ValueError): + parse_hhmm(bad) + + +def test_parse_hhmm_ok(): + assert parse_hhmm("07:00") == (7, 0) + assert parse_hhmm(" 7:05 ") == (7, 5) + + +# ------------------------------------------------------------------------------ WeeklyClock +def test_clock_rejects_bad_config(): + with pytest.raises(ValueError): + WeeklyClock(run_weekday=7) + with pytest.raises(ValueError): + WeeklyClock(run_time="25:00") + + +def test_next_run_before_and_after_the_weekly_slot(): + clock = WeeklyClock(0, "07:00") + before = datetime(2026, 9, 7, 6, 59, tzinfo=KST) # Monday, one minute before the run + week, run_at = clock.next_run(before) + assert (week, run_at) == ("2026-W36", datetime(2026, 9, 7, 7, 0, tzinfo=KST)) + + after = datetime(2026, 9, 7, 7, 0, tzinfo=KST) # exactly at the slot -> next week's run + week, run_at = clock.next_run(after) + assert (week, run_at) == ("2026-W37", datetime(2026, 9, 14, 7, 0, tzinfo=KST)) + + +def test_next_run_midweek_and_custom_weekday(): + wed = datetime(2026, 9, 9, 12, 0, tzinfo=KST) + assert WeeklyClock(0, "07:00").next_run(wed) == ( + "2026-W37", + datetime(2026, 9, 14, 7, 0, tzinfo=KST), + ) + # Thursday 08:30 runs cover the week that ended the previous Sunday + assert WeeklyClock(3, "08:30").next_run(wed) == ( + "2026-W36", + datetime(2026, 9, 10, 8, 30, tzinfo=KST), + ) + + +def test_next_run_requires_tz_aware(): + with pytest.raises(ValueError): + WeeklyClock().next_run(datetime(2026, 9, 7, 12, 0)) # noqa: DTZ001 - naive on purpose + + +def test_run_at_for_is_consistent_with_next_run(): + clock = WeeklyClock(0, "07:00") + assert clock.run_at_for("2026-W36") == datetime(2026, 9, 7, 7, 0, tzinfo=KST) + clock2 = WeeklyClock(3, "08:30") + assert clock2.run_at_for("2026-W36") == datetime(2026, 9, 10, 8, 30, tzinfo=KST) + + +def test_last_run_and_latest_completed_week(): + clock = WeeklyClock(0, "07:00") + monday_noon = datetime(2026, 9, 7, 12, 0, tzinfo=KST) + assert clock.last_run(monday_noon) == ("2026-W36", datetime(2026, 9, 7, 7, 0, tzinfo=KST)) + assert clock.latest_completed_week(monday_noon) == "2026-W36" + # mid-week: the last run is still the past Monday + wed = datetime(2026, 9, 9, 9, 0, tzinfo=KST) + assert clock.last_run(wed) == ("2026-W36", datetime(2026, 9, 7, 7, 0, tzinfo=KST)) + + +@pytest.mark.parametrize( + ("now_delta", "expected"), + [ + (timedelta(hours=1), "2026-W36"), # 1h after the slot: inside the window + (timedelta(hours=71), "2026-W36"), # still inside the 72h window + (timedelta(hours=73), None), # outside + ], +) +def test_catchup_candidate_window(now_delta, expected): + clock = WeeklyClock(0, "07:00") + run_at = datetime(2026, 9, 7, 7, 0, tzinfo=KST) + assert clock.catchup_candidate(run_at + now_delta, 72) == expected + + +def test_covered_week(): + clock = WeeklyClock(0, "07:00") + assert clock.covered_week(date(2026, 9, 7)) == "2026-W36" + assert clock.covered_week(date(2026, 9, 14)) == "2026-W37"