From d9dda79282c89fa70c2075a84a4a3bd73b2c96a5 Mon Sep 17 00:00:00 2001 From: jsahn Date: Wed, 2 Sep 2026 17:21:41 +0900 Subject: [PATCH] Add per-paper summary verification and cap safeguards - Per-paper summary batches are fact-checked against abstracts (verify -> revise once on major issues -> re-verify), controlled by SEBRIEF_VERIFY_ENABLED - The paper cap never drops papers already published in this week's digest (--force re-runs could otherwise erase them from every digest) - SEBRIEF_MAX_PAPERS default 40 -> 60 (observed weekly volume ~47) Co-Authored-By: Claude Fable 5 --- .env.example | 7 ++++-- README.md | 17 ++++++++----- sebrief/config.py | 4 ++- sebrief/llm/base.py | 12 +++++++++ sebrief/llm/minimax.py | 24 ++++++++++++++++++ sebrief/llm/prompts.py | 35 +++++++++++++++++++++++++- sebrief/pipeline.py | 18 ++++++++++---- sebrief/store.py | 5 ++++ sebrief/summarize.py | 47 ++++++++++++++++++++++++++++++++--- tests/_fakes.py | 22 +++++++++++++++++ tests/test_config.py | 4 ++- tests/test_pipeline.py | 22 +++++++++++++++++ tests/test_summarize.py | 55 ++++++++++++++++++++++++++++++++++++++--- 13 files changed, 249 insertions(+), 23 deletions(-) diff --git a/.env.example b/.env.example index bf304be..82485b3 100644 --- a/.env.example +++ b/.env.example @@ -10,8 +10,11 @@ SEBRIEF_OPENALEX_MAILTO= # 수집 정책 # 수집 창을 주 시작보다 며칠 앞당길지 (늦게 색인된 논문 보완; DOI 중복은 자동 스킵) SEBRIEF_BACKFILL_DAYS=7 -# 한 다이제스트에 담는 최대 논문 수 (초과분은 다음 주로 이월) -SEBRIEF_MAX_PAPERS=40 +# 한 다이제스트에 담는 최대 논문 수 (폭주 방어 안전판; 초과분은 다음 주로 이월) +SEBRIEF_MAX_PAPERS=60 +# 논문 한 편씩 구조화 요약 (한줄요약·문제·방법·결과·시사점; DOI당 한 번 생성 후 재사용) +SEBRIEF_PAPER_SUMMARY_ENABLED=true +SEBRIEF_PAPER_SUMMARY_BATCH_SIZE=8 # 보조 RSS 피드 CSV, 항목 형식 "이름|URL" (예: e-Informatica SEJ) SEBRIEF_RSS_FEEDS= # MDPI 기본 블랙리스트에 더해 차단할 ISSN CSV diff --git a/README.md b/README.md index 7051890..fba67e5 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,10 @@ tailnet 대시보드)하는 상주 프로그램입니다. 설계 문서는 [SEBR US2KR의 파이프라인 철학을 그대로 따르는 별도 프로젝트입니다. ``` -수집(OpenAlex · RSS) → SQLite 저장(DOI 중복 방지) → MiniMax 요약(JSON 스키마) - → LLM 교차 검증(초록 대조 팩트체크, major 이슈면 1회 자동 수정) → Markdown 렌더 → 파일 아카이브 + 대시보드 +수집(OpenAlex · RSS) → SQLite 저장(DOI 중복 방지) + → 논문별 요약(한줄요약·문제·방법·결과·시사점, DOI당 한 번 생성 후 재사용) + 배치 팩트체크 + → 주간 다이제스트 요약(JSON 스키마) → LLM 교차 검증(초록 대조 팩트체크, major면 1회 자동 수정) + → Markdown 렌더 → 파일 아카이브 + 대시보드 ``` ## 수집 소스 (sebrief/sources.py) @@ -78,18 +80,21 @@ cp .env.example .env # MINIMAX_API_KEY 필수 | `MINIMAX_MODEL` / `MINIMAX_FALLBACK_MODELS` | `MiniMax-M3` / `M2.7,M2.5` | 실패 시 순차 폴백 | | `SEBRIEF_OPENALEX_MAILTO` | — | 이메일을 넣으면 OpenAlex polite pool (더 빠르고 안정적) | | `SEBRIEF_BACKFILL_DAYS` | `7` | 수집 창을 주 시작보다 며칠 앞당길지 (늦은 색인 보완) | -| `SEBRIEF_MAX_PAPERS` | `40` | 다이제스트당 최대 논문 수 (초과분은 다음 주 이월) | +| `SEBRIEF_MAX_PAPERS` | `60` | 다이제스트당 최대 논문 수 — 폭주 방어 안전판 (초과분은 다음 주 이월) | +| `SEBRIEF_PAPER_SUMMARY_ENABLED` | `true` | 논문 한 편씩 구조화 요약 생성 (초록 없는 논문은 스킵, 실패 배치는 다음 실행에서 재시도) | +| `SEBRIEF_PAPER_SUMMARY_BATCH_SIZE` | `8` | 논문별 요약 LLM 호출당 논문 수 | | `SEBRIEF_RSS_FEEDS` | — | 보조 RSS CSV, 항목 `이름\|URL` | | `SEBRIEF_EXTRA_BLACKLIST_ISSNS` | — | 기본 MDPI 블랙리스트에 더해 차단할 ISSN CSV | | `SEBRIEF_RUN_WEEKDAY` / `SEBRIEF_RUN_TIME_KST` | `0` / `07:00` | 실행 요일(0=월)·시각 | | `SEBRIEF_CATCHUP_WINDOW_HOURS` | `72` | 재시작 시 따라잡기 허용 창 | -| `SEBRIEF_VERIFY_ENABLED` | `true` | 요약 후 LLM 팩트체크(초록과 대조). major 이슈 발견 시 1회 자동 수정 후 재검증. 검증 실패는 발행을 막지 않고 결과는 JSON·대시보드에 기록 | +| `SEBRIEF_VERIFY_ENABLED` | `true` | 요약 후 LLM 팩트체크(초록과 대조) — 주간 다이제스트와 논문별 요약 배치 모두. major 이슈 발견 시 1회 자동 수정 후 재검증. 검증 실패는 발행을 막지 않고 결과는 JSON·대시보드에 기록 | | `SEBRIEF_LOG_LEVEL` | `INFO` | 로그 레벨 | ## 산출물 -- `reports/YYYY-Wnn.md`, `.json` — ISO 주차 기준 아카이브 -- `data/sebrief.sqlite3` — 논문(DOI 중복 방지·동결)·다이제스트·발행 기록 +- `reports/YYYY-Wnn.md`, `.json` — ISO 주차 기준 아카이브. 주간 총평·하이라이트·주제별 그룹에 더해 + **논문별 요약 카드**(한줄요약·문제·방법·결과·시사점)를 전 논문에 대해 포함 +- `data/sebrief.sqlite3` — 논문(DOI 중복 방지·동결)·논문별 요약·다이제스트·발행 기록 - `logs/sebrief.log` — 일자별 회전, API 키 마스킹 ## 대시보드 (dashboard/) diff --git a/sebrief/config.py b/sebrief/config.py index 4cef55c..6608b03 100644 --- a/sebrief/config.py +++ b/sebrief/config.py @@ -41,7 +41,9 @@ class Settings(BaseSettings): # OpenAlex are still picked up; DOI dedup in the store keeps them from being summarised twice. backfill_days: int = Field(default=7, alias="SEBRIEF_BACKFILL_DAYS") # Papers beyond this cap are left undigested and roll over to the next week's digest. - max_papers: int = Field(default=40, alias="SEBRIEF_MAX_PAPERS") + # 폭주 방어용 안전판: 쿼리 이상·일괄 등록으로 한 주에 수백 편이 들어와도 다이제스트 LLM 호출이 + # 프롬프트/출력 한도를 넘지 않게 막는다. 실측 주간 볼륨(~47편, JOSS 포함)에 여유를 둔 값. + max_papers: int = Field(default=60, alias="SEBRIEF_MAX_PAPERS") # Supplementary RSS feeds, CSV of "이름|URL" (e.g. e-Informatica SEJ). Entries without a DOI are skipped. rss_feeds: Annotated[list[str], NoDecode] = Field(default_factory=list, alias="SEBRIEF_RSS_FEEDS") # Extra ISSNs to block on top of sebrief.sources.BLACKLIST_ISSNS (MDPI journals etc.). diff --git a/sebrief/llm/base.py b/sebrief/llm/base.py index 866f1e3..e9258bb 100644 --- a/sebrief/llm/base.py +++ b/sebrief/llm/base.py @@ -20,6 +20,18 @@ class LLMProvider(Protocol): """Per-paper structured summaries for one batch. Raise LLMError on total failure.""" ... + def verify_paper_summaries( + self, papers: list[Paper], items: list[PaperSummaryItem] + ) -> tuple[Verification, str]: + """Fact-check one batch of per-paper summaries against the abstracts. Raise LLMError on total failure.""" + ... + + def revise_paper_summaries( + self, papers: list[Paper], items: list[PaperSummaryItem], issues: list[VerificationIssue] + ) -> tuple[list[PaperSummaryItem], str]: + """Return a corrected batch addressing `issues`. Raise LLMError on total failure.""" + ... + def verify(self, payload: WeekPayload, digest: Digest) -> tuple[Verification, str]: """Fact-check `digest` against the payload. Raise LLMError on total failure.""" ... diff --git a/sebrief/llm/minimax.py b/sebrief/llm/minimax.py index 23e6528..701fd28 100644 --- a/sebrief/llm/minimax.py +++ b/sebrief/llm/minimax.py @@ -14,10 +14,13 @@ from pydantic import BaseModel, ValidationError from sebrief.config import Settings from sebrief.llm.base import LLMError from sebrief.llm.prompts import ( + PAPER_REVISE_INSTRUCTION, PAPER_SUMMARY_SYSTEM_PROMPT, + PAPER_VERIFY_SYSTEM_PROMPT, REVISE_INSTRUCTION, SYSTEM_PROMPT, VERIFY_SYSTEM_PROMPT, + build_paper_verify_message, build_papers_message, build_user_message, build_verify_message, @@ -230,6 +233,27 @@ class MiniMaxProvider: batch, model = self._run(messages, PaperSummaryBatch, "paper_summaries", task="summarize_papers") return batch.summaries, model + def verify_paper_summaries( + self, papers: list[Paper], items: list[PaperSummaryItem] + ) -> tuple[Verification, str]: + messages: list[dict[str, Any]] = [ + {"role": "system", "content": PAPER_VERIFY_SYSTEM_PROMPT}, + {"role": "user", "content": build_paper_verify_message(papers, items)}, + ] + return self._run(messages, Verification, "verification", task="verify_papers") + + def revise_paper_summaries( + self, papers: list[Paper], items: list[PaperSummaryItem], issues: list[VerificationIssue] + ) -> tuple[list[PaperSummaryItem], str]: + messages: list[dict[str, Any]] = [ + {"role": "system", "content": PAPER_SUMMARY_SYSTEM_PROMPT}, + {"role": "user", "content": build_papers_message(papers)}, + {"role": "assistant", "content": PaperSummaryBatch(summaries=items).model_dump_json()}, + {"role": "user", "content": PAPER_REVISE_INSTRUCTION + format_issues(issues)}, + ] + batch, model = self._run(messages, PaperSummaryBatch, "paper_summaries", task="revise_papers") + return batch.summaries, model + def verify(self, payload: WeekPayload, digest: Digest) -> tuple[Verification, str]: """Fact-check `digest` against the payload. Raise LLMError on total failure.""" messages: list[dict[str, Any]] = [ diff --git a/sebrief/llm/prompts.py b/sebrief/llm/prompts.py index 55ccb31..9c98c38 100644 --- a/sebrief/llm/prompts.py +++ b/sebrief/llm/prompts.py @@ -7,7 +7,7 @@ deterministic, human-readable serialisation of a `WeekPayload`. from __future__ import annotations -from sebrief.models import Digest, Paper, VerificationIssue, WeekPayload +from sebrief.models import Digest, Paper, PaperSummaryBatch, PaperSummaryItem, VerificationIssue, WeekPayload # Maximum characters of an abstract carried into the prompt (token budget control). ABSTRACT_MAX_CHARS = 1800 @@ -87,6 +87,29 @@ PAPER_SUMMARY_SYSTEM_PROMPT: str = f"""당신은 소프트웨어공학(SE) 논 {_PAPER_SUMMARY_SKELETON} """ +PAPER_VERIFY_SYSTEM_PROMPT: str = """당신은 소프트웨어공학 논문 요약의 팩트체커입니다. 논문별 요약 초안이 함께 제공된 +각 논문의 제목·초록·메타데이터와 모순되지 않는지만 검사합니다. 문체·구성은 평가하지 않습니다. + +## 검사 규칙 +1. 판단 근거는 사용자 메시지에 제공된 논문 목록뿐입니다. 당신의 배경지식과 달라 보여도, 제공된 + 초록과 일치하면 문제가 아닙니다. 각 요약은 doi가 가리키는 논문의 초록과만 대조합니다. +2. major: 요약이 해당 논문의 초록과 모순되거나, 초록에 없는 수치·실험 결과·결론을 사실처럼 + 서술하거나, 초록의 수치를 다르게 옮겼거나, 다른 논문의 내용을 섞어 서술한 경우. +3. minor: 초록만으로 확인되지 않는 인과관계를 단정, 저자 주장을 검증된 사실처럼 표현, 수치의 + 성격을 잘못 표기(예: 중위수를 평균으로), 모호하지만 틀렸다고 단정할 수 없는 서술. +4. 각 이슈의 claim에는 어느 DOI의 요약인지와 함께 요약 원문을 그대로 인용하고, problem에는 + 초록의 실제 내용과 함께 한 문장으로 설명합니다. +5. major 이슈가 하나도 없으면 ok=true 입니다. 사소한 트집을 만들지 말고, 문제가 없으면 빈 + issues로 반환합니다. +6. 출력은 지정된 JSON 스키마만 반환합니다. 스키마 밖 텍스트·설명·코드 펜스를 추가하지 않습니다. +""" + +PAPER_REVISE_INSTRUCTION: str = ( + "위 논문별 요약 초안에서 아래 검증 이슈가 발견되었습니다. 각 논문의 초록에 맞게 해당 서술만 고치고," + " 문제없는 요약은 그대로 유지하여 동일한 JSON 스키마(summaries 배열, 목록의 모든 논문 포함)로" + " 전체를 다시 출력하세요. 설명 없이 JSON 객체 하나만 출력합니다.\n\n검증 이슈:\n" +) + VERIFY_SYSTEM_PROMPT: str = """당신은 소프트웨어공학 주간 다이제스트의 팩트체커입니다. 다이제스트 초안이 함께 제공된 논문 목록(제목·초록·메타데이터)과 모순되지 않는지만 검사합니다. 문체·구성·의견은 평가하지 않습니다. @@ -153,6 +176,16 @@ def build_papers_message(papers: list[Paper]) -> str: return "\n".join(out) +def build_paper_verify_message(papers: list[Paper], items: list[PaperSummaryItem]) -> str: + """User message for the per-paper fact-check pass: the batch serialisation plus the draft summaries.""" + return ( + build_papers_message(papers) + + "\n## 검증 대상 논문별 요약 (JSON)\n" + + PaperSummaryBatch(summaries=items).model_dump_json(indent=2) + + "\n" + ) + + def build_user_message(payload: WeekPayload) -> str: """Serialise the payload into a deterministic Korean text block for the user message.""" out: list[str] = [] diff --git a/sebrief/pipeline.py b/sebrief/pipeline.py index 3df3840..cad52bf 100644 --- a/sebrief/pipeline.py +++ b/sebrief/pipeline.py @@ -78,10 +78,18 @@ def build_payload(week: str, settings: Settings, store: Store) -> WeekPayload: pending = run_safely("store", lambda: store.pending_papers(week), [], gaps) if len(pending) > settings.max_papers: - gaps.append( - f"papers: {len(pending)}편 수집, {settings.max_papers}편만 요약 (초과분은 다음 주로 이월)" - ) - pending = pending[: settings.max_papers] + # The cap only ever drops papers that have not been published yet. Papers already stamped + # with THIS week (--force re-run) must stay: dropping one would erase it from the published + # digest while its digest_week stamp keeps it out of every future digest. + published = run_safely("store", lambda: store.digested_dois(week), set(), gaps) + keep = [p for p in pending if p.doi in published] + room = max(settings.max_papers - len(keep), 0) + rollover = [p for p in pending if p.doi not in published] + dropped = len(rollover) - room + if dropped > 0: + gaps.append(f"papers: {len(pending)}편 수집, {dropped}편은 다음 주로 이월") + selected = {p.doi for p in keep} | {p.doi for p in rollover[:room]} + pending = [p for p in pending if p.doi in selected] payload = WeekPayload( week=week, @@ -129,7 +137,7 @@ def _summarize_papers(payload: WeekPayload, settings: Settings, store: Store) -> try: provider = build_provider(settings) summaries, notes = summarize_papers( - payload.papers, provider, settings.paper_summary_batch_size + payload.papers, provider, settings.paper_summary_batch_size, verify=settings.verify_enabled ) except Exception as exc: # noqa: BLE001 - per-paper summaries must never block the digest log.warning("paper summarisation failed entirely: %s: %s", type(exc).__name__, exc) diff --git a/sebrief/store.py b/sebrief/store.py index 9558a24..0a3fa58 100644 --- a/sebrief/store.py +++ b/sebrief/store.py @@ -175,6 +175,11 @@ class Store: ) return [_row_to_paper(row) for row in cur.fetchall()] + def digested_dois(self, week: str) -> set[str]: + """DOIs already stamped with this digest week (non-empty only on --force re-runs).""" + cur = self._conn.execute("SELECT doi FROM papers WHERE digest_week = ?", (week,)) + return {row["doi"] for row in cur} + def mark_digested(self, dois: list[str], week: str) -> None: with self._conn: self._conn.executemany( diff --git a/sebrief/summarize.py b/sebrief/summarize.py index e192bae..ee5bc3c 100644 --- a/sebrief/summarize.py +++ b/sebrief/summarize.py @@ -7,7 +7,7 @@ import logging from sebrief.config import Settings from sebrief.llm.base import LLMError, LLMProvider from sebrief.llm.minimax import MiniMaxProvider -from sebrief.models import Digest, Paper, PaperSummary, Verification, WeekPayload +from sebrief.models import Digest, Paper, PaperSummary, PaperSummaryItem, Verification, WeekPayload log = logging.getLogger(__name__) @@ -31,13 +31,14 @@ def summarize_payload(payload: WeekPayload, provider: LLMProvider) -> tuple[Dige def summarize_papers( - papers: list[Paper], provider: LLMProvider, batch_size: int = 8 + papers: list[Paper], provider: LLMProvider, batch_size: int = 8, verify: bool = True ) -> tuple[dict[str, PaperSummary], list[str]]: """Per-paper summaries for papers that still lack one, in batches of `batch_size`. Returns (doi -> summary, gap notes). A failed batch is skipped with a note - partial results are fine because summaries are persisted per DOI and retried on the next run. Papers without an - abstract are never summarised (제목만으로 요약하면 창작이 됨). + abstract are never summarised (제목만으로 요약하면 창작이 됨). With `verify`, each batch is + fact-checked against the abstracts and revised once on major issues (다이제스트 검증과 동일 정책). """ todo = [p for p in papers if p.summary is None and p.abstract.strip()] skipped = sum(1 for p in papers if p.summary is None and not p.abstract.strip()) @@ -53,6 +54,8 @@ def summarize_papers( log.warning("paper summary batch %d/%d failed: %s", n, len(batches), exc) notes.append(f"paper-summaries: batch {n}/{len(batches)} 실패 ({len(batch)}편, 다음 실행에서 재시도)") continue + if verify: + items = _checked_paper_batch(batch, items, provider) wanted = {p.doi for p in batch} matched = 0 for item in items: @@ -69,6 +72,44 @@ def summarize_papers( return results, notes +def _checked_paper_batch( + batch: list[Paper], items: list[PaperSummaryItem], provider: LLMProvider +) -> list[PaperSummaryItem]: + """Fact-check one batch; on major issues revise once and re-check. Any failure keeps the originals.""" + try: + raw_verification, model = provider.verify_paper_summaries(batch, items) + except LLMError as exc: + log.warning("paper batch verification failed; keeping the unverified batch: %s", exc) + return items + verification = _consistent(raw_verification) + log.info( + "paper batch verification by %s: ok=%s issues=%d (major=%s)", + model, + verification.ok, + len(verification.issues), + verification.has_major, + ) + if not verification.has_major: + return items + for issue in verification.issues: + log.warning("paper summary issue [%s] %s -> %s", issue.severity, issue.claim, issue.problem) + try: + revised, model = provider.revise_paper_summaries(batch, items, verification.issues) + log.info("paper batch revised by %s after verification issues", model) + re_verification, model = provider.verify_paper_summaries(batch, revised) + re_verification = _consistent(re_verification) + log.info( + "paper batch re-verification by %s: ok=%s issues=%d", + model, + re_verification.ok, + len(re_verification.issues), + ) + return revised + except LLMError as exc: + log.warning("paper batch revision failed; keeping the original batch: %s", exc) + return items + + def _consistent(verification: Verification) -> Verification: """Force `ok` to mean "no major issues" - verifier models are inconsistent about minor-only cases.""" ok = not verification.has_major diff --git a/tests/_fakes.py b/tests/_fakes.py index caf13ed..a453f03 100644 --- a/tests/_fakes.py +++ b/tests/_fakes.py @@ -138,6 +138,7 @@ class FakeStore: def __init__(self, *, pending: list[Paper] | None = None) -> None: self.upserted: list[list[Paper]] = [] self.saved_summaries: list[dict] = [] + self.published_dois: set[str] = set() self.pending = pending if pending is not None else [] self.digests: dict[str, object] = {} self.digested: list[tuple[list[str], str]] = [] @@ -157,6 +158,9 @@ class FakeStore: def mark_digested(self, dois: list[str], week: str) -> None: self.digested.append((list(dois), week)) + def digested_dois(self, week: str) -> set[str]: + return set(self.published_dois) + def has_digest(self, week: str) -> bool: return week in self.digests @@ -198,6 +202,8 @@ class FakeProvider: fail_summarize: bool = False, fail_revise: bool = False, fail_paper_summaries: bool = False, + paper_verifications: list | None = None, + revised_papers: list | None = None, ) -> None: self.digest = digest or make_digest() self.verifications = list(verifications or []) @@ -205,6 +211,8 @@ class FakeProvider: self.fail_summarize = fail_summarize self.fail_revise = fail_revise self.fail_paper_summaries = fail_paper_summaries + self.paper_verifications = list(paper_verifications or []) + self.revised_papers = revised_papers self.calls: list[str] = [] def summarize(self, payload): @@ -227,6 +235,20 @@ class FakeProvider: ] return items, "fake-model" + def verify_paper_summaries(self, papers, items): + self.calls.append("verify_papers") + if not self.paper_verifications: + from sebrief.models import Verification + + return Verification(ok=True, issues=[]), "fake-model" + return self.paper_verifications.pop(0), "fake-model" + + def revise_paper_summaries(self, papers, items, issues): + self.calls.append("revise_papers") + if self.revised_papers is not None: + return list(self.revised_papers), "fake-model" + return list(items), "fake-model" + def verify(self, payload, digest): self.calls.append("verify") if not self.verifications: diff --git a/tests/test_config.py b/tests/test_config.py index 89f8e90..0f7703d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -21,9 +21,11 @@ def test_defaults(): assert s.run_weekday == 0 assert s.run_time_kst == "07:00" assert s.catchup_window_hours == 72 - assert s.max_papers == 40 + assert s.max_papers == 60 assert s.backfill_days == 7 assert s.verify_enabled is True + assert s.paper_summary_enabled is True + assert s.paper_summary_batch_size == 8 assert s.rss_feeds == [] assert s.db_path == Path("data") / "sebrief.sqlite3" diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index bec0f94..a948e0b 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -77,9 +77,31 @@ def test_build_payload_caps_papers_and_notes_rollover(env): env["settings"] = env["settings"].model_copy(update={"max_papers": 3}) payload = build_payload(WEEK, env["settings"], env["store"]) assert len(payload.papers) == 3 + assert [p.doi for p in payload.papers] == ["10.1/p0", "10.1/p1", "10.1/p2"] assert any("이월" in g for g in payload.data_gaps) +def test_cap_never_drops_papers_already_in_this_weeks_digest(env): + # --force re-run: p3/p4 were published in this week's digest; the cap must drop rollover + # candidates (p0..p2) instead, even though they sort first. + env["store"].pending = [make_paper(f"10.1/p{i}") for i in range(5)] + env["store"].published_dois = {"10.1/p3", "10.1/p4"} + env["settings"] = env["settings"].model_copy(update={"max_papers": 3}) + payload = build_payload(WEEK, env["settings"], env["store"]) + dois = [p.doi for p in payload.papers] + assert "10.1/p3" in dois and "10.1/p4" in dois + assert len(dois) == 3 and dois == ["10.1/p0", "10.1/p3", "10.1/p4"] + + +def test_cap_keeps_all_published_even_beyond_max(env): + env["store"].pending = [make_paper(f"10.1/p{i}") for i in range(4)] + env["store"].published_dois = {f"10.1/p{i}" for i in range(4)} + env["settings"] = env["settings"].model_copy(update={"max_papers": 2}) + payload = build_payload(WEEK, env["settings"], env["store"]) + assert len(payload.papers) == 4 # a published paper is never dropped + assert not any("이월" in g for g in payload.data_gaps) + + # -------------------------------------------------------------------------------- run_digest def test_run_digest_happy_path_saves_marks_and_delivers(env): result = run_digest(WEEK, env["settings"]) diff --git a/tests/test_summarize.py b/tests/test_summarize.py index 13ab86e..9ae6b67 100644 --- a/tests/test_summarize.py +++ b/tests/test_summarize.py @@ -28,7 +28,7 @@ def test_summarize_dedups_gaps(): def test_summarize_papers_batches_and_maps_by_doi(): papers = [make_paper(f"10.1/p{i}") for i in range(5)] provider = FakeProvider() - summaries, notes = summarize_papers(papers, provider, batch_size=2) + summaries, notes = summarize_papers(papers, provider, batch_size=2, verify=False) assert notes == [] assert set(summaries) == {p.doi for p in papers} assert provider.calls == ["summarize_papers:2", "summarize_papers:2", "summarize_papers:1"] @@ -40,7 +40,7 @@ def test_summarize_papers_skips_already_summarized_and_abstractless(): no_abstract = make_paper("10.1/empty", abstract=" ") todo = make_paper("10.1/todo") provider = FakeProvider() - summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8) + summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8, verify=False) assert set(summaries) == {"10.1/todo"} assert provider.calls == ["summarize_papers:1"] @@ -53,7 +53,7 @@ def test_summarize_papers_failed_batch_is_noted_and_others_survive(): return super().summarize_papers(papers) papers = [make_paper(f"10.1/p{i}") for i in range(4)] - summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2) + summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2, verify=False) assert set(summaries) == {"10.1/p2", "10.1/p3"} assert len(notes) == 1 and "batch 1/2" in notes[0] and "재시도" in notes[0] @@ -65,10 +65,57 @@ def test_summarize_papers_discards_unknown_dois(): items.append(PaperSummaryItem(doi="10.9/invented", **make_summary().model_dump())) return items, model - summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8) + summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8, verify=False) assert set(summaries) == {"10.1/a"} and notes == [] +def test_paper_batch_verified_ok_keeps_originals(): + provider = FakeProvider() + summaries, notes = summarize_papers([make_paper("10.1/a")], provider, batch_size=8) + assert set(summaries) == {"10.1/a"} and notes == [] + assert provider.calls == ["summarize_papers:1", "verify_papers"] + + +def test_paper_batch_major_issue_revises_once_and_recheck(): + major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")]) + clean = Verification(ok=True, issues=[]) + revised = [PaperSummaryItem(doi="10.1/a", **make_summary(tldr="수정된 한 줄.").model_dump())] + provider = FakeProvider(paper_verifications=[major, clean], revised_papers=revised) + summaries, notes = summarize_papers([make_paper("10.1/a")], provider, batch_size=8) + assert summaries["10.1/a"].tldr == "수정된 한 줄." + assert notes == [] + assert provider.calls == ["summarize_papers:1", "verify_papers", "revise_papers", "verify_papers"] + + +def test_paper_batch_minor_only_skips_revision(): + minor = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="minor")]) + provider = FakeProvider(paper_verifications=[minor]) + summaries, _ = summarize_papers([make_paper("10.1/a")], provider, batch_size=8) + assert set(summaries) == {"10.1/a"} + assert provider.calls == ["summarize_papers:1", "verify_papers"] + + +def test_paper_batch_verify_failure_keeps_unverified_batch(): + class BrokenVerify(FakeProvider): + def verify_paper_summaries(self, papers, items): + raise LLMError("verifier down") + + summaries, notes = summarize_papers([make_paper("10.1/a")], BrokenVerify(), batch_size=8) + assert set(summaries) == {"10.1/a"} and notes == [] + + +def test_paper_batch_revise_failure_keeps_originals(): + major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")]) + + class BrokenRevise(FakeProvider): + def revise_paper_summaries(self, papers, items, issues): + raise LLMError("revise down") + + provider = BrokenRevise(paper_verifications=[major]) + summaries, _ = summarize_papers([make_paper("10.1/a")], provider, batch_size=8) + assert summaries["10.1/a"].tldr == "10.1/a 한 줄 요약입니다." + + def test_verify_ok_returns_original(): provider = FakeProvider(verifications=[Verification(ok=True, issues=[])]) digest = make_digest()