Add per-paper summary verification and cap safeguards

- Per-paper summary batches are fact-checked against abstracts (verify ->
  revise once on major issues -> re-verify), controlled by SEBRIEF_VERIFY_ENABLED
- The paper cap never drops papers already published in this week's digest
  (--force re-runs could otherwise erase them from every digest)
- SEBRIEF_MAX_PAPERS default 40 -> 60 (observed weekly volume ~47)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jsahn
2026-09-02 17:21:41 +09:00
co-authored by Claude Fable 5
parent c0b140d27f
commit d9dda79282
13 changed files with 249 additions and 23 deletions
+51 -4
View File
@@ -28,7 +28,7 @@ def test_summarize_dedups_gaps():
def test_summarize_papers_batches_and_maps_by_doi():
papers = [make_paper(f"10.1/p{i}") for i in range(5)]
provider = FakeProvider()
summaries, notes = summarize_papers(papers, provider, batch_size=2)
summaries, notes = summarize_papers(papers, provider, batch_size=2, verify=False)
assert notes == []
assert set(summaries) == {p.doi for p in papers}
assert provider.calls == ["summarize_papers:2", "summarize_papers:2", "summarize_papers:1"]
@@ -40,7 +40,7 @@ def test_summarize_papers_skips_already_summarized_and_abstractless():
no_abstract = make_paper("10.1/empty", abstract=" ")
todo = make_paper("10.1/todo")
provider = FakeProvider()
summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8)
summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8, verify=False)
assert set(summaries) == {"10.1/todo"}
assert provider.calls == ["summarize_papers:1"]
@@ -53,7 +53,7 @@ def test_summarize_papers_failed_batch_is_noted_and_others_survive():
return super().summarize_papers(papers)
papers = [make_paper(f"10.1/p{i}") for i in range(4)]
summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2)
summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2, verify=False)
assert set(summaries) == {"10.1/p2", "10.1/p3"}
assert len(notes) == 1 and "batch 1/2" in notes[0] and "재시도" in notes[0]
@@ -65,10 +65,57 @@ def test_summarize_papers_discards_unknown_dois():
items.append(PaperSummaryItem(doi="10.9/invented", **make_summary().model_dump()))
return items, model
summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8)
summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8, verify=False)
assert set(summaries) == {"10.1/a"} and notes == []
def test_paper_batch_verified_ok_keeps_originals():
provider = FakeProvider()
summaries, notes = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
assert set(summaries) == {"10.1/a"} and notes == []
assert provider.calls == ["summarize_papers:1", "verify_papers"]
def test_paper_batch_major_issue_revises_once_and_recheck():
major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")])
clean = Verification(ok=True, issues=[])
revised = [PaperSummaryItem(doi="10.1/a", **make_summary(tldr="수정된 한 줄.").model_dump())]
provider = FakeProvider(paper_verifications=[major, clean], revised_papers=revised)
summaries, notes = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
assert summaries["10.1/a"].tldr == "수정된 한 줄."
assert notes == []
assert provider.calls == ["summarize_papers:1", "verify_papers", "revise_papers", "verify_papers"]
def test_paper_batch_minor_only_skips_revision():
minor = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="minor")])
provider = FakeProvider(paper_verifications=[minor])
summaries, _ = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
assert set(summaries) == {"10.1/a"}
assert provider.calls == ["summarize_papers:1", "verify_papers"]
def test_paper_batch_verify_failure_keeps_unverified_batch():
class BrokenVerify(FakeProvider):
def verify_paper_summaries(self, papers, items):
raise LLMError("verifier down")
summaries, notes = summarize_papers([make_paper("10.1/a")], BrokenVerify(), batch_size=8)
assert set(summaries) == {"10.1/a"} and notes == []
def test_paper_batch_revise_failure_keeps_originals():
major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")])
class BrokenRevise(FakeProvider):
def revise_paper_summaries(self, papers, items, issues):
raise LLMError("revise down")
provider = BrokenRevise(paper_verifications=[major])
summaries, _ = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
assert summaries["10.1/a"].tldr == "10.1/a 한 줄 요약입니다."
def test_verify_ok_returns_original():
provider = FakeProvider(verifications=[Verification(ok=True, issues=[])])
digest = make_digest()