Add per-paper summary verification and cap safeguards
- Per-paper summary batches are fact-checked against abstracts (verify -> revise once on major issues -> re-verify), controlled by SEBRIEF_VERIFY_ENABLED - The paper cap never drops papers already published in this week's digest (--force re-runs could otherwise erase them from every digest) - SEBRIEF_MAX_PAPERS default 40 -> 60 (observed weekly volume ~47) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+51
-4
@@ -28,7 +28,7 @@ def test_summarize_dedups_gaps():
|
||||
def test_summarize_papers_batches_and_maps_by_doi():
|
||||
papers = [make_paper(f"10.1/p{i}") for i in range(5)]
|
||||
provider = FakeProvider()
|
||||
summaries, notes = summarize_papers(papers, provider, batch_size=2)
|
||||
summaries, notes = summarize_papers(papers, provider, batch_size=2, verify=False)
|
||||
assert notes == []
|
||||
assert set(summaries) == {p.doi for p in papers}
|
||||
assert provider.calls == ["summarize_papers:2", "summarize_papers:2", "summarize_papers:1"]
|
||||
@@ -40,7 +40,7 @@ def test_summarize_papers_skips_already_summarized_and_abstractless():
|
||||
no_abstract = make_paper("10.1/empty", abstract=" ")
|
||||
todo = make_paper("10.1/todo")
|
||||
provider = FakeProvider()
|
||||
summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8)
|
||||
summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8, verify=False)
|
||||
assert set(summaries) == {"10.1/todo"}
|
||||
assert provider.calls == ["summarize_papers:1"]
|
||||
|
||||
@@ -53,7 +53,7 @@ def test_summarize_papers_failed_batch_is_noted_and_others_survive():
|
||||
return super().summarize_papers(papers)
|
||||
|
||||
papers = [make_paper(f"10.1/p{i}") for i in range(4)]
|
||||
summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2)
|
||||
summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2, verify=False)
|
||||
assert set(summaries) == {"10.1/p2", "10.1/p3"}
|
||||
assert len(notes) == 1 and "batch 1/2" in notes[0] and "재시도" in notes[0]
|
||||
|
||||
@@ -65,10 +65,57 @@ def test_summarize_papers_discards_unknown_dois():
|
||||
items.append(PaperSummaryItem(doi="10.9/invented", **make_summary().model_dump()))
|
||||
return items, model
|
||||
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8)
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8, verify=False)
|
||||
assert set(summaries) == {"10.1/a"} and notes == []
|
||||
|
||||
|
||||
def test_paper_batch_verified_ok_keeps_originals():
|
||||
provider = FakeProvider()
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
|
||||
assert set(summaries) == {"10.1/a"} and notes == []
|
||||
assert provider.calls == ["summarize_papers:1", "verify_papers"]
|
||||
|
||||
|
||||
def test_paper_batch_major_issue_revises_once_and_recheck():
|
||||
major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")])
|
||||
clean = Verification(ok=True, issues=[])
|
||||
revised = [PaperSummaryItem(doi="10.1/a", **make_summary(tldr="수정된 한 줄.").model_dump())]
|
||||
provider = FakeProvider(paper_verifications=[major, clean], revised_papers=revised)
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
|
||||
assert summaries["10.1/a"].tldr == "수정된 한 줄."
|
||||
assert notes == []
|
||||
assert provider.calls == ["summarize_papers:1", "verify_papers", "revise_papers", "verify_papers"]
|
||||
|
||||
|
||||
def test_paper_batch_minor_only_skips_revision():
|
||||
minor = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="minor")])
|
||||
provider = FakeProvider(paper_verifications=[minor])
|
||||
summaries, _ = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
|
||||
assert set(summaries) == {"10.1/a"}
|
||||
assert provider.calls == ["summarize_papers:1", "verify_papers"]
|
||||
|
||||
|
||||
def test_paper_batch_verify_failure_keeps_unverified_batch():
|
||||
class BrokenVerify(FakeProvider):
|
||||
def verify_paper_summaries(self, papers, items):
|
||||
raise LLMError("verifier down")
|
||||
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], BrokenVerify(), batch_size=8)
|
||||
assert set(summaries) == {"10.1/a"} and notes == []
|
||||
|
||||
|
||||
def test_paper_batch_revise_failure_keeps_originals():
|
||||
major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")])
|
||||
|
||||
class BrokenRevise(FakeProvider):
|
||||
def revise_paper_summaries(self, papers, items, issues):
|
||||
raise LLMError("revise down")
|
||||
|
||||
provider = BrokenRevise(paper_verifications=[major])
|
||||
summaries, _ = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
|
||||
assert summaries["10.1/a"].tldr == "10.1/a 한 줄 요약입니다."
|
||||
|
||||
|
||||
def test_verify_ok_returns_original():
|
||||
provider = FakeProvider(verifications=[Verification(ok=True, issues=[])])
|
||||
digest = make_digest()
|
||||
|
||||
Reference in New Issue
Block a user