Add per-paper summary verification and cap safeguards
- Per-paper summary batches are fact-checked against abstracts (verify -> revise once on major issues -> re-verify), controlled by SEBRIEF_VERIFY_ENABLED - The paper cap never drops papers already published in this week's digest (--force re-runs could otherwise erase them from every digest) - SEBRIEF_MAX_PAPERS default 40 -> 60 (observed weekly volume ~47) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -138,6 +138,7 @@ class FakeStore:
|
||||
def __init__(self, *, pending: list[Paper] | None = None) -> None:
|
||||
self.upserted: list[list[Paper]] = []
|
||||
self.saved_summaries: list[dict] = []
|
||||
self.published_dois: set[str] = set()
|
||||
self.pending = pending if pending is not None else []
|
||||
self.digests: dict[str, object] = {}
|
||||
self.digested: list[tuple[list[str], str]] = []
|
||||
@@ -157,6 +158,9 @@ class FakeStore:
|
||||
def mark_digested(self, dois: list[str], week: str) -> None:
|
||||
self.digested.append((list(dois), week))
|
||||
|
||||
def digested_dois(self, week: str) -> set[str]:
|
||||
return set(self.published_dois)
|
||||
|
||||
def has_digest(self, week: str) -> bool:
|
||||
return week in self.digests
|
||||
|
||||
@@ -198,6 +202,8 @@ class FakeProvider:
|
||||
fail_summarize: bool = False,
|
||||
fail_revise: bool = False,
|
||||
fail_paper_summaries: bool = False,
|
||||
paper_verifications: list | None = None,
|
||||
revised_papers: list | None = None,
|
||||
) -> None:
|
||||
self.digest = digest or make_digest()
|
||||
self.verifications = list(verifications or [])
|
||||
@@ -205,6 +211,8 @@ class FakeProvider:
|
||||
self.fail_summarize = fail_summarize
|
||||
self.fail_revise = fail_revise
|
||||
self.fail_paper_summaries = fail_paper_summaries
|
||||
self.paper_verifications = list(paper_verifications or [])
|
||||
self.revised_papers = revised_papers
|
||||
self.calls: list[str] = []
|
||||
|
||||
def summarize(self, payload):
|
||||
@@ -227,6 +235,20 @@ class FakeProvider:
|
||||
]
|
||||
return items, "fake-model"
|
||||
|
||||
def verify_paper_summaries(self, papers, items):
|
||||
self.calls.append("verify_papers")
|
||||
if not self.paper_verifications:
|
||||
from sebrief.models import Verification
|
||||
|
||||
return Verification(ok=True, issues=[]), "fake-model"
|
||||
return self.paper_verifications.pop(0), "fake-model"
|
||||
|
||||
def revise_paper_summaries(self, papers, items, issues):
|
||||
self.calls.append("revise_papers")
|
||||
if self.revised_papers is not None:
|
||||
return list(self.revised_papers), "fake-model"
|
||||
return list(items), "fake-model"
|
||||
|
||||
def verify(self, payload, digest):
|
||||
self.calls.append("verify")
|
||||
if not self.verifications:
|
||||
|
||||
@@ -21,9 +21,11 @@ def test_defaults():
|
||||
assert s.run_weekday == 0
|
||||
assert s.run_time_kst == "07:00"
|
||||
assert s.catchup_window_hours == 72
|
||||
assert s.max_papers == 40
|
||||
assert s.max_papers == 60
|
||||
assert s.backfill_days == 7
|
||||
assert s.verify_enabled is True
|
||||
assert s.paper_summary_enabled is True
|
||||
assert s.paper_summary_batch_size == 8
|
||||
assert s.rss_feeds == []
|
||||
assert s.db_path == Path("data") / "sebrief.sqlite3"
|
||||
|
||||
|
||||
@@ -77,9 +77,31 @@ def test_build_payload_caps_papers_and_notes_rollover(env):
|
||||
env["settings"] = env["settings"].model_copy(update={"max_papers": 3})
|
||||
payload = build_payload(WEEK, env["settings"], env["store"])
|
||||
assert len(payload.papers) == 3
|
||||
assert [p.doi for p in payload.papers] == ["10.1/p0", "10.1/p1", "10.1/p2"]
|
||||
assert any("이월" in g for g in payload.data_gaps)
|
||||
|
||||
|
||||
def test_cap_never_drops_papers_already_in_this_weeks_digest(env):
|
||||
# --force re-run: p3/p4 were published in this week's digest; the cap must drop rollover
|
||||
# candidates (p0..p2) instead, even though they sort first.
|
||||
env["store"].pending = [make_paper(f"10.1/p{i}") for i in range(5)]
|
||||
env["store"].published_dois = {"10.1/p3", "10.1/p4"}
|
||||
env["settings"] = env["settings"].model_copy(update={"max_papers": 3})
|
||||
payload = build_payload(WEEK, env["settings"], env["store"])
|
||||
dois = [p.doi for p in payload.papers]
|
||||
assert "10.1/p3" in dois and "10.1/p4" in dois
|
||||
assert len(dois) == 3 and dois == ["10.1/p0", "10.1/p3", "10.1/p4"]
|
||||
|
||||
|
||||
def test_cap_keeps_all_published_even_beyond_max(env):
|
||||
env["store"].pending = [make_paper(f"10.1/p{i}") for i in range(4)]
|
||||
env["store"].published_dois = {f"10.1/p{i}" for i in range(4)}
|
||||
env["settings"] = env["settings"].model_copy(update={"max_papers": 2})
|
||||
payload = build_payload(WEEK, env["settings"], env["store"])
|
||||
assert len(payload.papers) == 4 # a published paper is never dropped
|
||||
assert not any("이월" in g for g in payload.data_gaps)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------------------- run_digest
|
||||
def test_run_digest_happy_path_saves_marks_and_delivers(env):
|
||||
result = run_digest(WEEK, env["settings"])
|
||||
|
||||
+51
-4
@@ -28,7 +28,7 @@ def test_summarize_dedups_gaps():
|
||||
def test_summarize_papers_batches_and_maps_by_doi():
|
||||
papers = [make_paper(f"10.1/p{i}") for i in range(5)]
|
||||
provider = FakeProvider()
|
||||
summaries, notes = summarize_papers(papers, provider, batch_size=2)
|
||||
summaries, notes = summarize_papers(papers, provider, batch_size=2, verify=False)
|
||||
assert notes == []
|
||||
assert set(summaries) == {p.doi for p in papers}
|
||||
assert provider.calls == ["summarize_papers:2", "summarize_papers:2", "summarize_papers:1"]
|
||||
@@ -40,7 +40,7 @@ def test_summarize_papers_skips_already_summarized_and_abstractless():
|
||||
no_abstract = make_paper("10.1/empty", abstract=" ")
|
||||
todo = make_paper("10.1/todo")
|
||||
provider = FakeProvider()
|
||||
summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8)
|
||||
summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8, verify=False)
|
||||
assert set(summaries) == {"10.1/todo"}
|
||||
assert provider.calls == ["summarize_papers:1"]
|
||||
|
||||
@@ -53,7 +53,7 @@ def test_summarize_papers_failed_batch_is_noted_and_others_survive():
|
||||
return super().summarize_papers(papers)
|
||||
|
||||
papers = [make_paper(f"10.1/p{i}") for i in range(4)]
|
||||
summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2)
|
||||
summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2, verify=False)
|
||||
assert set(summaries) == {"10.1/p2", "10.1/p3"}
|
||||
assert len(notes) == 1 and "batch 1/2" in notes[0] and "재시도" in notes[0]
|
||||
|
||||
@@ -65,10 +65,57 @@ def test_summarize_papers_discards_unknown_dois():
|
||||
items.append(PaperSummaryItem(doi="10.9/invented", **make_summary().model_dump()))
|
||||
return items, model
|
||||
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8)
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8, verify=False)
|
||||
assert set(summaries) == {"10.1/a"} and notes == []
|
||||
|
||||
|
||||
def test_paper_batch_verified_ok_keeps_originals():
|
||||
provider = FakeProvider()
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
|
||||
assert set(summaries) == {"10.1/a"} and notes == []
|
||||
assert provider.calls == ["summarize_papers:1", "verify_papers"]
|
||||
|
||||
|
||||
def test_paper_batch_major_issue_revises_once_and_recheck():
|
||||
major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")])
|
||||
clean = Verification(ok=True, issues=[])
|
||||
revised = [PaperSummaryItem(doi="10.1/a", **make_summary(tldr="수정된 한 줄.").model_dump())]
|
||||
provider = FakeProvider(paper_verifications=[major, clean], revised_papers=revised)
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
|
||||
assert summaries["10.1/a"].tldr == "수정된 한 줄."
|
||||
assert notes == []
|
||||
assert provider.calls == ["summarize_papers:1", "verify_papers", "revise_papers", "verify_papers"]
|
||||
|
||||
|
||||
def test_paper_batch_minor_only_skips_revision():
|
||||
minor = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="minor")])
|
||||
provider = FakeProvider(paper_verifications=[minor])
|
||||
summaries, _ = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
|
||||
assert set(summaries) == {"10.1/a"}
|
||||
assert provider.calls == ["summarize_papers:1", "verify_papers"]
|
||||
|
||||
|
||||
def test_paper_batch_verify_failure_keeps_unverified_batch():
|
||||
class BrokenVerify(FakeProvider):
|
||||
def verify_paper_summaries(self, papers, items):
|
||||
raise LLMError("verifier down")
|
||||
|
||||
summaries, notes = summarize_papers([make_paper("10.1/a")], BrokenVerify(), batch_size=8)
|
||||
assert set(summaries) == {"10.1/a"} and notes == []
|
||||
|
||||
|
||||
def test_paper_batch_revise_failure_keeps_originals():
|
||||
major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")])
|
||||
|
||||
class BrokenRevise(FakeProvider):
|
||||
def revise_paper_summaries(self, papers, items, issues):
|
||||
raise LLMError("revise down")
|
||||
|
||||
provider = BrokenRevise(paper_verifications=[major])
|
||||
summaries, _ = summarize_papers([make_paper("10.1/a")], provider, batch_size=8)
|
||||
assert summaries["10.1/a"].tldr == "10.1/a 한 줄 요약입니다."
|
||||
|
||||
|
||||
def test_verify_ok_returns_original():
|
||||
provider = FakeProvider(verifications=[Verification(ok=True, issues=[])])
|
||||
digest = make_digest()
|
||||
|
||||
Reference in New Issue
Block a user