106 lines
4.7 KiB
Python
106 lines
4.7 KiB
Python
"""Summarise glue: gap merging, per-paper batching, verification consistency, revise-once policy."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from _fakes import FakeProvider, make_digest, make_paper, make_payload, make_summary
|
|
|
|
from sebrief.llm.base import LLMError
|
|
from sebrief.models import PaperSummaryItem, Verification, VerificationIssue
|
|
from sebrief.summarize import summarize_papers, summarize_payload, verify_and_revise
|
|
|
|
|
|
def test_summarize_merges_payload_gaps_into_digest():
|
|
payload = make_payload(gaps=["openalex: down"])
|
|
provider = FakeProvider(digest=make_digest(data_gaps=["LLM이 적은 누락"]))
|
|
digest, model = summarize_payload(payload, provider)
|
|
assert model == "fake-model"
|
|
assert digest.data_gaps == ["openalex: down", "LLM이 적은 누락"]
|
|
|
|
|
|
def test_summarize_dedups_gaps():
|
|
payload = make_payload(gaps=["g1"])
|
|
provider = FakeProvider(digest=make_digest(data_gaps=["g1", "g2"]))
|
|
digest, _ = summarize_payload(payload, provider)
|
|
assert digest.data_gaps == ["g1", "g2"]
|
|
|
|
|
|
# -------------------------------------------------------------------- per-paper summaries
|
|
def test_summarize_papers_batches_and_maps_by_doi():
|
|
papers = [make_paper(f"10.1/p{i}") for i in range(5)]
|
|
provider = FakeProvider()
|
|
summaries, notes = summarize_papers(papers, provider, batch_size=2)
|
|
assert notes == []
|
|
assert set(summaries) == {p.doi for p in papers}
|
|
assert provider.calls == ["summarize_papers:2", "summarize_papers:2", "summarize_papers:1"]
|
|
assert summaries["10.1/p0"].tldr == "10.1/p0 한 줄 요약입니다."
|
|
|
|
|
|
def test_summarize_papers_skips_already_summarized_and_abstractless():
|
|
done = make_paper("10.1/done", summary=make_summary())
|
|
no_abstract = make_paper("10.1/empty", abstract=" ")
|
|
todo = make_paper("10.1/todo")
|
|
provider = FakeProvider()
|
|
summaries, _ = summarize_papers([done, no_abstract, todo], provider, batch_size=8)
|
|
assert set(summaries) == {"10.1/todo"}
|
|
assert provider.calls == ["summarize_papers:1"]
|
|
|
|
|
|
def test_summarize_papers_failed_batch_is_noted_and_others_survive():
|
|
class FlakyProvider(FakeProvider):
|
|
def summarize_papers(self, papers):
|
|
if any(p.doi == "10.1/p0" for p in papers):
|
|
raise LLMError("boom")
|
|
return super().summarize_papers(papers)
|
|
|
|
papers = [make_paper(f"10.1/p{i}") for i in range(4)]
|
|
summaries, notes = summarize_papers(papers, FlakyProvider(), batch_size=2)
|
|
assert set(summaries) == {"10.1/p2", "10.1/p3"}
|
|
assert len(notes) == 1 and "batch 1/2" in notes[0] and "재시도" in notes[0]
|
|
|
|
|
|
def test_summarize_papers_discards_unknown_dois():
|
|
class InventingProvider(FakeProvider):
|
|
def summarize_papers(self, papers):
|
|
items, model = super().summarize_papers(papers)
|
|
items.append(PaperSummaryItem(doi="10.9/invented", **make_summary().model_dump()))
|
|
return items, model
|
|
|
|
summaries, notes = summarize_papers([make_paper("10.1/a")], InventingProvider(), batch_size=8)
|
|
assert set(summaries) == {"10.1/a"} and notes == []
|
|
|
|
|
|
def test_verify_ok_returns_original():
|
|
provider = FakeProvider(verifications=[Verification(ok=True, issues=[])])
|
|
digest = make_digest()
|
|
out, verification = verify_and_revise(make_payload(), digest, provider)
|
|
assert out is digest and verification.ok
|
|
assert provider.calls == ["verify"]
|
|
|
|
|
|
def test_minor_only_issues_force_ok_true_and_skip_revision():
|
|
minor = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="minor")])
|
|
provider = FakeProvider(verifications=[minor])
|
|
_out, verification = verify_and_revise(make_payload(), make_digest(), provider)
|
|
assert verification.ok is True and len(verification.issues) == 1
|
|
assert provider.calls == ["verify"]
|
|
|
|
|
|
def test_major_issue_revises_once_and_reverifies():
|
|
major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")])
|
|
clean = Verification(ok=True, issues=[])
|
|
revised = make_digest(overview="수정된 총평입니다.")
|
|
provider = FakeProvider(verifications=[major, clean], revised=revised)
|
|
out, verification = verify_and_revise(make_payload(), make_digest(), provider)
|
|
assert out.overview == "수정된 총평입니다."
|
|
assert verification.ok is True
|
|
assert provider.calls == ["verify", "revise", "verify"]
|
|
|
|
|
|
def test_failed_revision_keeps_original_and_verdict():
|
|
major = Verification(ok=False, issues=[VerificationIssue(claim="c", problem="p", severity="major")])
|
|
provider = FakeProvider(verifications=[major], fail_revise=True)
|
|
digest = make_digest()
|
|
out, verification = verify_and_revise(make_payload(), digest, provider)
|
|
assert out is digest
|
|
assert verification.has_major and verification.ok is False
|