Add an LLM fact-check pass over the generated briefing

After summarisation the report is verified against the collected payload (numbers, directions, unsupported claims) by a second LLM call with a Verification JSON schema. Major issues trigger one revision and a re-check; minor-only verdicts are normalized to ok. Failures never block delivery. The verdict is stored in the result JSON and shown on the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-09-01 10:37:39 +09:00
parent 449b6d8ca3
commit ed48f4c96b
13 changed files with 499 additions and 47 deletions
@@ -25,8 +25,17 @@ from us2kr.llm.minimax import (
    report_response_format,
 )
 from us2kr.llm.prompts import SYSTEM_PROMPT, build_user_message
-from us2kr.models import EarningsItem, EconEvent, MarketPayload, NewsItem, Quote, Report
-from us2kr.summarize import build_provider, summarize_payload
+from us2kr.models import (
+    EarningsItem,
+    EconEvent,
+    MarketPayload,
+    NewsItem,
+    Quote,
+    Report,
+    Verification,
+    VerificationIssue,
+)
+from us2kr.summarize import build_provider, summarize_payload, verify_and_revise

 ET = ZoneInfo("America/New_York")
 KST = ZoneInfo("Asia/Seoul")
@@ -558,3 +567,132 @@ def test_summarize_payload_no_gaps_keeps_report() -> None:
    payload = make_payload().model_copy(update={"data_gaps": []})
    report, _ = summarize_payload(payload, StubProvider())
    assert report.data_gaps == []
+
+
+# ---------------------------------------------------------------------- verification
+GOOD_VERIFICATION: dict[str, Any] = {
+    "ok": False,
+    "issues": [
+        {
+            "claim": "S&P 500이 1.5% 하락했습니다.",
+            "problem": "제공된 데이터는 +0.81% 상승입니다.",
+            "severity": "major",
+        }
+    ],
+}
+
+
+def test_verify_sends_payload_and_draft_and_parses_verdict() -> None:
+    provider, client = make_provider([make_completion(json.dumps({"ok": True, "issues": []}))])
+    verification, model = provider.verify(make_payload(), Report.model_validate(GOOD_REPORT))
+
+    assert verification.ok is True and verification.issues == [] and model == "MiniMax-M3"
+    call = client.calls[0]
+    assert call["response_format"]["json_schema"]["name"] == "verification"
+    assert "팩트체커" in call["messages"][0]["content"]
+    user = call["messages"][1]["content"]
+    assert "## 검증 대상 브리핑 초안 (JSON)" in user
+    assert "반도체 강세에 나스닥 상승 마감" in user  # the draft under review
+    assert "S&P 500" in user  # the source data to check against
+
+
+def test_verify_verdict_has_major_property() -> None:
+    provider, _ = make_provider([make_completion(json.dumps(GOOD_VERIFICATION, ensure_ascii=False))])
+    verification, _ = provider.verify(make_payload(), Report.model_validate(GOOD_REPORT))
+    assert verification.ok is False and verification.has_major is True
+
+
+def test_verify_exhausts_models_with_task_tag() -> None:
+    provider, client = make_provider([make_status_error(500)] * 3)
+    with pytest.raises(LLMError, match=r"\(verify\)"):
+        provider.verify(make_payload(), Report.model_validate(GOOD_REPORT))
+    assert client.models_called == ["MiniMax-M3", "MiniMax-M2.7", "MiniMax-M2.5"]
+
+
+def test_revise_replays_draft_and_issues() -> None:
+    revised = {**GOOD_REPORT, "headline": "수정된 헤드라인"}
+    provider, client = make_provider([make_completion(json.dumps(revised, ensure_ascii=False))])
+    issues = [
+        VerificationIssue(claim="S&P 500이 1.5% 하락", problem="데이터는 +0.81%", severity="major")
+    ]
+    report, model = provider.revise(make_payload(), Report.model_validate(GOOD_REPORT), issues)
+
+    assert report.headline == "수정된 헤드라인" and model == "MiniMax-M3"
+    call = client.calls[0]
+    assert call["response_format"]["json_schema"]["name"] == "report"
+    messages = call["messages"]
+    assert messages[0]["content"] == SYSTEM_PROMPT  # same author persona as summarize
+    assert messages[2]["role"] == "assistant"  # the draft being corrected
+    assert "검증 이슈" in messages[3]["content"] and "[major]" in messages[3]["content"]
+
+
+# ------------------------------------------------------------------ verify_and_revise
+class GlueProvider:
+    """LLMProvider stub with scripted verify verdicts and one revise result."""
+
+    def __init__(
+        self,
+        verdicts: list[Verification],
+        revised: Report | None = None,
+        revise_fails: bool = False,
+    ) -> None:
+        self._verdicts = list(verdicts)
+        self._revised = revised
+        self._revise_fails = revise_fails
+        self.verify_calls = 0
+        self.revise_calls = 0
+
+    def summarize(self, payload: MarketPayload) -> tuple[Report, str]:
+        raise AssertionError("summarize must not be called")
+
+    def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]:
+        self.verify_calls += 1
+        return self._verdicts.pop(0), "stub"
+
+    def revise(self, payload, report, issues) -> tuple[Report, str]:
+        self.revise_calls += 1
+        if self._revise_fails:
+            raise LLMError("revise down")
+        assert self._revised is not None
+        return self._revised, "stub"
+
+
+def _issue(severity: str) -> VerificationIssue:
+    return VerificationIssue(claim="c", problem="p", severity=severity)  # type: ignore[arg-type]
+
+
+def test_verify_and_revise_clean_verdict_short_circuits() -> None:
+    provider = GlueProvider([Verification(ok=True)])
+    report = Report.model_validate(GOOD_REPORT)
+    out, verification = verify_and_revise(make_payload(), report, provider)
+    assert out is report and verification.ok is True
+    assert provider.verify_calls == 1 and provider.revise_calls == 0
+
+
+def test_verify_and_revise_minor_issues_do_not_revise_and_normalize_ok() -> None:
+    verdict = Verification(ok=False, issues=[_issue("minor")])  # verifiers over-report minor as not-ok
+    provider = GlueProvider([verdict])
+    report = Report.model_validate(GOOD_REPORT)
+    out, verification = verify_and_revise(make_payload(), report, provider)
+    assert out is report
+    assert verification.ok is True and len(verification.issues) == 1  # ok normalized to "no major"
+    assert provider.revise_calls == 0
+
+
+def test_verify_and_revise_major_issue_revises_and_rechecks() -> None:
+    revised = Report.model_validate({**GOOD_REPORT, "headline": "고친 헤드라인"})
+    provider = GlueProvider(
+        [Verification(ok=False, issues=[_issue("major")]), Verification(ok=True)], revised=revised
+    )
+    out, verification = verify_and_revise(make_payload(), Report.model_validate(GOOD_REPORT), provider)
+    assert out is revised and verification.ok is True
+    assert provider.verify_calls == 2 and provider.revise_calls == 1
+
+
+def test_verify_and_revise_revision_failure_keeps_original() -> None:
+    verdict = Verification(ok=False, issues=[_issue("major")])
+    provider = GlueProvider([verdict], revise_fails=True)
+    report = Report.model_validate(GOOD_REPORT)
+    out, verification = verify_and_revise(make_payload(), report, provider)
+    assert out is report and verification is verdict
+    assert provider.verify_calls == 1 and provider.revise_calls == 1