Add an LLM fact-check pass over the generated briefing

After summarisation the report is verified against the collected payload (numbers, directions, unsupported claims) by a second LLM call with a Verification JSON schema. Major issues trigger one revision and a re-check; minor-only verdicts are normalized to ok. Failures never block delivery. The verdict is stored in the result JSON and shown on the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-09-01 10:37:39 +09:00
parent 449b6d8ca3
commit ed48f4c96b
13 changed files with 499 additions and 47 deletions
@@ -25,8 +25,17 @@ from us2kr.llm.minimax import (
    report_response_format,
 )
 from us2kr.llm.prompts import SYSTEM_PROMPT, build_user_message
-from us2kr.models import EarningsItem, EconEvent, MarketPayload, NewsItem, Quote, Report
-from us2kr.summarize import build_provider, summarize_payload
+from us2kr.models import (
+    EarningsItem,
+    EconEvent,
+    MarketPayload,
+    NewsItem,
+    Quote,
+    Report,
+    Verification,
+    VerificationIssue,
+)
+from us2kr.summarize import build_provider, summarize_payload, verify_and_revise

 ET = ZoneInfo("America/New_York")
 KST = ZoneInfo("Asia/Seoul")
@@ -558,3 +567,132 @@ def test_summarize_payload_no_gaps_keeps_report() -> None:
    payload = make_payload().model_copy(update={"data_gaps": []})
    report, _ = summarize_payload(payload, StubProvider())
    assert report.data_gaps == []
+
+
+# ---------------------------------------------------------------------- verification
+GOOD_VERIFICATION: dict[str, Any] = {
+    "ok": False,
+    "issues": [
+        {
+            "claim": "S&P 500이 1.5% 하락했습니다.",
+            "problem": "제공된 데이터는 +0.81% 상승입니다.",
+            "severity": "major",
+        }
+    ],
+}
+
+
+def test_verify_sends_payload_and_draft_and_parses_verdict() -> None:
+    provider, client = make_provider([make_completion(json.dumps({"ok": True, "issues": []}))])
+    verification, model = provider.verify(make_payload(), Report.model_validate(GOOD_REPORT))
+
+    assert verification.ok is True and verification.issues == [] and model == "MiniMax-M3"
+    call = client.calls[0]
+    assert call["response_format"]["json_schema"]["name"] == "verification"
+    assert "팩트체커" in call["messages"][0]["content"]
+    user = call["messages"][1]["content"]
+    assert "## 검증 대상 브리핑 초안 (JSON)" in user
+    assert "반도체 강세에 나스닥 상승 마감" in user  # the draft under review
+    assert "S&P 500" in user  # the source data to check against
+
+
+def test_verify_verdict_has_major_property() -> None:
+    provider, _ = make_provider([make_completion(json.dumps(GOOD_VERIFICATION, ensure_ascii=False))])
+    verification, _ = provider.verify(make_payload(), Report.model_validate(GOOD_REPORT))
+    assert verification.ok is False and verification.has_major is True
+
+
+def test_verify_exhausts_models_with_task_tag() -> None:
+    provider, client = make_provider([make_status_error(500)] * 3)
+    with pytest.raises(LLMError, match=r"\(verify\)"):
+        provider.verify(make_payload(), Report.model_validate(GOOD_REPORT))
+    assert client.models_called == ["MiniMax-M3", "MiniMax-M2.7", "MiniMax-M2.5"]
+
+
+def test_revise_replays_draft_and_issues() -> None:
+    revised = {**GOOD_REPORT, "headline": "수정된 헤드라인"}
+    provider, client = make_provider([make_completion(json.dumps(revised, ensure_ascii=False))])
+    issues = [
+        VerificationIssue(claim="S&P 500이 1.5% 하락", problem="데이터는 +0.81%", severity="major")
+    ]
+    report, model = provider.revise(make_payload(), Report.model_validate(GOOD_REPORT), issues)
+
+    assert report.headline == "수정된 헤드라인" and model == "MiniMax-M3"
+    call = client.calls[0]
+    assert call["response_format"]["json_schema"]["name"] == "report"
+    messages = call["messages"]
+    assert messages[0]["content"] == SYSTEM_PROMPT  # same author persona as summarize
+    assert messages[2]["role"] == "assistant"  # the draft being corrected
+    assert "검증 이슈" in messages[3]["content"] and "[major]" in messages[3]["content"]
+
+
+# ------------------------------------------------------------------ verify_and_revise
+class GlueProvider:
+    """LLMProvider stub with scripted verify verdicts and one revise result."""
+
+    def __init__(
+        self,
+        verdicts: list[Verification],
+        revised: Report | None = None,
+        revise_fails: bool = False,
+    ) -> None:
+        self._verdicts = list(verdicts)
+        self._revised = revised
+        self._revise_fails = revise_fails
+        self.verify_calls = 0
+        self.revise_calls = 0
+
+    def summarize(self, payload: MarketPayload) -> tuple[Report, str]:
+        raise AssertionError("summarize must not be called")
+
+    def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]:
+        self.verify_calls += 1
+        return self._verdicts.pop(0), "stub"
+
+    def revise(self, payload, report, issues) -> tuple[Report, str]:
+        self.revise_calls += 1
+        if self._revise_fails:
+            raise LLMError("revise down")
+        assert self._revised is not None
+        return self._revised, "stub"
+
+
+def _issue(severity: str) -> VerificationIssue:
+    return VerificationIssue(claim="c", problem="p", severity=severity)  # type: ignore[arg-type]
+
+
+def test_verify_and_revise_clean_verdict_short_circuits() -> None:
+    provider = GlueProvider([Verification(ok=True)])
+    report = Report.model_validate(GOOD_REPORT)
+    out, verification = verify_and_revise(make_payload(), report, provider)
+    assert out is report and verification.ok is True
+    assert provider.verify_calls == 1 and provider.revise_calls == 0
+
+
+def test_verify_and_revise_minor_issues_do_not_revise_and_normalize_ok() -> None:
+    verdict = Verification(ok=False, issues=[_issue("minor")])  # verifiers over-report minor as not-ok
+    provider = GlueProvider([verdict])
+    report = Report.model_validate(GOOD_REPORT)
+    out, verification = verify_and_revise(make_payload(), report, provider)
+    assert out is report
+    assert verification.ok is True and len(verification.issues) == 1  # ok normalized to "no major"
+    assert provider.revise_calls == 0
+
+
+def test_verify_and_revise_major_issue_revises_and_rechecks() -> None:
+    revised = Report.model_validate({**GOOD_REPORT, "headline": "고친 헤드라인"})
+    provider = GlueProvider(
+        [Verification(ok=False, issues=[_issue("major")]), Verification(ok=True)], revised=revised
+    )
+    out, verification = verify_and_revise(make_payload(), Report.model_validate(GOOD_REPORT), provider)
+    assert out is revised and verification.ok is True
+    assert provider.verify_calls == 2 and provider.revise_calls == 1
+
+
+def test_verify_and_revise_revision_failure_keeps_original() -> None:
+    verdict = Verification(ok=False, issues=[_issue("major")])
+    provider = GlueProvider([verdict], revise_fails=True)
+    report = Report.model_validate(GOOD_REPORT)
+    out, verification = verify_and_revise(make_payload(), report, provider)
+    assert out is report and verification is verdict
+    assert provider.verify_calls == 1 and provider.revise_calls == 1
@@ -12,7 +12,7 @@ from us2kr import pipeline
 from us2kr.config import Settings
 from us2kr.llm.base import LLMError
 from us2kr.market_time import MarketClock
-from us2kr.models import EarningsItem, EconEvent, NewsItem, Quote, Report
+from us2kr.models import EarningsItem, EconEvent, NewsItem, Quote, Report, Verification

 KST = ZoneInfo("Asia/Seoul")
 ET = ZoneInfo("America/New_York")
@@ -103,6 +103,10 @@ class Env:
            self.summarize_payloads.append(payload)
            return sample_report(), "MiniMax-M3"

+        def verify_and_revise(payload, report, provider):
+            count("verify")
+            return report, Verification(ok=True)
+
        def render_markdown(payload, report, model_used):
            count("render")
            tag = "degraded" if report is None else report.headline
@@ -116,6 +120,7 @@ class Env:
        monkeypatch.setattr(pipeline, "fetch_earnings", fetch_earnings)
        monkeypatch.setattr(pipeline, "build_provider", lambda settings: object())
        monkeypatch.setattr(pipeline, "summarize_payload", summarize_payload)
+        monkeypatch.setattr(pipeline, "verify_and_revise", verify_and_revise)
        monkeypatch.setattr(pipeline, "render_markdown", render_markdown)
        monkeypatch.setattr(pipeline, "build_deliverers", lambda settings: self.deliverers)
        monkeypatch.setattr(pipeline, "_now", lambda: NOW)
@@ -540,3 +545,46 @@ def test_superseded_helper_with_real_clock():
        krx_date = clock.krx_date_for(session)
        assert clock.is_krx_session(krx_date) is False
        assert pipeline._superseded_before_krx_open(session, krx_date, clock) is expected, session
+
+
+# ---------------------------------------------------------------------- verification
+def test_run_briefing_records_verification(env):
+    result = pipeline.run_briefing(SESSION, env.settings)
+    assert env.calls["verify"] == 1
+    assert result.verification is not None and result.verification.ok is True
+    # the verdict is persisted with the report
+    assert '"verification"' in env.store.saved_results[0].model_dump_json()
+
+
+def test_run_briefing_verification_can_be_disabled(monkeypatch, tmp_path):
+    env = Env(monkeypatch, tmp_path, US2KR_VERIFY_ENABLED="false")
+    result = pipeline.run_briefing(SESSION, env.settings)
+    assert "verify" not in env.calls
+    assert result.verification is None and result.report is not None
+
+
+def test_run_briefing_verification_failure_is_non_fatal(env):
+    def boom(payload, report, provider):
+        raise RuntimeError("verifier down")
+
+    env.monkeypatch.setattr(pipeline, "verify_and_revise", boom)
+    result = pipeline.run_briefing(SESSION, env.settings)
+    assert result.verification is None
+    assert result.report is not None and result.degraded is False and result.error is None
+
+
+def test_run_briefing_no_llm_skips_verification(env):
+    result = pipeline.run_briefing(SESSION, env.settings, no_llm=True)
+    assert "verify" not in env.calls and result.verification is None
+
+
+def test_run_briefing_verification_replaces_report_with_revision(env):
+    revised = sample_report().model_copy(update={"headline": "검증 후 수정된 총평"})
+
+    def verify_and_revise(payload, report, provider):
+        return revised, Verification(ok=True)
+
+    env.monkeypatch.setattr(pipeline, "verify_and_revise", verify_and_revise)
+    result = pipeline.run_briefing(SESSION, env.settings)
+    assert result.report is revised
+    assert "검증 후 수정된 총평" in result.markdown