Add an LLM fact-check pass over the generated briefing

After summarisation the report is verified against the collected
payload (numbers, directions, unsupported claims) by a second LLM call
with a Verification JSON schema. Major issues trigger one revision and
a re-check; minor-only verdicts are normalized to ok. Failures never
block delivery. The verdict is stored in the result JSON and shown on
the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jsahn
2026-09-01 10:37:39 +09:00
co-authored by Claude Fable 5
parent 449b6d8ca3
commit ed48f4c96b
13 changed files with 499 additions and 47 deletions
+44 -2
View File
@@ -5,9 +5,9 @@ from __future__ import annotations
import logging
from us2kr.config import Settings
from us2kr.llm.base import LLMProvider
from us2kr.llm.base import LLMError, LLMProvider
from us2kr.llm.minimax import MiniMaxProvider
from us2kr.models import MarketPayload, Report
from us2kr.models import MarketPayload, Report, Verification
log = logging.getLogger(__name__)
@@ -28,3 +28,45 @@ def summarize_payload(payload: MarketPayload, provider: LLMProvider) -> tuple[Re
report = report.model_copy(update={"data_gaps": merged})
log.info("summary produced by %s (%d data gaps)", model_used, len(merged))
return report, model_used
def _consistent(verification: Verification) -> Verification:
"""Force `ok` to mean "no major issues" - verifier models are inconsistent about minor-only cases."""
ok = not verification.has_major
return verification if verification.ok == ok else verification.model_copy(update={"ok": ok})
def verify_and_revise(
payload: MarketPayload, report: Report, provider: LLMProvider
) -> tuple[Report, Verification]:
"""Fact-check the report; on major issues ask for one revision and re-check it.
Returns (possibly revised report, verification of the returned report). Raises LLMError only when
the *initial* verification cannot be produced; a failed revision keeps the original report.
"""
raw_verification, model = provider.verify(payload, report)
verification = _consistent(raw_verification)
log.info(
"verification by %s: ok=%s issues=%d (major=%s)",
model,
verification.ok,
len(verification.issues),
verification.has_major,
)
if not verification.has_major:
return report, verification
for issue in verification.issues:
log.warning("verification issue [%s] %s -> %s", issue.severity, issue.claim, issue.problem)
try:
revised, model = provider.revise(payload, report, verification.issues)
log.info("report revised by %s after verification issues", model)
re_verification, model = provider.verify(payload, revised)
re_verification = _consistent(re_verification)
log.info(
"re-verification by %s: ok=%s issues=%d", model, re_verification.ok, len(re_verification.issues)
)
return revised, re_verification
except LLMError as exc:
log.warning("revision failed; keeping the original report and verdict: %s", exc)
return report, verification