Add an LLM fact-check pass over the generated briefing
After summarisation the report is verified against the collected payload (numbers, directions, unsupported claims) by a second LLM call with a Verification JSON schema. Major issues trigger one revision and a re-check; minor-only verdicts are normalized to ok. Failures never block delivery. The verdict is stored in the result JSON and shown on the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+44
-2
@@ -5,9 +5,9 @@ from __future__ import annotations
|
||||
import logging
|
||||
|
||||
from us2kr.config import Settings
|
||||
from us2kr.llm.base import LLMProvider
|
||||
from us2kr.llm.base import LLMError, LLMProvider
|
||||
from us2kr.llm.minimax import MiniMaxProvider
|
||||
from us2kr.models import MarketPayload, Report
|
||||
from us2kr.models import MarketPayload, Report, Verification
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,3 +28,45 @@ def summarize_payload(payload: MarketPayload, provider: LLMProvider) -> tuple[Re
|
||||
report = report.model_copy(update={"data_gaps": merged})
|
||||
log.info("summary produced by %s (%d data gaps)", model_used, len(merged))
|
||||
return report, model_used
|
||||
|
||||
|
||||
def _consistent(verification: Verification) -> Verification:
|
||||
"""Force `ok` to mean "no major issues" - verifier models are inconsistent about minor-only cases."""
|
||||
ok = not verification.has_major
|
||||
return verification if verification.ok == ok else verification.model_copy(update={"ok": ok})
|
||||
|
||||
|
||||
def verify_and_revise(
|
||||
payload: MarketPayload, report: Report, provider: LLMProvider
|
||||
) -> tuple[Report, Verification]:
|
||||
"""Fact-check the report; on major issues ask for one revision and re-check it.
|
||||
|
||||
Returns (possibly revised report, verification of the returned report). Raises LLMError only when
|
||||
the *initial* verification cannot be produced; a failed revision keeps the original report.
|
||||
"""
|
||||
raw_verification, model = provider.verify(payload, report)
|
||||
verification = _consistent(raw_verification)
|
||||
log.info(
|
||||
"verification by %s: ok=%s issues=%d (major=%s)",
|
||||
model,
|
||||
verification.ok,
|
||||
len(verification.issues),
|
||||
verification.has_major,
|
||||
)
|
||||
if not verification.has_major:
|
||||
return report, verification
|
||||
|
||||
for issue in verification.issues:
|
||||
log.warning("verification issue [%s] %s -> %s", issue.severity, issue.claim, issue.problem)
|
||||
try:
|
||||
revised, model = provider.revise(payload, report, verification.issues)
|
||||
log.info("report revised by %s after verification issues", model)
|
||||
re_verification, model = provider.verify(payload, revised)
|
||||
re_verification = _consistent(re_verification)
|
||||
log.info(
|
||||
"re-verification by %s: ok=%s issues=%d", model, re_verification.ok, len(re_verification.issues)
|
||||
)
|
||||
return revised, re_verification
|
||||
except LLMError as exc:
|
||||
log.warning("revision failed; keeping the original report and verdict: %s", exc)
|
||||
return report, verification
|
||||
|
||||
Reference in New Issue
Block a user