Add an LLM fact-check pass over the generated briefing
After summarisation the report is verified against the collected payload (numbers, directions, unsupported claims) by a second LLM call with a Verification JSON schema. Major issues trigger one revision and a re-check; minor-only verdicts are normalized to ok. Failures never block delivery. The verdict is stored in the result JSON and shown on the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+18
-2
@@ -23,10 +23,10 @@ from us2kr.config import KST, Settings
|
||||
from us2kr.deliver import build_deliverers
|
||||
from us2kr.llm.base import LLMError
|
||||
from us2kr.market_time import MarketClock
|
||||
from us2kr.models import BriefingResult, MarketPayload, Report
|
||||
from us2kr.models import BriefingResult, MarketPayload, Report, Verification
|
||||
from us2kr.render import render_markdown
|
||||
from us2kr.store import Store
|
||||
from us2kr.summarize import build_provider, summarize_payload
|
||||
from us2kr.summarize import build_provider, summarize_payload, verify_and_revise
|
||||
from us2kr.symbols import SYMBOL_GROUPS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -162,6 +162,18 @@ def _summarize(payload: MarketPayload, settings: Settings) -> tuple[Report | Non
|
||||
return None, None, f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
def _verify(
|
||||
payload: MarketPayload, report: Report, settings: Settings
|
||||
) -> tuple[Report, Verification | None]:
|
||||
"""Fact-check (and possibly revise) the report. Any failure keeps the report and returns None."""
|
||||
try:
|
||||
provider = build_provider(settings)
|
||||
return verify_and_revise(payload, report, provider)
|
||||
except Exception as exc: # noqa: BLE001 - verification must never block the briefing
|
||||
log.warning("verification failed, delivering unverified report: %s: %s", type(exc).__name__, exc)
|
||||
return report, None
|
||||
|
||||
|
||||
def _superseded_before_krx_open(session: date, krx_date: date, clock: MarketClock) -> bool:
|
||||
"""True when another US session closes before KRX next trades, so this briefing would be stale.
|
||||
|
||||
@@ -271,6 +283,7 @@ def run_briefing(
|
||||
report: Report | None = None
|
||||
model_used: str | None = None
|
||||
error: str | None = None
|
||||
verification: Verification | None = None
|
||||
degraded = False
|
||||
if no_llm:
|
||||
degraded = True
|
||||
@@ -278,6 +291,8 @@ def run_briefing(
|
||||
else:
|
||||
report, model_used, error = _summarize(payload, settings)
|
||||
degraded = report is None
|
||||
if report is not None and settings.verify_enabled:
|
||||
report, verification = _verify(payload, report, settings)
|
||||
|
||||
markdown = render_markdown(payload, report, model_used)
|
||||
result = BriefingResult(
|
||||
@@ -287,6 +302,7 @@ def run_briefing(
|
||||
model_used=model_used,
|
||||
degraded=degraded,
|
||||
error=error,
|
||||
verification=verification,
|
||||
)
|
||||
if dry_run:
|
||||
print(markdown)
|
||||
|
||||
Reference in New Issue
Block a user