jsahnandClaude Fable 5
ed48f4c96b
After summarisation the report is verified against the collected payload (numbers, directions, unsupported claims) by a second LLM call with a Verification JSON schema. Major issues trigger one revision and a re-check; minor-only verdicts are normalized to ok. Failures never block delivery. The verdict is stored in the result JSON and shown on the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
73 lines
2.9 KiB
Python
73 lines
2.9 KiB
Python
"""Glue between the pipeline and the LLM layer."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
from us2kr.config import Settings
|
|
from us2kr.llm.base import LLMError, LLMProvider
|
|
from us2kr.llm.minimax import MiniMaxProvider
|
|
from us2kr.models import MarketPayload, Report, Verification
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def build_provider(settings: Settings) -> LLMProvider:
|
|
"""Construct the configured LLM provider (MiniMax only for now)."""
|
|
return MiniMaxProvider(settings)
|
|
|
|
|
|
def summarize_payload(payload: MarketPayload, provider: LLMProvider) -> tuple[Report, str]:
|
|
"""Summarise via the provider and guarantee `report.data_gaps` includes every payload gap."""
|
|
report, model_used = provider.summarize(payload)
|
|
merged: list[str] = []
|
|
for gap in [*payload.data_gaps, *report.data_gaps]:
|
|
if gap and gap not in merged:
|
|
merged.append(gap)
|
|
if merged != list(report.data_gaps):
|
|
report = report.model_copy(update={"data_gaps": merged})
|
|
log.info("summary produced by %s (%d data gaps)", model_used, len(merged))
|
|
return report, model_used
|
|
|
|
|
|
def _consistent(verification: Verification) -> Verification:
|
|
"""Force `ok` to mean "no major issues" - verifier models are inconsistent about minor-only cases."""
|
|
ok = not verification.has_major
|
|
return verification if verification.ok == ok else verification.model_copy(update={"ok": ok})
|
|
|
|
|
|
def verify_and_revise(
|
|
payload: MarketPayload, report: Report, provider: LLMProvider
|
|
) -> tuple[Report, Verification]:
|
|
"""Fact-check the report; on major issues ask for one revision and re-check it.
|
|
|
|
Returns (possibly revised report, verification of the returned report). Raises LLMError only when
|
|
the *initial* verification cannot be produced; a failed revision keeps the original report.
|
|
"""
|
|
raw_verification, model = provider.verify(payload, report)
|
|
verification = _consistent(raw_verification)
|
|
log.info(
|
|
"verification by %s: ok=%s issues=%d (major=%s)",
|
|
model,
|
|
verification.ok,
|
|
len(verification.issues),
|
|
verification.has_major,
|
|
)
|
|
if not verification.has_major:
|
|
return report, verification
|
|
|
|
for issue in verification.issues:
|
|
log.warning("verification issue [%s] %s -> %s", issue.severity, issue.claim, issue.problem)
|
|
try:
|
|
revised, model = provider.revise(payload, report, verification.issues)
|
|
log.info("report revised by %s after verification issues", model)
|
|
re_verification, model = provider.verify(payload, revised)
|
|
re_verification = _consistent(re_verification)
|
|
log.info(
|
|
"re-verification by %s: ok=%s issues=%d", model, re_verification.ok, len(re_verification.issues)
|
|
)
|
|
return revised, re_verification
|
|
except LLMError as exc:
|
|
log.warning("revision failed; keeping the original report and verdict: %s", exc)
|
|
return report, verification
|