Add an LLM fact-check pass over the generated briefing

After summarisation the report is verified against the collected
payload (numbers, directions, unsupported claims) by a second LLM call
with a Verification JSON schema. Major issues trigger one revision and
a re-check; minor-only verdicts are normalized to ok. Failures never
block delivery. The verdict is stored in the result JSON and shown on
the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

us2kr/config.py

@@ -65,6 +65,9 @@ class Settings(BaseSettings):
     news_lookback_hours: int = Field(default=18, alias="US2KR_NEWS_LOOKBACK_HOURS")
     news_max_items: int = Field(default=40, alias="US2KR_NEWS_MAX_ITEMS")
 
+    # --- LLM verification (fact-check pass over the generated report) ---
+    verify_enabled: bool = Field(default=True, alias="US2KR_VERIFY_ENABLED")
+
     log_level: str = Field(default="INFO", alias="US2KR_LOG_LEVEL")
 
     @field_validator("minimax_fallback_models", "krx_extra_holidays", "searxng_queries", mode="before")

us2kr/llm/base.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import Protocol
 
-from us2kr.models import MarketPayload, Report
+from us2kr.models import MarketPayload, Report, Verification, VerificationIssue
 
 
 class LLMError(RuntimeError):
@@ -15,3 +15,13 @@ class LLMProvider(Protocol):
     def summarize(self, payload: MarketPayload) -> tuple[Report, str]:
         """Return (validated Report, model id that produced it). Raise LLMError on total failure."""
         ...
+
+    def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]:
+        """Fact-check `report` against the payload. Raise LLMError on total failure."""
+        ...
+
+    def revise(
+        self, payload: MarketPayload, report: Report, issues: list[VerificationIssue]
+    ) -> tuple[Report, str]:
+        """Return a corrected Report addressing `issues`. Raise LLMError on total failure."""
+        ...

us2kr/llm/minimax.py

@@ -9,12 +9,19 @@ import re
 from typing import Any, get_origin
 
 import openai
-from pydantic import ValidationError
+from pydantic import BaseModel, ValidationError
 
 from us2kr.config import Settings
 from us2kr.llm.base import LLMError
-from us2kr.llm.prompts import SYSTEM_PROMPT, build_user_message
-from us2kr.models import MarketPayload, Report
+from us2kr.llm.prompts import (
+    REVISE_INSTRUCTION,
+    SYSTEM_PROMPT,
+    VERIFY_SYSTEM_PROMPT,
+    build_user_message,
+    build_verify_message,
+    format_issues,
+)
+from us2kr.models import MarketPayload, Report, Verification, VerificationIssue
 
 log = logging.getLogger(__name__)
 
@@ -46,16 +53,24 @@ def _strip_titles(node: Any) -> None:
             _strip_titles(value)
 
 
-def report_json_schema() -> dict[str, Any]:
-    """JSON schema for `Report` suitable for `response_format` (no titles, closed object)."""
-    schema = copy.deepcopy(Report.model_json_schema())
+def model_json_schema(model_cls: type[BaseModel]) -> dict[str, Any]:
+    """JSON schema for a pydantic model suitable for `response_format` (no titles, closed object)."""
+    schema = copy.deepcopy(model_cls.model_json_schema())
     _strip_titles(schema)
     schema["additionalProperties"] = False
     return schema
 
 
+def model_response_format(model_cls: type[BaseModel], name: str) -> dict[str, Any]:
+    return {"type": "json_schema", "json_schema": {"name": name, "schema": model_json_schema(model_cls)}}
+
+
+def report_json_schema() -> dict[str, Any]:
+    return model_json_schema(Report)
+
+
 def report_response_format() -> dict[str, Any]:
-    return {"type": "json_schema", "json_schema": {"name": "report", "schema": report_json_schema()}}
+    return model_response_format(Report, "report")
 
 
 def _first_json_object(text: str) -> str | None:
@@ -108,9 +123,15 @@ def extract_json_text(raw: str) -> str:
     return text
 
 
-_LIST_FIELDS: frozenset[str] = frozenset(
-    name for name, info in Report.model_fields.items() if get_origin(info.annotation) is list
-)
+def _list_fields(model_cls: type[BaseModel]) -> frozenset[str]:
+    """Names of the model's plain string-list fields (targets for the string->list coercion)."""
+    return frozenset(
+        name
+        for name, info in model_cls.model_fields.items()
+        if get_origin(info.annotation) is list and info.annotation == list[str]
+    )
+
+
 _BULLET_SPLIT = re.compile(r"(?:^|\n)\s*(?:[-*•]|\d+[.)])\s+")
 _SENTENCE_SPLIT = re.compile(r"(?<=[.!?。])\s+")
 
@@ -125,8 +146,8 @@ def _split_prose(text: str) -> list[str]:
     return parts or [text.strip()]
 
 
-def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
-    """Fix the common MiniMax schema slip of emitting list fields as a single string.
+def coerce_model_data(data: Any, model_cls: type[BaseModel]) -> tuple[Any, list[str]]:
+    """Fix the common MiniMax schema slip of emitting string-list fields as a single string.
 
     Returns (data, names of coerced fields). Anything that is not a dict is returned untouched so that
     pydantic can report the real error.
@@ -134,7 +155,7 @@ def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
     if not isinstance(data, dict):
         return data, []
     coerced: list[str] = []
-    for name in _LIST_FIELDS:
+    for name in _list_fields(model_cls):
         value = data.get(name)
         if isinstance(value, str):
             data[name] = _split_prose(value)
@@ -142,17 +163,25 @@ def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
     return data, coerced
 
 
-def parse_report(raw: str) -> Report:
-    """Turn raw model content into a validated Report (raises ValueError / ValidationError)."""
+def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
+    return coerce_model_data(data, Report)
+
+
+def parse_model[T: BaseModel](raw: str, model_cls: type[T]) -> T:
+    """Turn raw model content into a validated instance (raises ValueError / ValidationError)."""
     text = extract_json_text(raw)
     try:
         data = json.loads(text)
     except json.JSONDecodeError:
-        return Report.model_validate_json(text)  # let pydantic produce the canonical error
-    data, coerced = coerce_report_data(data)
+        return model_cls.model_validate_json(text)  # let pydantic produce the canonical error
+    data, coerced = coerce_model_data(data, model_cls)
     if coerced:
-        log.warning("coerced string -> list for report fields: %s", ", ".join(sorted(coerced)))
-    return Report.model_validate(data)
+        log.warning("coerced string -> list for %s fields: %s", model_cls.__name__, ", ".join(sorted(coerced)))
+    return model_cls.model_validate(data)
+
+
+def parse_report(raw: str) -> Report:
+    return parse_model(raw, Report)
 
 
 class MiniMaxProvider:
@@ -174,7 +203,6 @@ class MiniMaxProvider:
                 max_retries=2,
             )
         self._client = client
-        self._response_format = report_response_format()
 
     @property
     def models(self) -> list[str]:
@@ -182,28 +210,59 @@ class MiniMaxProvider:
 
     # ------------------------------------------------------------------ public
     def summarize(self, payload: MarketPayload) -> tuple[Report, str]:
-        user_message = build_user_message(payload)
-        base_messages: list[dict[str, Any]] = [
+        messages: list[dict[str, Any]] = [
             {"role": "system", "content": SYSTEM_PROMPT},
-            {"role": "user", "content": user_message},
+            {"role": "user", "content": build_user_message(payload)},
         ]
-        attempts: list[str] = []
-        for model in self._models:
-            result = self._attempt_model(model, base_messages, attempts)
-            if result is not None:
-                return result, model
-        raise LLMError("all MiniMax models failed: " + " | ".join(attempts))
+        return self._run(messages, Report, "report", task="summarize")
+
+    def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]:
+        """Fact-check `report` against the payload. Raise LLMError on total failure."""
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": VERIFY_SYSTEM_PROMPT},
+            {"role": "user", "content": build_verify_message(payload, report)},
+        ]
+        return self._run(messages, Verification, "verification", task="verify")
+
+    def revise(
+        self, payload: MarketPayload, report: Report, issues: list[VerificationIssue]
+    ) -> tuple[Report, str]:
+        """Ask for a corrected Report given verification issues. Raise LLMError on total failure."""
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": build_user_message(payload)},
+            {"role": "assistant", "content": report.model_dump_json()},
+            {"role": "user", "content": REVISE_INSTRUCTION + format_issues(issues)},
+        ]
+        return self._run(messages, Report, "report", task="revise")
 
     # ----------------------------------------------------------------- helpers
-    def _attempt_model(
-        self, model: str, messages: list[dict[str, Any]], attempts: list[str]
-    ) -> Report | None:
+    def _run[T: BaseModel](
+        self, messages: list[dict[str, Any]], model_cls: type[T], schema_name: str, task: str
+    ) -> tuple[T, str]:
+        """Try every configured model in order; return (parsed instance, model id)."""
+        response_format = model_response_format(model_cls, schema_name)
+        attempts: list[str] = []
+        for model in self._models:
+            result = self._attempt_model(model, messages, attempts, model_cls, response_format)
+            if result is not None:
+                return result, model
+        raise LLMError(f"all MiniMax models failed ({task}): " + " | ".join(attempts))
+
+    def _attempt_model[T: BaseModel](
+        self,
+        model: str,
+        messages: list[dict[str, Any]],
+        attempts: list[str],
+        model_cls: type[T],
+        response_format: dict[str, Any],
+    ) -> T | None:
         """One model: initial call plus at most one repair retry. Returns None when the model failed."""
-        content = self._call(model, messages, attempts)
+        content = self._call(model, messages, attempts, response_format)
         if content is None:
             return None
         try:
-            return parse_report(content)
+            return parse_model(content, model_cls)
         except (ValidationError, ValueError) as exc:
             error_text = _short(str(exc), 800)
             attempts.append(f"{model}: invalid report ({_short(str(exc), 160)})")
@@ -218,18 +277,23 @@ class MiniMaxProvider:
             {"role": "assistant", "content": _short(content, 12000)},
             {"role": "user", "content": REPAIR_INSTRUCTION + error_text},
         ]
-        content = self._call(model, repair_messages, attempts, label="repair")
+        content = self._call(model, repair_messages, attempts, response_format, label="repair")
         if content is None:
             return None
         try:
-            return parse_report(content)
+            return parse_model(content, model_cls)
         except (ValidationError, ValueError) as exc:
             attempts.append(f"{model} (repair): invalid report ({_short(str(exc), 160)})")
             log.warning("model %s repair retry still invalid: %s", model, _short(str(exc), 300))
             return None
 
     def _call(
-        self, model: str, messages: list[dict[str, Any]], attempts: list[str], label: str = "initial"
+        self,
+        model: str,
+        messages: list[dict[str, Any]],
+        attempts: list[str],
+        response_format: dict[str, Any],
+        label: str = "initial",
     ) -> str | None:
         """Perform one chat completion. Returns content text, or None (attempt recorded) on failure."""
         tag = model if label == "initial" else f"{model} ({label})"
@@ -241,7 +305,7 @@ class MiniMaxProvider:
                 model=model,
                 messages=messages,
                 max_completion_tokens=self._settings.minimax_max_completion_tokens,
-                response_format=self._response_format,
+                response_format=response_format,
                 extra_body=extra_body,
             )
         except OPENAI_ERRORS as exc:
@@ -297,8 +361,12 @@ def _short(text: str, limit: int) -> str:
 __all__ = [
     "OPENAI_ERRORS",
     "MiniMaxProvider",
+    "coerce_model_data",
     "coerce_report_data",
     "extract_json_text",
+    "model_json_schema",
+    "model_response_format",
+    "parse_model",
     "parse_report",
     "report_json_schema",
     "report_response_format",

us2kr/llm/prompts.py

@@ -164,6 +164,48 @@ def _bullets(items: list[str]) -> list[str]:
     return [f"- {item}" for item in items] or ["(없음)"]
 
 
+VERIFY_SYSTEM_PROMPT: str = """당신은 미국 시장 브리핑의 팩트체커입니다. 브리핑 초안이 함께 제공된 원본 데이터와
+모순되지 않는지만 검사합니다. 문체·구성·의견은 평가하지 않습니다.
+
+## 검사 규칙
+1. 판단 근거는 사용자 메시지에 제공된 데이터(시세·히스토리 노트·뉴스·경제지표·실적)뿐입니다.
+   당신의 배경지식이나 최신 시황과 달라 보여도, 제공된 데이터와 일치하면 문제가 아닙니다.
+2. major: 브리핑의 숫자가 데이터와 다르거나(반올림 오차 ±0.05는 허용), 상승/하락 방향이 반대이거나,
+   데이터에 없는 숫자·사건을 사실처럼 서술한 경우.
+3. minor: 뉴스 헤드라인만으로 확인되지 않는 인과관계를 단정, 제공된 표기와 다른 종목명, 모호하지만
+   틀렸다고 단정할 수 없는 서술.
+4. 각 이슈의 claim에는 브리핑 원문을 그대로 인용하고, problem에는 데이터의 실제 값과 함께 한 문장으로
+   설명합니다.
+5. major 이슈가 하나도 없으면 ok=true 입니다. 사소한 트집을 만들지 말고, 문제가 없으면 빈 issues로
+   반환합니다.
+6. 출력은 지정된 JSON 스키마만 반환합니다. 스키마 밖 텍스트·설명·코드 펜스를 추가하지 않습니다.
+"""
+
+REVISE_INSTRUCTION: str = (
+    "위 브리핑 초안에서 아래 검증 이슈가 발견되었습니다. 제공된 원본 데이터에 맞게 해당 서술만 고치고,"
+    " 문제없는 부분은 그대로 유지하여 동일한 JSON 스키마로 전체 브리핑을 다시 출력하세요."
+    " 설명 없이 JSON 객체 하나만 출력합니다.\n\n검증 이슈:\n"
+)
+
+
+def build_verify_message(payload: MarketPayload, report: Report) -> str:
+    """User message for the fact-check pass: the payload serialisation plus the draft report."""
+    return (
+        build_user_message(payload)
+        + "\n## 검증 대상 브리핑 초안 (JSON)\n"
+        + report.model_dump_json(indent=2)
+        + "\n"
+    )
+
+
+def format_issues(issues: list) -> str:
+    """Issues -> numbered plain-text block for the revise instruction."""
+    lines = []
+    for i, issue in enumerate(issues, 1):
+        lines.append(f"{i}. [{issue.severity}] 서술: {issue.claim}\n   문제: {issue.problem}")
+    return "\n".join(lines) or "(없음)"
+
+
 def build_user_message(payload: MarketPayload) -> str:
     """Serialise the payload into a deterministic Korean text block for the user message."""
     out: list[str] = []

us2kr/models.py

@@ -91,6 +91,25 @@ class Report(BaseModel):
     data_gaps: list[str] = Field(default_factory=list, description="수집 실패로 반영하지 못한 항목")
 
 
+class VerificationIssue(BaseModel):
+    claim: str = Field(description="문제가 된 브리핑 속 서술 (원문 그대로 인용)")
+    problem: str = Field(description="제공된 데이터와 어떻게 다른지 한 문장")
+    severity: Literal["minor", "major"] = Field(
+        default="minor", description="major: 수치·방향이 데이터와 모순 / minor: 근거 불충분·표현 문제"
+    )
+
+
+class Verification(BaseModel):
+    """LLM fact-check verdict for a generated Report, judged only against the payload."""
+
+    ok: bool = Field(description="major 이슈가 하나도 없으면 true")
+    issues: list[VerificationIssue] = Field(default_factory=list, description="발견된 문제 목록 (없으면 빈 배열)")
+
+    @property
+    def has_major(self) -> bool:
+        return any(i.severity == "major" for i in self.issues)
+
+
 class BriefingResult(BaseModel):
     payload: MarketPayload
     report: Report | None
@@ -98,3 +117,4 @@ class BriefingResult(BaseModel):
     model_used: str | None = None
     degraded: bool = False  # True when LLM failed and only the numeric table was produced
     error: str | None = None
+    verification: Verification | None = None  # None: not run (disabled / degraded / verifier failed)

us2kr/pipeline.py

@@ -23,10 +23,10 @@ from us2kr.config import KST, Settings
 from us2kr.deliver import build_deliverers
 from us2kr.llm.base import LLMError
 from us2kr.market_time import MarketClock
-from us2kr.models import BriefingResult, MarketPayload, Report
+from us2kr.models import BriefingResult, MarketPayload, Report, Verification
 from us2kr.render import render_markdown
 from us2kr.store import Store
-from us2kr.summarize import build_provider, summarize_payload
+from us2kr.summarize import build_provider, summarize_payload, verify_and_revise
 from us2kr.symbols import SYMBOL_GROUPS
 
 log = logging.getLogger(__name__)
@@ -162,6 +162,18 @@ def _summarize(payload: MarketPayload, settings: Settings) -> tuple[Report | Non
         return None, None, f"{type(exc).__name__}: {exc}"
 
 
+def _verify(
+    payload: MarketPayload, report: Report, settings: Settings
+) -> tuple[Report, Verification | None]:
+    """Fact-check (and possibly revise) the report. Any failure keeps the report and returns None."""
+    try:
+        provider = build_provider(settings)
+        return verify_and_revise(payload, report, provider)
+    except Exception as exc:  # noqa: BLE001 - verification must never block the briefing
+        log.warning("verification failed, delivering unverified report: %s: %s", type(exc).__name__, exc)
+        return report, None
+
+
 def _superseded_before_krx_open(session: date, krx_date: date, clock: MarketClock) -> bool:
     """True when another US session closes before KRX next trades, so this briefing would be stale.
 
@@ -271,6 +283,7 @@ def run_briefing(
         report: Report | None = None
         model_used: str | None = None
         error: str | None = None
+        verification: Verification | None = None
         degraded = False
         if no_llm:
             degraded = True
@@ -278,6 +291,8 @@ def run_briefing(
         else:
             report, model_used, error = _summarize(payload, settings)
             degraded = report is None
+            if report is not None and settings.verify_enabled:
+                report, verification = _verify(payload, report, settings)
 
         markdown = render_markdown(payload, report, model_used)
         result = BriefingResult(
@@ -287,6 +302,7 @@ def run_briefing(
             model_used=model_used,
             degraded=degraded,
             error=error,
+            verification=verification,
         )
         if dry_run:
             print(markdown)

us2kr/summarize.py

@@ -5,9 +5,9 @@ from __future__ import annotations
 import logging
 
 from us2kr.config import Settings
-from us2kr.llm.base import LLMProvider
+from us2kr.llm.base import LLMError, LLMProvider
 from us2kr.llm.minimax import MiniMaxProvider
-from us2kr.models import MarketPayload, Report
+from us2kr.models import MarketPayload, Report, Verification
 
 log = logging.getLogger(__name__)
 
@@ -28,3 +28,45 @@ def summarize_payload(payload: MarketPayload, provider: LLMProvider) -> tuple[Re
         report = report.model_copy(update={"data_gaps": merged})
     log.info("summary produced by %s (%d data gaps)", model_used, len(merged))
     return report, model_used
+
+
+def _consistent(verification: Verification) -> Verification:
+    """Force `ok` to mean "no major issues" - verifier models are inconsistent about minor-only cases."""
+    ok = not verification.has_major
+    return verification if verification.ok == ok else verification.model_copy(update={"ok": ok})
+
+
+def verify_and_revise(
+    payload: MarketPayload, report: Report, provider: LLMProvider
+) -> tuple[Report, Verification]:
+    """Fact-check the report; on major issues ask for one revision and re-check it.
+
+    Returns (possibly revised report, verification of the returned report). Raises LLMError only when
+    the *initial* verification cannot be produced; a failed revision keeps the original report.
+    """
+    raw_verification, model = provider.verify(payload, report)
+    verification = _consistent(raw_verification)
+    log.info(
+        "verification by %s: ok=%s issues=%d (major=%s)",
+        model,
+        verification.ok,
+        len(verification.issues),
+        verification.has_major,
+    )
+    if not verification.has_major:
+        return report, verification
+
+    for issue in verification.issues:
+        log.warning("verification issue [%s] %s -> %s", issue.severity, issue.claim, issue.problem)
+    try:
+        revised, model = provider.revise(payload, report, verification.issues)
+        log.info("report revised by %s after verification issues", model)
+        re_verification, model = provider.verify(payload, revised)
+        re_verification = _consistent(re_verification)
+        log.info(
+            "re-verification by %s: ok=%s issues=%d", model, re_verification.ok, len(re_verification.issues)
+        )
+        return revised, re_verification
+    except LLMError as exc:
+        log.warning("revision failed; keeping the original report and verdict: %s", exc)
+        return report, verification