Add an LLM fact-check pass over the generated briefing

After summarisation the report is verified against the collected
payload (numbers, directions, unsupported claims) by a second LLM call
with a Verification JSON schema. Major issues trigger one revision and
a re-check; minor-only verdicts are normalized to ok. Failures never
block delivery. The verdict is stored in the result JSON and shown on
the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

us2kr/llm/base.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import Protocol
 
-from us2kr.models import MarketPayload, Report
+from us2kr.models import MarketPayload, Report, Verification, VerificationIssue
 
 
 class LLMError(RuntimeError):
@@ -15,3 +15,13 @@ class LLMProvider(Protocol):
     def summarize(self, payload: MarketPayload) -> tuple[Report, str]:
         """Return (validated Report, model id that produced it). Raise LLMError on total failure."""
         ...
+
+    def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]:
+        """Fact-check `report` against the payload. Raise LLMError on total failure."""
+        ...
+
+    def revise(
+        self, payload: MarketPayload, report: Report, issues: list[VerificationIssue]
+    ) -> tuple[Report, str]:
+        """Return a corrected Report addressing `issues`. Raise LLMError on total failure."""
+        ...

us2kr/llm/minimax.py

@@ -9,12 +9,19 @@ import re
 from typing import Any, get_origin
 
 import openai
-from pydantic import ValidationError
+from pydantic import BaseModel, ValidationError
 
 from us2kr.config import Settings
 from us2kr.llm.base import LLMError
-from us2kr.llm.prompts import SYSTEM_PROMPT, build_user_message
-from us2kr.models import MarketPayload, Report
+from us2kr.llm.prompts import (
+    REVISE_INSTRUCTION,
+    SYSTEM_PROMPT,
+    VERIFY_SYSTEM_PROMPT,
+    build_user_message,
+    build_verify_message,
+    format_issues,
+)
+from us2kr.models import MarketPayload, Report, Verification, VerificationIssue
 
 log = logging.getLogger(__name__)
 
@@ -46,16 +53,24 @@ def _strip_titles(node: Any) -> None:
             _strip_titles(value)
 
 
-def report_json_schema() -> dict[str, Any]:
-    """JSON schema for `Report` suitable for `response_format` (no titles, closed object)."""
-    schema = copy.deepcopy(Report.model_json_schema())
+def model_json_schema(model_cls: type[BaseModel]) -> dict[str, Any]:
+    """JSON schema for a pydantic model suitable for `response_format` (no titles, closed object)."""
+    schema = copy.deepcopy(model_cls.model_json_schema())
     _strip_titles(schema)
     schema["additionalProperties"] = False
     return schema
 
 
+def model_response_format(model_cls: type[BaseModel], name: str) -> dict[str, Any]:
+    return {"type": "json_schema", "json_schema": {"name": name, "schema": model_json_schema(model_cls)}}
+
+
+def report_json_schema() -> dict[str, Any]:
+    return model_json_schema(Report)
+
+
 def report_response_format() -> dict[str, Any]:
-    return {"type": "json_schema", "json_schema": {"name": "report", "schema": report_json_schema()}}
+    return model_response_format(Report, "report")
 
 
 def _first_json_object(text: str) -> str | None:
@@ -108,9 +123,15 @@ def extract_json_text(raw: str) -> str:
     return text
 
 
-_LIST_FIELDS: frozenset[str] = frozenset(
-    name for name, info in Report.model_fields.items() if get_origin(info.annotation) is list
-)
+def _list_fields(model_cls: type[BaseModel]) -> frozenset[str]:
+    """Names of the model's plain string-list fields (targets for the string->list coercion)."""
+    return frozenset(
+        name
+        for name, info in model_cls.model_fields.items()
+        if get_origin(info.annotation) is list and info.annotation == list[str]
+    )
+
+
 _BULLET_SPLIT = re.compile(r"(?:^|\n)\s*(?:[-*•]|\d+[.)])\s+")
 _SENTENCE_SPLIT = re.compile(r"(?<=[.!?。])\s+")
 
@@ -125,8 +146,8 @@ def _split_prose(text: str) -> list[str]:
     return parts or [text.strip()]
 
 
-def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
-    """Fix the common MiniMax schema slip of emitting list fields as a single string.
+def coerce_model_data(data: Any, model_cls: type[BaseModel]) -> tuple[Any, list[str]]:
+    """Fix the common MiniMax schema slip of emitting string-list fields as a single string.
 
     Returns (data, names of coerced fields). Anything that is not a dict is returned untouched so that
     pydantic can report the real error.
@@ -134,7 +155,7 @@ def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
     if not isinstance(data, dict):
         return data, []
     coerced: list[str] = []
-    for name in _LIST_FIELDS:
+    for name in _list_fields(model_cls):
         value = data.get(name)
         if isinstance(value, str):
             data[name] = _split_prose(value)
@@ -142,17 +163,25 @@ def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
     return data, coerced
 
 
-def parse_report(raw: str) -> Report:
-    """Turn raw model content into a validated Report (raises ValueError / ValidationError)."""
+def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
+    return coerce_model_data(data, Report)
+
+
+def parse_model[T: BaseModel](raw: str, model_cls: type[T]) -> T:
+    """Turn raw model content into a validated instance (raises ValueError / ValidationError)."""
     text = extract_json_text(raw)
     try:
         data = json.loads(text)
     except json.JSONDecodeError:
-        return Report.model_validate_json(text)  # let pydantic produce the canonical error
-    data, coerced = coerce_report_data(data)
+        return model_cls.model_validate_json(text)  # let pydantic produce the canonical error
+    data, coerced = coerce_model_data(data, model_cls)
     if coerced:
-        log.warning("coerced string -> list for report fields: %s", ", ".join(sorted(coerced)))
-    return Report.model_validate(data)
+        log.warning("coerced string -> list for %s fields: %s", model_cls.__name__, ", ".join(sorted(coerced)))
+    return model_cls.model_validate(data)
+
+
+def parse_report(raw: str) -> Report:
+    return parse_model(raw, Report)
 
 
 class MiniMaxProvider:
@@ -174,7 +203,6 @@ class MiniMaxProvider:
                 max_retries=2,
             )
         self._client = client
-        self._response_format = report_response_format()
 
     @property
     def models(self) -> list[str]:
@@ -182,28 +210,59 @@ class MiniMaxProvider:
 
     # ------------------------------------------------------------------ public
     def summarize(self, payload: MarketPayload) -> tuple[Report, str]:
-        user_message = build_user_message(payload)
-        base_messages: list[dict[str, Any]] = [
+        messages: list[dict[str, Any]] = [
             {"role": "system", "content": SYSTEM_PROMPT},
-            {"role": "user", "content": user_message},
+            {"role": "user", "content": build_user_message(payload)},
         ]
-        attempts: list[str] = []
-        for model in self._models:
-            result = self._attempt_model(model, base_messages, attempts)
-            if result is not None:
-                return result, model
-        raise LLMError("all MiniMax models failed: " + " | ".join(attempts))
+        return self._run(messages, Report, "report", task="summarize")
+
+    def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]:
+        """Fact-check `report` against the payload. Raise LLMError on total failure."""
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": VERIFY_SYSTEM_PROMPT},
+            {"role": "user", "content": build_verify_message(payload, report)},
+        ]
+        return self._run(messages, Verification, "verification", task="verify")
+
+    def revise(
+        self, payload: MarketPayload, report: Report, issues: list[VerificationIssue]
+    ) -> tuple[Report, str]:
+        """Ask for a corrected Report given verification issues. Raise LLMError on total failure."""
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": build_user_message(payload)},
+            {"role": "assistant", "content": report.model_dump_json()},
+            {"role": "user", "content": REVISE_INSTRUCTION + format_issues(issues)},
+        ]
+        return self._run(messages, Report, "report", task="revise")
 
     # ----------------------------------------------------------------- helpers
-    def _attempt_model(
-        self, model: str, messages: list[dict[str, Any]], attempts: list[str]
-    ) -> Report | None:
+    def _run[T: BaseModel](
+        self, messages: list[dict[str, Any]], model_cls: type[T], schema_name: str, task: str
+    ) -> tuple[T, str]:
+        """Try every configured model in order; return (parsed instance, model id)."""
+        response_format = model_response_format(model_cls, schema_name)
+        attempts: list[str] = []
+        for model in self._models:
+            result = self._attempt_model(model, messages, attempts, model_cls, response_format)
+            if result is not None:
+                return result, model
+        raise LLMError(f"all MiniMax models failed ({task}): " + " | ".join(attempts))
+
+    def _attempt_model[T: BaseModel](
+        self,
+        model: str,
+        messages: list[dict[str, Any]],
+        attempts: list[str],
+        model_cls: type[T],
+        response_format: dict[str, Any],
+    ) -> T | None:
         """One model: initial call plus at most one repair retry. Returns None when the model failed."""
-        content = self._call(model, messages, attempts)
+        content = self._call(model, messages, attempts, response_format)
         if content is None:
             return None
         try:
-            return parse_report(content)
+            return parse_model(content, model_cls)
         except (ValidationError, ValueError) as exc:
             error_text = _short(str(exc), 800)
             attempts.append(f"{model}: invalid report ({_short(str(exc), 160)})")
@@ -218,18 +277,23 @@ class MiniMaxProvider:
             {"role": "assistant", "content": _short(content, 12000)},
             {"role": "user", "content": REPAIR_INSTRUCTION + error_text},
         ]
-        content = self._call(model, repair_messages, attempts, label="repair")
+        content = self._call(model, repair_messages, attempts, response_format, label="repair")
         if content is None:
             return None
         try:
-            return parse_report(content)
+            return parse_model(content, model_cls)
         except (ValidationError, ValueError) as exc:
             attempts.append(f"{model} (repair): invalid report ({_short(str(exc), 160)})")
             log.warning("model %s repair retry still invalid: %s", model, _short(str(exc), 300))
             return None
 
     def _call(
-        self, model: str, messages: list[dict[str, Any]], attempts: list[str], label: str = "initial"
+        self,
+        model: str,
+        messages: list[dict[str, Any]],
+        attempts: list[str],
+        response_format: dict[str, Any],
+        label: str = "initial",
     ) -> str | None:
         """Perform one chat completion. Returns content text, or None (attempt recorded) on failure."""
         tag = model if label == "initial" else f"{model} ({label})"
@@ -241,7 +305,7 @@ class MiniMaxProvider:
                 model=model,
                 messages=messages,
                 max_completion_tokens=self._settings.minimax_max_completion_tokens,
-                response_format=self._response_format,
+                response_format=response_format,
                 extra_body=extra_body,
             )
         except OPENAI_ERRORS as exc:
@@ -297,8 +361,12 @@ def _short(text: str, limit: int) -> str:
 __all__ = [
     "OPENAI_ERRORS",
     "MiniMaxProvider",
+    "coerce_model_data",
     "coerce_report_data",
     "extract_json_text",
+    "model_json_schema",
+    "model_response_format",
+    "parse_model",
     "parse_report",
     "report_json_schema",
     "report_response_format",

us2kr/llm/prompts.py

@@ -164,6 +164,48 @@ def _bullets(items: list[str]) -> list[str]:
     return [f"- {item}" for item in items] or ["(없음)"]
 
 
+VERIFY_SYSTEM_PROMPT: str = """당신은 미국 시장 브리핑의 팩트체커입니다. 브리핑 초안이 함께 제공된 원본 데이터와
+모순되지 않는지만 검사합니다. 문체·구성·의견은 평가하지 않습니다.
+
+## 검사 규칙
+1. 판단 근거는 사용자 메시지에 제공된 데이터(시세·히스토리 노트·뉴스·경제지표·실적)뿐입니다.
+   당신의 배경지식이나 최신 시황과 달라 보여도, 제공된 데이터와 일치하면 문제가 아닙니다.
+2. major: 브리핑의 숫자가 데이터와 다르거나(반올림 오차 ±0.05는 허용), 상승/하락 방향이 반대이거나,
+   데이터에 없는 숫자·사건을 사실처럼 서술한 경우.
+3. minor: 뉴스 헤드라인만으로 확인되지 않는 인과관계를 단정, 제공된 표기와 다른 종목명, 모호하지만
+   틀렸다고 단정할 수 없는 서술.
+4. 각 이슈의 claim에는 브리핑 원문을 그대로 인용하고, problem에는 데이터의 실제 값과 함께 한 문장으로
+   설명합니다.
+5. major 이슈가 하나도 없으면 ok=true 입니다. 사소한 트집을 만들지 말고, 문제가 없으면 빈 issues로
+   반환합니다.
+6. 출력은 지정된 JSON 스키마만 반환합니다. 스키마 밖 텍스트·설명·코드 펜스를 추가하지 않습니다.
+"""
+
+REVISE_INSTRUCTION: str = (
+    "위 브리핑 초안에서 아래 검증 이슈가 발견되었습니다. 제공된 원본 데이터에 맞게 해당 서술만 고치고,"
+    " 문제없는 부분은 그대로 유지하여 동일한 JSON 스키마로 전체 브리핑을 다시 출력하세요."
+    " 설명 없이 JSON 객체 하나만 출력합니다.\n\n검증 이슈:\n"
+)
+
+
+def build_verify_message(payload: MarketPayload, report: Report) -> str:
+    """User message for the fact-check pass: the payload serialisation plus the draft report."""
+    return (
+        build_user_message(payload)
+        + "\n## 검증 대상 브리핑 초안 (JSON)\n"
+        + report.model_dump_json(indent=2)
+        + "\n"
+    )
+
+
+def format_issues(issues: list) -> str:
+    """Issues -> numbered plain-text block for the revise instruction."""
+    lines = []
+    for i, issue in enumerate(issues, 1):
+        lines.append(f"{i}. [{issue.severity}] 서술: {issue.claim}\n   문제: {issue.problem}")
+    return "\n".join(lines) or "(없음)"
+
+
 def build_user_message(payload: MarketPayload) -> str:
     """Serialise the payload into a deterministic Korean text block for the user message."""
     out: list[str] = []