jsahn/US2KR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add an LLM fact-check pass over the generated briefing

Browse Source 
After summarisation the report is verified against the collected
payload (numbers, directions, unsupported claims) by a second LLM call
with a Verification JSON schema. Major issues trigger one revision and
a re-check; minor-only verdicts are normalized to ok. Failures never
block delivery. The verdict is stored in the result JSON and shown on
the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jsahn
2026-09-01 10:37:39 +09:00
co-authored by Claude Fable 5
parent 449b6d8ca3
commit ed48f4c96b
13 changed files with 499 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
us2kr/llm/minimax.py
+106 -38
View File
@@ -9,12 +9,19 @@ import re
from typing import Any, get_origin
import openai
from pydantic import ValidationError
from pydantic import BaseModel, ValidationError
from us2kr.config import Settings
from us2kr.llm.base import LLMError
from us2kr.llm.prompts import SYSTEM_PROMPT, build_user_message
from us2kr.models import MarketPayload, Report
from us2kr.llm.prompts import (
REVISE_INSTRUCTION,
SYSTEM_PROMPT,
VERIFY_SYSTEM_PROMPT,
build_user_message,
build_verify_message,
format_issues,
)
from us2kr.models import MarketPayload, Report, Verification, VerificationIssue
log = logging.getLogger(__name__)
@@ -46,16 +53,24 @@ def _strip_titles(node: Any) -> None:
_strip_titles(value)
def report_json_schema() -> dict[str, Any]:
"""JSON schema for `Report` suitable for `response_format` (no titles, closed object)."""
schema = copy.deepcopy(Report.model_json_schema())
def model_json_schema(model_cls: type[BaseModel]) -> dict[str, Any]:
"""JSON schema for a pydantic model suitable for `response_format` (no titles, closed object)."""
schema = copy.deepcopy(model_cls.model_json_schema())
_strip_titles(schema)
schema["additionalProperties"] = False
return schema
def model_response_format(model_cls: type[BaseModel], name: str) -> dict[str, Any]:
return {"type": "json_schema", "json_schema": {"name": name, "schema": model_json_schema(model_cls)}}
def report_json_schema() -> dict[str, Any]:
return model_json_schema(Report)
def report_response_format() -> dict[str, Any]:
return {"type": "json_schema", "json_schema": {"name": "report", "schema": report_json_schema()}}
return model_response_format(Report, "report")
def _first_json_object(text: str) -> str | None:
@@ -108,9 +123,15 @@ def extract_json_text(raw: str) -> str:
return text
_LIST_FIELDS: frozenset[str] = frozenset(
name for name, info in Report.model_fields.items() if get_origin(info.annotation) is list
)
def _list_fields(model_cls: type[BaseModel]) -> frozenset[str]:
"""Names of the model's plain string-list fields (targets for the string->list coercion)."""
return frozenset(
name
for name, info in model_cls.model_fields.items()
if get_origin(info.annotation) is list and info.annotation == list[str]
)
_BULLET_SPLIT = re.compile(r"(?:^|\n)\s*(?:[-*•]|\d+[.)])\s+")
_SENTENCE_SPLIT = re.compile(r"(?<=[.!?。])\s+")
@@ -125,8 +146,8 @@ def _split_prose(text: str) -> list[str]:
return parts or [text.strip()]
def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
"""Fix the common MiniMax schema slip of emitting list fields as a single string.
def coerce_model_data(data: Any, model_cls: type[BaseModel]) -> tuple[Any, list[str]]:
"""Fix the common MiniMax schema slip of emitting string-list fields as a single string.
Returns (data, names of coerced fields). Anything that is not a dict is returned untouched so that
pydantic can report the real error.
@@ -134,7 +155,7 @@ def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
if not isinstance(data, dict):
return data, []
coerced: list[str] = []
for name in _LIST_FIELDS:
for name in _list_fields(model_cls):
value = data.get(name)
if isinstance(value, str):
data[name] = _split_prose(value)
@@ -142,17 +163,25 @@ def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
return data, coerced
def parse_report(raw: str) -> Report:
"""Turn raw model content into a validated Report (raises ValueError / ValidationError)."""
def coerce_report_data(data: Any) -> tuple[Any, list[str]]:
return coerce_model_data(data, Report)
def parse_model[T: BaseModel](raw: str, model_cls: type[T]) -> T:
"""Turn raw model content into a validated instance (raises ValueError / ValidationError)."""
text = extract_json_text(raw)
try:
data = json.loads(text)
except json.JSONDecodeError:
return Report.model_validate_json(text) # let pydantic produce the canonical error
data, coerced = coerce_report_data(data)
return model_cls.model_validate_json(text) # let pydantic produce the canonical error
data, coerced = coerce_model_data(data, model_cls)
if coerced:
log.warning("coerced string -> list for report fields: %s", ", ".join(sorted(coerced)))
return Report.model_validate(data)
log.warning("coerced string -> list for %s fields: %s", model_cls.__name__, ", ".join(sorted(coerced)))
return model_cls.model_validate(data)
def parse_report(raw: str) -> Report:
return parse_model(raw, Report)
class MiniMaxProvider:
@@ -174,7 +203,6 @@ class MiniMaxProvider:
max_retries=2,
)
self._client = client
self._response_format = report_response_format()
@property
def models(self) -> list[str]:
@@ -182,28 +210,59 @@ class MiniMaxProvider:
# ------------------------------------------------------------------ public
def summarize(self, payload: MarketPayload) -> tuple[Report, str]:
user_message = build_user_message(payload)
base_messages: list[dict[str, Any]] = [
messages: list[dict[str, Any]] = [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_message},
{"role": "user", "content": build_user_message(payload)},
]
attempts: list[str] = []
for model in self._models:
result = self._attempt_model(model, base_messages, attempts)
if result is not None:
return result, model
raise LLMError("all MiniMax models failed: " + " | ".join(attempts))
return self._run(messages, Report, "report", task="summarize")
def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]:
"""Fact-check `report` against the payload. Raise LLMError on total failure."""
messages: list[dict[str, Any]] = [
{"role": "system", "content": VERIFY_SYSTEM_PROMPT},
{"role": "user", "content": build_verify_message(payload, report)},
]
return self._run(messages, Verification, "verification", task="verify")
def revise(
self, payload: MarketPayload, report: Report, issues: list[VerificationIssue]
) -> tuple[Report, str]:
"""Ask for a corrected Report given verification issues. Raise LLMError on total failure."""
messages: list[dict[str, Any]] = [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": build_user_message(payload)},
{"role": "assistant", "content": report.model_dump_json()},
{"role": "user", "content": REVISE_INSTRUCTION + format_issues(issues)},
]
return self._run(messages, Report, "report", task="revise")
# ----------------------------------------------------------------- helpers
def _attempt_model(
self, model: str, messages: list[dict[str, Any]], attempts: list[str]
) -> Report | None:
def _run[T: BaseModel](
self, messages: list[dict[str, Any]], model_cls: type[T], schema_name: str, task: str
) -> tuple[T, str]:
"""Try every configured model in order; return (parsed instance, model id)."""
response_format = model_response_format(model_cls, schema_name)
attempts: list[str] = []
for model in self._models:
result = self._attempt_model(model, messages, attempts, model_cls, response_format)
if result is not None:
return result, model
raise LLMError(f"all MiniMax models failed ({task}): " + " | ".join(attempts))
def _attempt_model[T: BaseModel](
self,
model: str,
messages: list[dict[str, Any]],
attempts: list[str],
model_cls: type[T],
response_format: dict[str, Any],
) -> T | None:
"""One model: initial call plus at most one repair retry. Returns None when the model failed."""
content = self._call(model, messages, attempts)
content = self._call(model, messages, attempts, response_format)
if content is None:
return None
try:
return parse_report(content)
return parse_model(content, model_cls)
except (ValidationError, ValueError) as exc:
error_text = _short(str(exc), 800)
attempts.append(f"{model}: invalid report ({_short(str(exc), 160)})")
@@ -218,18 +277,23 @@ class MiniMaxProvider:
{"role": "assistant", "content": _short(content, 12000)},
{"role": "user", "content": REPAIR_INSTRUCTION + error_text},
]
content = self._call(model, repair_messages, attempts, label="repair")
content = self._call(model, repair_messages, attempts, response_format, label="repair")
if content is None:
return None
try:
return parse_report(content)
return parse_model(content, model_cls)
except (ValidationError, ValueError) as exc:
attempts.append(f"{model} (repair): invalid report ({_short(str(exc), 160)})")
log.warning("model %s repair retry still invalid: %s", model, _short(str(exc), 300))
return None
def _call(
self, model: str, messages: list[dict[str, Any]], attempts: list[str], label: str = "initial"
self,
model: str,
messages: list[dict[str, Any]],
attempts: list[str],
response_format: dict[str, Any],
label: str = "initial",
) -> str | None:
"""Perform one chat completion. Returns content text, or None (attempt recorded) on failure."""
tag = model if label == "initial" else f"{model} ({label})"
@@ -241,7 +305,7 @@ class MiniMaxProvider:
model=model,
messages=messages,
max_completion_tokens=self._settings.minimax_max_completion_tokens,
response_format=self._response_format,
response_format=response_format,
extra_body=extra_body,
)
except OPENAI_ERRORS as exc:
@@ -297,8 +361,12 @@ def _short(text: str, limit: int) -> str:
__all__ = [
"OPENAI_ERRORS",
"MiniMaxProvider",
"coerce_model_data",
"coerce_report_data",
"extract_json_text",
"model_json_schema",
"model_response_format",
"parse_model",
"parse_report",
"report_json_schema",
"report_response_format",