From ed48f4c96baa4eab78adc064fdee6021da4ddb7c Mon Sep 17 00:00:00 2001 From: jsahn Date: Tue, 1 Sep 2026 10:37:39 +0900 Subject: [PATCH] Add an LLM fact-check pass over the generated briefing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After summarisation the report is verified against the collected payload (numbers, directions, unsupported claims) by a second LLM call with a Verification JSON schema. Major issues trigger one revision and a re-check; minor-only verdicts are normalized to ok. Failures never block delivery. The verdict is stored in the result JSON and shown on the dashboard (badge + 검증 메모). Disable with US2KR_VERIFY_ENABLED. Co-Authored-By: Claude Fable 5 --- README.md | 4 +- dashboard/src/client/main.ts | 27 +++++++ dashboard/src/shared/types.ts | 13 +++ dashboard/web/styles.css | 23 ++++++ tests/test_llm.py | 142 ++++++++++++++++++++++++++++++++- tests/test_pipeline.py | 50 +++++++++++- us2kr/config.py | 3 + us2kr/llm/base.py | 12 ++- us2kr/llm/minimax.py | 144 +++++++++++++++++++++++++--------- us2kr/llm/prompts.py | 42 ++++++++++ us2kr/models.py | 20 +++++ us2kr/pipeline.py | 20 ++++- us2kr/summarize.py | 46 ++++++++++- 13 files changed, 499 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 0d5de57..44bf609 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ 설계 문서는 [PLAN.md](PLAN.md)를 참고하세요. ``` -수집(yfinance · RSS · FRED) → SQLite 저장 → MiniMax-M3 요약(JSON 스키마) → Markdown 렌더 → Telegram / 파일 +수집(yfinance · RSS · SearXNG · FRED) → SQLite 저장 → MiniMax 요약(JSON 스키마) + → LLM 교차 검증(수치·방향 팩트체크, major 이슈면 1회 자동 수정) → Markdown 렌더 → Telegram / 파일 ``` ## 실행 시각 @@ -83,6 +84,7 @@ LaunchAgent(`container system start` + `container start us2kr`)가 담당합니 | `US2KR_NEWS_LOOKBACK_HOURS` / `US2KR_NEWS_MAX_ITEMS` | `18` / `40` | 뉴스 수집 범위 | | `SEARXNG_BASE_URL` | — | 있으면 RSS 외에 SearXNG 뉴스 검색 결과도 병합 (인스턴스에 `json` 포맷 활성화 필요) | | `US2KR_SEARXNG_QUERIES` | 기본 검색어 3개 | SearXNG 검색어 CSV (기본: `us2kr/collectors/searx.py`의 `DEFAULT_QUERIES`) | +| `US2KR_VERIFY_ENABLED` | `true` | 요약 후 LLM 팩트체크(수치·방향을 수집 데이터와 대조). major 이슈 발견 시 1회 자동 수정 후 재검증. 검증 실패는 발송을 막지 않고 결과는 JSON·대시보드에 기록 | | `US2KR_KRX_EXTRA_HOLIDAYS` | — | 캘린더에 없는 한국 임시휴장일 CSV (예: `2026-06-03`) | ## 산출물 diff --git a/dashboard/src/client/main.ts b/dashboard/src/client/main.ts index e357fd8..ae37c6b 100644 --- a/dashboard/src/client/main.ts +++ b/dashboard/src/client/main.ts @@ -190,6 +190,15 @@ function metaChips(result: BriefingResult): HTMLElement[] { ]; if (!p.krx_open_today) chips.push(el("span", "badge badge-warn", "한국 휴장")); if (result.degraded) chips.push(el("span", "badge badge-warn", "요약 없음")); + const v = result.verification; + if (v) { + const hasMajor = v.issues.some((i) => i.severity === "major"); + chips.push( + v.issues.length === 0 + ? el("span", "badge badge-ok", "검증 통과") + : el("span", `badge ${hasMajor ? "badge-error" : "badge-warn"}`, `검증 이슈 ${v.issues.length}`), + ); + } for (const gap of p.data_gaps) chips.push(el("span", "badge badge-warn", gap)); if (result.error) chips.push(el("span", "badge badge-error", result.error)); return chips; @@ -222,6 +231,24 @@ function renderSummaryCard(result: BriefingResult): void { } if (grid.childElementCount > 0) card.append(grid); host.append(card); + + const issues = result.verification?.issues ?? []; + if (issues.length > 0) { + const memo = el("section", "verify-memo"); + memo.append(el("h3", undefined, "검증 메모")); + const list = el("ul"); + for (const issue of issues) { + const item = el("li"); + item.append( + el("span", `verify-severity ${issue.severity}`, issue.severity === "major" ? "중대" : "경미"), + el("span", "verify-claim", issue.claim), + el("span", "verify-problem", ` — ${issue.problem}`), + ); + list.append(item); + } + memo.append(list); + host.append(memo); + } } async function showBriefing(krxDate: string): Promise { diff --git a/dashboard/src/shared/types.ts b/dashboard/src/shared/types.ts index 8a25465..551dc09 100644 --- a/dashboard/src/shared/types.ts +++ b/dashboard/src/shared/types.ts @@ -73,6 +73,17 @@ export interface Report { data_gaps: string[]; } +export interface VerificationIssue { + claim: string; + problem: string; + severity: "minor" | "major"; +} + +export interface Verification { + ok: boolean; + issues: VerificationIssue[]; +} + export interface BriefingResult { payload: MarketPayload; report: Report | null; @@ -80,6 +91,8 @@ export interface BriefingResult { model_used: string | null; degraded: boolean; error: string | null; + /** Absent/null: verification not run (disabled, degraded, verifier failed, or pre-feature archive). */ + verification?: Verification | null; } // ---------------------------------------------------------------- API shapes diff --git a/dashboard/web/styles.css b/dashboard/web/styles.css index 3c3d9f2..d587599 100644 --- a/dashboard/web/styles.css +++ b/dashboard/web/styles.css @@ -227,6 +227,29 @@ button[aria-pressed="true"] { background: var(--grid); } .summary-block ul { margin: 0; padding-left: 18px; } .summary-block li { margin: 2px 0; } +.verify-memo { + border: 1px dashed var(--baseline); + border-radius: 10px; + padding: 10px 14px; + margin: 0 0 14px; + font-size: 12.5px; +} +.verify-memo h3 { font-size: 12.5px; margin: 0 0 4px; color: var(--muted); font-weight: 650; } +.verify-memo ul { margin: 0; padding-left: 4px; list-style: none; } +.verify-memo li { margin: 3px 0; } +.verify-severity { + display: inline-block; + font-size: 11px; + font-weight: 600; + border-radius: 20px; + padding: 0 7px; + margin-right: 6px; +} +.verify-severity.major { background: var(--badge-err-bg); color: var(--badge-err-ink); } +.verify-severity.minor { background: var(--badge-warn-bg); color: var(--badge-warn-ink); } +.verify-claim { color: var(--ink); } +.verify-problem { color: var(--ink-2); } + /* ------------------------------------------------------------------ report */ .report-meta { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; font-size: 12.5px; color: var(--ink-2); margin-bottom: 4px; } .markdown { max-width: 76ch; } diff --git a/tests/test_llm.py b/tests/test_llm.py index 2a1711b..d3f541f 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -25,8 +25,17 @@ from us2kr.llm.minimax import ( report_response_format, ) from us2kr.llm.prompts import SYSTEM_PROMPT, build_user_message -from us2kr.models import EarningsItem, EconEvent, MarketPayload, NewsItem, Quote, Report -from us2kr.summarize import build_provider, summarize_payload +from us2kr.models import ( + EarningsItem, + EconEvent, + MarketPayload, + NewsItem, + Quote, + Report, + Verification, + VerificationIssue, +) +from us2kr.summarize import build_provider, summarize_payload, verify_and_revise ET = ZoneInfo("America/New_York") KST = ZoneInfo("Asia/Seoul") @@ -558,3 +567,132 @@ def test_summarize_payload_no_gaps_keeps_report() -> None: payload = make_payload().model_copy(update={"data_gaps": []}) report, _ = summarize_payload(payload, StubProvider()) assert report.data_gaps == [] + + +# ---------------------------------------------------------------------- verification +GOOD_VERIFICATION: dict[str, Any] = { + "ok": False, + "issues": [ + { + "claim": "S&P 500이 1.5% 하락했습니다.", + "problem": "제공된 데이터는 +0.81% 상승입니다.", + "severity": "major", + } + ], +} + + +def test_verify_sends_payload_and_draft_and_parses_verdict() -> None: + provider, client = make_provider([make_completion(json.dumps({"ok": True, "issues": []}))]) + verification, model = provider.verify(make_payload(), Report.model_validate(GOOD_REPORT)) + + assert verification.ok is True and verification.issues == [] and model == "MiniMax-M3" + call = client.calls[0] + assert call["response_format"]["json_schema"]["name"] == "verification" + assert "팩트체커" in call["messages"][0]["content"] + user = call["messages"][1]["content"] + assert "## 검증 대상 브리핑 초안 (JSON)" in user + assert "반도체 강세에 나스닥 상승 마감" in user # the draft under review + assert "S&P 500" in user # the source data to check against + + +def test_verify_verdict_has_major_property() -> None: + provider, _ = make_provider([make_completion(json.dumps(GOOD_VERIFICATION, ensure_ascii=False))]) + verification, _ = provider.verify(make_payload(), Report.model_validate(GOOD_REPORT)) + assert verification.ok is False and verification.has_major is True + + +def test_verify_exhausts_models_with_task_tag() -> None: + provider, client = make_provider([make_status_error(500)] * 3) + with pytest.raises(LLMError, match=r"\(verify\)"): + provider.verify(make_payload(), Report.model_validate(GOOD_REPORT)) + assert client.models_called == ["MiniMax-M3", "MiniMax-M2.7", "MiniMax-M2.5"] + + +def test_revise_replays_draft_and_issues() -> None: + revised = {**GOOD_REPORT, "headline": "수정된 헤드라인"} + provider, client = make_provider([make_completion(json.dumps(revised, ensure_ascii=False))]) + issues = [ + VerificationIssue(claim="S&P 500이 1.5% 하락", problem="데이터는 +0.81%", severity="major") + ] + report, model = provider.revise(make_payload(), Report.model_validate(GOOD_REPORT), issues) + + assert report.headline == "수정된 헤드라인" and model == "MiniMax-M3" + call = client.calls[0] + assert call["response_format"]["json_schema"]["name"] == "report" + messages = call["messages"] + assert messages[0]["content"] == SYSTEM_PROMPT # same author persona as summarize + assert messages[2]["role"] == "assistant" # the draft being corrected + assert "검증 이슈" in messages[3]["content"] and "[major]" in messages[3]["content"] + + +# ------------------------------------------------------------------ verify_and_revise +class GlueProvider: + """LLMProvider stub with scripted verify verdicts and one revise result.""" + + def __init__( + self, + verdicts: list[Verification], + revised: Report | None = None, + revise_fails: bool = False, + ) -> None: + self._verdicts = list(verdicts) + self._revised = revised + self._revise_fails = revise_fails + self.verify_calls = 0 + self.revise_calls = 0 + + def summarize(self, payload: MarketPayload) -> tuple[Report, str]: + raise AssertionError("summarize must not be called") + + def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]: + self.verify_calls += 1 + return self._verdicts.pop(0), "stub" + + def revise(self, payload, report, issues) -> tuple[Report, str]: + self.revise_calls += 1 + if self._revise_fails: + raise LLMError("revise down") + assert self._revised is not None + return self._revised, "stub" + + +def _issue(severity: str) -> VerificationIssue: + return VerificationIssue(claim="c", problem="p", severity=severity) # type: ignore[arg-type] + + +def test_verify_and_revise_clean_verdict_short_circuits() -> None: + provider = GlueProvider([Verification(ok=True)]) + report = Report.model_validate(GOOD_REPORT) + out, verification = verify_and_revise(make_payload(), report, provider) + assert out is report and verification.ok is True + assert provider.verify_calls == 1 and provider.revise_calls == 0 + + +def test_verify_and_revise_minor_issues_do_not_revise_and_normalize_ok() -> None: + verdict = Verification(ok=False, issues=[_issue("minor")]) # verifiers over-report minor as not-ok + provider = GlueProvider([verdict]) + report = Report.model_validate(GOOD_REPORT) + out, verification = verify_and_revise(make_payload(), report, provider) + assert out is report + assert verification.ok is True and len(verification.issues) == 1 # ok normalized to "no major" + assert provider.revise_calls == 0 + + +def test_verify_and_revise_major_issue_revises_and_rechecks() -> None: + revised = Report.model_validate({**GOOD_REPORT, "headline": "고친 헤드라인"}) + provider = GlueProvider( + [Verification(ok=False, issues=[_issue("major")]), Verification(ok=True)], revised=revised + ) + out, verification = verify_and_revise(make_payload(), Report.model_validate(GOOD_REPORT), provider) + assert out is revised and verification.ok is True + assert provider.verify_calls == 2 and provider.revise_calls == 1 + + +def test_verify_and_revise_revision_failure_keeps_original() -> None: + verdict = Verification(ok=False, issues=[_issue("major")]) + provider = GlueProvider([verdict], revise_fails=True) + report = Report.model_validate(GOOD_REPORT) + out, verification = verify_and_revise(make_payload(), report, provider) + assert out is report and verification is verdict + assert provider.verify_calls == 1 and provider.revise_calls == 1 diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 793a13b..c6105d6 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -12,7 +12,7 @@ from us2kr import pipeline from us2kr.config import Settings from us2kr.llm.base import LLMError from us2kr.market_time import MarketClock -from us2kr.models import EarningsItem, EconEvent, NewsItem, Quote, Report +from us2kr.models import EarningsItem, EconEvent, NewsItem, Quote, Report, Verification KST = ZoneInfo("Asia/Seoul") ET = ZoneInfo("America/New_York") @@ -103,6 +103,10 @@ class Env: self.summarize_payloads.append(payload) return sample_report(), "MiniMax-M3" + def verify_and_revise(payload, report, provider): + count("verify") + return report, Verification(ok=True) + def render_markdown(payload, report, model_used): count("render") tag = "degraded" if report is None else report.headline @@ -116,6 +120,7 @@ class Env: monkeypatch.setattr(pipeline, "fetch_earnings", fetch_earnings) monkeypatch.setattr(pipeline, "build_provider", lambda settings: object()) monkeypatch.setattr(pipeline, "summarize_payload", summarize_payload) + monkeypatch.setattr(pipeline, "verify_and_revise", verify_and_revise) monkeypatch.setattr(pipeline, "render_markdown", render_markdown) monkeypatch.setattr(pipeline, "build_deliverers", lambda settings: self.deliverers) monkeypatch.setattr(pipeline, "_now", lambda: NOW) @@ -540,3 +545,46 @@ def test_superseded_helper_with_real_clock(): krx_date = clock.krx_date_for(session) assert clock.is_krx_session(krx_date) is False assert pipeline._superseded_before_krx_open(session, krx_date, clock) is expected, session + + +# ---------------------------------------------------------------------- verification +def test_run_briefing_records_verification(env): + result = pipeline.run_briefing(SESSION, env.settings) + assert env.calls["verify"] == 1 + assert result.verification is not None and result.verification.ok is True + # the verdict is persisted with the report + assert '"verification"' in env.store.saved_results[0].model_dump_json() + + +def test_run_briefing_verification_can_be_disabled(monkeypatch, tmp_path): + env = Env(monkeypatch, tmp_path, US2KR_VERIFY_ENABLED="false") + result = pipeline.run_briefing(SESSION, env.settings) + assert "verify" not in env.calls + assert result.verification is None and result.report is not None + + +def test_run_briefing_verification_failure_is_non_fatal(env): + def boom(payload, report, provider): + raise RuntimeError("verifier down") + + env.monkeypatch.setattr(pipeline, "verify_and_revise", boom) + result = pipeline.run_briefing(SESSION, env.settings) + assert result.verification is None + assert result.report is not None and result.degraded is False and result.error is None + + +def test_run_briefing_no_llm_skips_verification(env): + result = pipeline.run_briefing(SESSION, env.settings, no_llm=True) + assert "verify" not in env.calls and result.verification is None + + +def test_run_briefing_verification_replaces_report_with_revision(env): + revised = sample_report().model_copy(update={"headline": "검증 후 수정된 총평"}) + + def verify_and_revise(payload, report, provider): + return revised, Verification(ok=True) + + env.monkeypatch.setattr(pipeline, "verify_and_revise", verify_and_revise) + result = pipeline.run_briefing(SESSION, env.settings) + assert result.report is revised + assert "검증 후 수정된 총평" in result.markdown diff --git a/us2kr/config.py b/us2kr/config.py index 8f818d6..d4a6df5 100644 --- a/us2kr/config.py +++ b/us2kr/config.py @@ -65,6 +65,9 @@ class Settings(BaseSettings): news_lookback_hours: int = Field(default=18, alias="US2KR_NEWS_LOOKBACK_HOURS") news_max_items: int = Field(default=40, alias="US2KR_NEWS_MAX_ITEMS") + # --- LLM verification (fact-check pass over the generated report) --- + verify_enabled: bool = Field(default=True, alias="US2KR_VERIFY_ENABLED") + log_level: str = Field(default="INFO", alias="US2KR_LOG_LEVEL") @field_validator("minimax_fallback_models", "krx_extra_holidays", "searxng_queries", mode="before") diff --git a/us2kr/llm/base.py b/us2kr/llm/base.py index c610e1d..48652f5 100644 --- a/us2kr/llm/base.py +++ b/us2kr/llm/base.py @@ -4,7 +4,7 @@ from __future__ import annotations from typing import Protocol -from us2kr.models import MarketPayload, Report +from us2kr.models import MarketPayload, Report, Verification, VerificationIssue class LLMError(RuntimeError): @@ -15,3 +15,13 @@ class LLMProvider(Protocol): def summarize(self, payload: MarketPayload) -> tuple[Report, str]: """Return (validated Report, model id that produced it). Raise LLMError on total failure.""" ... + + def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]: + """Fact-check `report` against the payload. Raise LLMError on total failure.""" + ... + + def revise( + self, payload: MarketPayload, report: Report, issues: list[VerificationIssue] + ) -> tuple[Report, str]: + """Return a corrected Report addressing `issues`. Raise LLMError on total failure.""" + ... diff --git a/us2kr/llm/minimax.py b/us2kr/llm/minimax.py index 535d971..8898c2e 100644 --- a/us2kr/llm/minimax.py +++ b/us2kr/llm/minimax.py @@ -9,12 +9,19 @@ import re from typing import Any, get_origin import openai -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from us2kr.config import Settings from us2kr.llm.base import LLMError -from us2kr.llm.prompts import SYSTEM_PROMPT, build_user_message -from us2kr.models import MarketPayload, Report +from us2kr.llm.prompts import ( + REVISE_INSTRUCTION, + SYSTEM_PROMPT, + VERIFY_SYSTEM_PROMPT, + build_user_message, + build_verify_message, + format_issues, +) +from us2kr.models import MarketPayload, Report, Verification, VerificationIssue log = logging.getLogger(__name__) @@ -46,16 +53,24 @@ def _strip_titles(node: Any) -> None: _strip_titles(value) -def report_json_schema() -> dict[str, Any]: - """JSON schema for `Report` suitable for `response_format` (no titles, closed object).""" - schema = copy.deepcopy(Report.model_json_schema()) +def model_json_schema(model_cls: type[BaseModel]) -> dict[str, Any]: + """JSON schema for a pydantic model suitable for `response_format` (no titles, closed object).""" + schema = copy.deepcopy(model_cls.model_json_schema()) _strip_titles(schema) schema["additionalProperties"] = False return schema +def model_response_format(model_cls: type[BaseModel], name: str) -> dict[str, Any]: + return {"type": "json_schema", "json_schema": {"name": name, "schema": model_json_schema(model_cls)}} + + +def report_json_schema() -> dict[str, Any]: + return model_json_schema(Report) + + def report_response_format() -> dict[str, Any]: - return {"type": "json_schema", "json_schema": {"name": "report", "schema": report_json_schema()}} + return model_response_format(Report, "report") def _first_json_object(text: str) -> str | None: @@ -108,9 +123,15 @@ def extract_json_text(raw: str) -> str: return text -_LIST_FIELDS: frozenset[str] = frozenset( - name for name, info in Report.model_fields.items() if get_origin(info.annotation) is list -) +def _list_fields(model_cls: type[BaseModel]) -> frozenset[str]: + """Names of the model's plain string-list fields (targets for the string->list coercion).""" + return frozenset( + name + for name, info in model_cls.model_fields.items() + if get_origin(info.annotation) is list and info.annotation == list[str] + ) + + _BULLET_SPLIT = re.compile(r"(?:^|\n)\s*(?:[-*•]|\d+[.)])\s+") _SENTENCE_SPLIT = re.compile(r"(?<=[.!?。])\s+") @@ -125,8 +146,8 @@ def _split_prose(text: str) -> list[str]: return parts or [text.strip()] -def coerce_report_data(data: Any) -> tuple[Any, list[str]]: - """Fix the common MiniMax schema slip of emitting list fields as a single string. +def coerce_model_data(data: Any, model_cls: type[BaseModel]) -> tuple[Any, list[str]]: + """Fix the common MiniMax schema slip of emitting string-list fields as a single string. Returns (data, names of coerced fields). Anything that is not a dict is returned untouched so that pydantic can report the real error. @@ -134,7 +155,7 @@ def coerce_report_data(data: Any) -> tuple[Any, list[str]]: if not isinstance(data, dict): return data, [] coerced: list[str] = [] - for name in _LIST_FIELDS: + for name in _list_fields(model_cls): value = data.get(name) if isinstance(value, str): data[name] = _split_prose(value) @@ -142,17 +163,25 @@ def coerce_report_data(data: Any) -> tuple[Any, list[str]]: return data, coerced -def parse_report(raw: str) -> Report: - """Turn raw model content into a validated Report (raises ValueError / ValidationError).""" +def coerce_report_data(data: Any) -> tuple[Any, list[str]]: + return coerce_model_data(data, Report) + + +def parse_model[T: BaseModel](raw: str, model_cls: type[T]) -> T: + """Turn raw model content into a validated instance (raises ValueError / ValidationError).""" text = extract_json_text(raw) try: data = json.loads(text) except json.JSONDecodeError: - return Report.model_validate_json(text) # let pydantic produce the canonical error - data, coerced = coerce_report_data(data) + return model_cls.model_validate_json(text) # let pydantic produce the canonical error + data, coerced = coerce_model_data(data, model_cls) if coerced: - log.warning("coerced string -> list for report fields: %s", ", ".join(sorted(coerced))) - return Report.model_validate(data) + log.warning("coerced string -> list for %s fields: %s", model_cls.__name__, ", ".join(sorted(coerced))) + return model_cls.model_validate(data) + + +def parse_report(raw: str) -> Report: + return parse_model(raw, Report) class MiniMaxProvider: @@ -174,7 +203,6 @@ class MiniMaxProvider: max_retries=2, ) self._client = client - self._response_format = report_response_format() @property def models(self) -> list[str]: @@ -182,28 +210,59 @@ class MiniMaxProvider: # ------------------------------------------------------------------ public def summarize(self, payload: MarketPayload) -> tuple[Report, str]: - user_message = build_user_message(payload) - base_messages: list[dict[str, Any]] = [ + messages: list[dict[str, Any]] = [ {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": user_message}, + {"role": "user", "content": build_user_message(payload)}, ] - attempts: list[str] = [] - for model in self._models: - result = self._attempt_model(model, base_messages, attempts) - if result is not None: - return result, model - raise LLMError("all MiniMax models failed: " + " | ".join(attempts)) + return self._run(messages, Report, "report", task="summarize") + + def verify(self, payload: MarketPayload, report: Report) -> tuple[Verification, str]: + """Fact-check `report` against the payload. Raise LLMError on total failure.""" + messages: list[dict[str, Any]] = [ + {"role": "system", "content": VERIFY_SYSTEM_PROMPT}, + {"role": "user", "content": build_verify_message(payload, report)}, + ] + return self._run(messages, Verification, "verification", task="verify") + + def revise( + self, payload: MarketPayload, report: Report, issues: list[VerificationIssue] + ) -> tuple[Report, str]: + """Ask for a corrected Report given verification issues. Raise LLMError on total failure.""" + messages: list[dict[str, Any]] = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": build_user_message(payload)}, + {"role": "assistant", "content": report.model_dump_json()}, + {"role": "user", "content": REVISE_INSTRUCTION + format_issues(issues)}, + ] + return self._run(messages, Report, "report", task="revise") # ----------------------------------------------------------------- helpers - def _attempt_model( - self, model: str, messages: list[dict[str, Any]], attempts: list[str] - ) -> Report | None: + def _run[T: BaseModel]( + self, messages: list[dict[str, Any]], model_cls: type[T], schema_name: str, task: str + ) -> tuple[T, str]: + """Try every configured model in order; return (parsed instance, model id).""" + response_format = model_response_format(model_cls, schema_name) + attempts: list[str] = [] + for model in self._models: + result = self._attempt_model(model, messages, attempts, model_cls, response_format) + if result is not None: + return result, model + raise LLMError(f"all MiniMax models failed ({task}): " + " | ".join(attempts)) + + def _attempt_model[T: BaseModel]( + self, + model: str, + messages: list[dict[str, Any]], + attempts: list[str], + model_cls: type[T], + response_format: dict[str, Any], + ) -> T | None: """One model: initial call plus at most one repair retry. Returns None when the model failed.""" - content = self._call(model, messages, attempts) + content = self._call(model, messages, attempts, response_format) if content is None: return None try: - return parse_report(content) + return parse_model(content, model_cls) except (ValidationError, ValueError) as exc: error_text = _short(str(exc), 800) attempts.append(f"{model}: invalid report ({_short(str(exc), 160)})") @@ -218,18 +277,23 @@ class MiniMaxProvider: {"role": "assistant", "content": _short(content, 12000)}, {"role": "user", "content": REPAIR_INSTRUCTION + error_text}, ] - content = self._call(model, repair_messages, attempts, label="repair") + content = self._call(model, repair_messages, attempts, response_format, label="repair") if content is None: return None try: - return parse_report(content) + return parse_model(content, model_cls) except (ValidationError, ValueError) as exc: attempts.append(f"{model} (repair): invalid report ({_short(str(exc), 160)})") log.warning("model %s repair retry still invalid: %s", model, _short(str(exc), 300)) return None def _call( - self, model: str, messages: list[dict[str, Any]], attempts: list[str], label: str = "initial" + self, + model: str, + messages: list[dict[str, Any]], + attempts: list[str], + response_format: dict[str, Any], + label: str = "initial", ) -> str | None: """Perform one chat completion. Returns content text, or None (attempt recorded) on failure.""" tag = model if label == "initial" else f"{model} ({label})" @@ -241,7 +305,7 @@ class MiniMaxProvider: model=model, messages=messages, max_completion_tokens=self._settings.minimax_max_completion_tokens, - response_format=self._response_format, + response_format=response_format, extra_body=extra_body, ) except OPENAI_ERRORS as exc: @@ -297,8 +361,12 @@ def _short(text: str, limit: int) -> str: __all__ = [ "OPENAI_ERRORS", "MiniMaxProvider", + "coerce_model_data", "coerce_report_data", "extract_json_text", + "model_json_schema", + "model_response_format", + "parse_model", "parse_report", "report_json_schema", "report_response_format", diff --git a/us2kr/llm/prompts.py b/us2kr/llm/prompts.py index 6aaba57..561040f 100644 --- a/us2kr/llm/prompts.py +++ b/us2kr/llm/prompts.py @@ -164,6 +164,48 @@ def _bullets(items: list[str]) -> list[str]: return [f"- {item}" for item in items] or ["(없음)"] +VERIFY_SYSTEM_PROMPT: str = """당신은 미국 시장 브리핑의 팩트체커입니다. 브리핑 초안이 함께 제공된 원본 데이터와 +모순되지 않는지만 검사합니다. 문체·구성·의견은 평가하지 않습니다. + +## 검사 규칙 +1. 판단 근거는 사용자 메시지에 제공된 데이터(시세·히스토리 노트·뉴스·경제지표·실적)뿐입니다. + 당신의 배경지식이나 최신 시황과 달라 보여도, 제공된 데이터와 일치하면 문제가 아닙니다. +2. major: 브리핑의 숫자가 데이터와 다르거나(반올림 오차 ±0.05는 허용), 상승/하락 방향이 반대이거나, + 데이터에 없는 숫자·사건을 사실처럼 서술한 경우. +3. minor: 뉴스 헤드라인만으로 확인되지 않는 인과관계를 단정, 제공된 표기와 다른 종목명, 모호하지만 + 틀렸다고 단정할 수 없는 서술. +4. 각 이슈의 claim에는 브리핑 원문을 그대로 인용하고, problem에는 데이터의 실제 값과 함께 한 문장으로 + 설명합니다. +5. major 이슈가 하나도 없으면 ok=true 입니다. 사소한 트집을 만들지 말고, 문제가 없으면 빈 issues로 + 반환합니다. +6. 출력은 지정된 JSON 스키마만 반환합니다. 스키마 밖 텍스트·설명·코드 펜스를 추가하지 않습니다. +""" + +REVISE_INSTRUCTION: str = ( + "위 브리핑 초안에서 아래 검증 이슈가 발견되었습니다. 제공된 원본 데이터에 맞게 해당 서술만 고치고," + " 문제없는 부분은 그대로 유지하여 동일한 JSON 스키마로 전체 브리핑을 다시 출력하세요." + " 설명 없이 JSON 객체 하나만 출력합니다.\n\n검증 이슈:\n" +) + + +def build_verify_message(payload: MarketPayload, report: Report) -> str: + """User message for the fact-check pass: the payload serialisation plus the draft report.""" + return ( + build_user_message(payload) + + "\n## 검증 대상 브리핑 초안 (JSON)\n" + + report.model_dump_json(indent=2) + + "\n" + ) + + +def format_issues(issues: list) -> str: + """Issues -> numbered plain-text block for the revise instruction.""" + lines = [] + for i, issue in enumerate(issues, 1): + lines.append(f"{i}. [{issue.severity}] 서술: {issue.claim}\n 문제: {issue.problem}") + return "\n".join(lines) or "(없음)" + + def build_user_message(payload: MarketPayload) -> str: """Serialise the payload into a deterministic Korean text block for the user message.""" out: list[str] = [] diff --git a/us2kr/models.py b/us2kr/models.py index 68532dd..d936e85 100644 --- a/us2kr/models.py +++ b/us2kr/models.py @@ -91,6 +91,25 @@ class Report(BaseModel): data_gaps: list[str] = Field(default_factory=list, description="수집 실패로 반영하지 못한 항목") +class VerificationIssue(BaseModel): + claim: str = Field(description="문제가 된 브리핑 속 서술 (원문 그대로 인용)") + problem: str = Field(description="제공된 데이터와 어떻게 다른지 한 문장") + severity: Literal["minor", "major"] = Field( + default="minor", description="major: 수치·방향이 데이터와 모순 / minor: 근거 불충분·표현 문제" + ) + + +class Verification(BaseModel): + """LLM fact-check verdict for a generated Report, judged only against the payload.""" + + ok: bool = Field(description="major 이슈가 하나도 없으면 true") + issues: list[VerificationIssue] = Field(default_factory=list, description="발견된 문제 목록 (없으면 빈 배열)") + + @property + def has_major(self) -> bool: + return any(i.severity == "major" for i in self.issues) + + class BriefingResult(BaseModel): payload: MarketPayload report: Report | None @@ -98,3 +117,4 @@ class BriefingResult(BaseModel): model_used: str | None = None degraded: bool = False # True when LLM failed and only the numeric table was produced error: str | None = None + verification: Verification | None = None # None: not run (disabled / degraded / verifier failed) diff --git a/us2kr/pipeline.py b/us2kr/pipeline.py index daaf2b2..33f8289 100644 --- a/us2kr/pipeline.py +++ b/us2kr/pipeline.py @@ -23,10 +23,10 @@ from us2kr.config import KST, Settings from us2kr.deliver import build_deliverers from us2kr.llm.base import LLMError from us2kr.market_time import MarketClock -from us2kr.models import BriefingResult, MarketPayload, Report +from us2kr.models import BriefingResult, MarketPayload, Report, Verification from us2kr.render import render_markdown from us2kr.store import Store -from us2kr.summarize import build_provider, summarize_payload +from us2kr.summarize import build_provider, summarize_payload, verify_and_revise from us2kr.symbols import SYMBOL_GROUPS log = logging.getLogger(__name__) @@ -162,6 +162,18 @@ def _summarize(payload: MarketPayload, settings: Settings) -> tuple[Report | Non return None, None, f"{type(exc).__name__}: {exc}" +def _verify( + payload: MarketPayload, report: Report, settings: Settings +) -> tuple[Report, Verification | None]: + """Fact-check (and possibly revise) the report. Any failure keeps the report and returns None.""" + try: + provider = build_provider(settings) + return verify_and_revise(payload, report, provider) + except Exception as exc: # noqa: BLE001 - verification must never block the briefing + log.warning("verification failed, delivering unverified report: %s: %s", type(exc).__name__, exc) + return report, None + + def _superseded_before_krx_open(session: date, krx_date: date, clock: MarketClock) -> bool: """True when another US session closes before KRX next trades, so this briefing would be stale. @@ -271,6 +283,7 @@ def run_briefing( report: Report | None = None model_used: str | None = None error: str | None = None + verification: Verification | None = None degraded = False if no_llm: degraded = True @@ -278,6 +291,8 @@ def run_briefing( else: report, model_used, error = _summarize(payload, settings) degraded = report is None + if report is not None and settings.verify_enabled: + report, verification = _verify(payload, report, settings) markdown = render_markdown(payload, report, model_used) result = BriefingResult( @@ -287,6 +302,7 @@ def run_briefing( model_used=model_used, degraded=degraded, error=error, + verification=verification, ) if dry_run: print(markdown) diff --git a/us2kr/summarize.py b/us2kr/summarize.py index c724dda..ff83bde 100644 --- a/us2kr/summarize.py +++ b/us2kr/summarize.py @@ -5,9 +5,9 @@ from __future__ import annotations import logging from us2kr.config import Settings -from us2kr.llm.base import LLMProvider +from us2kr.llm.base import LLMError, LLMProvider from us2kr.llm.minimax import MiniMaxProvider -from us2kr.models import MarketPayload, Report +from us2kr.models import MarketPayload, Report, Verification log = logging.getLogger(__name__) @@ -28,3 +28,45 @@ def summarize_payload(payload: MarketPayload, provider: LLMProvider) -> tuple[Re report = report.model_copy(update={"data_gaps": merged}) log.info("summary produced by %s (%d data gaps)", model_used, len(merged)) return report, model_used + + +def _consistent(verification: Verification) -> Verification: + """Force `ok` to mean "no major issues" - verifier models are inconsistent about minor-only cases.""" + ok = not verification.has_major + return verification if verification.ok == ok else verification.model_copy(update={"ok": ok}) + + +def verify_and_revise( + payload: MarketPayload, report: Report, provider: LLMProvider +) -> tuple[Report, Verification]: + """Fact-check the report; on major issues ask for one revision and re-check it. + + Returns (possibly revised report, verification of the returned report). Raises LLMError only when + the *initial* verification cannot be produced; a failed revision keeps the original report. + """ + raw_verification, model = provider.verify(payload, report) + verification = _consistent(raw_verification) + log.info( + "verification by %s: ok=%s issues=%d (major=%s)", + model, + verification.ok, + len(verification.issues), + verification.has_major, + ) + if not verification.has_major: + return report, verification + + for issue in verification.issues: + log.warning("verification issue [%s] %s -> %s", issue.severity, issue.claim, issue.problem) + try: + revised, model = provider.revise(payload, report, verification.issues) + log.info("report revised by %s after verification issues", model) + re_verification, model = provider.verify(payload, revised) + re_verification = _consistent(re_verification) + log.info( + "re-verification by %s: ok=%s issues=%d", model, re_verification.ok, len(re_verification.issues) + ) + return revised, re_verification + except LLMError as exc: + log.warning("revision failed; keeping the original report and verdict: %s", exc) + return report, verification