"""LLM layer: JSON extraction/coercion, prompt building, provider fallback and repair retry.""" from __future__ import annotations import json from types import SimpleNamespace import pytest from _fakes import make_digest, make_payload from sebrief.config import Settings from sebrief.llm.base import LLMError from sebrief.llm.minimax import ( MiniMaxProvider, coerce_model_data, digest_json_schema, extract_json_text, parse_digest, ) from sebrief.llm.prompts import build_user_message, build_verify_message, format_issues from sebrief.models import Digest, VerificationIssue VALID_DIGEST_JSON = make_digest().model_dump_json() # ------------------------------------------------------------------------ extraction/parsing def test_extract_json_text_plain_and_fenced(): assert extract_json_text('{"a": 1}') == '{"a": 1}' assert extract_json_text('```json\n{"a": 1}\n```') == '{"a": 1}' def test_extract_json_text_strips_think_blocks(): raw = f"reasoning {{nested}} {VALID_DIGEST_JSON}" assert json.loads(extract_json_text(raw)) == json.loads(VALID_DIGEST_JSON) def test_extract_json_text_unterminated_think_falls_back_to_first_object(): raw = f"never closed... {VALID_DIGEST_JSON}" assert json.loads(extract_json_text(raw))["overview"].startswith("이번 주") def test_extract_json_text_raises_without_object(): with pytest.raises(ValueError): extract_json_text("no json here") def test_parse_digest_valid(): digest = parse_digest(VALID_DIGEST_JSON) assert isinstance(digest, Digest) assert digest.highlights[0].doi == "10.1145/1234567" def test_coerce_string_list_fields(): data = json.loads(VALID_DIGEST_JSON) data["data_gaps"] = "누락 1. 누락 2." coerced, fields = coerce_model_data(data, Digest) assert fields == ["data_gaps"] assert coerced["data_gaps"] == ["누락 1.", "누락 2."] # nested object lists are left for pydantic to validate digest = Digest.model_validate(coerced) assert len(digest.data_gaps) == 2 def test_digest_json_schema_is_closed_and_untitled(): schema = digest_json_schema() assert schema["additionalProperties"] is False assert "title" not in schema assert "overview" in schema["properties"] # ---------------------------------------------------------------------------- CJK guard def test_parse_rejects_hanja_and_kana_in_prose(): data = json.loads(VALID_DIGEST_JSON) data["overview"] = "이번 주는 実態 조사가 활발했습니다." with pytest.raises(ValueError, match="한자·가나"): parse_digest(json.dumps(data, ensure_ascii=False)) data["overview"] = "일본어 かな 누수입니다." with pytest.raises(ValueError, match="한자·가나"): parse_digest(json.dumps(data, ensure_ascii=False)) def test_parse_allows_cjk_in_verbatim_title(): data = json.loads(VALID_DIGEST_JSON) data["highlights"][0]["title"] = "Understanding 中文 Identifiers in Code" digest = parse_digest(json.dumps(data, ensure_ascii=False)) assert "中文" in digest.highlights[0].title def test_hanja_reply_is_repaired_on_the_same_model(): dirty = json.loads(VALID_DIGEST_JSON) dirty["overview"] = "소프트웨어 実態 분석 주간입니다." provider, client = make_provider([json.dumps(dirty, ensure_ascii=False), VALID_DIGEST_JSON]) digest, model = provider.summarize(make_payload()) assert model == "m1" and "実態" not in digest.overview repair = client.calls[1]["messages"][-1]["content"] assert "한자·가나" in repair and "実態" in repair # ------------------------------------------------------------------------------- prompts def test_build_user_message_contains_papers_and_gaps(): payload = make_payload(gaps=["rss: down"]) msg = build_user_message(payload) assert "## 대상 주차" in msg and "2026-W36" in msg assert "DOI: 10.1145/1234567" in msg assert "Test-Driven Digest Generation" in msg assert "- rss: down" in msg def test_build_user_message_truncates_long_abstracts(): payload = make_payload(papers=[make_payload().papers[0].model_copy(update={"abstract": "x" * 5000})]) msg = build_user_message(payload) assert "x" * 1800 not in msg and "…" in msg def test_build_verify_message_appends_draft(): payload = make_payload() msg = build_verify_message(payload, make_digest()) assert msg.startswith(build_user_message(payload)) assert "## 검증 대상 다이제스트 초안 (JSON)" in msg def test_format_issues(): issues = [VerificationIssue(claim="c", problem="p", severity="major")] text = format_issues(issues) assert "[major]" in text and "서술: c" in text and "문제: p" in text assert format_issues([]) == "(없음)" # ------------------------------------------------------------------------------- provider class FakeClient: """chat.completions.create stub answering from a queue of contents (or exceptions).""" def __init__(self, replies: list): self.replies = list(replies) self.calls: list[dict] = [] self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create)) def _create(self, **kwargs): self.calls.append(kwargs) reply = self.replies.pop(0) if isinstance(reply, Exception): raise reply finish = "length" if reply == "" else "stop" message = SimpleNamespace(content=None if reply == "" else reply) choice = SimpleNamespace(message=message, finish_reason=finish) return SimpleNamespace(choices=[choice], model=kwargs["model"], usage=None) def make_provider(replies: list, models: str = "m1", fallbacks: str = "m2") -> tuple[MiniMaxProvider, FakeClient]: settings = Settings( _env_file=None, MINIMAX_API_KEY="sk-test-key", MINIMAX_MODEL=models, MINIMAX_FALLBACK_MODELS=fallbacks, ) client = FakeClient(replies) return MiniMaxProvider(settings, client=client), client def test_summarize_happy_path(): provider, client = make_provider([VALID_DIGEST_JSON]) digest, model = provider.summarize(make_payload()) assert model == "m1" and digest.overview assert client.calls[0]["response_format"]["json_schema"]["name"] == "digest" def test_invalid_reply_triggers_repair_then_fallback_model(): # m1: garbage twice (initial + repair) -> m2 succeeds provider, client = make_provider(["not json", "{\"still\": \"wrong\"}", VALID_DIGEST_JSON]) _digest, model = provider.summarize(make_payload()) assert model == "m2" assert len(client.calls) == 3 repair_call = client.calls[1] assert any("JSON 스키마 검증에 실패" in m["content"] for m in repair_call["messages"] if m["role"] == "user") def test_truncated_and_empty_replies_move_to_next_model(): provider, _ = make_provider(["", VALID_DIGEST_JSON]) _, model = provider.summarize(make_payload()) assert model == "m2" provider, _ = make_provider(["", VALID_DIGEST_JSON]) _, model = provider.summarize(make_payload()) assert model == "m2" def test_all_models_failed_raises_llm_error(): provider, _ = make_provider(["nope", "nope", "nope", "nope"]) # 2 models x (initial + repair) with pytest.raises(LLMError, match="all MiniMax models failed"): provider.summarize(make_payload()) def test_summarize_papers_uses_batch_schema_and_prompt(): reply = ( '{"summaries": [{"doi": "10.1145/1234567", "tldr": "한 줄.", "problem": "p", ' '"approach": "a", "findings": "f", "implication": "i"}]}' ) provider, client = make_provider([reply]) items, model = provider.summarize_papers(make_payload().papers) assert model == "m1" and len(items) == 1 and items[0].doi == "10.1145/1234567" call = client.calls[0] assert call["response_format"]["json_schema"]["name"] == "paper_summaries" assert call["messages"][0]["content"].startswith("당신은 소프트웨어공학(SE) 논문 요약자") assert "DOI: 10.1145/1234567" in call["messages"][1]["content"] def test_verify_and_revise_use_expected_message_shapes(): verification_json = '{"ok": false, "issues": [{"claim": "c", "problem": "p", "severity": "major"}]}' provider, client = make_provider([verification_json]) verification, _ = provider.verify(make_payload(), make_digest()) assert verification.has_major assert client.calls[0]["messages"][0]["content"].startswith("당신은 소프트웨어공학 주간 다이제스트의 팩트체커") provider, client = make_provider([VALID_DIGEST_JSON]) issues = [VerificationIssue(claim="c", problem="p", severity="major")] provider.revise(make_payload(), make_digest(), issues) roles = [m["role"] for m in client.calls[0]["messages"]] assert roles == ["system", "user", "assistant", "user"] assert "검증 이슈" in client.calls[0]["messages"][-1]["content"] def test_provider_requires_api_key_and_models(): with pytest.raises(LLMError, match="MINIMAX_API_KEY"): MiniMaxProvider(Settings(_env_file=None, MINIMAX_API_KEY="")) with pytest.raises(LLMError, match="no MiniMax model"): MiniMaxProvider( Settings(_env_file=None, MINIMAX_API_KEY="sk-x", MINIMAX_MODEL="", MINIMAX_FALLBACK_MODELS=""), client=object(), )