sebrief/tests/test_llm.py

"""LLM layer: JSON extraction/coercion, prompt building, provider fallback and repair retry."""

from __future__ import annotations

import json
from types import SimpleNamespace

import pytest
from _fakes import make_digest, make_payload

from sebrief.config import Settings
from sebrief.llm.base import LLMError
from sebrief.llm.minimax import (
    MiniMaxProvider,
    coerce_model_data,
    digest_json_schema,
    extract_json_text,
    parse_digest,
)
from sebrief.llm.prompts import build_user_message, build_verify_message, format_issues
from sebrief.models import Digest, VerificationIssue

VALID_DIGEST_JSON = make_digest().model_dump_json()


# ------------------------------------------------------------------------ extraction/parsing
def test_extract_json_text_plain_and_fenced():
    assert extract_json_text('{"a": 1}') == '{"a": 1}'
    assert extract_json_text('```json\n{"a": 1}\n```') == '{"a": 1}'


def test_extract_json_text_strips_think_blocks():
    raw = f"<think>reasoning {{nested}} </think>{VALID_DIGEST_JSON}"
    assert json.loads(extract_json_text(raw)) == json.loads(VALID_DIGEST_JSON)


def test_extract_json_text_unterminated_think_falls_back_to_first_object():
    raw = f"<think>never closed... {VALID_DIGEST_JSON}"
    assert json.loads(extract_json_text(raw))["overview"].startswith("이번 주")


def test_extract_json_text_raises_without_object():
    with pytest.raises(ValueError):
        extract_json_text("no json here")


def test_parse_digest_valid():
    digest = parse_digest(VALID_DIGEST_JSON)
    assert isinstance(digest, Digest)
    assert digest.highlights[0].doi == "10.1145/1234567"


def test_coerce_string_list_fields():
    data = json.loads(VALID_DIGEST_JSON)
    data["data_gaps"] = "누락 1. 누락 2."
    coerced, fields = coerce_model_data(data, Digest)
    assert fields == ["data_gaps"]
    assert coerced["data_gaps"] == ["누락 1.", "누락 2."]
    # nested object lists are left for pydantic to validate
    digest = Digest.model_validate(coerced)
    assert len(digest.data_gaps) == 2


def test_digest_json_schema_is_closed_and_untitled():
    schema = digest_json_schema()
    assert schema["additionalProperties"] is False
    assert "title" not in schema
    assert "overview" in schema["properties"]


# ---------------------------------------------------------------------------- CJK guard
def test_parse_rejects_hanja_and_kana_in_prose():
    data = json.loads(VALID_DIGEST_JSON)
    data["overview"] = "이번 주는 実態 조사가 활발했습니다."
    with pytest.raises(ValueError, match="한자·가나"):
        parse_digest(json.dumps(data, ensure_ascii=False))
    data["overview"] = "일본어 かな 누수입니다."
    with pytest.raises(ValueError, match="한자·가나"):
        parse_digest(json.dumps(data, ensure_ascii=False))


def test_parse_allows_cjk_in_verbatim_title():
    data = json.loads(VALID_DIGEST_JSON)
    data["highlights"][0]["title"] = "Understanding 中文 Identifiers in Code"
    digest = parse_digest(json.dumps(data, ensure_ascii=False))
    assert "中文" in digest.highlights[0].title


def test_hanja_reply_is_repaired_on_the_same_model():
    dirty = json.loads(VALID_DIGEST_JSON)
    dirty["overview"] = "소프트웨어 実態 분석 주간입니다."
    provider, client = make_provider([json.dumps(dirty, ensure_ascii=False), VALID_DIGEST_JSON])
    digest, model = provider.summarize(make_payload())
    assert model == "m1" and "実態" not in digest.overview
    repair = client.calls[1]["messages"][-1]["content"]
    assert "한자·가나" in repair and "実態" in repair


# ------------------------------------------------------------------------------- prompts
def test_build_user_message_contains_papers_and_gaps():
    payload = make_payload(gaps=["rss: down"])
    msg = build_user_message(payload)
    assert "## 대상 주차" in msg and "2026-W36" in msg
    assert "DOI: 10.1145/1234567" in msg
    assert "Test-Driven Digest Generation" in msg
    assert "- rss: down" in msg


def test_build_user_message_truncates_long_abstracts():
    payload = make_payload(papers=[make_payload().papers[0].model_copy(update={"abstract": "x" * 5000})])
    msg = build_user_message(payload)
    assert "x" * 1800 not in msg and "…" in msg


def test_build_verify_message_appends_draft():
    payload = make_payload()
    msg = build_verify_message(payload, make_digest())
    assert msg.startswith(build_user_message(payload))
    assert "## 검증 대상 다이제스트 초안 (JSON)" in msg


def test_format_issues():
    issues = [VerificationIssue(claim="c", problem="p", severity="major")]
    text = format_issues(issues)
    assert "[major]" in text and "서술: c" in text and "문제: p" in text
    assert format_issues([]) == "(없음)"


# ------------------------------------------------------------------------------- provider
class FakeClient:
    """chat.completions.create stub answering from a queue of contents (or exceptions)."""

    def __init__(self, replies: list):
        self.replies = list(replies)
        self.calls: list[dict] = []
        self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create))

    def _create(self, **kwargs):
        self.calls.append(kwargs)
        reply = self.replies.pop(0)
        if isinstance(reply, Exception):
            raise reply
        finish = "length" if reply == "<TRUNCATED>" else "stop"
        message = SimpleNamespace(content=None if reply == "<EMPTY>" else reply)
        choice = SimpleNamespace(message=message, finish_reason=finish)
        return SimpleNamespace(choices=[choice], model=kwargs["model"], usage=None)


def make_provider(replies: list, models: str = "m1", fallbacks: str = "m2") -> tuple[MiniMaxProvider, FakeClient]:
    settings = Settings(
        _env_file=None,
        MINIMAX_API_KEY="sk-test-key",
        MINIMAX_MODEL=models,
        MINIMAX_FALLBACK_MODELS=fallbacks,
    )
    client = FakeClient(replies)
    return MiniMaxProvider(settings, client=client), client


def test_summarize_happy_path():
    provider, client = make_provider([VALID_DIGEST_JSON])
    digest, model = provider.summarize(make_payload())
    assert model == "m1" and digest.overview
    assert client.calls[0]["response_format"]["json_schema"]["name"] == "digest"


def test_invalid_reply_triggers_repair_then_fallback_model():
    # m1: garbage twice (initial + repair) -> m2 succeeds
    provider, client = make_provider(["not json", "{\"still\": \"wrong\"}", VALID_DIGEST_JSON])
    _digest, model = provider.summarize(make_payload())
    assert model == "m2"
    assert len(client.calls) == 3
    repair_call = client.calls[1]
    assert any("JSON 스키마 검증에 실패" in m["content"] for m in repair_call["messages"] if m["role"] == "user")


def test_truncated_and_empty_replies_move_to_next_model():
    provider, _ = make_provider(["<TRUNCATED>", VALID_DIGEST_JSON])
    _, model = provider.summarize(make_payload())
    assert model == "m2"

    provider, _ = make_provider(["<EMPTY>", VALID_DIGEST_JSON])
    _, model = provider.summarize(make_payload())
    assert model == "m2"


def test_all_models_failed_raises_llm_error():
    provider, _ = make_provider(["nope", "nope", "nope", "nope"])  # 2 models x (initial + repair)
    with pytest.raises(LLMError, match="all MiniMax models failed"):
        provider.summarize(make_payload())


def test_summarize_papers_uses_batch_schema_and_prompt():
    reply = (
        '{"summaries": [{"doi": "10.1145/1234567", "tldr": "한 줄.", "problem": "p", '
        '"approach": "a", "findings": "f", "implication": "i"}]}'
    )
    provider, client = make_provider([reply])
    items, model = provider.summarize_papers(make_payload().papers)
    assert model == "m1" and len(items) == 1 and items[0].doi == "10.1145/1234567"
    call = client.calls[0]
    assert call["response_format"]["json_schema"]["name"] == "paper_summaries"
    assert call["messages"][0]["content"].startswith("당신은 소프트웨어공학(SE) 논문 요약자")
    assert "DOI: 10.1145/1234567" in call["messages"][1]["content"]


def test_verify_and_revise_use_expected_message_shapes():
    verification_json = '{"ok": false, "issues": [{"claim": "c", "problem": "p", "severity": "major"}]}'
    provider, client = make_provider([verification_json])
    verification, _ = provider.verify(make_payload(), make_digest())
    assert verification.has_major
    assert client.calls[0]["messages"][0]["content"].startswith("당신은 소프트웨어공학 주간 다이제스트의 팩트체커")

    provider, client = make_provider([VALID_DIGEST_JSON])
    issues = [VerificationIssue(claim="c", problem="p", severity="major")]
    provider.revise(make_payload(), make_digest(), issues)
    roles = [m["role"] for m in client.calls[0]["messages"]]
    assert roles == ["system", "user", "assistant", "user"]
    assert "검증 이슈" in client.calls[0]["messages"][-1]["content"]


def test_provider_requires_api_key_and_models():
    with pytest.raises(LLMError, match="MINIMAX_API_KEY"):
        MiniMaxProvider(Settings(_env_file=None, MINIMAX_API_KEY=""))
    with pytest.raises(LLMError, match="no MiniMax model"):
        MiniMaxProvider(
            Settings(_env_file=None, MINIMAX_API_KEY="sk-x", MINIMAX_MODEL="", MINIMAX_FALLBACK_MODELS=""),
            client=object(),
        )