Reject CJK ideographs/kana leaking into Korean prose

MiniMax occasionally mixes hanja/kana into Korean text (実態, 软件, かな).
Parsed outputs are now scanned (verbatim fields exempt: title/doi/claim) and
offending replies flow through the existing repair retry; prompts spell the
rule out with examples.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jsahn
2026-09-02 20:25:20 +09:00
co-authored by Claude Fable 5
parent 15521956fe
commit a6ccf04fb2
3 changed files with 72 additions and 7 deletions
+28
View File
@@ -68,6 +68,34 @@ def test_digest_json_schema_is_closed_and_untitled():
assert "overview" in schema["properties"]
# ---------------------------------------------------------------------------- CJK guard
def test_parse_rejects_hanja_and_kana_in_prose():
data = json.loads(VALID_DIGEST_JSON)
data["overview"] = "이번 주는 実態 조사가 활발했습니다."
with pytest.raises(ValueError, match="한자·가나"):
parse_digest(json.dumps(data, ensure_ascii=False))
data["overview"] = "일본어 かな 누수입니다."
with pytest.raises(ValueError, match="한자·가나"):
parse_digest(json.dumps(data, ensure_ascii=False))
def test_parse_allows_cjk_in_verbatim_title():
data = json.loads(VALID_DIGEST_JSON)
data["highlights"][0]["title"] = "Understanding 中文 Identifiers in Code"
digest = parse_digest(json.dumps(data, ensure_ascii=False))
assert "中文" in digest.highlights[0].title
def test_hanja_reply_is_repaired_on_the_same_model():
dirty = json.loads(VALID_DIGEST_JSON)
dirty["overview"] = "소프트웨어 実態 분석 주간입니다."
provider, client = make_provider([json.dumps(dirty, ensure_ascii=False), VALID_DIGEST_JSON])
digest, model = provider.summarize(make_payload())
assert model == "m1" and "実態" not in digest.overview
repair = client.calls[1]["messages"][-1]["content"]
assert "한자·가나" in repair and "実態" in repair
# ------------------------------------------------------------------------------- prompts
def test_build_user_message_contains_papers_and_gaps():
payload = make_payload(gaps=["rss: down"])