Supplement slow-indexed IEEE journals from Crossref (TSE fresh papers)

OpenAlex indexes TSE ~2-3 months late (nothing after 2026-06-26 as of Sep 2), so TSE never appeared in weekly digests. Sources can now be flagged crossref_fresh: their recent papers are discovered via Crossref (created-date window, paper-level CC licence = OA) and listed without abstracts; a digested paper's missing abstract is filled once OpenAlex indexes the DOI, so a later --force re-run can add its per-paper summary. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-09-02 21:00:32 +09:00
parent a6ccf04fb2
commit 3f1d16cae1
8 changed files with 307 additions and 4 deletions
@@ -0,0 +1,115 @@
+"""Crossref supplement: item mapping, paper-level CC-licence OA filter, failure isolation."""
+
+from __future__ import annotations
+
+from datetime import date
+
+import pytest
+
+from sebrief.collectors import crossref
+from sebrief.collectors.crossref import fetch_crossref_papers, paper_from_item
+from sebrief.config import Settings
+from sebrief.sources import Source, crossref_fresh_sources
+
+SETTINGS = Settings(_env_file=None)
+START, END = date(2026, 8, 24), date(2026, 9, 6)
+TSE = crossref_fresh_sources()[0]
+
+CC_ITEM = {
+    "DOI": "10.1109/TSE.2026.3729363",
+    "title": ["An Empirical Study of <i>Flaky</i> Tests"],
+    "author": [{"given": "Alice", "family": "Kim"}, {"family": "Lee"}],
+    "created": {"date-parts": [[2026, 8, 31]]},
+    "license": [{"URL": "https://creativecommons.org/licenses/by/4.0/"}],
+}
+PAYWALLED_ITEM = {
+    "DOI": "10.1109/TSE.2026.3729587",
+    "title": ["Navigating Complexity"],
+    "created": {"date-parts": [[2026, 9, 1]]},
+    "license": [{"URL": "https://ieeexplore.ieee.org/Xplorehelp/downloads"}],
+}
+
+
+def test_crossref_fresh_sources_contains_tse():
+    assert [s.key for s in crossref_fresh_sources()] == ["tse"]
+
+
+def test_paper_from_item_maps_fields_and_strips_tags():
+    paper = paper_from_item(CC_ITEM, TSE)
+    assert paper is not None
+    assert paper.doi == "10.1109/tse.2026.3729363"
+    assert paper.title == "An Empirical Study of Flaky Tests"
+    assert paper.authors == ["Alice Kim", "Lee"]
+    assert paper.venue == "IEEE Transactions on Software Engineering"
+    assert paper.published_date == date(2026, 8, 31)
+    assert paper.abstract == ""  # IEEE는 Crossref에 초록을 예치하지 않음
+    assert paper.oa_url == "https://doi.org/10.1109/tse.2026.3729363"
+    assert paper.source == "crossref:tse"
+
+
+def test_paper_from_item_rejects_missing_doi_or_title():
+    assert paper_from_item({"title": ["T"]}, TSE) is None
+    assert paper_from_item({"DOI": "10.1/x", "title": []}, TSE) is None
+
+
+class FakeResponse:
+    def __init__(self, items):
+        self._items = items
+
+    def raise_for_status(self):
+        pass
+
+    def json(self):
+        return {"message": {"items": self._items}}
+
+
+def test_fetch_keeps_only_cc_licensed_papers(monkeypatch):
+    calls = []
+
+    def fake_get(url, params=None, timeout=None, headers=None):
+        calls.append({"url": url, "params": params})
+        return FakeResponse([CC_ITEM, PAYWALLED_ITEM, {"DOI": "10.1/nolicense", "title": ["X"]}])
+
+    monkeypatch.setattr(crossref.requests, "get", fake_get)
+    papers = fetch_crossref_papers(START, END, SETTINGS)
+    assert [p.doi for p in papers] == ["10.1109/tse.2026.3729363"]
+    assert "/journals/0098-5589/works" in calls[0]["url"]
+    assert calls[0]["params"]["filter"] == "from-created-date:2026-08-24,until-created-date:2026-09-06"
+
+
+def test_fetch_dedups_by_doi(monkeypatch):
+    monkeypatch.setattr(crossref.requests, "get", lambda *a, **k: FakeResponse([CC_ITEM, CC_ITEM]))
+    papers = fetch_crossref_papers(START, END, SETTINGS)
+    assert len(papers) == 1
+
+
+def test_all_sources_failed_raises(monkeypatch):
+    def boom(*a, **k):
+        raise RuntimeError("crossref down")
+
+    monkeypatch.setattr(crossref.requests, "get", boom)
+    with pytest.raises(RuntimeError, match="all crossref queries failed"):
+        fetch_crossref_papers(START, END, SETTINGS)
+
+
+def test_blacklisted_source_is_dropped(monkeypatch):
+    monkeypatch.setattr(crossref.requests, "get", lambda *a, **k: FakeResponse([CC_ITEM]))
+    settings = Settings(_env_file=None, SEBRIEF_EXTRA_BLACKLIST_ISSNS="0098-5589")
+    assert fetch_crossref_papers(START, END, settings) == []
+
+
+def test_mailto_forwarded(monkeypatch):
+    calls = []
+
+    def fake_get(url, params=None, timeout=None, headers=None):
+        calls.append(params)
+        return FakeResponse([])
+
+    monkeypatch.setattr(crossref.requests, "get", fake_get)
+    fetch_crossref_papers(START, END, Settings(_env_file=None, SEBRIEF_OPENALEX_MAILTO="ops@example.org"))
+    assert calls[0]["mailto"] == "ops@example.org"
+
+
+def test_source_without_crossref_flag_not_queried():
+    plain = Source("x", "X Journal", ("1111-2222",))
+    assert plain.crossref_fresh is False
@@ -22,9 +22,11 @@ def env(monkeypatch, tmp_path):
        "rss": [],
        "settings": Settings(_env_file=None, SEBRIEF_DATA_DIR=str(tmp_path / "data")),
    }
+    ctx["crossref"] = []
    monkeypatch.setattr(pipeline, "_now", lambda: NOW)
    monkeypatch.setattr(pipeline, "Store", lambda path: ctx["store"])
    monkeypatch.setattr(pipeline, "fetch_openalex_papers", lambda start, end, s: ctx["openalex"])
+    monkeypatch.setattr(pipeline, "fetch_crossref_papers", lambda start, end, s: ctx["crossref"])
    monkeypatch.setattr(pipeline, "fetch_rss_papers", lambda feeds, start, end, bl: ctx["rss"])
    monkeypatch.setattr(pipeline, "build_provider", lambda s: ctx["provider"])
    monkeypatch.setattr(pipeline, "build_deliverers", lambda s: ctx["deliverers"])
@@ -66,6 +68,16 @@ def test_build_payload_merges_rss_dedup_by_doi(env):
    assert upserted[0].source == "openalex"  # openalex metadata wins on the duplicate


+def test_build_payload_merges_crossref_openalex_wins_duplicates(env):
+    dup = make_paper(source="crossref:tse", venue="TSE", abstract="")  # same DOI as openalex
+    fresh = make_paper("10.1109/tse.2026.1", source="crossref:tse", abstract="")
+    env["crossref"] = [dup, fresh]
+    build_payload(WEEK, env["settings"], env["store"])
+    [upserted] = env["store"].upserted
+    assert [p.doi for p in upserted] == ["10.1145/1234567", "10.1109/tse.2026.1"]
+    assert upserted[0].source == "openalex"  # richer OpenAlex metadata wins the duplicate
+
+
 def test_build_payload_invalid_feed_spec_is_a_gap(env):
    env["settings"] = env["settings"].model_copy(update={"rss_feeds": ["broken-spec"]})
    payload = build_payload(WEEK, env["settings"], env["store"])
@@ -38,6 +38,17 @@ def test_upsert_refreshes_undigested_metadata(tmp_path):
        assert paper.abstract == "now with abstract"


+def test_digested_paper_gains_missing_abstract_but_stays_frozen(tmp_path):
+    with make_store(tmp_path) as store:
+        store.upsert_papers([make_paper("10.1/a", title="fresh via crossref", abstract="")])
+        store.mark_digested(["10.1/a"], WEEK)
+        # OpenAlex later indexes the DOI with an abstract (and a normalised title)
+        store.upsert_papers([make_paper("10.1/a", title="OpenAlex Title", abstract="now indexed")])
+        [paper] = store.pending_papers(WEEK)
+        assert paper.abstract == "now indexed"  # 초록만 보강
+        assert paper.title == "fresh via crossref"  # 그 외 메타데이터는 동결 유지
+
+
 def test_digested_papers_are_frozen_and_skipped(tmp_path):
    with make_store(tmp_path) as store:
        store.upsert_papers([make_paper("10.1/a", title="original")])