Supplement slow-indexed IEEE journals from Crossref (TSE fresh papers)

OpenAlex indexes TSE ~2-3 months late (nothing after 2026-06-26 as of Sep 2),
so TSE never appeared in weekly digests. Sources can now be flagged
crossref_fresh: their recent papers are discovered via Crossref (created-date
window, paper-level CC licence = OA) and listed without abstracts; a digested
paper's missing abstract is filled once OpenAlex indexes the DOI, so a later
--force re-run can add its per-paper summary.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

README.md

@@ -17,7 +17,8 @@ US2KR의 파이프라인 철학을 그대로 따르는 별도 프로젝트입니
 | 소스 | 방식 |
 |---|---|
 | TOSEM (완전 OA) | ISSN 화이트리스트, 전량 |
-| TSE · EMSE · IST · JSS (하이브리드) | ISSN 화이트리스트 + **논문 단위** `is_oa` 필터 |
+| TSE (하이브리드, IEEE) | ISSN 화이트리스트 + OA 필터. OpenAlex 색인이 2~3개월 늦어 **Crossref로 신간 보조 수집**(논문 단위 CC 라이선스 = OA 판별). Crossref에는 초록이 없어 신간은 "초록 미제공"으로 실리고, OpenAlex 색인 시 초록이 보강됨 |
+| EMSE · IST · JSS (하이브리드) | ISSN 화이트리스트 + **논문 단위** `is_oa` 필터 |
 | SoftwareX · JOSS | 전량 (도구/소프트웨어 논문) |
 | IEEE Access · PeerJ CS (메가저널) | ISSN + OpenAlex Software 토픽(`topics.subfield.id:1712`) + **키워드 이중 필터** |
 | e-Informatica SEJ 등 | `SEBRIEF_RSS_FEEDS`로 RSS 보조 (DOI 없는 항목은 스킵) |

sebrief/collectors/crossref.py

@@ -0,0 +1,133 @@
+"""Crossref supplement: fresh papers from journals OpenAlex indexes months late (IEEE - TSE 등).
+
+Crossref는 발행사가 DOI를 등록하는 즉시 반영되지만 IEEE는 초록을 예치하지 않으므로, 여기서 수집된
+논문은 초록 없이(제목·링크·발행일) 다이제스트에 실린다. OA 판별은 논문 단위 라이선스로 한다:
+하이브리드 저널에서 CC 라이선스가 등록된 논문 = OA (Unpaywall 없이 판별 가능, 2026-09 실측).
+
+IEEE early access는 Crossref의 published 날짜가 연 단위 정밀도라 쓸 수 없어, 등록일(created,
+조기 공개일과 사실상 동일)을 발행일로 쓴다. Only if *every* source query fails does
+`fetch_crossref_papers` raise.
+"""
+
+from __future__ import annotations
+
+import html
+import logging
+import re
+from datetime import date
+from typing import Any
+
+import requests
+
+from sebrief.collectors.openalex import normalize_doi
+from sebrief.config import Settings
+from sebrief.models import Paper
+from sebrief.sources import Source, crossref_fresh_sources, is_blacklisted
+
+log = logging.getLogger(__name__)
+
+BASE_URL = "https://api.crossref.org"
+USER_AGENT = "sebrief/0.1 (OA software engineering papers weekly digest)"
+TIMEOUT_SECONDS = 30.0
+ROWS = 100
+SELECT = "DOI,title,author,created,license,container-title"
+
+_TAG_RE = re.compile(r"<[^>]+>")
+_WS_RE = re.compile(r"\s+")
+
+
+def _is_cc_licensed(item: dict[str, Any]) -> bool:
+    """Paper-level OA in a hybrid journal: a Creative Commons licence is registered for the DOI."""
+    return any("creativecommons" in (lic.get("URL") or "").lower() for lic in item.get("license") or [])
+
+
+def _created_date(item: dict[str, Any]) -> date | None:
+    parts = ((item.get("created") or {}).get("date-parts") or [[None]])[0]
+    if not parts or parts[0] is None:
+        return None
+    try:
+        return date(parts[0], parts[1] if len(parts) > 1 else 1, parts[2] if len(parts) > 2 else 1)
+    except ValueError:
+        return None
+
+
+def paper_from_item(item: dict[str, Any], source: Source) -> Paper | None:
+    """Map one Crossref work item to a Paper; None when it has no usable DOI or title."""
+    doi = normalize_doi(item.get("DOI"))
+    titles = item.get("title") or []
+    title = _WS_RE.sub(" ", html.unescape(_TAG_RE.sub("", titles[0] if titles else ""))).strip()
+    if not doi or not title:
+        return None
+    authors = [
+        name
+        for a in item.get("author") or []
+        if (name := " ".join(part for part in (a.get("given"), a.get("family")) if part).strip())
+    ]
+    return Paper(
+        doi=doi,
+        title=title,
+        abstract="",  # IEEE는 Crossref에 초록을 예치하지 않음 - OpenAlex 색인 시 보강됨
+        authors=authors,
+        venue=source.name,
+        published_date=_created_date(item),
+        oa_url=f"https://doi.org/{doi}",
+        topics=[],
+        source=f"crossref:{source.key}",
+    )
+
+
+def _fetch_source(source: Source, start: date, end: date, settings: Settings) -> list[Paper]:
+    params: dict[str, str] = {
+        "filter": f"from-created-date:{start.isoformat()},until-created-date:{end.isoformat()}",
+        "rows": str(ROWS),
+        "select": SELECT,
+    }
+    if settings.openalex_mailto:
+        params["mailto"] = settings.openalex_mailto
+    resp = requests.get(
+        f"{BASE_URL}/journals/{source.issns[0]}/works",
+        params=params,
+        timeout=TIMEOUT_SECONDS,
+        headers={"User-Agent": USER_AGENT},
+    )
+    resp.raise_for_status()
+    items = (resp.json().get("message") or {}).get("items") or []
+    papers: list[Paper] = []
+    for item in items:
+        if not _is_cc_licensed(item):
+            continue  # 하이브리드 저널의 비 OA 논문
+        if is_blacklisted(source.issns, None, settings.extra_blacklist_issns):
+            continue
+        paper = paper_from_item(item, source)
+        if paper is not None:
+            papers.append(paper)
+    log.info("crossref %s: %d items -> %d OA papers in %s..%s", source.key, len(items), len(papers), start, end)
+    return papers
+
+
+def fetch_crossref_papers(start: date, end: date, settings: Settings) -> list[Paper]:
+    """Fresh OA papers (paper-level CC licence) from the crossref_fresh sources, dedup by DOI.
+
+    Raises RuntimeError only when every source query failed (and at least one source is configured).
+    """
+    sources = crossref_fresh_sources()
+    collected: list[Paper] = []
+    failures: list[str] = []
+    for source in sources:
+        try:
+            collected.extend(_fetch_source(source, start, end, settings))
+        except Exception as exc:  # noqa: BLE001 - per-source isolation
+            log.warning("crossref %s failed: %s: %s", source.key, type(exc).__name__, str(exc)[:200])
+            failures.append(f"{source.key}: {type(exc).__name__}")
+
+    if sources and failures and len(failures) == len(sources):
+        raise RuntimeError("all crossref queries failed: " + "; ".join(failures))
+
+    unique: list[Paper] = []
+    seen: set[str] = set()
+    for paper in collected:
+        if paper.doi in seen:
+            continue
+        seen.add(paper.doi)
+        unique.append(paper)
+    return unique

sebrief/pipeline.py

@@ -12,6 +12,7 @@ from datetime import datetime, timedelta
 from zoneinfo import ZoneInfo
 
 from sebrief.collectors.base import run_safely
+from sebrief.collectors.crossref import fetch_crossref_papers
 from sebrief.collectors.openalex import fetch_openalex_papers
 from sebrief.collectors.rss import fetch_rss_papers, parse_feed_specs
 from sebrief.config import KST, Settings
@@ -60,6 +61,14 @@ def build_payload(week: str, settings: Settings, store: Store) -> WeekPayload:
         "openalex", lambda: fetch_openalex_papers(collect_from, week_end, settings), [], gaps
     )
 
+    # OpenAlex 색인이 늦는 저널(TSE 등)의 신간 보조 수집 - OpenAlex가 먼저이므로 중복 DOI는
+    # 더 풍부한 OpenAlex 메타데이터가 이긴다.
+    crossref_papers = run_safely(
+        "crossref", lambda: fetch_crossref_papers(collect_from, week_end, settings), [], gaps
+    )
+    if crossref_papers:
+        papers = _dedup_by_doi([*papers, *crossref_papers])
+
     feeds, invalid = parse_feed_specs(settings.rss_feeds)
     for spec in invalid:
         gaps.append(f"rss: invalid feed spec {spec!r} (expected 이름|URL)")

sebrief/sources.py

@@ -21,13 +21,22 @@ class Source:
     name: str
     issns: tuple[str, ...]
     se_filter: bool = False  # True: mega-journal - needs the SE topic/keyword double filter
+    # True: OpenAlex 색인이 몇 달씩 늦는 저널(IEEE) - Crossref에서 신간을 보조 수집한다.
+    # Crossref에는 초록이 없어 해당 논문은 "초록 미제공"으로 실리고, 나중에 OpenAlex가 색인하면
+    # 초록이 보강된다 (store.upsert_papers의 빈 초록 채움 규칙).
+    crossref_fresh: bool = False
 
 
 SOURCES: tuple[Source, ...] = (
     # 완전 OA (2026-01 ACM Open 전환) - 전량 수집
     Source("tosem", "ACM Transactions on Software Engineering and Methodology", ("1049-331X", "1557-7392")),
     # 하이브리드 - 논문 단위 OA 필터로 수집
-    Source("tse", "IEEE Transactions on Software Engineering", ("0098-5589", "1939-3520", "2326-3881")),
+    Source(
+        "tse",
+        "IEEE Transactions on Software Engineering",
+        ("0098-5589", "1939-3520", "2326-3881"),
+        crossref_fresh=True,  # OpenAlex의 TSE 색인이 2~3개월 지연됨 (2026-09 실측)
+    ),
     Source("emse", "Empirical Software Engineering", ("1382-3256", "1573-7616")),
     Source("ist", "Information and Software Technology", ("0950-5849",)),
     Source("jss", "Journal of Systems and Software", ("0164-1212", "1873-1228")),
@@ -118,6 +127,11 @@ def issns_for(se_filter: bool) -> tuple[str, ...]:
     return tuple(out)
 
 
+def crossref_fresh_sources() -> tuple[Source, ...]:
+    """Sources whose fresh papers are supplemented from Crossref (slow OpenAlex indexing)."""
+    return tuple(src for src in SOURCES if src.crossref_fresh)
+
+
 def is_blacklisted(
     issns: Iterable[str], publisher: str | None, extra_issns: Iterable[str] = ()
 ) -> bool:

sebrief/store.py

@@ -111,10 +111,18 @@ class Store:
         with self._conn:
             for p in papers:
                 row = self._conn.execute(
-                    "SELECT digest_week FROM papers WHERE doi = ?", (p.doi,)
+                    "SELECT digest_week, abstract FROM papers WHERE doi = ?", (p.doi,)
                 ).fetchone()
                 if row is not None and row["digest_week"] is not None:
-                    continue  # already digested - frozen
+                    # Already digested - frozen, with one exception: a paper that entered without an
+                    # abstract (Crossref 신간) gains it once OpenAlex indexes the DOI, so a later
+                    # --force re-run can produce its per-paper summary.
+                    if not row["abstract"] and p.abstract.strip():
+                        self._conn.execute(
+                            "UPDATE papers SET abstract = ? WHERE doi = ?", (p.abstract, p.doi)
+                        )
+                        log.info("filled missing abstract for digested paper %s", p.doi)
+                    continue
                 values = (
                     p.title,
                     p.abstract,

tests/test_crossref.py

@@ -0,0 +1,115 @@
+"""Crossref supplement: item mapping, paper-level CC-licence OA filter, failure isolation."""
+
+from __future__ import annotations
+
+from datetime import date
+
+import pytest
+
+from sebrief.collectors import crossref
+from sebrief.collectors.crossref import fetch_crossref_papers, paper_from_item
+from sebrief.config import Settings
+from sebrief.sources import Source, crossref_fresh_sources
+
+SETTINGS = Settings(_env_file=None)
+START, END = date(2026, 8, 24), date(2026, 9, 6)
+TSE = crossref_fresh_sources()[0]
+
+CC_ITEM = {
+    "DOI": "10.1109/TSE.2026.3729363",
+    "title": ["An Empirical Study of <i>Flaky</i> Tests"],
+    "author": [{"given": "Alice", "family": "Kim"}, {"family": "Lee"}],
+    "created": {"date-parts": [[2026, 8, 31]]},
+    "license": [{"URL": "https://creativecommons.org/licenses/by/4.0/"}],
+}
+PAYWALLED_ITEM = {
+    "DOI": "10.1109/TSE.2026.3729587",
+    "title": ["Navigating Complexity"],
+    "created": {"date-parts": [[2026, 9, 1]]},
+    "license": [{"URL": "https://ieeexplore.ieee.org/Xplorehelp/downloads"}],
+}
+
+
+def test_crossref_fresh_sources_contains_tse():
+    assert [s.key for s in crossref_fresh_sources()] == ["tse"]
+
+
+def test_paper_from_item_maps_fields_and_strips_tags():
+    paper = paper_from_item(CC_ITEM, TSE)
+    assert paper is not None
+    assert paper.doi == "10.1109/tse.2026.3729363"
+    assert paper.title == "An Empirical Study of Flaky Tests"
+    assert paper.authors == ["Alice Kim", "Lee"]
+    assert paper.venue == "IEEE Transactions on Software Engineering"
+    assert paper.published_date == date(2026, 8, 31)
+    assert paper.abstract == ""  # IEEE는 Crossref에 초록을 예치하지 않음
+    assert paper.oa_url == "https://doi.org/10.1109/tse.2026.3729363"
+    assert paper.source == "crossref:tse"
+
+
+def test_paper_from_item_rejects_missing_doi_or_title():
+    assert paper_from_item({"title": ["T"]}, TSE) is None
+    assert paper_from_item({"DOI": "10.1/x", "title": []}, TSE) is None
+
+
+class FakeResponse:
+    def __init__(self, items):
+        self._items = items
+
+    def raise_for_status(self):
+        pass
+
+    def json(self):
+        return {"message": {"items": self._items}}
+
+
+def test_fetch_keeps_only_cc_licensed_papers(monkeypatch):
+    calls = []
+
+    def fake_get(url, params=None, timeout=None, headers=None):
+        calls.append({"url": url, "params": params})
+        return FakeResponse([CC_ITEM, PAYWALLED_ITEM, {"DOI": "10.1/nolicense", "title": ["X"]}])
+
+    monkeypatch.setattr(crossref.requests, "get", fake_get)
+    papers = fetch_crossref_papers(START, END, SETTINGS)
+    assert [p.doi for p in papers] == ["10.1109/tse.2026.3729363"]
+    assert "/journals/0098-5589/works" in calls[0]["url"]
+    assert calls[0]["params"]["filter"] == "from-created-date:2026-08-24,until-created-date:2026-09-06"
+
+
+def test_fetch_dedups_by_doi(monkeypatch):
+    monkeypatch.setattr(crossref.requests, "get", lambda *a, **k: FakeResponse([CC_ITEM, CC_ITEM]))
+    papers = fetch_crossref_papers(START, END, SETTINGS)
+    assert len(papers) == 1
+
+
+def test_all_sources_failed_raises(monkeypatch):
+    def boom(*a, **k):
+        raise RuntimeError("crossref down")
+
+    monkeypatch.setattr(crossref.requests, "get", boom)
+    with pytest.raises(RuntimeError, match="all crossref queries failed"):
+        fetch_crossref_papers(START, END, SETTINGS)
+
+
+def test_blacklisted_source_is_dropped(monkeypatch):
+    monkeypatch.setattr(crossref.requests, "get", lambda *a, **k: FakeResponse([CC_ITEM]))
+    settings = Settings(_env_file=None, SEBRIEF_EXTRA_BLACKLIST_ISSNS="0098-5589")
+    assert fetch_crossref_papers(START, END, settings) == []
+
+
+def test_mailto_forwarded(monkeypatch):
+    calls = []
+
+    def fake_get(url, params=None, timeout=None, headers=None):
+        calls.append(params)
+        return FakeResponse([])
+
+    monkeypatch.setattr(crossref.requests, "get", fake_get)
+    fetch_crossref_papers(START, END, Settings(_env_file=None, SEBRIEF_OPENALEX_MAILTO="ops@example.org"))
+    assert calls[0]["mailto"] == "ops@example.org"
+
+
+def test_source_without_crossref_flag_not_queried():
+    plain = Source("x", "X Journal", ("1111-2222",))
+    assert plain.crossref_fresh is False

tests/test_pipeline.py

@@ -22,9 +22,11 @@ def env(monkeypatch, tmp_path):
         "rss": [],
         "settings": Settings(_env_file=None, SEBRIEF_DATA_DIR=str(tmp_path / "data")),
     }
+    ctx["crossref"] = []
     monkeypatch.setattr(pipeline, "_now", lambda: NOW)
     monkeypatch.setattr(pipeline, "Store", lambda path: ctx["store"])
     monkeypatch.setattr(pipeline, "fetch_openalex_papers", lambda start, end, s: ctx["openalex"])
+    monkeypatch.setattr(pipeline, "fetch_crossref_papers", lambda start, end, s: ctx["crossref"])
     monkeypatch.setattr(pipeline, "fetch_rss_papers", lambda feeds, start, end, bl: ctx["rss"])
     monkeypatch.setattr(pipeline, "build_provider", lambda s: ctx["provider"])
     monkeypatch.setattr(pipeline, "build_deliverers", lambda s: ctx["deliverers"])
@@ -66,6 +68,16 @@ def test_build_payload_merges_rss_dedup_by_doi(env):
     assert upserted[0].source == "openalex"  # openalex metadata wins on the duplicate
 
 
+def test_build_payload_merges_crossref_openalex_wins_duplicates(env):
+    dup = make_paper(source="crossref:tse", venue="TSE", abstract="")  # same DOI as openalex
+    fresh = make_paper("10.1109/tse.2026.1", source="crossref:tse", abstract="")
+    env["crossref"] = [dup, fresh]
+    build_payload(WEEK, env["settings"], env["store"])
+    [upserted] = env["store"].upserted
+    assert [p.doi for p in upserted] == ["10.1145/1234567", "10.1109/tse.2026.1"]
+    assert upserted[0].source == "openalex"  # richer OpenAlex metadata wins the duplicate
+
+
 def test_build_payload_invalid_feed_spec_is_a_gap(env):
     env["settings"] = env["settings"].model_copy(update={"rss_feeds": ["broken-spec"]})
     payload = build_payload(WEEK, env["settings"], env["store"])

tests/test_store.py

@@ -38,6 +38,17 @@ def test_upsert_refreshes_undigested_metadata(tmp_path):
         assert paper.abstract == "now with abstract"
 
 
+def test_digested_paper_gains_missing_abstract_but_stays_frozen(tmp_path):
+    with make_store(tmp_path) as store:
+        store.upsert_papers([make_paper("10.1/a", title="fresh via crossref", abstract="")])
+        store.mark_digested(["10.1/a"], WEEK)
+        # OpenAlex later indexes the DOI with an abstract (and a normalised title)
+        store.upsert_papers([make_paper("10.1/a", title="OpenAlex Title", abstract="now indexed")])
+        [paper] = store.pending_papers(WEEK)
+        assert paper.abstract == "now indexed"  # 초록만 보강
+        assert paper.title == "fresh via crossref"  # 그 외 메타데이터는 동결 유지
+
+
 def test_digested_papers_are_frozen_and_skipped(tmp_path):
     with make_store(tmp_path) as store:
         store.upsert_papers([make_paper("10.1/a", title="original")])