From 3f1d16cae19e9ad1dfa56db5e24b849b6361561b Mon Sep 17 00:00:00 2001 From: jsahn Date: Wed, 2 Sep 2026 21:00:32 +0900 Subject: [PATCH] Supplement slow-indexed IEEE journals from Crossref (TSE fresh papers) OpenAlex indexes TSE ~2-3 months late (nothing after 2026-06-26 as of Sep 2), so TSE never appeared in weekly digests. Sources can now be flagged crossref_fresh: their recent papers are discovered via Crossref (created-date window, paper-level CC licence = OA) and listed without abstracts; a digested paper's missing abstract is filled once OpenAlex indexes the DOI, so a later --force re-run can add its per-paper summary. Co-Authored-By: Claude Fable 5 --- README.md | 3 +- sebrief/collectors/crossref.py | 133 +++++++++++++++++++++++++++++++++ sebrief/pipeline.py | 9 +++ sebrief/sources.py | 16 +++- sebrief/store.py | 12 ++- tests/test_crossref.py | 115 ++++++++++++++++++++++++++++ tests/test_pipeline.py | 12 +++ tests/test_store.py | 11 +++ 8 files changed, 307 insertions(+), 4 deletions(-) create mode 100644 sebrief/collectors/crossref.py create mode 100644 tests/test_crossref.py diff --git a/README.md b/README.md index fba67e5..6e313d1 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ US2KR의 파이프라인 철학을 그대로 따르는 별도 프로젝트입니 | 소스 | 방식 | |---|---| | TOSEM (완전 OA) | ISSN 화이트리스트, 전량 | -| TSE · EMSE · IST · JSS (하이브리드) | ISSN 화이트리스트 + **논문 단위** `is_oa` 필터 | +| TSE (하이브리드, IEEE) | ISSN 화이트리스트 + OA 필터. OpenAlex 색인이 2~3개월 늦어 **Crossref로 신간 보조 수집**(논문 단위 CC 라이선스 = OA 판별). Crossref에는 초록이 없어 신간은 "초록 미제공"으로 실리고, OpenAlex 색인 시 초록이 보강됨 | +| EMSE · IST · JSS (하이브리드) | ISSN 화이트리스트 + **논문 단위** `is_oa` 필터 | | SoftwareX · JOSS | 전량 (도구/소프트웨어 논문) | | IEEE Access · PeerJ CS (메가저널) | ISSN + OpenAlex Software 토픽(`topics.subfield.id:1712`) + **키워드 이중 필터** | | e-Informatica SEJ 등 | `SEBRIEF_RSS_FEEDS`로 RSS 보조 (DOI 없는 항목은 스킵) | diff --git a/sebrief/collectors/crossref.py b/sebrief/collectors/crossref.py new file mode 100644 index 0000000..8b0d5ed --- /dev/null +++ b/sebrief/collectors/crossref.py @@ -0,0 +1,133 @@ +"""Crossref supplement: fresh papers from journals OpenAlex indexes months late (IEEE - TSE 등). + +Crossref는 발행사가 DOI를 등록하는 즉시 반영되지만 IEEE는 초록을 예치하지 않으므로, 여기서 수집된 +논문은 초록 없이(제목·링크·발행일) 다이제스트에 실린다. OA 판별은 논문 단위 라이선스로 한다: +하이브리드 저널에서 CC 라이선스가 등록된 논문 = OA (Unpaywall 없이 판별 가능, 2026-09 실측). + +IEEE early access는 Crossref의 published 날짜가 연 단위 정밀도라 쓸 수 없어, 등록일(created, +조기 공개일과 사실상 동일)을 발행일로 쓴다. Only if *every* source query fails does +`fetch_crossref_papers` raise. +""" + +from __future__ import annotations + +import html +import logging +import re +from datetime import date +from typing import Any + +import requests + +from sebrief.collectors.openalex import normalize_doi +from sebrief.config import Settings +from sebrief.models import Paper +from sebrief.sources import Source, crossref_fresh_sources, is_blacklisted + +log = logging.getLogger(__name__) + +BASE_URL = "https://api.crossref.org" +USER_AGENT = "sebrief/0.1 (OA software engineering papers weekly digest)" +TIMEOUT_SECONDS = 30.0 +ROWS = 100 +SELECT = "DOI,title,author,created,license,container-title" + +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def _is_cc_licensed(item: dict[str, Any]) -> bool: + """Paper-level OA in a hybrid journal: a Creative Commons licence is registered for the DOI.""" + return any("creativecommons" in (lic.get("URL") or "").lower() for lic in item.get("license") or []) + + +def _created_date(item: dict[str, Any]) -> date | None: + parts = ((item.get("created") or {}).get("date-parts") or [[None]])[0] + if not parts or parts[0] is None: + return None + try: + return date(parts[0], parts[1] if len(parts) > 1 else 1, parts[2] if len(parts) > 2 else 1) + except ValueError: + return None + + +def paper_from_item(item: dict[str, Any], source: Source) -> Paper | None: + """Map one Crossref work item to a Paper; None when it has no usable DOI or title.""" + doi = normalize_doi(item.get("DOI")) + titles = item.get("title") or [] + title = _WS_RE.sub(" ", html.unescape(_TAG_RE.sub("", titles[0] if titles else ""))).strip() + if not doi or not title: + return None + authors = [ + name + for a in item.get("author") or [] + if (name := " ".join(part for part in (a.get("given"), a.get("family")) if part).strip()) + ] + return Paper( + doi=doi, + title=title, + abstract="", # IEEE는 Crossref에 초록을 예치하지 않음 - OpenAlex 색인 시 보강됨 + authors=authors, + venue=source.name, + published_date=_created_date(item), + oa_url=f"https://doi.org/{doi}", + topics=[], + source=f"crossref:{source.key}", + ) + + +def _fetch_source(source: Source, start: date, end: date, settings: Settings) -> list[Paper]: + params: dict[str, str] = { + "filter": f"from-created-date:{start.isoformat()},until-created-date:{end.isoformat()}", + "rows": str(ROWS), + "select": SELECT, + } + if settings.openalex_mailto: + params["mailto"] = settings.openalex_mailto + resp = requests.get( + f"{BASE_URL}/journals/{source.issns[0]}/works", + params=params, + timeout=TIMEOUT_SECONDS, + headers={"User-Agent": USER_AGENT}, + ) + resp.raise_for_status() + items = (resp.json().get("message") or {}).get("items") or [] + papers: list[Paper] = [] + for item in items: + if not _is_cc_licensed(item): + continue # 하이브리드 저널의 비 OA 논문 + if is_blacklisted(source.issns, None, settings.extra_blacklist_issns): + continue + paper = paper_from_item(item, source) + if paper is not None: + papers.append(paper) + log.info("crossref %s: %d items -> %d OA papers in %s..%s", source.key, len(items), len(papers), start, end) + return papers + + +def fetch_crossref_papers(start: date, end: date, settings: Settings) -> list[Paper]: + """Fresh OA papers (paper-level CC licence) from the crossref_fresh sources, dedup by DOI. + + Raises RuntimeError only when every source query failed (and at least one source is configured). + """ + sources = crossref_fresh_sources() + collected: list[Paper] = [] + failures: list[str] = [] + for source in sources: + try: + collected.extend(_fetch_source(source, start, end, settings)) + except Exception as exc: # noqa: BLE001 - per-source isolation + log.warning("crossref %s failed: %s: %s", source.key, type(exc).__name__, str(exc)[:200]) + failures.append(f"{source.key}: {type(exc).__name__}") + + if sources and failures and len(failures) == len(sources): + raise RuntimeError("all crossref queries failed: " + "; ".join(failures)) + + unique: list[Paper] = [] + seen: set[str] = set() + for paper in collected: + if paper.doi in seen: + continue + seen.add(paper.doi) + unique.append(paper) + return unique diff --git a/sebrief/pipeline.py b/sebrief/pipeline.py index cad52bf..80f0285 100644 --- a/sebrief/pipeline.py +++ b/sebrief/pipeline.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta from zoneinfo import ZoneInfo from sebrief.collectors.base import run_safely +from sebrief.collectors.crossref import fetch_crossref_papers from sebrief.collectors.openalex import fetch_openalex_papers from sebrief.collectors.rss import fetch_rss_papers, parse_feed_specs from sebrief.config import KST, Settings @@ -60,6 +61,14 @@ def build_payload(week: str, settings: Settings, store: Store) -> WeekPayload: "openalex", lambda: fetch_openalex_papers(collect_from, week_end, settings), [], gaps ) + # OpenAlex 색인이 늦는 저널(TSE 등)의 신간 보조 수집 - OpenAlex가 먼저이므로 중복 DOI는 + # 더 풍부한 OpenAlex 메타데이터가 이긴다. + crossref_papers = run_safely( + "crossref", lambda: fetch_crossref_papers(collect_from, week_end, settings), [], gaps + ) + if crossref_papers: + papers = _dedup_by_doi([*papers, *crossref_papers]) + feeds, invalid = parse_feed_specs(settings.rss_feeds) for spec in invalid: gaps.append(f"rss: invalid feed spec {spec!r} (expected 이름|URL)") diff --git a/sebrief/sources.py b/sebrief/sources.py index 4aead56..cff7a68 100644 --- a/sebrief/sources.py +++ b/sebrief/sources.py @@ -21,13 +21,22 @@ class Source: name: str issns: tuple[str, ...] se_filter: bool = False # True: mega-journal - needs the SE topic/keyword double filter + # True: OpenAlex 색인이 몇 달씩 늦는 저널(IEEE) - Crossref에서 신간을 보조 수집한다. + # Crossref에는 초록이 없어 해당 논문은 "초록 미제공"으로 실리고, 나중에 OpenAlex가 색인하면 + # 초록이 보강된다 (store.upsert_papers의 빈 초록 채움 규칙). + crossref_fresh: bool = False SOURCES: tuple[Source, ...] = ( # 완전 OA (2026-01 ACM Open 전환) - 전량 수집 Source("tosem", "ACM Transactions on Software Engineering and Methodology", ("1049-331X", "1557-7392")), # 하이브리드 - 논문 단위 OA 필터로 수집 - Source("tse", "IEEE Transactions on Software Engineering", ("0098-5589", "1939-3520", "2326-3881")), + Source( + "tse", + "IEEE Transactions on Software Engineering", + ("0098-5589", "1939-3520", "2326-3881"), + crossref_fresh=True, # OpenAlex의 TSE 색인이 2~3개월 지연됨 (2026-09 실측) + ), Source("emse", "Empirical Software Engineering", ("1382-3256", "1573-7616")), Source("ist", "Information and Software Technology", ("0950-5849",)), Source("jss", "Journal of Systems and Software", ("0164-1212", "1873-1228")), @@ -118,6 +127,11 @@ def issns_for(se_filter: bool) -> tuple[str, ...]: return tuple(out) +def crossref_fresh_sources() -> tuple[Source, ...]: + """Sources whose fresh papers are supplemented from Crossref (slow OpenAlex indexing).""" + return tuple(src for src in SOURCES if src.crossref_fresh) + + def is_blacklisted( issns: Iterable[str], publisher: str | None, extra_issns: Iterable[str] = () ) -> bool: diff --git a/sebrief/store.py b/sebrief/store.py index 0a3fa58..3e03372 100644 --- a/sebrief/store.py +++ b/sebrief/store.py @@ -111,10 +111,18 @@ class Store: with self._conn: for p in papers: row = self._conn.execute( - "SELECT digest_week FROM papers WHERE doi = ?", (p.doi,) + "SELECT digest_week, abstract FROM papers WHERE doi = ?", (p.doi,) ).fetchone() if row is not None and row["digest_week"] is not None: - continue # already digested - frozen + # Already digested - frozen, with one exception: a paper that entered without an + # abstract (Crossref 신간) gains it once OpenAlex indexes the DOI, so a later + # --force re-run can produce its per-paper summary. + if not row["abstract"] and p.abstract.strip(): + self._conn.execute( + "UPDATE papers SET abstract = ? WHERE doi = ?", (p.abstract, p.doi) + ) + log.info("filled missing abstract for digested paper %s", p.doi) + continue values = ( p.title, p.abstract, diff --git a/tests/test_crossref.py b/tests/test_crossref.py new file mode 100644 index 0000000..ce2aa9e --- /dev/null +++ b/tests/test_crossref.py @@ -0,0 +1,115 @@ +"""Crossref supplement: item mapping, paper-level CC-licence OA filter, failure isolation.""" + +from __future__ import annotations + +from datetime import date + +import pytest + +from sebrief.collectors import crossref +from sebrief.collectors.crossref import fetch_crossref_papers, paper_from_item +from sebrief.config import Settings +from sebrief.sources import Source, crossref_fresh_sources + +SETTINGS = Settings(_env_file=None) +START, END = date(2026, 8, 24), date(2026, 9, 6) +TSE = crossref_fresh_sources()[0] + +CC_ITEM = { + "DOI": "10.1109/TSE.2026.3729363", + "title": ["An Empirical Study of Flaky Tests"], + "author": [{"given": "Alice", "family": "Kim"}, {"family": "Lee"}], + "created": {"date-parts": [[2026, 8, 31]]}, + "license": [{"URL": "https://creativecommons.org/licenses/by/4.0/"}], +} +PAYWALLED_ITEM = { + "DOI": "10.1109/TSE.2026.3729587", + "title": ["Navigating Complexity"], + "created": {"date-parts": [[2026, 9, 1]]}, + "license": [{"URL": "https://ieeexplore.ieee.org/Xplorehelp/downloads"}], +} + + +def test_crossref_fresh_sources_contains_tse(): + assert [s.key for s in crossref_fresh_sources()] == ["tse"] + + +def test_paper_from_item_maps_fields_and_strips_tags(): + paper = paper_from_item(CC_ITEM, TSE) + assert paper is not None + assert paper.doi == "10.1109/tse.2026.3729363" + assert paper.title == "An Empirical Study of Flaky Tests" + assert paper.authors == ["Alice Kim", "Lee"] + assert paper.venue == "IEEE Transactions on Software Engineering" + assert paper.published_date == date(2026, 8, 31) + assert paper.abstract == "" # IEEE는 Crossref에 초록을 예치하지 않음 + assert paper.oa_url == "https://doi.org/10.1109/tse.2026.3729363" + assert paper.source == "crossref:tse" + + +def test_paper_from_item_rejects_missing_doi_or_title(): + assert paper_from_item({"title": ["T"]}, TSE) is None + assert paper_from_item({"DOI": "10.1/x", "title": []}, TSE) is None + + +class FakeResponse: + def __init__(self, items): + self._items = items + + def raise_for_status(self): + pass + + def json(self): + return {"message": {"items": self._items}} + + +def test_fetch_keeps_only_cc_licensed_papers(monkeypatch): + calls = [] + + def fake_get(url, params=None, timeout=None, headers=None): + calls.append({"url": url, "params": params}) + return FakeResponse([CC_ITEM, PAYWALLED_ITEM, {"DOI": "10.1/nolicense", "title": ["X"]}]) + + monkeypatch.setattr(crossref.requests, "get", fake_get) + papers = fetch_crossref_papers(START, END, SETTINGS) + assert [p.doi for p in papers] == ["10.1109/tse.2026.3729363"] + assert "/journals/0098-5589/works" in calls[0]["url"] + assert calls[0]["params"]["filter"] == "from-created-date:2026-08-24,until-created-date:2026-09-06" + + +def test_fetch_dedups_by_doi(monkeypatch): + monkeypatch.setattr(crossref.requests, "get", lambda *a, **k: FakeResponse([CC_ITEM, CC_ITEM])) + papers = fetch_crossref_papers(START, END, SETTINGS) + assert len(papers) == 1 + + +def test_all_sources_failed_raises(monkeypatch): + def boom(*a, **k): + raise RuntimeError("crossref down") + + monkeypatch.setattr(crossref.requests, "get", boom) + with pytest.raises(RuntimeError, match="all crossref queries failed"): + fetch_crossref_papers(START, END, SETTINGS) + + +def test_blacklisted_source_is_dropped(monkeypatch): + monkeypatch.setattr(crossref.requests, "get", lambda *a, **k: FakeResponse([CC_ITEM])) + settings = Settings(_env_file=None, SEBRIEF_EXTRA_BLACKLIST_ISSNS="0098-5589") + assert fetch_crossref_papers(START, END, settings) == [] + + +def test_mailto_forwarded(monkeypatch): + calls = [] + + def fake_get(url, params=None, timeout=None, headers=None): + calls.append(params) + return FakeResponse([]) + + monkeypatch.setattr(crossref.requests, "get", fake_get) + fetch_crossref_papers(START, END, Settings(_env_file=None, SEBRIEF_OPENALEX_MAILTO="ops@example.org")) + assert calls[0]["mailto"] == "ops@example.org" + + +def test_source_without_crossref_flag_not_queried(): + plain = Source("x", "X Journal", ("1111-2222",)) + assert plain.crossref_fresh is False diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a948e0b..15f08cd 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -22,9 +22,11 @@ def env(monkeypatch, tmp_path): "rss": [], "settings": Settings(_env_file=None, SEBRIEF_DATA_DIR=str(tmp_path / "data")), } + ctx["crossref"] = [] monkeypatch.setattr(pipeline, "_now", lambda: NOW) monkeypatch.setattr(pipeline, "Store", lambda path: ctx["store"]) monkeypatch.setattr(pipeline, "fetch_openalex_papers", lambda start, end, s: ctx["openalex"]) + monkeypatch.setattr(pipeline, "fetch_crossref_papers", lambda start, end, s: ctx["crossref"]) monkeypatch.setattr(pipeline, "fetch_rss_papers", lambda feeds, start, end, bl: ctx["rss"]) monkeypatch.setattr(pipeline, "build_provider", lambda s: ctx["provider"]) monkeypatch.setattr(pipeline, "build_deliverers", lambda s: ctx["deliverers"]) @@ -66,6 +68,16 @@ def test_build_payload_merges_rss_dedup_by_doi(env): assert upserted[0].source == "openalex" # openalex metadata wins on the duplicate +def test_build_payload_merges_crossref_openalex_wins_duplicates(env): + dup = make_paper(source="crossref:tse", venue="TSE", abstract="") # same DOI as openalex + fresh = make_paper("10.1109/tse.2026.1", source="crossref:tse", abstract="") + env["crossref"] = [dup, fresh] + build_payload(WEEK, env["settings"], env["store"]) + [upserted] = env["store"].upserted + assert [p.doi for p in upserted] == ["10.1145/1234567", "10.1109/tse.2026.1"] + assert upserted[0].source == "openalex" # richer OpenAlex metadata wins the duplicate + + def test_build_payload_invalid_feed_spec_is_a_gap(env): env["settings"] = env["settings"].model_copy(update={"rss_feeds": ["broken-spec"]}) payload = build_payload(WEEK, env["settings"], env["store"]) diff --git a/tests/test_store.py b/tests/test_store.py index c0f81b1..104792e 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -38,6 +38,17 @@ def test_upsert_refreshes_undigested_metadata(tmp_path): assert paper.abstract == "now with abstract" +def test_digested_paper_gains_missing_abstract_but_stays_frozen(tmp_path): + with make_store(tmp_path) as store: + store.upsert_papers([make_paper("10.1/a", title="fresh via crossref", abstract="")]) + store.mark_digested(["10.1/a"], WEEK) + # OpenAlex later indexes the DOI with an abstract (and a normalised title) + store.upsert_papers([make_paper("10.1/a", title="OpenAlex Title", abstract="now indexed")]) + [paper] = store.pending_papers(WEEK) + assert paper.abstract == "now indexed" # 초록만 보강 + assert paper.title == "fresh via crossref" # 그 외 메타데이터는 동결 유지 + + def test_digested_papers_are_frozen_and_skipped(tmp_path): with make_store(tmp_path) as store: store.upsert_papers([make_paper("10.1/a", title="original")])