jsahnandClaude Fable 5
3f1d16cae1
OpenAlex indexes TSE ~2-3 months late (nothing after 2026-06-26 as of Sep 2), so TSE never appeared in weekly digests. Sources can now be flagged crossref_fresh: their recent papers are discovered via Crossref (created-date window, paper-level CC licence = OA) and listed without abstracts; a digested paper's missing abstract is filled once OpenAlex indexes the DOI, so a later --force re-run can add its per-paper summary. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
116 lines
4.1 KiB
Python
116 lines
4.1 KiB
Python
"""Crossref supplement: item mapping, paper-level CC-licence OA filter, failure isolation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import date
|
|
|
|
import pytest
|
|
|
|
from sebrief.collectors import crossref
|
|
from sebrief.collectors.crossref import fetch_crossref_papers, paper_from_item
|
|
from sebrief.config import Settings
|
|
from sebrief.sources import Source, crossref_fresh_sources
|
|
|
|
SETTINGS = Settings(_env_file=None)
|
|
START, END = date(2026, 8, 24), date(2026, 9, 6)
|
|
TSE = crossref_fresh_sources()[0]
|
|
|
|
CC_ITEM = {
|
|
"DOI": "10.1109/TSE.2026.3729363",
|
|
"title": ["An Empirical Study of <i>Flaky</i> Tests"],
|
|
"author": [{"given": "Alice", "family": "Kim"}, {"family": "Lee"}],
|
|
"created": {"date-parts": [[2026, 8, 31]]},
|
|
"license": [{"URL": "https://creativecommons.org/licenses/by/4.0/"}],
|
|
}
|
|
PAYWALLED_ITEM = {
|
|
"DOI": "10.1109/TSE.2026.3729587",
|
|
"title": ["Navigating Complexity"],
|
|
"created": {"date-parts": [[2026, 9, 1]]},
|
|
"license": [{"URL": "https://ieeexplore.ieee.org/Xplorehelp/downloads"}],
|
|
}
|
|
|
|
|
|
def test_crossref_fresh_sources_contains_tse():
|
|
assert [s.key for s in crossref_fresh_sources()] == ["tse"]
|
|
|
|
|
|
def test_paper_from_item_maps_fields_and_strips_tags():
|
|
paper = paper_from_item(CC_ITEM, TSE)
|
|
assert paper is not None
|
|
assert paper.doi == "10.1109/tse.2026.3729363"
|
|
assert paper.title == "An Empirical Study of Flaky Tests"
|
|
assert paper.authors == ["Alice Kim", "Lee"]
|
|
assert paper.venue == "IEEE Transactions on Software Engineering"
|
|
assert paper.published_date == date(2026, 8, 31)
|
|
assert paper.abstract == "" # IEEE는 Crossref에 초록을 예치하지 않음
|
|
assert paper.oa_url == "https://doi.org/10.1109/tse.2026.3729363"
|
|
assert paper.source == "crossref:tse"
|
|
|
|
|
|
def test_paper_from_item_rejects_missing_doi_or_title():
|
|
assert paper_from_item({"title": ["T"]}, TSE) is None
|
|
assert paper_from_item({"DOI": "10.1/x", "title": []}, TSE) is None
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(self, items):
|
|
self._items = items
|
|
|
|
def raise_for_status(self):
|
|
pass
|
|
|
|
def json(self):
|
|
return {"message": {"items": self._items}}
|
|
|
|
|
|
def test_fetch_keeps_only_cc_licensed_papers(monkeypatch):
|
|
calls = []
|
|
|
|
def fake_get(url, params=None, timeout=None, headers=None):
|
|
calls.append({"url": url, "params": params})
|
|
return FakeResponse([CC_ITEM, PAYWALLED_ITEM, {"DOI": "10.1/nolicense", "title": ["X"]}])
|
|
|
|
monkeypatch.setattr(crossref.requests, "get", fake_get)
|
|
papers = fetch_crossref_papers(START, END, SETTINGS)
|
|
assert [p.doi for p in papers] == ["10.1109/tse.2026.3729363"]
|
|
assert "/journals/0098-5589/works" in calls[0]["url"]
|
|
assert calls[0]["params"]["filter"] == "from-created-date:2026-08-24,until-created-date:2026-09-06"
|
|
|
|
|
|
def test_fetch_dedups_by_doi(monkeypatch):
|
|
monkeypatch.setattr(crossref.requests, "get", lambda *a, **k: FakeResponse([CC_ITEM, CC_ITEM]))
|
|
papers = fetch_crossref_papers(START, END, SETTINGS)
|
|
assert len(papers) == 1
|
|
|
|
|
|
def test_all_sources_failed_raises(monkeypatch):
|
|
def boom(*a, **k):
|
|
raise RuntimeError("crossref down")
|
|
|
|
monkeypatch.setattr(crossref.requests, "get", boom)
|
|
with pytest.raises(RuntimeError, match="all crossref queries failed"):
|
|
fetch_crossref_papers(START, END, SETTINGS)
|
|
|
|
|
|
def test_blacklisted_source_is_dropped(monkeypatch):
|
|
monkeypatch.setattr(crossref.requests, "get", lambda *a, **k: FakeResponse([CC_ITEM]))
|
|
settings = Settings(_env_file=None, SEBRIEF_EXTRA_BLACKLIST_ISSNS="0098-5589")
|
|
assert fetch_crossref_papers(START, END, settings) == []
|
|
|
|
|
|
def test_mailto_forwarded(monkeypatch):
|
|
calls = []
|
|
|
|
def fake_get(url, params=None, timeout=None, headers=None):
|
|
calls.append(params)
|
|
return FakeResponse([])
|
|
|
|
monkeypatch.setattr(crossref.requests, "get", fake_get)
|
|
fetch_crossref_papers(START, END, Settings(_env_file=None, SEBRIEF_OPENALEX_MAILTO="ops@example.org"))
|
|
assert calls[0]["mailto"] == "ops@example.org"
|
|
|
|
|
|
def test_source_without_crossref_flag_not_queried():
|
|
plain = Source("x", "X Journal", ("1111-2222",))
|
|
assert plain.crossref_fresh is False
|