Add SearXNG news collector merged into the briefing pipeline

Optional meta-search source (SEARXNG_BASE_URL): per-query isolation, time_range=day, merged and deduplicated with the RSS feeds. Briefings still publish on RSS alone if the instance is down. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-09-01 00:58:35 +09:00
parent 8bc43b3ce4
commit 920d9358a0
7 changed files with 383 additions and 14 deletions
@@ -192,6 +192,70 @@ def test_build_payload_history_failure_is_a_gap(env):
    assert any(g.startswith("history:") for g in payload.data_gaps)


+def test_build_payload_skips_searx_when_not_configured(env):
+    def boom(*args, **kwargs):
+        raise AssertionError("fetch_searx_news must not be called without SEARXNG_BASE_URL")
+
+    env.monkeypatch.setattr(pipeline, "fetch_searx_news", boom)
+    payload = pipeline.build_payload(SESSION, env.settings, env.clock, env.store)
+    assert payload.data_gaps == []
+
+
+def test_build_payload_merges_searx_news(monkeypatch, tmp_path):
+    env = Env(
+        monkeypatch,
+        tmp_path,
+        SEARXNG_BASE_URL="http://searx.local:8888",
+        US2KR_SEARXNG_QUERIES="markets, fed",
+        US2KR_NEWS_MAX_ITEMS="3",
+    )
+    rss = sample_news()  # 1 item, published 17:00 ET
+    searx_items = [
+        NewsItem(  # duplicate of the RSS story by link; same timestamp -> RSS copy wins the tie
+            title="Stocks rally (search copy)",
+            link="https://example.com/1#utm",
+            source="Reuters",
+            published=datetime(2026, 8, 25, 17, 0, tzinfo=ET),
+        ),
+        NewsItem(
+            title="Chip stocks jump",
+            link="https://example.com/chips",
+            source="Barron's",
+            published=datetime(2026, 8, 25, 18, 30, tzinfo=ET),
+        ),
+    ]
+
+    def fetch_searx_news(base_url, since, queries=None, **kwargs):
+        env.calls["searxng"] = env.calls.get("searxng", 0) + 1
+        assert base_url == "http://searx.local:8888"
+        assert queries == ["markets", "fed"]
+        assert since == env.news_since
+        return searx_items
+
+    env.monkeypatch.setattr(pipeline, "fetch_searx_news", fetch_searx_news)
+    payload = pipeline.build_payload(SESSION, env.settings, env.clock, env.store)
+
+    assert env.calls["searxng"] == 1
+    assert [n.title for n in payload.news] == ["Chip stocks jump", "Stocks rally"]
+    assert payload.data_gaps == []
+    # the archived news blob is the merged list
+    assert "Chip stocks jump" in env.store.raw[(SESSION, "news")]
+    assert rss[0].title in env.store.raw[(SESSION, "news")]
+
+
+def test_build_payload_searx_failure_keeps_rss_news(monkeypatch, tmp_path):
+    env = Env(monkeypatch, tmp_path, SEARXNG_BASE_URL="http://searx.local:8888")
+
+    def boom(base_url, since, queries=None, **kwargs):
+        raise RuntimeError("all searxng queries failed: markets: HTTPError")
+
+    env.monkeypatch.setattr(pipeline, "fetch_searx_news", boom)
+    payload = pipeline.build_payload(SESSION, env.settings, env.clock, env.store)
+
+    assert [n.title for n in payload.news] == ["Stocks rally"]
+    assert any(g.startswith("searxng: RuntimeError") for g in payload.data_gaps)
+
+
 # -------------------------------------------------------------------------------- run_briefing
 def test_run_briefing_happy_path_saves_then_delivers(env):
    result = pipeline.run_briefing(SESSION, env.settings)
@@ -0,0 +1,151 @@
+"""Tests for us2kr.collectors.searx (no network: requests.get is mocked)."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime, timedelta
+
+import pytest
+import requests
+
+from us2kr.collectors import searx
+from us2kr.collectors.searx import _parse_published, fetch_searx_news
+
+SINCE = datetime(2026, 8, 25, 12, 0, tzinfo=UTC)
+
+
+def _result(title: str, url: str, published: str | None = None, **extra) -> dict:
+    result = {"title": title, "url": url, "content": extra.pop("content", "")}
+    if published is not None:
+        result["publishedDate"] = published
+    result.update(extra)
+    return result
+
+
+class _Resp:
+    def __init__(self, results: list | None, status: int = 200, body: str | None = None):
+        self.status_code = status
+        self._body = body if body is not None else json.dumps({"results": results})
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            raise requests.HTTPError(f"{self.status_code} error")
+
+    def json(self):
+        return json.loads(self._body)
+
+
+def _iso(dt: datetime) -> str:
+    return dt.isoformat()
+
+
+def test_parse_published_variants():
+    assert _parse_published("2026-08-25T14:00:00+00:00") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC)
+    assert _parse_published("2026-08-25T14:00:00Z") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC)
+    # naive -> assumed UTC
+    assert _parse_published("2026-08-25T14:00:00") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC)
+    assert _parse_published(None) is None
+    assert _parse_published("") is None
+    assert _parse_published("yesterday-ish") is None
+    assert _parse_published(1234) is None
+
+
+def test_filters_maps_and_floors_undated(mocker):
+    fresh = SINCE + timedelta(hours=2)
+    results = [
+        _result("Fed pauses", "https://n.example/fed", _iso(fresh), content="<p>Held &amp; steady</p>", source="Reuters"),
+        _result("Old story", "https://n.example/old", _iso(SINCE - timedelta(hours=3))),
+        _result("Undated wrap", "https://n.example/wrap", None, engine="bing news"),
+        _result("", "https://n.example/notitle", _iso(fresh)),
+        _result("No link", "", _iso(fresh)),
+    ]
+    get = mocker.patch("us2kr.collectors.searx.requests.get", return_value=_Resp(results))
+
+    items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["markets"])
+
+    assert [i.title for i in items] == ["Fed pauses", "Undated wrap"]
+    assert items[0].summary == "Held & steady"
+    assert items[0].source == "Reuters"
+    assert items[0].published == fresh
+    # undated: kept, timestamped at the window floor, engine used as source fallback
+    assert items[1].published == SINCE
+    assert items[1].source == "bing news"
+
+    params = get.call_args.kwargs["params"]
+    assert params["format"] == "json"
+    assert params["categories"] == "news"
+    assert params["time_range"] == "day"
+    assert get.call_args.args[0] == "http://searx.local:8888/search"
+
+
+def test_naive_since_is_treated_as_utc(mocker):
+    fresh = SINCE + timedelta(hours=1)
+    mocker.patch(
+        "us2kr.collectors.searx.requests.get",
+        return_value=_Resp([_result("A", "https://n.example/a", _iso(fresh))]),
+    )
+    items = fetch_searx_news("http://searx.local:8888", since=SINCE.replace(tzinfo=None), queries=["q"])
+    assert len(items) == 1
+    assert items[0].published.tzinfo is not None
+
+
+def test_dedup_across_queries_and_max_per_query(mocker):
+    fresh = SINCE + timedelta(hours=1)
+    per_query = {
+        "q1": _Resp(
+            [
+                _result("Story A", "https://n.example/a/", _iso(fresh)),
+                _result("Story B", "https://n.example/b", _iso(fresh)),
+            ]
+        ),
+        # same article as q1's Story A modulo trailing slash / fragment
+        "q2": _Resp([_result("Story A again", "https://n.example/a#frag", _iso(fresh))]),
+    }
+    mocker.patch(
+        "us2kr.collectors.searx.requests.get",
+        side_effect=lambda url, params, **kw: per_query[params["q"]],
+    )
+    items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["q1", "q2"], max_per_query=1)
+    # max_per_query=1 keeps only Story A from q1; q2's duplicate link is dropped
+    assert [i.title for i in items] == ["Story A"]
+
+
+def test_partial_failure_is_tolerated(mocker, caplog):
+    fresh = SINCE + timedelta(hours=1)
+    per_query = {
+        "bad": _Resp(None, status=500),
+        "good": _Resp([_result("Kept", "https://n.example/k", _iso(fresh))]),
+    }
+    mocker.patch(
+        "us2kr.collectors.searx.requests.get",
+        side_effect=lambda url, params, **kw: per_query[params["q"]],
+    )
+    items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["bad", "good"])
+    assert [i.title for i in items] == ["Kept"]
+
+
+def test_all_queries_failing_raises(mocker):
+    mocker.patch("us2kr.collectors.searx.requests.get", return_value=_Resp(None, status=502))
+    with pytest.raises(RuntimeError, match="all searxng queries failed"):
+        fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["a", "b"])
+
+
+def test_malformed_response_counts_as_failure(mocker):
+    mocker.patch(
+        "us2kr.collectors.searx.requests.get",
+        return_value=_Resp(None, body=json.dumps({"unexpected": True})),
+    )
+    with pytest.raises(RuntimeError, match="all searxng queries failed"):
+        fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["only"])
+
+
+def test_default_queries_used_when_none(mocker):
+    calls: list[str] = []
+
+    def record(url, params, **kw):
+        calls.append(params["q"])
+        return _Resp([])
+
+    mocker.patch("us2kr.collectors.searx.requests.get", side_effect=record)
+    assert fetch_searx_news("http://searx.local:8888", since=SINCE) == []
+    assert calls == searx.DEFAULT_QUERIES