Add SearXNG news collector merged into the briefing pipeline

Optional meta-search source (SEARXNG_BASE_URL): per-query isolation,
time_range=day, merged and deduplicated with the RSS feeds. Briefings
still publish on RSS alone if the instance is down.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

us2kr/collectors/news.py

@@ -125,6 +125,23 @@ def _parse_feed(source: str, content: bytes, since: datetime) -> list[NewsItem]:
     return items
 
 
+def dedup_news(items: list[NewsItem]) -> list[NewsItem]:
+    """Newest first, deduplicated by normalised title and by link (fragment/trailing slash ignored)."""
+    seen_titles: set[str] = set()
+    seen_links: set[str] = set()
+    unique: list[NewsItem] = []
+    # Stable sort newest first before dedup so the freshest duplicate is the one kept.
+    for item in sorted(items, key=lambda n: n.published, reverse=True):
+        tkey = normalize_title(item.title)
+        lkey = item.link.split("#", 1)[0].rstrip("/").lower()
+        if tkey in seen_titles or lkey in seen_links:
+            continue
+        seen_titles.add(tkey)
+        seen_links.add(lkey)
+        unique.append(item)
+    return unique
+
+
 def fetch_news(since: datetime, max_items: int = 40) -> list[NewsItem]:
     """Collect headlines published at/after `since` from all feeds, deduplicated, newest first.
 
@@ -151,18 +168,7 @@ def fetch_news(since: datetime, max_items: int = 40) -> list[NewsItem]:
     if failures and len(failures) == len(FEEDS):
         raise RuntimeError("all news feeds failed: " + "; ".join(failures))
 
-    seen_titles: set[str] = set()
-    seen_links: set[str] = set()
-    unique: list[NewsItem] = []
-    # Stable sort newest first before dedup so the freshest duplicate is the one kept.
-    for item in sorted(collected, key=lambda n: n.published, reverse=True):
-        tkey = normalize_title(item.title)
-        lkey = item.link.split("#", 1)[0].rstrip("/").lower()
-        if tkey in seen_titles or lkey in seen_links:
-            continue
-        seen_titles.add(tkey)
-        seen_links.add(lkey)
-        unique.append(item)
+    unique = dedup_news(collected)
 
     log.info(
         "news: %d raw -> %d unique, returning %d (feeds failed: %d/%d)",

us2kr/collectors/searx.py

@@ -0,0 +1,121 @@
+"""SearXNG news collector: supplements the RSS feeds with meta-search results.
+
+Used only when ``SEARXNG_BASE_URL`` is configured. Each query is fetched independently so that
+one failing query never spoils the others. Only if *every* query fails does
+`fetch_searx_news` raise. The instance must have the JSON output format enabled
+(searxng settings.yml: ``search.formats`` includes ``json``).
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from urllib.parse import urljoin
+
+import requests
+
+from us2kr.collectors.news import USER_AGENT, strip_html
+from us2kr.models import NewsItem
+
+log = logging.getLogger(__name__)
+
+SEARX_TIMEOUT_SECONDS = 15.0
+MAX_PER_QUERY = 15
+
+# Complements the fixed RSS feeds: broad market wrap + macro + the Korea/semis angle the
+# briefing cares about. Override with US2KR_SEARXNG_QUERIES (CSV).
+DEFAULT_QUERIES: list[str] = [
+    "US stock market",
+    "Federal Reserve",
+    "semiconductor stocks",
+]
+
+
+def _parse_published(raw: object) -> datetime | None:
+    """ISO-8601 publishedDate -> tz-aware UTC datetime, or None when absent/garbled."""
+    if not raw or not isinstance(raw, str):
+        return None
+    try:
+        parsed = datetime.fromisoformat(raw)
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=UTC)
+    return parsed.astimezone(UTC)
+
+
+def _fetch_query(base_url: str, query: str) -> list[dict]:
+    resp = requests.get(
+        urljoin(base_url.rstrip("/") + "/", "search"),
+        params={"q": query, "format": "json", "categories": "news", "time_range": "day"},
+        timeout=SEARX_TIMEOUT_SECONDS,
+        headers={"User-Agent": USER_AGENT},
+    )
+    resp.raise_for_status()
+    results = resp.json().get("results")
+    if not isinstance(results, list):  # malformed server payload, not a caller type error
+        raise ValueError("malformed SearXNG response: no results list")  # noqa: TRY004
+    return results
+
+
+def fetch_searx_news(
+    base_url: str,
+    since: datetime,
+    queries: list[str] | None = None,
+    max_per_query: int = MAX_PER_QUERY,
+) -> list[NewsItem]:
+    """Search the news category for each query, returning items published at/after `since`.
+
+    Results without a publishedDate are kept (time_range=day already bounds their age) but
+    timestamped at `since`, so they sort last and never win deduplication against a dated copy.
+    Raises RuntimeError only when every query failed.
+    """
+    if since.tzinfo is None:
+        since = since.replace(tzinfo=UTC)
+    else:
+        since = since.astimezone(UTC)
+
+    queries = queries or DEFAULT_QUERIES
+    collected: list[NewsItem] = []
+    failures: list[str] = []
+    seen_links: set[str] = set()  # the same article routinely matches several queries
+    for query in queries:
+        try:
+            results = _fetch_query(base_url, query)
+        except Exception as exc:  # noqa: BLE001 - per-query isolation
+            log.warning("searxng query %r failed: %s: %s", query, type(exc).__name__, str(exc)[:200])
+            failures.append(f"{query}: {type(exc).__name__}")
+            continue
+        kept = undated = 0
+        for result in results:
+            title = (result.get("title") or "").strip()
+            link = (result.get("url") or "").strip()
+            if not title or not link:
+                continue
+            lkey = link.split("#", 1)[0].rstrip("/").lower()
+            if lkey in seen_links:
+                continue
+            published = _parse_published(result.get("publishedDate"))
+            if published is None:
+                undated += 1
+                published = since
+            elif published < since:
+                continue
+            seen_links.add(lkey)
+            collected.append(
+                NewsItem(
+                    title=title,
+                    summary=strip_html(result.get("content") or ""),
+                    link=link,
+                    source=result.get("source") or result.get("engine") or "SearXNG",
+                    published=published,
+                )
+            )
+            kept += 1
+            if kept >= max_per_query:
+                break
+        log.info("searxng query %r: %d kept (%d undated) of %d results", query, kept, undated, len(results))
+
+    if failures and len(failures) == len(queries):
+        raise RuntimeError("all searxng queries failed: " + "; ".join(failures))
+    return collected

us2kr/config.py

@@ -38,6 +38,13 @@ class Settings(BaseSettings):
     # --- Optional data keys ---
     fred_api_key: str | None = Field(default=None, alias="FRED_API_KEY")
 
+    # --- SearXNG (optional news search; instance must enable the json output format) ---
+    searxng_base_url: str | None = Field(default=None, alias="SEARXNG_BASE_URL")
+    # Empty -> collector defaults (us2kr.collectors.searx.DEFAULT_QUERIES).
+    searxng_queries: Annotated[list[str], NoDecode] = Field(
+        default_factory=list, alias="US2KR_SEARXNG_QUERIES"
+    )
+
     # --- Paths ---
     data_dir: Path = Field(default=Path("data"), alias="US2KR_DATA_DIR")
     reports_dir: Path = Field(default=Path("reports"), alias="US2KR_REPORTS_DIR")
@@ -60,7 +67,7 @@ class Settings(BaseSettings):
 
     log_level: str = Field(default="INFO", alias="US2KR_LOG_LEVEL")
 
-    @field_validator("minimax_fallback_models", "krx_extra_holidays", mode="before")
+    @field_validator("minimax_fallback_models", "krx_extra_holidays", "searxng_queries", mode="before")
     @classmethod
     def _split_csv(cls, v):
         if isinstance(v, str):

us2kr/pipeline.py

@@ -16,8 +16,9 @@ from pydantic import BaseModel
 
 from us2kr.collectors.base import run_safely
 from us2kr.collectors.events import fetch_earnings, fetch_events
-from us2kr.collectors.news import fetch_news
+from us2kr.collectors.news import dedup_news, fetch_news
 from us2kr.collectors.prices import fetch_quotes
+from us2kr.collectors.searx import fetch_searx_news
 from us2kr.config import KST, Settings
 from us2kr.deliver import build_deliverers
 from us2kr.llm.base import LLMError
@@ -79,6 +80,20 @@ def build_payload(session: date, settings: Settings, clock: MarketClock, store:
     news = run_safely(
         "news", lambda: fetch_news(since=news_since, max_items=settings.news_max_items), [], gaps
     )
+    if settings.searxng_base_url:
+        searx_news = run_safely(
+            "searxng",
+            lambda: fetch_searx_news(
+                settings.searxng_base_url,  # type: ignore[arg-type]
+                since=news_since,
+                queries=settings.searxng_queries or None,
+            ),
+            [],
+            gaps,
+        )
+        if searx_news:
+            # RSS first so it wins timestamp ties in dedup; cap stays at news_max_items overall.
+            news = dedup_news([*news, *searx_news])[: settings.news_max_items]
     events = run_safely("events", lambda: fetch_events(session, settings.fred_api_key), [], gaps)
     tickers = earnings_tickers()
     earnings = run_safely("earnings", lambda: fetch_earnings(session, tickers), [], gaps)