jsahnandClaude Fable 5
920d9358a0
Optional meta-search source (SEARXNG_BASE_URL): per-query isolation, time_range=day, merged and deduplicated with the RSS feeds. Briefings still publish on RSS alone if the instance is down. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
122 lines
4.3 KiB
Python
122 lines
4.3 KiB
Python
"""SearXNG news collector: supplements the RSS feeds with meta-search results.
|
|
|
|
Used only when ``SEARXNG_BASE_URL`` is configured. Each query is fetched independently so that
|
|
one failing query never spoils the others. Only if *every* query fails does
|
|
`fetch_searx_news` raise. The instance must have the JSON output format enabled
|
|
(searxng settings.yml: ``search.formats`` includes ``json``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from urllib.parse import urljoin
|
|
|
|
import requests
|
|
|
|
from us2kr.collectors.news import USER_AGENT, strip_html
|
|
from us2kr.models import NewsItem
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
SEARX_TIMEOUT_SECONDS = 15.0
|
|
MAX_PER_QUERY = 15
|
|
|
|
# Complements the fixed RSS feeds: broad market wrap + macro + the Korea/semis angle the
|
|
# briefing cares about. Override with US2KR_SEARXNG_QUERIES (CSV).
|
|
DEFAULT_QUERIES: list[str] = [
|
|
"US stock market",
|
|
"Federal Reserve",
|
|
"semiconductor stocks",
|
|
]
|
|
|
|
|
|
def _parse_published(raw: object) -> datetime | None:
|
|
"""ISO-8601 publishedDate -> tz-aware UTC datetime, or None when absent/garbled."""
|
|
if not raw or not isinstance(raw, str):
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(raw)
|
|
except ValueError:
|
|
return None
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=UTC)
|
|
return parsed.astimezone(UTC)
|
|
|
|
|
|
def _fetch_query(base_url: str, query: str) -> list[dict]:
|
|
resp = requests.get(
|
|
urljoin(base_url.rstrip("/") + "/", "search"),
|
|
params={"q": query, "format": "json", "categories": "news", "time_range": "day"},
|
|
timeout=SEARX_TIMEOUT_SECONDS,
|
|
headers={"User-Agent": USER_AGENT},
|
|
)
|
|
resp.raise_for_status()
|
|
results = resp.json().get("results")
|
|
if not isinstance(results, list): # malformed server payload, not a caller type error
|
|
raise ValueError("malformed SearXNG response: no results list") # noqa: TRY004
|
|
return results
|
|
|
|
|
|
def fetch_searx_news(
|
|
base_url: str,
|
|
since: datetime,
|
|
queries: list[str] | None = None,
|
|
max_per_query: int = MAX_PER_QUERY,
|
|
) -> list[NewsItem]:
|
|
"""Search the news category for each query, returning items published at/after `since`.
|
|
|
|
Results without a publishedDate are kept (time_range=day already bounds their age) but
|
|
timestamped at `since`, so they sort last and never win deduplication against a dated copy.
|
|
Raises RuntimeError only when every query failed.
|
|
"""
|
|
if since.tzinfo is None:
|
|
since = since.replace(tzinfo=UTC)
|
|
else:
|
|
since = since.astimezone(UTC)
|
|
|
|
queries = queries or DEFAULT_QUERIES
|
|
collected: list[NewsItem] = []
|
|
failures: list[str] = []
|
|
seen_links: set[str] = set() # the same article routinely matches several queries
|
|
for query in queries:
|
|
try:
|
|
results = _fetch_query(base_url, query)
|
|
except Exception as exc: # noqa: BLE001 - per-query isolation
|
|
log.warning("searxng query %r failed: %s: %s", query, type(exc).__name__, str(exc)[:200])
|
|
failures.append(f"{query}: {type(exc).__name__}")
|
|
continue
|
|
kept = undated = 0
|
|
for result in results:
|
|
title = (result.get("title") or "").strip()
|
|
link = (result.get("url") or "").strip()
|
|
if not title or not link:
|
|
continue
|
|
lkey = link.split("#", 1)[0].rstrip("/").lower()
|
|
if lkey in seen_links:
|
|
continue
|
|
published = _parse_published(result.get("publishedDate"))
|
|
if published is None:
|
|
undated += 1
|
|
published = since
|
|
elif published < since:
|
|
continue
|
|
seen_links.add(lkey)
|
|
collected.append(
|
|
NewsItem(
|
|
title=title,
|
|
summary=strip_html(result.get("content") or ""),
|
|
link=link,
|
|
source=result.get("source") or result.get("engine") or "SearXNG",
|
|
published=published,
|
|
)
|
|
)
|
|
kept += 1
|
|
if kept >= max_per_query:
|
|
break
|
|
log.info("searxng query %r: %d kept (%d undated) of %d results", query, kept, undated, len(results))
|
|
|
|
if failures and len(failures) == len(queries):
|
|
raise RuntimeError("all searxng queries failed: " + "; ".join(failures))
|
|
return collected
|