"""Tests for us2kr.collectors.searx (no network: requests.get is mocked).""" from __future__ import annotations import json from datetime import UTC, datetime, timedelta import pytest import requests from us2kr.collectors import searx from us2kr.collectors.searx import _parse_published, fetch_searx_news SINCE = datetime(2026, 8, 25, 12, 0, tzinfo=UTC) def _result(title: str, url: str, published: str | None = None, **extra) -> dict: result = {"title": title, "url": url, "content": extra.pop("content", "")} if published is not None: result["publishedDate"] = published result.update(extra) return result class _Resp: def __init__(self, results: list | None, status: int = 200, body: str | None = None): self.status_code = status self._body = body if body is not None else json.dumps({"results": results}) def raise_for_status(self) -> None: if self.status_code >= 400: raise requests.HTTPError(f"{self.status_code} error") def json(self): return json.loads(self._body) def _iso(dt: datetime) -> str: return dt.isoformat() def test_parse_published_variants(): assert _parse_published("2026-08-25T14:00:00+00:00") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC) assert _parse_published("2026-08-25T14:00:00Z") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC) # naive -> assumed UTC assert _parse_published("2026-08-25T14:00:00") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC) assert _parse_published(None) is None assert _parse_published("") is None assert _parse_published("yesterday-ish") is None assert _parse_published(1234) is None def test_filters_maps_and_floors_undated(mocker): fresh = SINCE + timedelta(hours=2) results = [ _result("Fed pauses", "https://n.example/fed", _iso(fresh), content="
Held & steady
", source="Reuters"), _result("Old story", "https://n.example/old", _iso(SINCE - timedelta(hours=3))), _result("Undated wrap", "https://n.example/wrap", None, engine="bing news"), _result("", "https://n.example/notitle", _iso(fresh)), _result("No link", "", _iso(fresh)), ] get = mocker.patch("us2kr.collectors.searx.requests.get", return_value=_Resp(results)) items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["markets"]) assert [i.title for i in items] == ["Fed pauses", "Undated wrap"] assert items[0].summary == "Held & steady" assert items[0].source == "Reuters" assert items[0].published == fresh # undated: kept, timestamped at the window floor, engine used as source fallback assert items[1].published == SINCE assert items[1].source == "bing news" params = get.call_args.kwargs["params"] assert params["format"] == "json" assert params["categories"] == "news" assert params["time_range"] == "day" assert get.call_args.args[0] == "http://searx.local:8888/search" def test_naive_since_is_treated_as_utc(mocker): fresh = SINCE + timedelta(hours=1) mocker.patch( "us2kr.collectors.searx.requests.get", return_value=_Resp([_result("A", "https://n.example/a", _iso(fresh))]), ) items = fetch_searx_news("http://searx.local:8888", since=SINCE.replace(tzinfo=None), queries=["q"]) assert len(items) == 1 assert items[0].published.tzinfo is not None def test_dedup_across_queries_and_max_per_query(mocker): fresh = SINCE + timedelta(hours=1) per_query = { "q1": _Resp( [ _result("Story A", "https://n.example/a/", _iso(fresh)), _result("Story B", "https://n.example/b", _iso(fresh)), ] ), # same article as q1's Story A modulo trailing slash / fragment "q2": _Resp([_result("Story A again", "https://n.example/a#frag", _iso(fresh))]), } mocker.patch( "us2kr.collectors.searx.requests.get", side_effect=lambda url, params, **kw: per_query[params["q"]], ) items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["q1", "q2"], max_per_query=1) # max_per_query=1 keeps only Story A from q1; q2's duplicate link is dropped assert [i.title for i in items] == ["Story A"] def test_partial_failure_is_tolerated(mocker, caplog): fresh = SINCE + timedelta(hours=1) per_query = { "bad": _Resp(None, status=500), "good": _Resp([_result("Kept", "https://n.example/k", _iso(fresh))]), } mocker.patch( "us2kr.collectors.searx.requests.get", side_effect=lambda url, params, **kw: per_query[params["q"]], ) items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["bad", "good"]) assert [i.title for i in items] == ["Kept"] def test_all_queries_failing_raises(mocker): mocker.patch("us2kr.collectors.searx.requests.get", return_value=_Resp(None, status=502)) with pytest.raises(RuntimeError, match="all searxng queries failed"): fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["a", "b"]) def test_malformed_response_counts_as_failure(mocker): mocker.patch( "us2kr.collectors.searx.requests.get", return_value=_Resp(None, body=json.dumps({"unexpected": True})), ) with pytest.raises(RuntimeError, match="all searxng queries failed"): fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["only"]) def test_default_queries_used_when_none(mocker): calls: list[str] = [] def record(url, params, **kw): calls.append(params["q"]) return _Resp([]) mocker.patch("us2kr.collectors.searx.requests.get", side_effect=record) assert fetch_searx_news("http://searx.local:8888", since=SINCE) == [] assert calls == searx.DEFAULT_QUERIES