US2KR/tests/test_searx.py

"""Tests for us2kr.collectors.searx (no network: requests.get is mocked)."""

from __future__ import annotations

import json
from datetime import UTC, datetime, timedelta

import pytest
import requests

from us2kr.collectors import searx
from us2kr.collectors.searx import _parse_published, fetch_searx_news

SINCE = datetime(2026, 8, 25, 12, 0, tzinfo=UTC)


def _result(title: str, url: str, published: str | None = None, **extra) -> dict:
    result = {"title": title, "url": url, "content": extra.pop("content", "")}
    if published is not None:
        result["publishedDate"] = published
    result.update(extra)
    return result


class _Resp:
    def __init__(self, results: list | None, status: int = 200, body: str | None = None):
        self.status_code = status
        self._body = body if body is not None else json.dumps({"results": results})

    def raise_for_status(self) -> None:
        if self.status_code >= 400:
            raise requests.HTTPError(f"{self.status_code} error")

    def json(self):
        return json.loads(self._body)


def _iso(dt: datetime) -> str:
    return dt.isoformat()


def test_parse_published_variants():
    assert _parse_published("2026-08-25T14:00:00+00:00") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC)
    assert _parse_published("2026-08-25T14:00:00Z") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC)
    # naive -> assumed UTC
    assert _parse_published("2026-08-25T14:00:00") == datetime(2026, 8, 25, 14, 0, tzinfo=UTC)
    assert _parse_published(None) is None
    assert _parse_published("") is None
    assert _parse_published("yesterday-ish") is None
    assert _parse_published(1234) is None


def test_filters_maps_and_floors_undated(mocker):
    fresh = SINCE + timedelta(hours=2)
    results = [
        _result("Fed pauses", "https://n.example/fed", _iso(fresh), content="<p>Held &amp; steady</p>", source="Reuters"),
        _result("Old story", "https://n.example/old", _iso(SINCE - timedelta(hours=3))),
        _result("Undated wrap", "https://n.example/wrap", None, engine="bing news"),
        _result("", "https://n.example/notitle", _iso(fresh)),
        _result("No link", "", _iso(fresh)),
    ]
    get = mocker.patch("us2kr.collectors.searx.requests.get", return_value=_Resp(results))

    items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["markets"])

    assert [i.title for i in items] == ["Fed pauses", "Undated wrap"]
    assert items[0].summary == "Held & steady"
    assert items[0].source == "Reuters"
    assert items[0].published == fresh
    # undated: kept, timestamped at the window floor, engine used as source fallback
    assert items[1].published == SINCE
    assert items[1].source == "bing news"

    params = get.call_args.kwargs["params"]
    assert params["format"] == "json"
    assert params["categories"] == "news"
    assert params["time_range"] == "day"
    assert get.call_args.args[0] == "http://searx.local:8888/search"


def test_naive_since_is_treated_as_utc(mocker):
    fresh = SINCE + timedelta(hours=1)
    mocker.patch(
        "us2kr.collectors.searx.requests.get",
        return_value=_Resp([_result("A", "https://n.example/a", _iso(fresh))]),
    )
    items = fetch_searx_news("http://searx.local:8888", since=SINCE.replace(tzinfo=None), queries=["q"])
    assert len(items) == 1
    assert items[0].published.tzinfo is not None


def test_dedup_across_queries_and_max_per_query(mocker):
    fresh = SINCE + timedelta(hours=1)
    per_query = {
        "q1": _Resp(
            [
                _result("Story A", "https://n.example/a/", _iso(fresh)),
                _result("Story B", "https://n.example/b", _iso(fresh)),
            ]
        ),
        # same article as q1's Story A modulo trailing slash / fragment
        "q2": _Resp([_result("Story A again", "https://n.example/a#frag", _iso(fresh))]),
    }
    mocker.patch(
        "us2kr.collectors.searx.requests.get",
        side_effect=lambda url, params, **kw: per_query[params["q"]],
    )
    items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["q1", "q2"], max_per_query=1)
    # max_per_query=1 keeps only Story A from q1; q2's duplicate link is dropped
    assert [i.title for i in items] == ["Story A"]


def test_partial_failure_is_tolerated(mocker, caplog):
    fresh = SINCE + timedelta(hours=1)
    per_query = {
        "bad": _Resp(None, status=500),
        "good": _Resp([_result("Kept", "https://n.example/k", _iso(fresh))]),
    }
    mocker.patch(
        "us2kr.collectors.searx.requests.get",
        side_effect=lambda url, params, **kw: per_query[params["q"]],
    )
    items = fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["bad", "good"])
    assert [i.title for i in items] == ["Kept"]


def test_all_queries_failing_raises(mocker):
    mocker.patch("us2kr.collectors.searx.requests.get", return_value=_Resp(None, status=502))
    with pytest.raises(RuntimeError, match="all searxng queries failed"):
        fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["a", "b"])


def test_malformed_response_counts_as_failure(mocker):
    mocker.patch(
        "us2kr.collectors.searx.requests.get",
        return_value=_Resp(None, body=json.dumps({"unexpected": True})),
    )
    with pytest.raises(RuntimeError, match="all searxng queries failed"):
        fetch_searx_news("http://searx.local:8888", since=SINCE, queries=["only"])


def test_default_queries_used_when_none(mocker):
    calls: list[str] = []

    def record(url, params, **kw):
        calls.append(params["q"])
        return _Resp([])

    mocker.patch("us2kr.collectors.searx.requests.get", side_effect=record)
    assert fetch_searx_news("http://searx.local:8888", since=SINCE) == []
    assert calls == searx.DEFAULT_QUERIES