US2KR/us2kr/collectors/news.py

"""News collector: curated RSS feeds via requests + feedparser.

Each feed is fetched independently so that one broken feed never spoils the others.
Only if *every* feed fails does `fetch_news` raise.
"""

from __future__ import annotations

import calendar
import html
import logging
import re
from datetime import UTC, datetime
from html.parser import HTMLParser

import feedparser
import requests

from us2kr.models import NewsItem

log = logging.getLogger(__name__)

USER_AGENT = "Mozilla/5.0 (compatible; US2KR/0.1; +https://github.com/us2kr) feed-reader"
FEED_TIMEOUT_SECONDS = 10.0
SUMMARY_MAX_CHARS = 300

# (source label, feed url). Order matters only for tie-breaking on identical timestamps.
FEEDS: list[tuple[str, str]] = [
    ("CNBC Top News", "https://www.cnbc.com/id/100003114/device/rss/rss.html"),
    ("CNBC Markets", "https://www.cnbc.com/id/20910258/device/rss/rss.html"),
    ("MarketWatch Top Stories", "https://feeds.content.dowjones.io/public/rss/mw_topstories"),
    ("MarketWatch MarketPulse", "https://feeds.content.dowjones.io/public/rss/mw_marketpulse"),
    ("Yahoo Finance", "https://finance.yahoo.com/news/rssindex"),
    ("Federal Reserve", "https://www.federalreserve.gov/feeds/press_all.xml"),
    ("Investing.com", "https://www.investing.com/rss/news_25.rss"),
]

_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
_WS_RE = re.compile(r"\s+")


class _TextExtractor(HTMLParser):
    """Minimal HTML -> text: keeps text nodes, drops tags/scripts/styles."""

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self._parts: list[str] = []
        self._skip = 0

    def handle_starttag(self, tag: str, attrs) -> None:
        if tag in ("script", "style"):
            self._skip += 1
        elif tag in ("br", "p", "div", "li"):
            self._parts.append(" ")

    def handle_endtag(self, tag: str) -> None:
        if tag in ("script", "style") and self._skip:
            self._skip -= 1
        elif tag in ("p", "div", "li"):
            self._parts.append(" ")

    def handle_data(self, data: str) -> None:
        if not self._skip:
            self._parts.append(data)

    def text(self) -> str:
        return "".join(self._parts)


def strip_html(raw: str | None, max_chars: int = SUMMARY_MAX_CHARS) -> str:
    """Remove tags/entities, collapse whitespace and truncate to `max_chars`."""
    if not raw:
        return ""
    parser = _TextExtractor()
    try:
        parser.feed(raw)
        parser.close()
        text = parser.text()
    except Exception:  # noqa: BLE001 - malformed HTML must not kill the collector
        text = re.sub(r"<[^>]+>", " ", raw)
    text = _WS_RE.sub(" ", html.unescape(text)).strip()
    if len(text) > max_chars:
        text = text[: max_chars - 1].rstrip() + "…"
    return text


def normalize_title(title: str) -> str:
    """Lowercase, drop punctuation and collapse whitespace - used as a dedup key."""
    return _WS_RE.sub(" ", _PUNCT_RE.sub("", title.lower())).strip()


def _entry_datetime(entry) -> datetime | None:
    """Return tz-aware UTC datetime from published_parsed / updated_parsed, else None."""
    for key in ("published_parsed", "updated_parsed", "created_parsed"):
        tm = entry.get(key)
        if tm:
            try:
                return datetime.fromtimestamp(calendar.timegm(tm), tz=UTC)
            except (OverflowError, ValueError, TypeError):
                continue
    return None


def _fetch_feed_bytes(url: str) -> bytes:
    resp = requests.get(url, timeout=FEED_TIMEOUT_SECONDS, headers={"User-Agent": USER_AGENT})
    resp.raise_for_status()
    return resp.content


def _parse_feed(source: str, content: bytes, since: datetime) -> list[NewsItem]:
    parsed = feedparser.parse(content)
    if parsed.get("bozo") and not parsed.entries:
        raise ValueError(f"unparseable feed: {parsed.get('bozo_exception')}")
    items: list[NewsItem] = []
    for entry in parsed.entries:
        title = _WS_RE.sub(" ", html.unescape(entry.get("title") or "")).strip()
        link = (entry.get("link") or "").strip()
        if not title or not link:
            continue
        published = _entry_datetime(entry)
        if published is None or published < since:
            continue
        summary = strip_html(entry.get("summary") or entry.get("description") or "")
        items.append(NewsItem(title=title, summary=summary, link=link, source=source, published=published))
    return items


def dedup_news(items: list[NewsItem]) -> list[NewsItem]:
    """Newest first, deduplicated by normalised title and by link (fragment/trailing slash ignored)."""
    seen_titles: set[str] = set()
    seen_links: set[str] = set()
    unique: list[NewsItem] = []
    # Stable sort newest first before dedup so the freshest duplicate is the one kept.
    for item in sorted(items, key=lambda n: n.published, reverse=True):
        tkey = normalize_title(item.title)
        lkey = item.link.split("#", 1)[0].rstrip("/").lower()
        if tkey in seen_titles or lkey in seen_links:
            continue
        seen_titles.add(tkey)
        seen_links.add(lkey)
        unique.append(item)
    return unique


def fetch_news(since: datetime, max_items: int = 40) -> list[NewsItem]:
    """Collect headlines published at/after `since` from all feeds, deduplicated, newest first.

    Raises RuntimeError only when every feed failed.
    """
    if since.tzinfo is None:
        since = since.replace(tzinfo=UTC)
    else:
        since = since.astimezone(UTC)

    collected: list[NewsItem] = []
    failures: list[str] = []
    for source, url in FEEDS:
        try:
            content = _fetch_feed_bytes(url)
            items = _parse_feed(source, content, since)
        except Exception as exc:  # noqa: BLE001 - per-feed isolation
            log.warning("news feed %s failed: %s: %s", source, type(exc).__name__, str(exc)[:200])
            failures.append(f"{source}: {type(exc).__name__}")
            continue
        log.info("news feed %s: %d items since %s", source, len(items), since.isoformat())
        collected.extend(items)

    if failures and len(failures) == len(FEEDS):
        raise RuntimeError("all news feeds failed: " + "; ".join(failures))

    unique = dedup_news(collected)

    log.info(
        "news: %d raw -> %d unique, returning %d (feeds failed: %d/%d)",
        len(collected),
        len(unique),
        min(len(unique), max_items),
        len(failures),
        len(FEEDS),
    )
    return unique[:max_items]