Initial commit

us2kr/collectors/news.py

@@ -0,0 +1,175 @@
+"""News collector: curated RSS feeds via requests + feedparser.
+
+Each feed is fetched independently so that one broken feed never spoils the others.
+Only if *every* feed fails does `fetch_news` raise.
+"""
+
+from __future__ import annotations
+
+import calendar
+import html
+import logging
+import re
+from datetime import UTC, datetime
+from html.parser import HTMLParser
+
+import feedparser
+import requests
+
+from us2kr.models import NewsItem
+
+log = logging.getLogger(__name__)
+
+USER_AGENT = "Mozilla/5.0 (compatible; US2KR/0.1; +https://github.com/us2kr) feed-reader"
+FEED_TIMEOUT_SECONDS = 10.0
+SUMMARY_MAX_CHARS = 300
+
+# (source label, feed url). Order matters only for tie-breaking on identical timestamps.
+FEEDS: list[tuple[str, str]] = [
+    ("CNBC Top News", "https://www.cnbc.com/id/100003114/device/rss/rss.html"),
+    ("CNBC Markets", "https://www.cnbc.com/id/20910258/device/rss/rss.html"),
+    ("MarketWatch Top Stories", "https://feeds.content.dowjones.io/public/rss/mw_topstories"),
+    ("MarketWatch MarketPulse", "https://feeds.content.dowjones.io/public/rss/mw_marketpulse"),
+    ("Yahoo Finance", "https://finance.yahoo.com/news/rssindex"),
+    ("Federal Reserve", "https://www.federalreserve.gov/feeds/press_all.xml"),
+    ("Investing.com", "https://www.investing.com/rss/news_25.rss"),
+]
+
+_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
+_WS_RE = re.compile(r"\s+")
+
+
+class _TextExtractor(HTMLParser):
+    """Minimal HTML -> text: keeps text nodes, drops tags/scripts/styles."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._parts: list[str] = []
+        self._skip = 0
+
+    def handle_starttag(self, tag: str, attrs) -> None:
+        if tag in ("script", "style"):
+            self._skip += 1
+        elif tag in ("br", "p", "div", "li"):
+            self._parts.append(" ")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in ("script", "style") and self._skip:
+            self._skip -= 1
+        elif tag in ("p", "div", "li"):
+            self._parts.append(" ")
+
+    def handle_data(self, data: str) -> None:
+        if not self._skip:
+            self._parts.append(data)
+
+    def text(self) -> str:
+        return "".join(self._parts)
+
+
+def strip_html(raw: str | None, max_chars: int = SUMMARY_MAX_CHARS) -> str:
+    """Remove tags/entities, collapse whitespace and truncate to `max_chars`."""
+    if not raw:
+        return ""
+    parser = _TextExtractor()
+    try:
+        parser.feed(raw)
+        parser.close()
+        text = parser.text()
+    except Exception:  # noqa: BLE001 - malformed HTML must not kill the collector
+        text = re.sub(r"<[^>]+>", " ", raw)
+    text = _WS_RE.sub(" ", html.unescape(text)).strip()
+    if len(text) > max_chars:
+        text = text[: max_chars - 1].rstrip() + "…"
+    return text
+
+
+def normalize_title(title: str) -> str:
+    """Lowercase, drop punctuation and collapse whitespace - used as a dedup key."""
+    return _WS_RE.sub(" ", _PUNCT_RE.sub("", title.lower())).strip()
+
+
+def _entry_datetime(entry) -> datetime | None:
+    """Return tz-aware UTC datetime from published_parsed / updated_parsed, else None."""
+    for key in ("published_parsed", "updated_parsed", "created_parsed"):
+        tm = entry.get(key)
+        if tm:
+            try:
+                return datetime.fromtimestamp(calendar.timegm(tm), tz=UTC)
+            except (OverflowError, ValueError, TypeError):
+                continue
+    return None
+
+
+def _fetch_feed_bytes(url: str) -> bytes:
+    resp = requests.get(url, timeout=FEED_TIMEOUT_SECONDS, headers={"User-Agent": USER_AGENT})
+    resp.raise_for_status()
+    return resp.content
+
+
+def _parse_feed(source: str, content: bytes, since: datetime) -> list[NewsItem]:
+    parsed = feedparser.parse(content)
+    if parsed.get("bozo") and not parsed.entries:
+        raise ValueError(f"unparseable feed: {parsed.get('bozo_exception')}")
+    items: list[NewsItem] = []
+    for entry in parsed.entries:
+        title = _WS_RE.sub(" ", html.unescape(entry.get("title") or "")).strip()
+        link = (entry.get("link") or "").strip()
+        if not title or not link:
+            continue
+        published = _entry_datetime(entry)
+        if published is None or published < since:
+            continue
+        summary = strip_html(entry.get("summary") or entry.get("description") or "")
+        items.append(NewsItem(title=title, summary=summary, link=link, source=source, published=published))
+    return items
+
+
+def fetch_news(since: datetime, max_items: int = 40) -> list[NewsItem]:
+    """Collect headlines published at/after `since` from all feeds, deduplicated, newest first.
+
+    Raises RuntimeError only when every feed failed.
+    """
+    if since.tzinfo is None:
+        since = since.replace(tzinfo=UTC)
+    else:
+        since = since.astimezone(UTC)
+
+    collected: list[NewsItem] = []
+    failures: list[str] = []
+    for source, url in FEEDS:
+        try:
+            content = _fetch_feed_bytes(url)
+            items = _parse_feed(source, content, since)
+        except Exception as exc:  # noqa: BLE001 - per-feed isolation
+            log.warning("news feed %s failed: %s: %s", source, type(exc).__name__, str(exc)[:200])
+            failures.append(f"{source}: {type(exc).__name__}")
+            continue
+        log.info("news feed %s: %d items since %s", source, len(items), since.isoformat())
+        collected.extend(items)
+
+    if failures and len(failures) == len(FEEDS):
+        raise RuntimeError("all news feeds failed: " + "; ".join(failures))
+
+    seen_titles: set[str] = set()
+    seen_links: set[str] = set()
+    unique: list[NewsItem] = []
+    # Stable sort newest first before dedup so the freshest duplicate is the one kept.
+    for item in sorted(collected, key=lambda n: n.published, reverse=True):
+        tkey = normalize_title(item.title)
+        lkey = item.link.split("#", 1)[0].rstrip("/").lower()
+        if tkey in seen_titles or lkey in seen_links:
+            continue
+        seen_titles.add(tkey)
+        seen_links.add(lkey)
+        unique.append(item)
+
+    log.info(
+        "news: %d raw -> %d unique, returning %d (feeds failed: %d/%d)",
+        len(collected),
+        len(unique),
+        min(len(unique), max_items),
+        len(failures),
+        len(FEEDS),
+    )
+    return unique[:max_items]