313 lines
12 KiB
Python
313 lines
12 KiB
Python
"""Economic calendar (FRED release dates) and earnings (yfinance) collectors.
|
|
|
|
Both are best-effort: FRED is skipped entirely without an API key, and earnings lookups are
|
|
bounded by a ticker cap and a wall-clock budget so a slow Yahoo never stalls the briefing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import math
|
|
import time
|
|
from datetime import date, datetime, timedelta
|
|
from datetime import time as dtime
|
|
from zoneinfo import ZoneInfo
|
|
|
|
import requests
|
|
import yfinance as yf
|
|
|
|
from us2kr.models import EarningsItem, EconEvent
|
|
from us2kr.symbols import display_name
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
ET = ZoneInfo("America/New_York")
|
|
|
|
FRED_RELEASE_DATES_URL = "https://api.stlouisfed.org/fred/releases/dates"
|
|
FRED_TIMEOUT_SECONDS = 10.0
|
|
|
|
# Substring of the FRED release name (lower-cased) -> (Korean label, importance, typical release time ET).
|
|
# Typical times are the customary BLS/BEA/Fed embargo times; None when the time varies.
|
|
_KNOWN_RELEASES: list[tuple[str, str, str, dtime | None]] = [
|
|
("consumer price index", "미국 소비자물가지수(CPI)", "high", dtime(8, 30)),
|
|
("producer price index", "미국 생산자물가지수(PPI)", "high", dtime(8, 30)),
|
|
("employment situation", "미국 고용보고서(비농업 고용·실업률)", "high", dtime(8, 30)),
|
|
("gross domestic product", "미국 GDP", "high", dtime(8, 30)),
|
|
("fomc", "FOMC", "high", dtime(14, 0)),
|
|
("federal open market committee", "FOMC", "high", dtime(14, 0)),
|
|
("advance monthly sales for retail", "미국 소매판매", "high", dtime(8, 30)),
|
|
("personal income and outlays", "미국 개인소득·지출(PCE 물가)", "high", dtime(8, 30)),
|
|
("unemployment insurance weekly claims", "주간 신규 실업수당 청구건수", "medium", dtime(8, 30)),
|
|
("job openings and labor turnover", "구인·이직보고서(JOLTS)", "medium", dtime(10, 0)),
|
|
("employment cost index", "고용비용지수(ECI)", "medium", dtime(8, 30)),
|
|
("industrial production", "산업생산·설비가동률", "medium", dtime(9, 15)),
|
|
("durable goods", "내구재 주문", "medium", dtime(8, 30)),
|
|
("new residential construction", "주택착공·건축허가", "medium", dtime(8, 30)),
|
|
("new residential sales", "신규주택판매", "low", dtime(10, 0)),
|
|
("surveys of consumers", "미시간대 소비자심리", "medium", dtime(10, 0)),
|
|
("beige book", "베이지북", "medium", dtime(14, 0)),
|
|
("import and export price", "수출입물가", "low", dtime(8, 30)),
|
|
("productivity and costs", "생산성·단위노동비용", "low", dtime(8, 30)),
|
|
]
|
|
|
|
EARNINGS_MAX_TICKERS = 15
|
|
EARNINGS_TIME_BUDGET_SECONDS = 30.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Economic releases (FRED)
|
|
# ---------------------------------------------------------------------------
|
|
def _classify_release(name: str) -> tuple[str, str, dtime | None] | None:
|
|
lowered = name.lower()
|
|
for needle, ko, importance, at in _KNOWN_RELEASES:
|
|
if needle in lowered:
|
|
return ko, importance, at
|
|
return None
|
|
|
|
|
|
def _without_url(exc: requests.RequestException) -> requests.RequestException:
|
|
"""Same exception type as ``exc`` with a message that never contains the request URL / API key."""
|
|
response = getattr(exc, "response", None)
|
|
status = getattr(response, "status_code", None)
|
|
detail = f"HTTP {status}" if status is not None else "request failed"
|
|
message = f"FRED release calendar: {type(exc).__name__}: {detail}"
|
|
try:
|
|
return type(exc)(message, response=response)
|
|
except TypeError: # exotic subclass signature (e.g. JSONDecodeError)
|
|
return requests.RequestException(message, response=response)
|
|
|
|
|
|
def fetch_events(session_date: date, fred_api_key: str | None) -> list[EconEvent]:
|
|
"""Well-known US releases dated `session_date` (released) or the next day (upcoming).
|
|
|
|
Returns [] when no FRED key is configured. Raises on HTTP/parse failure so the pipeline's
|
|
run_safely can record a data gap.
|
|
"""
|
|
if not fred_api_key:
|
|
log.info("events: FRED_API_KEY not set, skipping economic calendar")
|
|
return []
|
|
|
|
end_date = session_date + timedelta(days=1)
|
|
params = {
|
|
"realtime_start": session_date.isoformat(),
|
|
"realtime_end": end_date.isoformat(),
|
|
"include_release_dates_with_no_data": "true",
|
|
"limit": "1000",
|
|
"api_key": fred_api_key,
|
|
"file_type": "json",
|
|
}
|
|
try:
|
|
resp = requests.get(FRED_RELEASE_DATES_URL, params=params, timeout=FRED_TIMEOUT_SECONDS)
|
|
resp.raise_for_status()
|
|
except requests.RequestException as exc:
|
|
# requests embeds the full URL (including api_key=...) in HTTPError/ConnectionError messages,
|
|
# which would otherwise end up in tracebacks and in the briefing's data_gaps line.
|
|
raise _without_url(exc) from None
|
|
data = resp.json()
|
|
rows = data.get("release_dates") or []
|
|
|
|
events: list[EconEvent] = []
|
|
seen: set[tuple[str, date]] = set()
|
|
for row in rows:
|
|
try:
|
|
rel_date = date.fromisoformat(str(row.get("date", ""))[:10])
|
|
except ValueError:
|
|
continue
|
|
if rel_date < session_date or rel_date > end_date:
|
|
continue
|
|
classified = _classify_release(str(row.get("release_name", "")))
|
|
if classified is None:
|
|
continue
|
|
ko, importance, at = classified
|
|
if (ko, rel_date) in seen:
|
|
continue
|
|
seen.add((ko, rel_date))
|
|
when = datetime.combine(rel_date, at, tzinfo=ET) if at else None
|
|
events.append(
|
|
EconEvent(
|
|
name=ko,
|
|
time=when,
|
|
country="US",
|
|
importance=importance, # type: ignore[arg-type]
|
|
kind="released" if rel_date == session_date else "upcoming",
|
|
)
|
|
)
|
|
|
|
events.sort(key=lambda e: (e.kind != "released", e.time or datetime.max.replace(tzinfo=ET), e.name))
|
|
log.info(
|
|
"events: %d FRED release rows -> %d known events for %s/%s",
|
|
len(rows),
|
|
len(events),
|
|
session_date,
|
|
end_date,
|
|
)
|
|
return events
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Earnings (yfinance)
|
|
# ---------------------------------------------------------------------------
|
|
def _is_equity_ticker(ticker: str) -> bool:
|
|
return not (ticker.startswith("^") or "=" in ticker or ticker.endswith("-USD") or "." in ticker)
|
|
|
|
|
|
def next_us_business_day(d: date) -> date:
|
|
"""Next XNYS session after `d` (falls back to a weekday roll if the calendar is unavailable)."""
|
|
try:
|
|
import exchange_calendars as xcals
|
|
|
|
cal = xcals.get_calendar("XNYS", start=d - timedelta(days=7), end=d + timedelta(days=30))
|
|
return cal.date_to_session(d + timedelta(days=1), direction="next").date()
|
|
except Exception as exc: # noqa: BLE001 - calendar is a nicety here, not a requirement
|
|
log.debug("XNYS calendar unavailable (%s); using weekday roll", exc)
|
|
nxt = d + timedelta(days=1)
|
|
while nxt.weekday() >= 5:
|
|
nxt += timedelta(days=1)
|
|
return nxt
|
|
|
|
|
|
def _to_float(value) -> float | None:
|
|
try:
|
|
f = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return None if math.isnan(f) else f
|
|
|
|
|
|
def _calendar_dates(cal: dict) -> list[date]:
|
|
"""Dates from Ticker.calendar['Earnings Date'].
|
|
|
|
Caveat: yfinance derives these with datetime.fromtimestamp() in the *machine's* local zone, so on a
|
|
KST host an after-close (16:00 ET) report shows up one day late. Only used as a fallback.
|
|
"""
|
|
raw = cal.get("Earnings Date") or []
|
|
if not isinstance(raw, (list, tuple)):
|
|
raw = [raw]
|
|
out: list[date] = []
|
|
for v in raw:
|
|
if isinstance(v, datetime):
|
|
out.append(v.date())
|
|
elif isinstance(v, date):
|
|
out.append(v)
|
|
elif hasattr(v, "date"): # pandas.Timestamp
|
|
try:
|
|
out.append(v.date())
|
|
except Exception as exc: # noqa: BLE001
|
|
log.debug("earnings: unparseable calendar date %r: %s", v, exc)
|
|
return out
|
|
|
|
|
|
def _timing_from_hour(hour: int) -> str:
|
|
if hour < 12:
|
|
return "bmo"
|
|
if hour >= 16:
|
|
return "amc"
|
|
return "unknown"
|
|
|
|
|
|
class _NoEarningsData(Exception):
|
|
"""get_earnings_dates() returned nothing usable - caller should fall back to Ticker.calendar."""
|
|
|
|
|
|
def _match_from_earnings_dates(tk, ticker: str, targets: set[date]) -> EarningsItem | None:
|
|
"""Primary source: Ticker.get_earnings_dates() rows are tz-aware ET, so date and bmo/amc are exact.
|
|
|
|
Returns None when rows exist but none fall on a target date; raises _NoEarningsData when the
|
|
scrape yielded no rows at all.
|
|
"""
|
|
df = tk.get_earnings_dates(limit=12)
|
|
if df is None or getattr(df, "empty", True):
|
|
raise _NoEarningsData(ticker)
|
|
for ts, row in df.iterrows():
|
|
try:
|
|
ts_et = ts.tz_convert(ET) if getattr(ts, "tzinfo", None) is not None else ts
|
|
except Exception: # noqa: BLE001
|
|
ts_et = ts
|
|
report_date = ts_et.date()
|
|
if report_date not in targets:
|
|
continue
|
|
return EarningsItem(
|
|
symbol=ticker,
|
|
name=display_name(ticker),
|
|
report_date=report_date,
|
|
timing=_timing_from_hour(int(ts_et.hour)), # type: ignore[arg-type]
|
|
eps_estimate=_to_float(row.get("EPS Estimate")),
|
|
eps_actual=_to_float(row.get("Reported EPS")),
|
|
)
|
|
return None
|
|
|
|
|
|
def _match_from_calendar(tk, ticker: str, targets: set[date]) -> EarningsItem | None:
|
|
"""Fallback source when the earnings-dates scrape is unavailable; timing stays unknown."""
|
|
cal = tk.calendar or {}
|
|
match = next((d for d in sorted(_calendar_dates(cal)) if d in targets), None)
|
|
if match is None:
|
|
return None
|
|
return EarningsItem(
|
|
symbol=ticker,
|
|
name=display_name(ticker),
|
|
report_date=match,
|
|
eps_estimate=_to_float(cal.get("Earnings Average")),
|
|
revenue_estimate=_to_float(cal.get("Revenue Average")),
|
|
)
|
|
|
|
|
|
def _lookup_ticker(ticker: str, targets: set[date]) -> EarningsItem | None:
|
|
tk = yf.Ticker(ticker)
|
|
item: EarningsItem | None = None
|
|
try:
|
|
item = _match_from_earnings_dates(tk, ticker, targets)
|
|
except Exception as exc: # noqa: BLE001 - scrape missing/broken: fall back to the calendar module
|
|
log.debug("earnings: %s earnings_dates unavailable (%s); trying calendar", ticker, type(exc).__name__)
|
|
return _match_from_calendar(tk, ticker, targets)
|
|
if item is None:
|
|
return None
|
|
# Enrich with consensus revenue (and EPS if the scrape lacked it) from the calendar module.
|
|
try:
|
|
cal = tk.calendar or {}
|
|
item.revenue_estimate = _to_float(cal.get("Revenue Average"))
|
|
if item.eps_estimate is None:
|
|
item.eps_estimate = _to_float(cal.get("Earnings Average"))
|
|
except Exception as exc: # noqa: BLE001 - revenue consensus is optional
|
|
log.debug("earnings: %s calendar enrichment failed: %s", ticker, exc)
|
|
return item
|
|
|
|
|
|
def fetch_earnings(session_date: date, tickers: list[str]) -> list[EarningsItem]:
|
|
"""Tracked tickers reporting on `session_date` (after close) or the next US session (before open).
|
|
|
|
Best-effort: each ticker is isolated with try/except, at most EARNINGS_MAX_TICKERS are queried and
|
|
the whole scan stops once EARNINGS_TIME_BUDGET_SECONDS is spent.
|
|
"""
|
|
targets = {session_date, next_us_business_day(session_date)}
|
|
equities = [t for t in tickers if _is_equity_ticker(t)]
|
|
candidates = equities[:EARNINGS_MAX_TICKERS]
|
|
if len(candidates) < len(equities):
|
|
log.info("earnings: capped tickers %d -> %d", len(equities), EARNINGS_MAX_TICKERS)
|
|
|
|
t0 = time.monotonic()
|
|
found: list[EarningsItem] = []
|
|
scanned = 0
|
|
for ticker in candidates:
|
|
if time.monotonic() - t0 > EARNINGS_TIME_BUDGET_SECONDS:
|
|
log.warning("earnings: time budget exhausted after %d/%d tickers", scanned, len(candidates))
|
|
break
|
|
scanned += 1
|
|
try:
|
|
item = _lookup_ticker(ticker, targets)
|
|
except Exception as exc: # noqa: BLE001 - per-ticker isolation
|
|
log.warning("earnings: %s failed: %s: %s", ticker, type(exc).__name__, str(exc)[:120])
|
|
continue
|
|
if item is not None:
|
|
found.append(item)
|
|
|
|
found.sort(key=lambda e: (e.report_date, e.timing, e.symbol))
|
|
log.info(
|
|
"earnings: scanned %d tickers in %.1fs, %d reporting on %s",
|
|
scanned,
|
|
time.monotonic() - t0,
|
|
len(found),
|
|
sorted(targets),
|
|
)
|
|
return found
|