feat(phase-1a): external data fetchers + refresh pipeline + scheduler
10종목 시드 + pykrx OHLCV / 외인·기관 거래대금, KIS read-only EOD, OpenDART
공시, 네이버 금융 뉴스 스크레이퍼, 구글 뉴스 RSS, yfinance 거시(KOSPI/KOSDAQ/
USDKRW/US10Y) fetcher 를 추가하고 refresh_one / daily_batch / backfill /
APScheduler(16:00 KST) 파이프라인으로 묶음.
- backend/app/seed: 10종목 시드 (대형/고변동/테마/플랫폼/방어)
- backend/app/fetch: pykrx, kis, dart, news, macro, symbols_seed
- backend/app/pipelines: refresh_one, daily_batch, backfill(CLI), scheduler
- backend/app/api/refresh.py: POST /api/refresh/{code}?lookback_days=N
- backend/app/main.py: lifespan 으로 스케줄러 기동/종료, /health/keys 추가
- README: .env 복사 안내 보강
스모크 테스트 (실제 키 사용) 결과:
KIS token : ok (token 346자 발급)
KIS daily : 005930 11rows
DART list : 005930 30일 10건
Naver news : 005930 12건
Google RSS : "삼성전자" 92건
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
83
backend/app/fetch/macro.py
Normal file
83
backend/app/fetch/macro.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""거시/지수 지표: yfinance 로 KOSPI / KOSDAQ / USDKRW / US10Y."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, timedelta
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db.connection import get_engine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# yfinance ticker -> macro_daily.key
|
||||
TICKER_MAP: dict[str, str] = {
|
||||
"^KS11": "kospi",
|
||||
"^KQ11": "kosdaq",
|
||||
"KRW=X": "usdkrw",
|
||||
"^TNX": "us10y",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MacroResult:
|
||||
key: str
|
||||
inserted: int
|
||||
updated: int
|
||||
error: str | None = None
|
||||
|
||||
def status(self) -> str:
|
||||
return "failed" if self.error else "ok"
|
||||
|
||||
|
||||
def fetch_macro_daily(*, years: int = 5) -> list[MacroResult]:
|
||||
end = date.today()
|
||||
start = end - timedelta(days=365 * years)
|
||||
try:
|
||||
import yfinance as yf
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return [MacroResult(key="*", inserted=0, updated=0, error=f"yfinance import failed: {exc}")]
|
||||
|
||||
results: list[MacroResult] = []
|
||||
engine = get_engine()
|
||||
for ticker, key in TICKER_MAP.items():
|
||||
try:
|
||||
df = yf.download(ticker, start=start.isoformat(), end=(end + timedelta(days=1)).isoformat(),
|
||||
progress=False, auto_adjust=False)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("yfinance failed ticker=%s", ticker)
|
||||
results.append(MacroResult(key=key, inserted=0, updated=0, error=str(exc)))
|
||||
continue
|
||||
if df is None or df.empty:
|
||||
results.append(MacroResult(key=key, inserted=0, updated=0, error="empty"))
|
||||
continue
|
||||
# Close 컬럼만 사용
|
||||
close_col = "Close" if "Close" in df.columns else df.columns[0]
|
||||
ins = upd = 0
|
||||
with engine.begin() as conn:
|
||||
for idx, row in df.iterrows():
|
||||
day = idx.date() if hasattr(idx, "date") else date.fromisoformat(str(idx)[:10])
|
||||
val = row[close_col]
|
||||
try:
|
||||
val_f = float(val.iloc[0] if hasattr(val, "iloc") else val)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
res = conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO macro_daily (date, key, value)
|
||||
VALUES (:date, :key, :value)
|
||||
ON CONFLICT (date, key) DO UPDATE SET value = EXCLUDED.value
|
||||
RETURNING (xmax = 0) AS inserted
|
||||
"""
|
||||
),
|
||||
{"date": day, "key": key, "value": val_f},
|
||||
)
|
||||
r = res.first()
|
||||
if r and r[0]:
|
||||
ins += 1
|
||||
else:
|
||||
upd += 1
|
||||
results.append(MacroResult(key=key, inserted=ins, updated=upd))
|
||||
return results
|
||||
Reference in New Issue
Block a user