@@ -276,20 +276,20 @@ python main.py #if run on Mac, please use 'pythonw main.py'
@@ -299,26 +299,26 @@ python main.py #if run on Mac, please use 'pythonw main.py'
@@ -316,16 +316,16 @@ python main.py
|
|
|
@@ -362,51 +362,51 @@ python main.py
|
GBDT
BOOST
LR
|
CNN
RNN
|
- TFT (IJoF'2019)
- GATs (NIPS'2017)
- SFM (KDD'2017)
+ TFT (IJoF'2019)
+ GATs (NIPS'2017) qlib
+ SFM (KDD'2017) qlib
|
- Transformer (NeurIPS'2017)
- TCTS (ICML'2021)
- TRA (KDD'2021)
- TCN (KDD'2018)
- IGMTF (KDD'2021)
- HIST (KDD'2018)
- Localformer ('2021)
+ Transformer (NeurIPS'2017)
+ TCTS (ICML'2021) qlib
+ TRA (KDD'2021) qlib
+ TCN (KDD'2018) qlib
+ IGMTF (KDD'2021) qlib
+ HIST (KDD'2018) qlib
+ Localformer ('2021) qlib
|
ChatGPT
@@ -418,9 +418,11 @@ python main.py
+> 注:标有 qlib 的模型未包含在本仓库中,链接指向 [microsoft/qlib](https://github.com/microsoft/qlib/tree/main/examples/benchmarks) 上游 benchmark;标有 未包含 的模型暂无实现。本仓库自带的模型位于 [pytrader/strategies/benchmarks](../pytrader/strategies/benchmarks)。
+
### Benchmark and Model zoo
-Results and models are available in the [model zoo](docs/03-智能策略/model_zoo.md). AI strategies is shown at [here](./pytrader/strategies/), local run ``python backend/pytrader/strategies/workflow_by_code.py``, also provide [](https://mybinder.org/v2/gh/UFund-Me/Qbot/blob/main/backend/pytrader/strategies/workflow_by_code.ipynb/HEAD)
+Results and models are available in the [model zoo](03-智能策略/model_zoo.md). AI strategies is shown at [here](../pytrader/strategies/), local run ``python pytrader/strategies/workflow_by_code.py``, also provide [](https://mybinder.org/v2/gh/UFund-Me/Qbot/HEAD?labpath=pytrader%2Fstrategies%2Fworkflow_by_code.ipynb)
👉 点击展开查看具体AI模型benchmark结果
@@ -450,7 +452,7 @@ Results and models are available in the [model zoo](docs/03-智能策略/model_z
| HIST | ✓ | ✗ | pytorch | ✗ | ✗ | Wentao Xu, et al.2021 | ✗ |
-**Note:** All the about **300+ models, methods of 40+ papers** in quant.ai supported by [Model Zoo](./docs/03-智能策略/model_zoo.md) can be trained or used in this codebase.
+**Note:** All the about **300+ models, methods of 40+ papers** in quant.ai supported by [Model Zoo](./03-智能策略/model_zoo.md) can be trained or used in this codebase.
@@ -582,7 +584,7 @@ Qbot买卖强弱指标(收费)
本项目编写了详细的策略原理说明和平台搭建到使用的详细文档,尤其适合量化小白。欢迎加群交流!
-[在线文档](https://ufund-me.github.io/Qbot/#/) | [❓ 常见问题](https://ufund-me.github.io/Qbot/#/04-%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98/FQA) | [Jupyter Notebook](./pytrader/strategies/notebook)
+[在线文档](https://ufund-me.github.io/Qbot/#/) | [❓ 常见问题](https://ufund-me.github.io/Qbot/#/04-%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98/FQA) | [Jupyter Notebook](../pytrader/strategies/notebook)
## Quantstats Report
@@ -651,7 +653,7 @@ dagit -h 0.0.0.0 -p 3000
## Contributing
-We appreciate all contributions to improve Qbot. Please refer to [CONTRIBUTING](.github/CONTRIBUTING) for the contributing guideline.
+We appreciate all contributions to improve Qbot. Please refer to [CONTRIBUTING](../.github/CONTRIBUTING.md) for the contributing guideline.
## 🍮 Community
- Github discussions 💬 or issues 💭
@@ -743,7 +745,7 @@ If you like the project, you can become a sponsor at [Open Collective](https://o
## LICENSE
-
+
署名-非商业性使用-相同方式共享 4.0 国际
diff --git a/docs/notebook/Strategy_runner.py b/docs/notebook/Strategy_runner.py
index 77f6b227..3518cdda 100644
--- a/docs/notebook/Strategy_runner.py
+++ b/docs/notebook/Strategy_runner.py
@@ -10,7 +10,12 @@
data_path = './data/'
if not os.path.exists(data_path):
os.makedirs(data_path)
-mytoken = '565ee6d69fb85cb0bc7fdfc8dea4b8ce1f18366f30e8d23a253bb9cf'
+mytoken = os.environ.get("TUSHARE_TOKEN")
+if not mytoken:
+ raise RuntimeError(
+ "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. "
+ "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)."
+ )
class Strategy_runner:
diff --git a/docs/notebook/stock_api.py b/docs/notebook/stock_api.py
index 3c360a4e..9de927a2 100644
--- a/docs/notebook/stock_api.py
+++ b/docs/notebook/stock_api.py
@@ -1,6 +1,13 @@
+import os
+
import tushare as ts
-mytoken = '565ee6d69fb85cb0bc7fdfc8dea4b8ce1f18366f30e8d23a253bb9cf'
+mytoken = os.environ.get("TUSHARE_TOKEN")
+if not mytoken:
+ raise RuntimeError(
+ "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. "
+ "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)."
+ )
ts.set_token(mytoken)
pro = ts.pro_api()
diff --git a/pyfunds/backtest/xalpha/universal.py b/pyfunds/backtest/xalpha/universal.py
index 9d873103..509a4162 100644
--- a/pyfunds/backtest/xalpha/universal.py
+++ b/pyfunds/backtest/xalpha/universal.py
@@ -1248,7 +1248,14 @@ def _get_daily(
return df
-def get_xueqiu_rt(code, token="a664afb60c7036c7947578ac1a5860c4cfb6b3b5"):
+def get_xueqiu_rt(code, token=None):
+ if not token:
+ token = os.environ.get("XUEQIU_TOKEN")
+ if not token:
+ raise RuntimeError(
+ "No Xueqiu token given. Pass token=get_token() or set the "
+ "XUEQIU_TOKEN environment variable."
+ )
if code.startswith("HK") and code[2:].isdigit():
code = code[2:]
url = "https://stock.xueqiu.com/v5/stock/quote.json?symbol={code}&extend=detail"
@@ -2667,7 +2674,12 @@ def get_bar_fromwsj(code, token=None, interval=3600):
# TODO: there is other freq tags, but I have no time to explore them, contributions are welcome:)
freq = trans.get(str(interval), interval)
if not token:
- token = "cecc4267a0194af89ca343805a3e57af"
+ token = os.environ.get("WSJ_TOKEN")
+ if not token:
+ raise RuntimeError(
+ "No WSJ token given. Pass token=... or set the WSJ_TOKEN "
+ "environment variable."
+ )
# the thing I am concerned here is whether token is refreshed
params = {
diff --git a/pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml b/pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml
new file mode 100644
index 00000000..6a6af111
--- /dev/null
+++ b/pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml
@@ -0,0 +1,79 @@
+# 台股版 LightGBM + Alpha158
+# 先用 qbot/data/tw_collector.py 建好 ~/.qlib/qlib_data/tw_data,再執行:
+# qrun pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml
+# 說明見 docs/03-智能策略/台股qlib工作流.md
+qlib_init:
+ provider_uri: "~/.qlib/qlib_data/tw_data"
+ region: tw # 交易單位 1000 股(一張),需 pyqlib >= 0.8.6
+market: &market top200 # 依過去 60 日成交值每季重選的前 200 檔
+benchmark: &benchmark TW0050 # 0050 還原權值(含息);TWII 是不含息的價格指數
+data_handler_config: &data_handler_config
+ start_time: 2008-01-01
+ end_time: 2025-12-31
+ fit_start_time: 2008-01-01
+ fit_end_time: 2017-12-31
+ instruments: *market
+port_analysis_config: &port_analysis_config
+ strategy:
+ class: TopkDropoutStrategy
+ module_path: qlib.contrib.strategy
+ kwargs:
+ signal:
+ topk: 50
+ n_drop: 5
+ backtest:
+ # 2015-06-01 起漲跌幅才從 7% 放寬到 10%,回測期間請放在之後
+ start_time: 2021-01-01
+ # 資料最後一天必須晚於 end_time(qlib 回測需要下一個交易日)
+ end_time: 2025-12-31
+ # 一張要 1000 股,帳戶太小時高價股會買不到;小資金可在 exchange_kwargs 加 trade_unit: 1 模擬零股
+ account: 1000000000
+ benchmark: *benchmark
+ exchange_kwargs:
+ # 漲停價會依升降單位向下取整,實際漲停幅度約 9.5%~10%,因此不用 0.1
+ limit_threshold: 0.095
+ deal_price: close
+ open_cost: 0.001425 # 手續費 0.1425%(券商折扣可自行調低)
+ close_cost: 0.004425 # 手續費 0.1425% + 證交稅 0.3%
+ min_cost: 20 # 最低手續費 20 元
+task:
+ model:
+ class: LGBModel
+ module_path: qlib.contrib.model.gbdt
+ kwargs:
+ loss: mse
+ colsample_bytree: 0.8879
+ learning_rate: 0.2
+ subsample: 0.8789
+ lambda_l1: 205.6999
+ lambda_l2: 580.9768
+ max_depth: 8
+ num_leaves: 210
+ num_threads: 20
+ dataset:
+ class: DatasetH
+ module_path: qlib.data.dataset
+ kwargs:
+ handler:
+ class: Alpha158
+ module_path: qlib.contrib.data.handler
+ kwargs: *data_handler_config
+ segments:
+ train: [2008-01-01, 2017-12-31]
+ valid: [2018-01-01, 2020-12-31]
+ test: [2021-01-01, 2025-12-31]
+ record:
+ - class: SignalRecord
+ module_path: qlib.workflow.record_temp
+ kwargs:
+ model:
+ dataset:
+ - class: SigAnaRecord
+ module_path: qlib.workflow.record_temp
+ kwargs:
+ ana_long_short: False
+ ann_scaler: 252
+ - class: PortAnaRecord
+ module_path: qlib.workflow.record_temp
+ kwargs:
+ config: *port_analysis_config
diff --git a/pytrader/strategies/k-line.py b/pytrader/strategies/k-line.py
index b8b52c6d..0ef71d84 100644
--- a/pytrader/strategies/k-line.py
+++ b/pytrader/strategies/k-line.py
@@ -1,5 +1,7 @@
# blog https://blog.csdn.net/qq_44176343/article/details/109903512
+import os
+
import tushare as ts
import matplotlib.pyplot as plt
import seaborn as sns
@@ -8,7 +10,12 @@
# pip install --upgrade mplfinance
import mpl_finance as mpf
-token = "6f747880359ef14fe2fd5fc0c2c08a4e09a47e7ac161d643ae7036c0"
+token = os.environ.get("TUSHARE_TOKEN")
+if not token:
+ raise RuntimeError(
+ "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. "
+ "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)."
+ )
pro = ts.pro_api(token)
df = ts.get_k_data('000002','2019-06-01','2019-09-30')
diff --git a/pytrader/strategies/multi_factor_strategy.py b/pytrader/strategies/multi_factor_strategy.py
index 6afd21c1..dda687dd 100644
--- a/pytrader/strategies/multi_factor_strategy.py
+++ b/pytrader/strategies/multi_factor_strategy.py
@@ -13,12 +13,25 @@
Licensed under the MIT License.
'''
+import os
+
import backtrader as bt
import tushare as ts
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
+
+def get_tushare_token():
+ token = os.environ.get("TUSHARE_TOKEN")
+ if not token:
+ raise RuntimeError(
+ "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. "
+ "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)."
+ )
+ return token
+
+
class PEFetcher:
def __init__(self, ts):
self.ts = ts
@@ -59,7 +72,7 @@ def get_factor_data(self, symbol, date):
def get_data(symbol, data_fetchers, start_date, end_date):
- ts.set_token('e96f18882532434b7692388cb028eb267d3f0d56845dc92eef06ea4a')
+ ts.set_token(get_tushare_token())
data = {}
dates = ts.trade_cal()
@@ -150,7 +163,7 @@ def next(self):
if __name__ == '__main__':
# 加载数据
- ts.set_token('e96f18882532434b7692388cb028eb267d3f0d56845dc92eef06ea4a')
+ ts.set_token(get_tushare_token())
pro = ts.pro_api()
data_fetchers = [
{'code': 'pe_ttm', 'data_fetcher': PEFetcher(pro)},
diff --git a/qbot/data/dump_bin.py b/qbot/data/dump_bin.py
index 3b377f68..70ef7213 100644
--- a/qbot/data/dump_bin.py
+++ b/qbot/data/dump_bin.py
@@ -149,9 +149,7 @@ def _get_date(
def _get_source_data(self, file_path: Path) -> pd.DataFrame:
df = pd.read_csv(str(file_path.resolve()), low_memory=False)
- df[self.date_field_name] = (
- df[self.date_field_name].astype(str).astype(np.datetime64)
- )
+ df[self.date_field_name] = pd.to_datetime(df[self.date_field_name].astype(str))
# df.drop_duplicates([self.date_field_name], inplace=True)
return df
@@ -225,8 +223,8 @@ def data_merge_calendar(
) -> pd.DataFrame:
# calendars
calendars_df = pd.DataFrame(data=calendars_list, columns=[self.date_field_name])
- calendars_df[self.date_field_name] = calendars_df[self.date_field_name].astype(
- np.datetime64
+ calendars_df[self.date_field_name] = pd.to_datetime(
+ calendars_df[self.date_field_name]
)
cal_df = calendars_df[
(calendars_df[self.date_field_name] >= df[self.date_field_name].min())
diff --git a/qbot/data/tw_collector.py b/qbot/data/tw_collector.py
new file mode 100644
index 00000000..edb23da7
--- /dev/null
+++ b/qbot/data/tw_collector.py
@@ -0,0 +1,521 @@
+"""台股日線資料下載,並轉成 qlib 格式。
+
+流程(可用 `run` 一次跑完):
+ download 從 Yahoo Finance 下載上市、上櫃股票及基準指數的日線
+ normalize 對齊交易日曆、還原權值(qlib 的 factor 慣例)、計算 change / vwap
+ dump 用 dump_bin.py 轉成 qlib 二進位格式,並產生 twse / tpex 股票池
+ universe 依過去成交值每季重選前 N 檔,產生動態股票池(例如 top200)
+
+範例:
+ python qbot/data/tw_collector.py run
+ python qbot/data/tw_collector.py run --symbols_file my_list.txt --top_n 50
+ python qbot/data/tw_collector.py download --start 2005-01-01 --skip_exists True
+
+股票代碼在 qlib 中一律寫成 TW + 代號(TW2330、TW6488),
+加權指數為 TWII,元大台灣50 為 TW0050。
+詳細說明見 docs/03-智能策略/台股qlib工作流.md
+"""
+
+import os
+import re
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+from loguru import logger
+
+DEFAULT_BASE_DIR = "~/.qlib/stock_data/tw"
+DEFAULT_QLIB_DIR = "~/.qlib/qlib_data/tw_data"
+SOURCE_DIR_NAME = "source"
+NORMALIZE_DIR_NAME = "normalize"
+STOCK_LIST_FILE = "stock_list.csv"
+FAILED_FILE = "failed.txt"
+
+# qlib 代碼 -> Yahoo 代碼
+BENCHMARKS = {"TWII": "^TWII", "TW0050": "0050.TW"}
+CALENDAR_SYMBOL = "TWII"
+# 指數沒有可靠的成交量,不套用「量為 0 視為停牌」的規則
+INDEX_SYMBOLS = {"TWII"}
+
+MARKET_SUFFIX = {"twse": ".TW", "tpex": ".TWO"}
+SUFFIX_MARKET = {v: k for k, v in MARKET_SUFFIX.items()}
+YAHOO_COLUMNS = {
+ "Open": "open",
+ "High": "high",
+ "Low": "low",
+ "Close": "close",
+ "Adj Close": "adjclose",
+ "Volume": "volume",
+ "Dividends": "dividends",
+ "Stock Splits": "stock_splits",
+}
+PRICE_FIELDS = ["open", "high", "low", "close"]
+QLIB_FIELDS = PRICE_FIELDS + ["volume", "vwap", "factor", "change"]
+
+FINMIND_URL = "https://api.finmindtrade.com/api/v4/data"
+STOCK_CODE_PATTERN = re.compile(r"[1-9]\d{3}")
+
+
+def _to_qlib_symbol(code: str) -> str:
+ return f"TW{code}"
+
+
+def _make_stock_list(
+ codes: List[str],
+ markets: List[str],
+ names: Optional[List[str]] = None,
+ industries: Optional[List[str]] = None,
+) -> pd.DataFrame:
+ n = len(codes)
+ df = pd.DataFrame(
+ {
+ "code": codes,
+ "market": markets,
+ "name": names if names is not None else [""] * n,
+ "industry": industries if industries is not None else [""] * n,
+ }
+ )
+ df["symbol"] = df["code"].map(_to_qlib_symbol)
+ df["yahoo"] = df["code"] + df["market"].map(MARKET_SUFFIX)
+ return df[["symbol", "code", "market", "yahoo", "name", "industry"]]
+
+
+def filter_stock_info(info: pd.DataFrame) -> pd.DataFrame:
+ """從 FinMind TaiwanStockInfo 留下上市、上櫃的普通股(4 碼、非 0 開頭)。
+
+ 0 開頭的是 ETF,興櫃(type=emerging)流動性太低也排除。
+ """
+ df = info.copy()
+ df["stock_id"] = df["stock_id"].astype(str).str.strip()
+ df = df[df["type"].isin(list(MARKET_SUFFIX))]
+ df = df[df["stock_id"].str.fullmatch(STOCK_CODE_PATTERN.pattern)]
+ df = df.drop_duplicates("stock_id").sort_values("stock_id")
+ return _make_stock_list(
+ df["stock_id"].tolist(),
+ df["type"].tolist(),
+ df["stock_name"].tolist(),
+ df["industry_category"].tolist(),
+ )
+
+
+def fetch_finmind_stock_list(token: Optional[str] = None, timeout: int = 30):
+ """抓 FinMind 的台股清單(TaiwanStockInfo)。token 可省略,但請求額度較低。"""
+ import requests
+
+ token = token or os.environ.get("FINMIND_TOKEN")
+ headers = {"Authorization": f"Bearer {token}"} if token else {}
+ resp = requests.get(
+ FINMIND_URL,
+ params={"dataset": "TaiwanStockInfo"},
+ headers=headers,
+ timeout=timeout,
+ )
+ resp.raise_for_status()
+ payload = resp.json()
+ if payload.get("status") != 200 or not payload.get("data"):
+ raise RuntimeError(f"FinMind TaiwanStockInfo failed: {payload.get('msg')}")
+ return filter_stock_info(pd.DataFrame(payload["data"]))
+
+
+def load_symbols_file(path) -> pd.DataFrame:
+ """讀取自訂股票清單,每行一檔,# 之後為註解。
+
+ 可寫 2330(預設上市)、2330.TW、6488.TWO,代號後面可接名稱。
+ """
+ codes, markets, names = [], [], []
+ for line in Path(path).expanduser().read_text(encoding="utf-8").splitlines():
+ line = line.split("#", 1)[0].strip()
+ if not line:
+ continue
+ parts = line.split(maxsplit=1)
+ token = parts[0].upper()
+ code, dot, suffix = token.partition(".")
+ market = SUFFIX_MARKET.get(f".{suffix}") if dot else "twse"
+ if market is None or not STOCK_CODE_PATTERN.fullmatch(code):
+ raise ValueError(f"Unrecognized symbol in {path}: {parts[0]}")
+ codes.append(code)
+ markets.append(market)
+ names.append(parts[1] if len(parts) > 1 else "")
+ df = _make_stock_list(codes, markets, names)
+ return df.drop_duplicates("symbol").reset_index(drop=True)
+
+
+def fetch_yahoo(ticker: str, start: str, end: str) -> pd.DataFrame:
+ """下載 Yahoo 日線(不還原),回傳 date + 小寫欄位;查無資料時回傳空表。"""
+ import yfinance as yf
+
+ df = yf.Ticker(ticker).history(
+ start=start, end=end, interval="1d", auto_adjust=False, actions=True
+ )
+ if df is None or df.empty:
+ return pd.DataFrame()
+ df = df.rename(columns=YAHOO_COLUMNS)
+ df.index = pd.to_datetime(df.index)
+ if df.index.tz is not None:
+ df.index = df.index.tz_localize(None)
+ df.index = df.index.normalize()
+ df.index.name = "date"
+ columns = [c for c in YAHOO_COLUMNS.values() if c in df.columns]
+ return df[columns].reset_index()
+
+
+def download(
+ base_dir: str = DEFAULT_BASE_DIR,
+ start: str = "2005-01-01",
+ end: Optional[str] = None,
+ symbols_file: Optional[str] = None,
+ finmind_token: Optional[str] = None,
+ max_workers: int = 4,
+ delay: float = 0.5,
+ retries: int = 3,
+ retry_wait: float = 5.0,
+ skip_exists: bool = False,
+):
+ """下載股票清單與日線到 base_dir/source。
+
+ end 含當天,預設為今天。沒給 symbols_file 時從 FinMind 抓全部上市、上櫃股票。
+ Yahoo 有流量限制,中斷後可加 --skip_exists True 只補抓缺的檔案。
+ """
+ base_dir = Path(base_dir).expanduser()
+ source_dir = base_dir.joinpath(SOURCE_DIR_NAME)
+ source_dir.mkdir(parents=True, exist_ok=True)
+
+ if symbols_file:
+ stocks = load_symbols_file(symbols_file)
+ else:
+ stocks = fetch_finmind_stock_list(finmind_token)
+ stocks.to_csv(base_dir.joinpath(STOCK_LIST_FILE), index=False)
+ logger.info(f"{len(stocks)} stocks in {base_dir.joinpath(STOCK_LIST_FILE)}")
+
+ # yfinance 的 end 不含當天
+ end = pd.Timestamp(end) if end else pd.Timestamp.today().normalize()
+ end = (end + pd.Timedelta(days=1)).strftime("%Y-%m-%d")
+ jobs = dict(BENCHMARKS)
+ jobs.update(stocks.set_index("symbol")["yahoo"].to_dict())
+
+ def _job(symbol: str, ticker: str) -> str:
+ path = source_dir.joinpath(f"{symbol}.csv")
+ if skip_exists and path.exists():
+ return "skip"
+ for attempt in range(retries):
+ try:
+ df = fetch_yahoo(ticker, start, end)
+ break
+ except Exception as e: # yfinance 會丟各種網路、限流例外
+ logger.warning(f"{ticker} attempt {attempt + 1} failed: {e}")
+ if attempt + 1 < retries:
+ time.sleep(retry_wait * 2**attempt)
+ else:
+ return "error"
+ time.sleep(delay)
+ if df.empty:
+ return "empty"
+ df.to_csv(path, index=False)
+ return "ok"
+
+ results: Dict[str, str] = {}
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ futures = {executor.submit(_job, s, t): s for s, t in jobs.items()}
+ for i, future in enumerate(as_completed(futures), 1):
+ results[futures[future]] = future.result()
+ if i % 100 == 0 or i == len(futures):
+ logger.info(f"download {i}/{len(futures)}")
+
+ failed = sorted(s for s, r in results.items() if r in ("error", "empty"))
+ base_dir.joinpath(FAILED_FILE).write_text("\n".join(failed), encoding="utf-8")
+ counts = pd.Series(results).value_counts().to_dict()
+ logger.info(f"download finished: {counts}")
+ if failed:
+ logger.warning(
+ f"{len(failed)} symbols failed, see {base_dir.joinpath(FAILED_FILE)}; "
+ "rerun with --skip_exists True to retry only the missing ones"
+ )
+ if CALENDAR_SYMBOL not in results or results[CALENDAR_SYMBOL] in ("error", "empty"):
+ logger.warning("TWII download failed; normalize will fall back to stock dates")
+
+
+def _source_files(base_dir: Path) -> Dict[str, Path]:
+ source_dir = base_dir.joinpath(SOURCE_DIR_NAME)
+ return {p.stem: p for p in sorted(source_dir.glob("*.csv"))}
+
+
+def _read_source(path: Path) -> pd.DataFrame:
+ df = pd.read_csv(path)
+ df["date"] = pd.to_datetime(df["date"])
+ return df
+
+
+def load_calendar(base_dir) -> pd.DatetimeIndex:
+ """以加權指數的交易日為日曆;沒有指數資料時改用所有股票有成交的日期。"""
+ files = _source_files(Path(base_dir).expanduser())
+ if CALENDAR_SYMBOL in files:
+ df = _read_source(files[CALENDAR_SYMBOL])
+ dates = df.loc[df["close"].notna(), "date"]
+ else:
+ logger.warning("TWII.csv not found, building calendar from stock dates")
+ dates = []
+ for symbol, path in files.items():
+ if symbol in BENCHMARKS:
+ continue
+ df = _read_source(path)
+ dates.extend(df.loc[df["volume"] > 0, "date"])
+ return pd.DatetimeIndex(sorted(set(pd.to_datetime(dates))), name="date")
+
+
+def normalize_symbol(
+ df: pd.DataFrame,
+ symbol: str,
+ calendar: Optional[pd.DatetimeIndex] = None,
+ is_index: bool = False,
+) -> pd.DataFrame:
+ """把 Yahoo 原始日線轉成 qlib 慣例(同 qlib yahoo collector 的 1d normalize)。
+
+ - 價格為還原權值後再除以首日收盤價,volume 反向調整
+ - factor = 還原價 / 原始價,qlib 用 close / factor 換回實際股價來計算整張交易
+ - change 為還原收盤價的日報酬,用來判斷漲跌停(除息日不會被當成大跌)
+ - vwap 用 (high + low + close) / 3 近似(Yahoo 沒有成交金額)
+ """
+ df = df.copy()
+ df["date"] = pd.to_datetime(df["date"])
+ df = df.drop_duplicates("date").set_index("date").sort_index()
+ if calendar is not None and len(calendar) > 0:
+ in_range = (calendar >= df.index.min()) & (calendar <= df.index.max())
+ df = df.reindex(calendar[in_range])
+ df.index.name = "date"
+ if df.empty:
+ return pd.DataFrame()
+ if "adjclose" not in df.columns:
+ df["adjclose"] = df["close"]
+
+ invalid = df["close"].isna() | (df["close"] <= 0)
+ if not is_index:
+ invalid |= df["volume"].isna() | (df["volume"] <= 0)
+ df.loc[invalid, PRICE_FIELDS + ["adjclose", "volume"]] = np.nan
+ if df["close"].isna().all():
+ return pd.DataFrame()
+
+ df["factor"] = (df["adjclose"] / df["close"]).ffill()
+ for col in PRICE_FIELDS:
+ df[col] = df[col] * df["factor"]
+ df["volume"] = df["volume"] / df["factor"]
+
+ close = df["close"].ffill()
+ df["change"] = close / close.shift(1) - 1
+ df.loc[invalid, "change"] = np.nan
+
+ first_close = df["close"].dropna().iloc[0]
+ for col in PRICE_FIELDS + ["factor"]:
+ df[col] = df[col] / first_close
+ df["volume"] = df["volume"] * first_close
+ df["vwap"] = (df["high"] + df["low"] + df["close"]) / 3
+
+ df["symbol"] = symbol
+ return df.reset_index()[["date", "symbol"] + QLIB_FIELDS]
+
+
+def normalize(base_dir: str = DEFAULT_BASE_DIR):
+ """把 base_dir/source 的原始資料轉到 base_dir/normalize。"""
+ base_dir = Path(base_dir).expanduser()
+ normalize_dir = base_dir.joinpath(NORMALIZE_DIR_NAME)
+ normalize_dir.mkdir(parents=True, exist_ok=True)
+ for old in normalize_dir.glob("*.csv"):
+ old.unlink()
+
+ calendar = load_calendar(base_dir)
+ logger.info(f"calendar: {len(calendar)} days")
+ count = 0
+ for symbol, path in _source_files(base_dir).items():
+ df = normalize_symbol(
+ _read_source(path), symbol, calendar, is_index=symbol in INDEX_SYMBOLS
+ )
+ if df.empty:
+ logger.warning(f"{symbol} has no valid rows, skipped")
+ continue
+ df.to_csv(normalize_dir.joinpath(f"{symbol}.csv"), index=False)
+ count += 1
+ logger.info(f"normalized {count} symbols into {normalize_dir}")
+
+
+def _instruments_dir(qlib_dir) -> Path:
+ path = Path(qlib_dir).expanduser().joinpath("instruments")
+ path.mkdir(parents=True, exist_ok=True)
+ return path
+
+
+def _read_instruments(path: Path) -> pd.DataFrame:
+ return pd.read_csv(
+ path, sep="\t", header=None, names=["symbol", "start", "end"], dtype=str
+ )
+
+
+def _write_instruments(df: pd.DataFrame, path: Path):
+ df[["symbol", "start", "end"]].to_csv(path, sep="\t", header=False, index=False)
+
+
+def write_market_instruments(
+ base_dir: str = DEFAULT_BASE_DIR, qlib_dir: str = DEFAULT_QLIB_DIR
+):
+ """從 all.txt 移除基準指數,並依股票清單拆出 twse.txt、tpex.txt。"""
+ inst_dir = _instruments_dir(qlib_dir)
+ all_path = inst_dir.joinpath("all.txt")
+ inst = _read_instruments(all_path)
+ inst = inst[~inst["symbol"].isin(list(BENCHMARKS))]
+ _write_instruments(inst, all_path)
+
+ stock_list_path = Path(base_dir).expanduser().joinpath(STOCK_LIST_FILE)
+ if not stock_list_path.exists():
+ return
+ stocks = pd.read_csv(stock_list_path, dtype=str)
+ for market in MARKET_SUFFIX:
+ symbols = set(stocks.loc[stocks["market"] == market, "symbol"])
+ _write_instruments(
+ inst[inst["symbol"].isin(symbols)], inst_dir.joinpath(f"{market}.txt")
+ )
+
+
+def dump(
+ base_dir: str = DEFAULT_BASE_DIR,
+ qlib_dir: str = DEFAULT_QLIB_DIR,
+ max_workers: int = 8,
+):
+ """用 dump_bin.py 把 base_dir/normalize 轉成 qlib 格式(會覆寫 qlib_dir 的日曆)。"""
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
+ from dump_bin import DumpDataAll
+
+ normalize_dir = Path(base_dir).expanduser().joinpath(NORMALIZE_DIR_NAME)
+ DumpDataAll(
+ csv_path=str(normalize_dir),
+ qlib_dir=qlib_dir,
+ max_workers=max_workers,
+ date_field_name="date",
+ symbol_field_name="symbol",
+ include_fields=",".join(QLIB_FIELDS),
+ )()
+ write_market_instruments(base_dir, qlib_dir)
+
+
+def compute_liquidity_universe(
+ values: pd.DataFrame, top_n: int = 200, window: int = 60, freq: str = "Q"
+) -> pd.DataFrame:
+ """依過去 window 日平均成交值,在每期最後一個交易日收盤後選出前 top_n 檔。
+
+ values:index 為交易日,columns 為代碼,上市前或下市後為 NaN,停牌日為 0。
+ 選股只用當天(含)以前的資料,成分從下一個交易日生效,到下一次重選日為止,
+ 所以沒有前視偏差。回傳 symbol / start / end,連續入選的期間會合併成一段。
+ """
+ dates = values.index
+ avg = values.rolling(window, min_periods=window).mean()
+ periods = dates.to_period(freq)
+ rebalance_dates = dates[~periods.duplicated(keep="last")]
+
+ spans: Dict[str, List[list]] = {}
+ for i, rebalance in enumerate(rebalance_dates):
+ pos = dates.get_loc(rebalance)
+ if pos + 1 >= len(dates):
+ break
+ ranked = avg.loc[rebalance].dropna()
+ ranked = ranked[ranked > 0].nlargest(top_n)
+ start = dates[pos + 1]
+ if i + 1 < len(rebalance_dates):
+ end = rebalance_dates[i + 1]
+ else:
+ end = dates[-1]
+ for symbol in ranked.index:
+ symbol_spans = spans.setdefault(symbol, [])
+ # 上一期也有入選(上一段剛好結束在這次重選日)就延長
+ if symbol_spans and symbol_spans[-1][1] == rebalance:
+ symbol_spans[-1][1] = end
+ else:
+ symbol_spans.append([start, end])
+
+ rows = [
+ (symbol, start, end)
+ for symbol, symbol_spans in sorted(spans.items())
+ for start, end in symbol_spans
+ ]
+ df = pd.DataFrame(rows, columns=["symbol", "start", "end"])
+ for col in ("start", "end"):
+ df[col] = pd.to_datetime(df[col]).dt.strftime("%Y-%m-%d")
+ return df
+
+
+def _load_trading_values(base_dir: Path, calendar: pd.DatetimeIndex) -> pd.DataFrame:
+ stock_list_path = base_dir.joinpath(STOCK_LIST_FILE)
+ stocks = set(pd.read_csv(stock_list_path, dtype=str)["symbol"])
+ values = {}
+ for symbol, path in _source_files(base_dir).items():
+ if symbol not in stocks:
+ continue
+ df = _read_source(path).drop_duplicates("date").set_index("date").sort_index()
+ # Yahoo 的 close 只做分割調整、未除息,乘上成交量即為當日成交值
+ value = (df["close"] * df["volume"]).clip(lower=0)
+ in_range = (calendar >= df.index.min()) & (calendar <= df.index.max())
+ values[symbol] = value.reindex(calendar[in_range]).fillna(0)
+ return pd.DataFrame(values).reindex(calendar)
+
+
+def universe(
+ base_dir: str = DEFAULT_BASE_DIR,
+ qlib_dir: str = DEFAULT_QLIB_DIR,
+ top_n: int = 200,
+ window: int = 60,
+ freq: str = "Q",
+ name: Optional[str] = None,
+):
+ """產生動態股票池 instruments/.txt(預設 top200,每季重選)。"""
+ base_dir = Path(base_dir).expanduser()
+ name = name or f"top{top_n}"
+ calendar = load_calendar(base_dir)
+ values = _load_trading_values(base_dir, calendar)
+ df = compute_liquidity_universe(values, top_n=top_n, window=window, freq=freq)
+ path = _instruments_dir(qlib_dir).joinpath(f"{name}.txt")
+ _write_instruments(df, path)
+ logger.info(f"universe {name}: {df['symbol'].nunique()} symbols -> {path}")
+
+
+def run(
+ base_dir: str = DEFAULT_BASE_DIR,
+ qlib_dir: str = DEFAULT_QLIB_DIR,
+ start: str = "2005-01-01",
+ end: Optional[str] = None,
+ symbols_file: Optional[str] = None,
+ finmind_token: Optional[str] = None,
+ max_workers: int = 4,
+ delay: float = 0.5,
+ skip_exists: bool = False,
+ top_n: int = 200,
+ window: int = 60,
+ freq: str = "Q",
+):
+ """依序執行 download、normalize、dump、universe。"""
+ download(
+ base_dir,
+ start=start,
+ end=end,
+ symbols_file=symbols_file,
+ finmind_token=finmind_token,
+ max_workers=max_workers,
+ delay=delay,
+ skip_exists=skip_exists,
+ )
+ normalize(base_dir)
+ dump(base_dir, qlib_dir)
+ universe(base_dir, qlib_dir, top_n=top_n, window=window, freq=freq)
+
+
+if __name__ == "__main__":
+ import fire
+
+ fire.Fire(
+ {
+ "run": run,
+ "download": download,
+ "normalize": normalize,
+ "dump": dump,
+ "universe": universe,
+ }
+ )
diff --git a/qbot/engine/trade/engine_apis/stocks/gmtrade_example.py b/qbot/engine/trade/engine_apis/stocks/gmtrade_example.py
index cd96dc2f..0d6e6725 100644
--- a/qbot/engine/trade/engine_apis/stocks/gmtrade_example.py
+++ b/qbot/engine/trade/engine_apis/stocks/gmtrade_example.py
@@ -16,16 +16,25 @@
"""
# 本示例运行于python3.6及以上版本
+import os
+
from gmtrade.api import * # noqa: F403
# token身份认证,掘金登录后可在仿真交易官网获取
-set_token("c793349a885556506e27c2081c73091b4e77f28b")
+GMTRADE_TOKEN = os.environ.get("GMTRADE_TOKEN")
+GMTRADE_ACCOUNT = os.environ.get("GMTRADE_ACCOUNT")
+if not GMTRADE_TOKEN or not GMTRADE_ACCOUNT:
+ raise RuntimeError(
+ "GMTRADE_TOKEN and GMTRADE_ACCOUNT must be set to your 掘金 simulation token "
+ "and account ID (see docs/Install_guide.md)."
+ )
+set_token(GMTRADE_TOKEN)
# 示例中为掘金官方仿真服务地址,如接入掘金终端,则填空
set_endpoint("api.myquant.cn:9000")
# 登录账户,账户ID由登录并申请仿真账户后,可复制获取;account_alias为账号别名,选填
-a1 = account(account_id="5e4cdda3-f2fb-11ed-ae27-00163e022aa6", account_alias="")
+a1 = account(account_id=GMTRADE_ACCOUNT, account_alias="")
login(a1) # 注意,可以输入账户也可以输入账户组成的list
diff --git a/qbot/strategies/arbr_strategy.py b/qbot/strategies/arbr_strategy.py
index 44f7a508..2808811c 100644
--- a/qbot/strategies/arbr_strategy.py
+++ b/qbot/strategies/arbr_strategy.py
@@ -11,6 +11,8 @@
# ****************************************************************************
+import os
+
import matplotlib.pyplot as plt
import numpy as np # noqa F401
import pandas as pd
@@ -27,7 +29,12 @@
# 设置token
-token = "6f747880359ef14fe2fd5fc0c2c08a4e09a47e7ac161d643ae7036c0"
+token = os.environ.get("TUSHARE_TOKEN")
+if not token:
+ raise RuntimeError(
+ "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. "
+ "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)."
+ )
pro = ts.pro_api(token)
index = {
diff --git a/tests/test_tw_collector.py b/tests/test_tw_collector.py
new file mode 100644
index 00000000..4b4474a6
--- /dev/null
+++ b/tests/test_tw_collector.py
@@ -0,0 +1,297 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+
+import importlib.util
+import shutil
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import numpy as np
+import pandas as pd
+
+sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("qbot", "data")))
+import tw_collector as tw # noqa: E402
+
+HAS_QLIB = importlib.util.find_spec("qlib") is not None
+
+
+def make_raw(dates, start_price, seed, ex_div=None, dividend=0.0, suspended=()):
+ """產生和 tw_collector.fetch_yahoo 同格式的原始日線。"""
+ rng = np.random.default_rng(seed)
+ close = start_price * np.cumprod(1 + rng.normal(0, 0.01, len(dates)))
+ adjclose = close.copy()
+ if ex_div is not None:
+ pos = dates.get_loc(pd.Timestamp(ex_div))
+ close[pos:] -= dividend
+ # Yahoo 的還原價:除息日前的價格乘上 (1 - 股息 / 除息前一日收盤)
+ adjclose = close.copy()
+ adjclose[:pos] *= 1 - dividend / close[pos - 1]
+ volume = rng.integers(1_000_000, 5_000_000, len(dates)).astype(float)
+ df = pd.DataFrame(
+ {
+ "date": dates,
+ "open": close * 0.995,
+ "high": close * 1.01,
+ "low": close * 0.99,
+ "close": close,
+ "adjclose": adjclose,
+ "volume": volume,
+ "dividends": 0.0,
+ "stock_splits": 0.0,
+ }
+ )
+ for day in suspended:
+ df.loc[df["date"] == pd.Timestamp(day), "volume"] = 0
+ return df
+
+
+class StockListTest(unittest.TestCase):
+ def test_filter_stock_info(self):
+ info = pd.DataFrame(
+ {
+ "industry_category": [
+ "半導體業",
+ "ETF",
+ "電子零組件業",
+ "半導體業",
+ "其他",
+ ],
+ "stock_id": ["2330", "0050", "6488", "2330", "7777"],
+ "stock_name": ["台積電", "元大台灣50", "環球晶", "台積電", "興櫃股"],
+ "type": ["twse", "twse", "tpex", "twse", "emerging"],
+ "date": ["2026-10-01"] * 5,
+ }
+ )
+ df = tw.filter_stock_info(info)
+ self.assertEqual(df["symbol"].tolist(), ["TW2330", "TW6488"])
+ self.assertEqual(df["yahoo"].tolist(), ["2330.TW", "6488.TWO"])
+ self.assertEqual(df["market"].tolist(), ["twse", "tpex"])
+
+ def test_load_symbols_file(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = Path(tmp, "list.txt")
+ path.write_text(
+ "# 範例\n2330 台積電\n6488.two\n\n2317.TW # 鴻海\n2330\n",
+ encoding="utf-8",
+ )
+ df = tw.load_symbols_file(path)
+ self.assertEqual(df["symbol"].tolist(), ["TW2330", "TW6488", "TW2317"])
+ self.assertEqual(df["yahoo"].tolist(), ["2330.TW", "6488.TWO", "2317.TW"])
+ self.assertEqual(df["name"].tolist()[0], "台積電")
+
+ path.write_text("0050\n", encoding="utf-8")
+ with self.assertRaises(ValueError):
+ tw.load_symbols_file(path)
+
+
+class FetchYahooTest(unittest.TestCase):
+ def test_columns_and_timezone(self):
+ index = pd.DatetimeIndex(
+ ["2024-01-02", "2024-01-03"], tz="Asia/Taipei", name="Date"
+ )
+ history = pd.DataFrame(
+ {
+ "Open": [590.0, 584.0],
+ "High": [593.0, 585.0],
+ "Low": [589.0, 576.0],
+ "Close": [593.0, 578.0],
+ "Adj Close": [570.1, 555.7],
+ "Volume": [26059058, 37106763],
+ "Dividends": [0.0, 0.0],
+ "Stock Splits": [0.0, 0.0],
+ },
+ index=index,
+ )
+ ticker = mock.Mock()
+ ticker.history.return_value = history
+ fake_yf = types.SimpleNamespace(Ticker=mock.Mock(return_value=ticker))
+ with mock.patch.dict(sys.modules, {"yfinance": fake_yf}):
+ df = tw.fetch_yahoo("2330.TW", "2024-01-01", "2024-01-04")
+ fake_yf.Ticker.assert_called_once_with("2330.TW")
+ self.assertEqual(ticker.history.call_args.kwargs["auto_adjust"], False)
+ self.assertEqual(
+ df.columns.tolist(), ["date"] + list(tw.YAHOO_COLUMNS.values())
+ )
+ self.assertEqual(
+ df["date"].tolist(), list(pd.to_datetime(["2024-01-02", "2024-01-03"]))
+ )
+ self.assertIsNone(df["date"].dt.tz)
+
+
+class NormalizeTest(unittest.TestCase):
+ def setUp(self):
+ self.dates = pd.bdate_range("2024-01-01", periods=30)
+ self.raw = make_raw(
+ self.dates,
+ 100.0,
+ seed=1,
+ ex_div=self.dates[10],
+ dividend=3.0,
+ suspended=[self.dates[20]],
+ )
+
+ def test_adjustment_is_reversible(self):
+ df = tw.normalize_symbol(self.raw, "TW2330", self.dates).set_index("date")
+ raw = self.raw.set_index("date")
+ valid = raw["volume"] > 0
+ # qlib 用 close / factor 換回實際股價,用 volume * factor 換回實際股數
+ np.testing.assert_allclose(
+ (df["close"] / df["factor"])[valid], raw["close"][valid]
+ )
+ np.testing.assert_allclose(
+ (df["volume"] * df["factor"])[valid], raw["volume"][valid]
+ )
+ self.assertAlmostEqual(df["close"].dropna().iloc[0], 1.0)
+
+ def test_change_ignores_dividend_and_suspension(self):
+ df = tw.normalize_symbol(self.raw, "TW2330", self.dates).set_index("date")
+ raw = self.raw.set_index("date")
+ ex_div = self.dates[10]
+ expected = raw.loc[ex_div, "adjclose"] / raw["adjclose"].shift(1)[ex_div] - 1
+ self.assertAlmostEqual(df.loc[ex_div, "change"], expected)
+ self.assertGreater(df.loc[ex_div, "change"], -0.05)
+
+ suspended = self.dates[20]
+ self.assertTrue(df.loc[suspended, ["close", "volume", "change"]].isna().all())
+ # 復牌日的 change 對比停牌前最後一個收盤價
+ resumed = self.dates[21]
+ expected = df.loc[resumed, "close"] / df.loc[self.dates[19], "close"] - 1
+ self.assertAlmostEqual(df.loc[resumed, "change"], expected)
+
+ def test_vwap_and_calendar_alignment(self):
+ calendar = self.dates.delete(5)
+ raw = self.raw.drop(index=7)
+ df = tw.normalize_symbol(raw, "TW2330", calendar).set_index("date")
+ self.assertNotIn(self.dates[5], df.index)
+ self.assertTrue(np.isnan(df.loc[self.dates[7], "close"]))
+ np.testing.assert_allclose(
+ df["vwap"].dropna(), ((df["high"] + df["low"] + df["close"]) / 3).dropna()
+ )
+
+ def test_index_keeps_zero_volume_rows(self):
+ raw = self.raw.assign(volume=0.0, adjclose=self.raw["close"])
+ df = tw.normalize_symbol(raw, "TWII", self.dates, is_index=True)
+ self.assertEqual(df["close"].notna().sum(), len(self.dates))
+ self.assertEqual(df["factor"].nunique(), 1)
+
+
+class UniverseTest(unittest.TestCase):
+ def test_quarterly_top_n_without_lookahead(self):
+ dates = pd.bdate_range("2020-01-01", "2020-12-31")
+ values = pd.DataFrame(index=dates)
+ values["A"] = 100.0
+ values["B"] = np.where(dates <= "2020-06-30", 90.0, 0.0)
+ values["C"] = np.where(dates < "2020-08-14", 10.0, 1000.0)
+ values["D"] = 50.0
+ df = tw.compute_liquidity_universe(values, top_n=2, window=20, freq="Q")
+ spans = {
+ s: g[["start", "end"]].values.tolist() for s, g in df.groupby("symbol")
+ }
+ self.assertEqual(spans["A"], [["2020-04-01", "2020-12-31"]])
+ self.assertEqual(spans["B"], [["2020-04-01", "2020-09-30"]])
+ # C 在 8 月中就放量,但要等 9/30 重選後才入選
+ self.assertEqual(spans["C"], [["2020-10-01", "2020-12-31"]])
+ self.assertNotIn("D", spans)
+
+ def test_unlisted_stock_waits_for_full_window(self):
+ dates = pd.bdate_range("2020-01-01", "2020-06-30")
+ values = pd.DataFrame(index=dates)
+ values["A"] = 1.0
+ values["NEW"] = np.where(dates >= "2020-03-20", 1000.0, np.nan)
+ df = tw.compute_liquidity_universe(values, top_n=1, window=20, freq="Q")
+ # 3/31 時 NEW 只有 8 天資料,不夠 20 日不能入選
+ self.assertEqual(df.loc[df["symbol"] == "A", "start"].tolist(), ["2020-04-01"])
+ self.assertNotIn("NEW", df["symbol"].tolist())
+
+
+@unittest.skipUnless(HAS_QLIB, "pyqlib is not installed")
+class PipelineTest(unittest.TestCase):
+ STOCKS = {
+ "2330": "twse",
+ "2317": "twse",
+ "2454": "twse",
+ "2881": "twse",
+ "6488": "tpex",
+ "5274": "tpex",
+ }
+
+ def setUp(self):
+ self.tmp = Path(tempfile.mkdtemp())
+ self.base_dir = self.tmp.joinpath("tw")
+ self.qlib_dir = self.tmp.joinpath("qlib")
+ self.dates = pd.bdate_range("2022-01-03", "2023-12-29")
+ self.raw = {"^TWII": make_raw(self.dates, 17000.0, seed=0)}
+ self.raw["0050.TW"] = make_raw(self.dates, 140.0, seed=1)
+ for i, (code, market) in enumerate(self.STOCKS.items()):
+ ticker = code + tw.MARKET_SUFFIX[market]
+ self.raw[ticker] = make_raw(
+ self.dates,
+ 50.0 * (i + 1),
+ seed=10 + i,
+ ex_div="2023-07-03",
+ dividend=2.0,
+ )
+
+ def tearDown(self):
+ shutil.rmtree(self.tmp)
+
+ def _fake_stock_list(self, token=None):
+ info = pd.DataFrame(
+ {
+ "industry_category": ["x"] * len(self.STOCKS),
+ "stock_id": list(self.STOCKS),
+ "stock_name": list(self.STOCKS),
+ "type": list(self.STOCKS.values()),
+ }
+ )
+ return tw.filter_stock_info(info)
+
+ def _fake_fetch(self, ticker, start, end):
+ df = self.raw[ticker]
+ return df[(df["date"] >= start) & (df["date"] < end)].copy()
+
+ def test_run_builds_qlib_data(self):
+ import qlib
+ from qlib.config import C
+ from qlib.data import D
+
+ with mock.patch.object(
+ tw, "fetch_finmind_stock_list", self._fake_stock_list
+ ), mock.patch.object(tw, "fetch_yahoo", self._fake_fetch):
+ tw.run(
+ base_dir=str(self.base_dir),
+ qlib_dir=str(self.qlib_dir),
+ start="2022-01-01",
+ end="2023-12-31",
+ delay=0,
+ top_n=3,
+ window=20,
+ )
+
+ inst_dir = self.qlib_dir.joinpath("instruments")
+ all_symbols = tw._read_instruments(inst_dir.joinpath("all.txt"))["symbol"]
+ self.assertEqual(sorted(all_symbols), sorted("TW" + c for c in self.STOCKS))
+ tpex = tw._read_instruments(inst_dir.joinpath("tpex.txt"))["symbol"]
+ self.assertEqual(sorted(tpex), ["TW5274", "TW6488"])
+
+ qlib.init(provider_uri=str(self.qlib_dir), region="tw")
+ self.assertEqual(C.trade_unit, 1000)
+ members = D.list_instruments(D.instruments("top3"), as_list=True)
+ self.assertTrue(0 < len(members) <= len(self.STOCKS))
+
+ fields = ["$close", "$factor", "$change", "$vwap"]
+ feat = D.features(["TW2330", "TW0050"], fields, "2022-01-01", "2023-12-31")
+ tsmc = feat.loc["TW2330"]
+ raw = self.raw["2330.TW"].set_index("date")
+ np.testing.assert_allclose(
+ (tsmc["$close"] / tsmc["$factor"]).values, raw["close"].values, rtol=1e-5
+ )
+ self.assertFalse(feat.loc["TW0050", "$close"].isna().any())
+
+
+if __name__ == "__main__":
+ unittest.main()
|