From 12b04cdb2f090edfbf11911f7788dc5d0a4efa2d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 14:41:43 +0000 Subject: [PATCH 1/4] [doc] Fix stale strategy and benchmark paths in README The strategy pool tables pointed at qbot/strategy/..., which does not exist. Point each entry at its real file under qbot/strategies/ or pytrader/strategies/ (stoch_rsi_strategy.py -> StochRSI.py, rsi_cci_strategy.py -> CCI.py, which combines RSI6 and CCI14). Benchmark models that live in pytrader/strategies/benchmarks/ link there; the rest (and LSTM, whose local folder only has an empty README) link to the upstream microsoft/qlib benchmark and are tagged "qlib". ImVoxelNet is in neither and is marked as not included. Also drop the nonexistent backend/ prefix from the Quick Start PYTHONPATH, the Model Zoo run command, the Binder link (now in the labpath format), the notebook link and the quantstats link, and point the repo stats image at .github/stats/metrics.svg where the stats script writes it. Apply the same fixes to docs/README.md, with links made relative to docs/. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01A74mvLr44PgMnpwcctRzHx --- README.md | 122 +++++++++++++++---------------- docs/README.md | 192 +++++++++++++++++++++++++------------------------ 2 files changed, 159 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index 2c50aa6a..08ccd6e0 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ cd Qbot pip install -r dev/requirements.txt -export PYTHONPATH=${PYTHONPATH}:$(pwd):$(pwd)/backend/multi-fact/mfm_learner +export PYTHONPATH=${PYTHONPATH}:$(pwd) python main.py #if run on Mac, please use 'pythonw main.py' ``` @@ -246,7 +246,7 @@ python main.py #if run on Mac, please use 'pythonw main.py' 以下即为,数据指标单因子或组合因子和通过深度学习、机器学习、强化学习挖掘到的交易因子,然后通过组合优化算法实现趋势交易、风险策略、alpha策略、动量轮动等等交易策略。 -策略库源代码路径:[qbot/strategy](qbot/strategy) +策略库源代码路径:[qbot/strategies](qbot/strategies)、[pytrader/strategies](pytrader/strategies)
@@ -276,20 +276,20 @@ python main.py #if run on Mac, please use 'pythonw main.py' @@ -299,26 +299,26 @@ python main.py #if run on Mac, please use 'pythonw main.py'
  • 因子组合
  • @@ -398,51 +398,51 @@ python main.py #if run on Mac, please use 'pythonw main.py'
  • GBDT
  • BOOST
  • LR
  • CNN
  • RNN
  • -
  • TFT (IJoF'2019)
  • -
  • GATs (NIPS'2017)
  • -
  • SFM (KDD'2017)
  • +
  • TFT (IJoF'2019)
  • +
  • GATs (NIPS'2017) qlib
  • +
  • SFM (KDD'2017) qlib
  • -
  • Transformer (NeurIPS'2017)
  • -
  • TCTS (ICML'2021)
  • -
  • TRA (KDD'2021)
  • -
  • TCN (KDD'2018)
  • -
  • IGMTF (KDD'2021)
  • -
  • HIST (KDD'2018)
  • -
  • Localformer ('2021)
  • +
  • Transformer (NeurIPS'2017)
  • +
  • TCTS (ICML'2021) qlib
  • +
  • TRA (KDD'2021) qlib
  • +
  • TCN (KDD'2018) qlib
  • +
  • IGMTF (KDD'2021) qlib
  • +
  • HIST (KDD'2018) qlib
  • +
  • Localformer ('2021) qlib
  • ChatGPT
  • @@ -454,9 +454,11 @@ python main.py #if run on Mac, please use 'pythonw main.py' +> 注:标有 qlib 的模型未包含在本仓库中,链接指向 [microsoft/qlib](https://github.com/microsoft/qlib/tree/main/examples/benchmarks) 上游 benchmark;标有 未包含 的模型暂无实现。本仓库自带的模型位于 [pytrader/strategies/benchmarks](pytrader/strategies/benchmarks)。 + ### Benchmark and Model zoo -Results and models are available in the [model zoo](docs/03-智能策略/model_zoo.md). AI strategies is shown at [here](./pytrader/strategies/), local run ``python backend/pytrader/strategies/workflow_by_code.py``, also provide [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/UFund-Me/Qbot/blob/main/backend/pytrader/strategies/workflow_by_code.ipynb/HEAD) +Results and models are available in the [model zoo](docs/03-智能策略/model_zoo.md). AI strategies is shown at [here](./pytrader/strategies/), local run ``python pytrader/strategies/workflow_by_code.py``, also provide [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/UFund-Me/Qbot/HEAD?labpath=pytrader%2Fstrategies%2Fworkflow_by_code.ipynb)
    👉 点击展开查看具体AI模型benchmark结果 @@ -658,13 +660,13 @@ Qbot买卖强弱指标(收费) 本项目编写了详细的策略原理说明和平台搭建到使用的详细文档,尤其适合量化小白。欢迎加群交流! -[在线文档](https://ufund-me.github.io/Qbot/#/) | [❓ 常见问题](https://ufund-me.github.io/Qbot/#/04-%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98/FQA) | [Jupyter Notebook](./backend/pytrader/strategies/notebook) +[在线文档](https://ufund-me.github.io/Qbot/#/) | [❓ 常见问题](https://ufund-me.github.io/Qbot/#/04-%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98/FQA) | [Jupyter Notebook](./pytrader/strategies/notebook) ## Quantstats Report ![Quantstats Report](https://user-images.githubusercontent.com/29084184/207054856-44d1815b-f92f-40a7-b82e-e4a6b3960f2f.png) -Click [HERE](backend/quantstats#visualize-stock-performance) to more detail. +Click [HERE](qbot/plugins/quantstats#visualize-stock-performance) to more detail. ### Some strategy backtest results: @@ -824,7 +826,7 @@ We appreciate all contributions to improve Qbot. Please refer to [CONTRIBUTING.m ## Repository Statistics -![Repo Stats](stats/metrics.svg) +![Repo Stats](.github/stats/metrics.svg) *Daily clones, total forks, and total stars over time. Updated daily at 00:00 UTC.* ## Sponsors & support diff --git a/docs/README.md b/docs/README.md index 97211f62..32496ef7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -88,7 +88,7 @@ git clone https://github.com/UFund-Me/Qbot --depth 1 cd Qbot pip install -r requirements.txt -export PYTHONPATH=${PYTHONPATH}:$(pwd):$(pwd)/backend/multi-fact/mfm_learner +export PYTHONPATH=${PYTHONPATH}:$(pwd) python main.py #if run on Mac, please use 'pythonw main.py' ``` @@ -204,13 +204,13 @@ python main.py ## Strategy pool -通过Qbot 可以积木式完成策略编写、多因子挖掘,实现数据开发、因子开发、组合优化、交易执行的[量化交易全流程](docs/01-新手指引/量化策略的分类和原理.md#1量化选股策略)。 +通过Qbot 可以积木式完成策略编写、多因子挖掘,实现数据开发、因子开发、组合优化、交易执行的[量化交易全流程](01-新手指引/量化策略的分类和原理.md#1量化选股策略)。 如果说策略是量化的核心 ,那么因子就是策略的核心。通过Qbot量化投研平台研究员可实现自动化因子挖掘,提取出具备预测能力的单因子,利用历史数据进行回测,如果回测结果显示该因子的预测能力达标,就提交到因子库。然后,对因子库里的因子进行有机组合,以形成预测模型,预测模型是整个量化策略的目标。 以下即为,数据指标单因子或组合因子和通过深度学习、机器学习、强化学习挖掘到的交易因子,然后通过组合优化算法实现趋势交易、风险策略、alpha策略、动量轮动等等交易策略。 -策略库源代码路径:[qbot/strategy](qbot/strategy) +策略库源代码路径:[qbot/strategies](../qbot/strategies)、[pytrader/strategies](../pytrader/strategies)
    @@ -238,76 +238,76 @@ python main.py
  • 传统指标(对应下方Qbot支持的指标 这里)
  • 因子组合
  • 经典策略
  • @@ -316,16 +316,16 @@ python main.py @@ -362,51 +362,51 @@ python main.py
  • GBDT
  • BOOST
  • LR
  • CNN
  • RNN
  • -
  • TFT (IJoF'2019)
  • -
  • GATs (NIPS'2017)
  • -
  • SFM (KDD'2017)
  • +
  • TFT (IJoF'2019)
  • +
  • GATs (NIPS'2017) qlib
  • +
  • SFM (KDD'2017) qlib
  • -
  • Transformer (NeurIPS'2017)
  • -
  • TCTS (ICML'2021)
  • -
  • TRA (KDD'2021)
  • -
  • TCN (KDD'2018)
  • -
  • IGMTF (KDD'2021)
  • -
  • HIST (KDD'2018)
  • -
  • Localformer ('2021)
  • +
  • Transformer (NeurIPS'2017)
  • +
  • TCTS (ICML'2021) qlib
  • +
  • TRA (KDD'2021) qlib
  • +
  • TCN (KDD'2018) qlib
  • +
  • IGMTF (KDD'2021) qlib
  • +
  • HIST (KDD'2018) qlib
  • +
  • Localformer ('2021) qlib
  • ChatGPT
  • @@ -418,9 +418,11 @@ python main.py +> 注:标有 qlib 的模型未包含在本仓库中,链接指向 [microsoft/qlib](https://github.com/microsoft/qlib/tree/main/examples/benchmarks) 上游 benchmark;标有 未包含 的模型暂无实现。本仓库自带的模型位于 [pytrader/strategies/benchmarks](../pytrader/strategies/benchmarks)。 + ### Benchmark and Model zoo -Results and models are available in the [model zoo](docs/03-智能策略/model_zoo.md). AI strategies is shown at [here](./pytrader/strategies/), local run ``python backend/pytrader/strategies/workflow_by_code.py``, also provide [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/UFund-Me/Qbot/blob/main/backend/pytrader/strategies/workflow_by_code.ipynb/HEAD) +Results and models are available in the [model zoo](03-智能策略/model_zoo.md). AI strategies is shown at [here](../pytrader/strategies/), local run ``python pytrader/strategies/workflow_by_code.py``, also provide [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/UFund-Me/Qbot/HEAD?labpath=pytrader%2Fstrategies%2Fworkflow_by_code.ipynb)
    👉 点击展开查看具体AI模型benchmark结果 @@ -450,7 +452,7 @@ Results and models are available in the [model zoo](docs/03-智能策略/model_z | HIST | ✓ | ✗ | pytorch | ✗ | ✗ | Wentao Xu, et al.2021 | ✗ | -**Note:** All the about **300+ models, methods of 40+ papers** in quant.ai supported by [Model Zoo](./docs/03-智能策略/model_zoo.md) can be trained or used in this codebase. +**Note:** All the about **300+ models, methods of 40+ papers** in quant.ai supported by [Model Zoo](./03-智能策略/model_zoo.md) can be trained or used in this codebase.
    @@ -582,7 +584,7 @@ Qbot买卖强弱指标(收费) 本项目编写了详细的策略原理说明和平台搭建到使用的详细文档,尤其适合量化小白。欢迎加群交流! -[在线文档](https://ufund-me.github.io/Qbot/#/) | [❓ 常见问题](https://ufund-me.github.io/Qbot/#/04-%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98/FQA) | [Jupyter Notebook](./pytrader/strategies/notebook) +[在线文档](https://ufund-me.github.io/Qbot/#/) | [❓ 常见问题](https://ufund-me.github.io/Qbot/#/04-%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98/FQA) | [Jupyter Notebook](../pytrader/strategies/notebook) ## Quantstats Report @@ -651,7 +653,7 @@ dagit -h 0.0.0.0 -p 3000 ## Contributing -We appreciate all contributions to improve Qbot. Please refer to [CONTRIBUTING](.github/CONTRIBUTING) for the contributing guideline. +We appreciate all contributions to improve Qbot. Please refer to [CONTRIBUTING](../.github/CONTRIBUTING.md) for the contributing guideline. ## 🍮 Community - Github discussions 💬 or issues 💭 @@ -743,7 +745,7 @@ If you like the project, you can become a sponsor at [Open Collective](https://o ## LICENSE - + 署名-非商业性使用-相同方式共享 4.0 国际 From 34646f0cd78bdd01014dcd7f15c7d7474170e4b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 14:43:31 +0000 Subject: [PATCH 2/4] Read API tokens from environment variables instead of hardcoding them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several scripts embedded Tushare, 掘金 (GM trade), Xueqiu and WSJ tokens as string literals. They now read TUSHARE_TOKEN, GMTRADE_TOKEN/GMTRADE_ACCOUNT, XUEQIU_TOKEN and WSJ_TOKEN from the environment and raise a clear RuntimeError when the variable is missing. The variables are documented in docs/Install_guide.md. The removed tokens remain in git history and should be rotated at the provider. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q3cFCJHBzDPiwhm4owkVpa --- docs/Install_guide.md | 20 +++++++++++++++++++ docs/notebook/Strategy_runner.py | 7 ++++++- docs/notebook/stock_api.py | 9 ++++++++- pyfunds/backtest/xalpha/universal.py | 16 +++++++++++++-- pytrader/strategies/k-line.py | 9 ++++++++- pytrader/strategies/multi_factor_strategy.py | 17 ++++++++++++++-- .../engine_apis/stocks/gmtrade_example.py | 13 ++++++++++-- qbot/strategies/arbr_strategy.py | 9 ++++++++- 8 files changed, 90 insertions(+), 10 deletions(-) diff --git a/docs/Install_guide.md b/docs/Install_guide.md index 790c834a..7cbf4ee6 100644 --- a/docs/Install_guide.md +++ b/docs/Install_guide.md @@ -139,6 +139,26 @@ docker exec -it bash +2.3 数据源 / 交易 API token(环境变量) + +部分策略和示例脚本通过环境变量读取 API token,代码中不再硬编码。运行前请先设置对应变量,未设置时脚本会直接报错并提示缺少哪个变量。 + +| 环境变量 | 用途 | 使用位置 | +| --- | --- | --- | +| `TUSHARE_TOKEN` | Tushare Pro token,在 [tushare.pro 个人主页](https://tushare.pro/user/token) 获取 | `pytrader/strategies/multi_factor_strategy.py`、`pytrader/strategies/k-line.py`、`qbot/strategies/arbr_strategy.py`、`docs/notebook/stock_api.py`、`docs/notebook/Strategy_runner.py` | +| `GMTRADE_TOKEN` | 掘金仿真交易 token,登录[掘金仿真交易](https://sim.myquant.cn/sim/help/#仿真账户登录)后获取 | `qbot/engine/trade/engine_apis/stocks/gmtrade_example.py` | +| `GMTRADE_ACCOUNT` | 掘金仿真账户 ID | `qbot/engine/trade/engine_apis/stocks/gmtrade_example.py` | +| `XUEQIU_TOKEN` | 雪球 `xq_a_token` cookie。仅在调用 `get_xueqiu_rt()` 且未传入 `token` 时需要;`get_rt()` 会自动通过 `get_token()` 匿名获取 | `pyfunds/backtest/xalpha/universal.py` | +| `WSJ_TOKEN` | WSJ / MarketWatch 行情接口的 EntitlementToken,`get_bar(..., _from="wsj")` 时需要 | `pyfunds/backtest/xalpha/universal.py` | + +```bash +export TUSHARE_TOKEN= +export GMTRADE_TOKEN= +export GMTRADE_ACCOUNT= +``` + +Windows(PowerShell)使用 `$env:TUSHARE_TOKEN=""`。请勿将 token 提交到代码仓库。 + ## Run:启动Qbot ```bash diff --git a/docs/notebook/Strategy_runner.py b/docs/notebook/Strategy_runner.py index 77f6b227..3518cdda 100644 --- a/docs/notebook/Strategy_runner.py +++ b/docs/notebook/Strategy_runner.py @@ -10,7 +10,12 @@ data_path = './data/' if not os.path.exists(data_path): os.makedirs(data_path) -mytoken = '565ee6d69fb85cb0bc7fdfc8dea4b8ce1f18366f30e8d23a253bb9cf' +mytoken = os.environ.get("TUSHARE_TOKEN") +if not mytoken: + raise RuntimeError( + "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. " + "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)." + ) class Strategy_runner: diff --git a/docs/notebook/stock_api.py b/docs/notebook/stock_api.py index 3c360a4e..9de927a2 100644 --- a/docs/notebook/stock_api.py +++ b/docs/notebook/stock_api.py @@ -1,6 +1,13 @@ +import os + import tushare as ts -mytoken = '565ee6d69fb85cb0bc7fdfc8dea4b8ce1f18366f30e8d23a253bb9cf' +mytoken = os.environ.get("TUSHARE_TOKEN") +if not mytoken: + raise RuntimeError( + "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. " + "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)." + ) ts.set_token(mytoken) pro = ts.pro_api() diff --git a/pyfunds/backtest/xalpha/universal.py b/pyfunds/backtest/xalpha/universal.py index 9d873103..509a4162 100644 --- a/pyfunds/backtest/xalpha/universal.py +++ b/pyfunds/backtest/xalpha/universal.py @@ -1248,7 +1248,14 @@ def _get_daily( return df -def get_xueqiu_rt(code, token="a664afb60c7036c7947578ac1a5860c4cfb6b3b5"): +def get_xueqiu_rt(code, token=None): + if not token: + token = os.environ.get("XUEQIU_TOKEN") + if not token: + raise RuntimeError( + "No Xueqiu token given. Pass token=get_token() or set the " + "XUEQIU_TOKEN environment variable." + ) if code.startswith("HK") and code[2:].isdigit(): code = code[2:] url = "https://stock.xueqiu.com/v5/stock/quote.json?symbol={code}&extend=detail" @@ -2667,7 +2674,12 @@ def get_bar_fromwsj(code, token=None, interval=3600): # TODO: there is other freq tags, but I have no time to explore them, contributions are welcome:) freq = trans.get(str(interval), interval) if not token: - token = "cecc4267a0194af89ca343805a3e57af" + token = os.environ.get("WSJ_TOKEN") + if not token: + raise RuntimeError( + "No WSJ token given. Pass token=... or set the WSJ_TOKEN " + "environment variable." + ) # the thing I am concerned here is whether token is refreshed params = { diff --git a/pytrader/strategies/k-line.py b/pytrader/strategies/k-line.py index b8b52c6d..0ef71d84 100644 --- a/pytrader/strategies/k-line.py +++ b/pytrader/strategies/k-line.py @@ -1,5 +1,7 @@ # blog https://blog.csdn.net/qq_44176343/article/details/109903512 +import os + import tushare as ts import matplotlib.pyplot as plt import seaborn as sns @@ -8,7 +10,12 @@ # pip install --upgrade mplfinance import mpl_finance as mpf -token = "6f747880359ef14fe2fd5fc0c2c08a4e09a47e7ac161d643ae7036c0" +token = os.environ.get("TUSHARE_TOKEN") +if not token: + raise RuntimeError( + "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. " + "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)." + ) pro = ts.pro_api(token) df = ts.get_k_data('000002','2019-06-01','2019-09-30') diff --git a/pytrader/strategies/multi_factor_strategy.py b/pytrader/strategies/multi_factor_strategy.py index 6afd21c1..dda687dd 100644 --- a/pytrader/strategies/multi_factor_strategy.py +++ b/pytrader/strategies/multi_factor_strategy.py @@ -13,12 +13,25 @@ Licensed under the MIT License. ''' +import os + import backtrader as bt import tushare as ts import numpy as np import pandas as pd from sklearn.linear_model import LinearRegression + +def get_tushare_token(): + token = os.environ.get("TUSHARE_TOKEN") + if not token: + raise RuntimeError( + "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. " + "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)." + ) + return token + + class PEFetcher: def __init__(self, ts): self.ts = ts @@ -59,7 +72,7 @@ def get_factor_data(self, symbol, date): def get_data(symbol, data_fetchers, start_date, end_date): - ts.set_token('e96f18882532434b7692388cb028eb267d3f0d56845dc92eef06ea4a') + ts.set_token(get_tushare_token()) data = {} dates = ts.trade_cal() @@ -150,7 +163,7 @@ def next(self): if __name__ == '__main__': # 加载数据 - ts.set_token('e96f18882532434b7692388cb028eb267d3f0d56845dc92eef06ea4a') + ts.set_token(get_tushare_token()) pro = ts.pro_api() data_fetchers = [ {'code': 'pe_ttm', 'data_fetcher': PEFetcher(pro)}, diff --git a/qbot/engine/trade/engine_apis/stocks/gmtrade_example.py b/qbot/engine/trade/engine_apis/stocks/gmtrade_example.py index cd96dc2f..0d6e6725 100644 --- a/qbot/engine/trade/engine_apis/stocks/gmtrade_example.py +++ b/qbot/engine/trade/engine_apis/stocks/gmtrade_example.py @@ -16,16 +16,25 @@ """ # 本示例运行于python3.6及以上版本 +import os + from gmtrade.api import * # noqa: F403 # token身份认证,掘金登录后可在仿真交易官网获取 -set_token("c793349a885556506e27c2081c73091b4e77f28b") +GMTRADE_TOKEN = os.environ.get("GMTRADE_TOKEN") +GMTRADE_ACCOUNT = os.environ.get("GMTRADE_ACCOUNT") +if not GMTRADE_TOKEN or not GMTRADE_ACCOUNT: + raise RuntimeError( + "GMTRADE_TOKEN and GMTRADE_ACCOUNT must be set to your 掘金 simulation token " + "and account ID (see docs/Install_guide.md)." + ) +set_token(GMTRADE_TOKEN) # 示例中为掘金官方仿真服务地址,如接入掘金终端,则填空 set_endpoint("api.myquant.cn:9000") # 登录账户,账户ID由登录并申请仿真账户后,可复制获取;account_alias为账号别名,选填 -a1 = account(account_id="5e4cdda3-f2fb-11ed-ae27-00163e022aa6", account_alias="") +a1 = account(account_id=GMTRADE_ACCOUNT, account_alias="") login(a1) # 注意,可以输入账户也可以输入账户组成的list diff --git a/qbot/strategies/arbr_strategy.py b/qbot/strategies/arbr_strategy.py index 44f7a508..2808811c 100644 --- a/qbot/strategies/arbr_strategy.py +++ b/qbot/strategies/arbr_strategy.py @@ -11,6 +11,8 @@ # **************************************************************************** +import os + import matplotlib.pyplot as plt import numpy as np # noqa F401 import pandas as pd @@ -27,7 +29,12 @@ # 设置token -token = "6f747880359ef14fe2fd5fc0c2c08a4e09a47e7ac161d643ae7036c0" +token = os.environ.get("TUSHARE_TOKEN") +if not token: + raise RuntimeError( + "TUSHARE_TOKEN is not set. Export your Tushare token first, e.g. " + "`export TUSHARE_TOKEN=` (see docs/Install_guide.md)." + ) pro = ts.pro_api(token) index = { From 588da731590797099b23044fd554751ead45a3a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 14:45:00 +0000 Subject: [PATCH 3/4] [ci] Fix repo-stats workflow script path The workflow ran scripts/update_repo_stats.py, but the script lives at .github/scripts/update_repo_stats.py, so every run failed before generating .github/stats/metrics.svg and the README chart never resolved. Also install requests and matplotlib directly instead of probing scripts/requirements.txt, which doesn't exist and would pull in an unrelated file if one were added to scripts/ later. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01A74mvLr44PgMnpwcctRzHx --- .github/workflows/repo-stats.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/repo-stats.yml b/.github/workflows/repo-stats.yml index 764552b4..8488762f 100644 --- a/.github/workflows/repo-stats.yml +++ b/.github/workflows/repo-stats.yml @@ -29,19 +29,14 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f scripts/requirements.txt ]; then - pip install -r scripts/requirements.txt - else - # Install minimal dependencies if requirements file doesn't exist - pip install requests matplotlib - fi + pip install requests matplotlib - name: Update repository statistics env: GITHUB_TOKEN: ${{ secrets.SECRET_TOKEN }} REPO: ${{ github.repository }} run: | - python scripts/update_repo_stats.py \ + python .github/scripts/update_repo_stats.py \ --token "$GITHUB_TOKEN" \ --repo "$REPO" \ --stats-dir ".github/stats" From 90d390030e6e757ac184c0cdbab66cf7ed0b30c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 15:17:37 +0000 Subject: [PATCH 4/4] Add Taiwan stock data collector and qlib workflow config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - qbot/data/tw_collector.py: download TWSE/TPEx daily bars from Yahoo (stock list from FinMind or a custom file), normalize to qlib's adjusted-price/factor convention aligned to the TAIEX calendar, dump to qlib format, and build a quarterly top-N-by-turnover universe without look-ahead - LightGBM + Alpha158 config for region tw: 1000-share lots, 0050 as a dividend-inclusive benchmark, 0.1425% fee + 0.3% sell tax, 20 TWD minimum fee, 9.5% limit threshold - dump_bin.py: replace astype(np.datetime64), which pandas >= 2 rejects - Guide in docs/03-智能策略/台股qlib工作流.md and tests with synthetic data Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vf3KKYsmLyF6drsFMgMBoY --- README.md | 2 + ...ib\345\267\245\344\275\234\346\265\201.md" | 93 ++++ .../workflow_config_lightgbm_Alpha158_tw.yaml | 79 +++ qbot/data/dump_bin.py | 8 +- qbot/data/tw_collector.py | 521 ++++++++++++++++++ tests/test_tw_collector.py | 297 ++++++++++ 6 files changed, 995 insertions(+), 5 deletions(-) create mode 100644 "docs/03-\346\231\272\350\203\275\347\255\226\347\225\245/\345\217\260\350\202\241qlib\345\267\245\344\275\234\346\265\201.md" create mode 100644 pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml create mode 100644 qbot/data/tw_collector.py create mode 100644 tests/test_tw_collector.py diff --git a/README.md b/README.md index 2c50aa6a..c1dddeee 100644 --- a/README.md +++ b/README.md @@ -456,6 +456,8 @@ python main.py #if run on Mac, please use 'pythonw main.py' ### Benchmark and Model zoo +台股版本(資料下載 + qlib 回測設定)見 [台股qlib工作流](docs/03-智能策略/台股qlib工作流.md)。 + Results and models are available in the [model zoo](docs/03-智能策略/model_zoo.md). AI strategies is shown at [here](./pytrader/strategies/), local run ``python backend/pytrader/strategies/workflow_by_code.py``, also provide [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/UFund-Me/Qbot/blob/main/backend/pytrader/strategies/workflow_by_code.ipynb/HEAD)
    👉 点击展开查看具体AI模型benchmark结果 diff --git "a/docs/03-\346\231\272\350\203\275\347\255\226\347\225\245/\345\217\260\350\202\241qlib\345\267\245\344\275\234\346\265\201.md" "b/docs/03-\346\231\272\350\203\275\347\255\226\347\225\245/\345\217\260\350\202\241qlib\345\267\245\344\275\234\346\265\201.md" new file mode 100644 index 00000000..a2c5e4a8 --- /dev/null +++ "b/docs/03-\346\231\272\350\203\275\347\255\226\347\225\245/\345\217\260\350\202\241qlib\345\267\245\344\275\234\346\265\201.md" @@ -0,0 +1,93 @@ +## 台股 qlib 工作流 + +用 Yahoo Finance 的台股日線建立 qlib 資料,再用 Alpha158 因子 + LightGBM 選股並回測。 + +- 資料腳本:[qbot/data/tw_collector.py](../../qbot/data/tw_collector.py) +- 回測設定:[workflow_config_lightgbm_Alpha158_tw.yaml](../../pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml) + +### 安裝 + +```bash +pip install "pyqlib>=0.8.6" lightgbm yfinance fire loguru requests +``` + +`region: tw` 需要 pyqlib 0.8.6 以上。 + +### 1. 建立資料 + +```bash +python qbot/data/tw_collector.py run +``` + +會依序執行下面四步,也可以分開跑: + +| 步驟 | 做什麼 | 輸出 | +|---|---|---| +| `download` | 從 FinMind 取得上市、上櫃普通股清單,從 Yahoo 下載 2005 年起的日線、加權指數與 0050 | `~/.qlib/stock_data/tw/source/` | +| `normalize` | 以加權指數的交易日為日曆對齊、還原權值、計算 `change` 與 `vwap` | `~/.qlib/stock_data/tw/normalize/` | +| `dump` | 轉成 qlib 二進位格式,並產生 `all`、`twse`、`tpex` 股票池 | `~/.qlib/qlib_data/tw_data/` | +| `universe` | 依過去 60 日平均成交值每季重選前 200 檔,產生 `top200` 股票池 | `instruments/top200.txt` | + +常用參數: + +```bash +# FinMind token(可省略,有 token 的請求額度較高),也可設環境變數 FINMIND_TOKEN +python qbot/data/tw_collector.py run --finmind_token + +# Yahoo 有流量限制,下載中斷後只補抓缺的檔案 +python qbot/data/tw_collector.py download --skip_exists True + +# 自訂股票清單:每行一檔,2330(預設上市)、2330.TW、6488.TWO,# 之後為註解 +python qbot/data/tw_collector.py run --symbols_file my_list.txt --top_n 50 + +# 改股票池:前 100 檔、每月重選 +python qbot/data/tw_collector.py universe --top_n 100 --freq M +``` + +更新資料時重新執行 `run` 即可(會重建整個資料夾)。下載失敗的代碼記錄在 `~/.qlib/stock_data/tw/failed.txt`。 + +qlib 中的代碼一律寫成 `TW` + 代號,例如 `TW2330`、`TW6488`;加權指數為 `TWII`,元大台灣50 為 `TW0050`。 + +### 2. 訓練與回測 + +```bash +qrun pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml +``` + +結果存在執行目錄下的 `mlruns/`,會印出 IC / Rank IC 與扣除成本前後的超額報酬。畫圖可參考 [pytrader/analyser/workflow.py](../../pytrader/analyser/workflow.py)。 + +### 設定說明 + +| 設定 | 值 | 原因 | +|---|---|---| +| `region` | `tw` | 交易單位 1000 股(一張)。買進一律整張;qlib 以還原價把股息視為再投入,持有部位經過除息後換算股數會略偏離整張,全部賣出時也就不是整張 | +| `market` | `top200` | 動態股票池,只用重選日(含)以前的成交值,從下一個交易日生效,沒有前視偏差;避免用「現在的台灣50成分股」回測過去 | +| `benchmark` | `TW0050` | 0050 的還原價含息;`TWII` 是不含息的價格指數,拿來當基準會讓超額報酬每年虛增約一個殖利率 | +| `limit_threshold` | `0.095` | 漲停價依升降單位向下取整,實際漲停幅度約 9.5%~10%,用 0.1 會漏掉大部分漲停 | +| `open_cost` / `close_cost` | `0.001425` / `0.004425` | 手續費 0.1425%,賣出另加證交稅 0.3%;有券商折扣可自行調低 | +| `min_cost` | `20` | 最低手續費 20 元 | +| `account` | `1000000000` | 高價股一張動輒數百萬,帳戶太小會買不到;小資金可在 `exchange_kwargs` 加 `trade_unit: 1` 模擬零股(2020-10-26 起開放盤中零股) | +| 回測期間 | 2021~2025 | 2015-06-01 前漲跌幅是 7%,`limit_threshold` 不適用 | + +### 改用其他模型 + +repo 內其他 benchmark(XGBoost、MLP、Transformer…)的 yaml 都是 A 股設定,照下面改即可套用台股: + +1. `qlib_init`:`provider_uri: "~/.qlib/qlib_data/tw_data"`、`region: tw` +2. `market: top200`、`benchmark: TW0050` +3. `exchange_kwargs`:`limit_threshold`、`open_cost`、`close_cost`、`min_cost` 改成上表的值 +4. 資料、訓練、回測的日期改成台股資料的範圍 +5. 策略參數用 `signal: `(repo 內舊 yaml 寫的 `model: ` / `dataset: ` 是舊版 qlib 的寫法) + +### 限制 + +- **倖存者偏差**:股票清單只有目前上市、上櫃的公司,Yahoo 也查不到已下市股票,回測績效會偏高 +- **VWAP 是近似值**:Yahoo 沒有成交金額,`vwap` 用 (最高 + 最低 + 收盤) / 3 +- **Yahoo 資料品質**:偶有錯價或缺漏,重要結論請用其他資料源交叉驗證 +- **漲跌停判斷是近似**:以 9.5% 判斷,少數漲幅 9.5%~10% 但沒漲停的日子會被當成漲停 +- **沒有基本面資料**:本工作流只有量價因子;財報、月營收、三大法人等需另外串接(例如 FinMind) + +### 常見問題 + +- **回測出現 `IndexError: index ... is out of bounds`**:資料最後一天必須晚於回測 `end_time`,qlib 回測需要下一個交易日 +- **`MlflowException: The filesystem tracking backend ... is in maintenance mode`**:新版 MLflow 預設不讓 qlib 用 `mlruns/` 資料夾,執行前設定 `export MLFLOW_ALLOW_FILE_STORE=true` diff --git a/pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml b/pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml new file mode 100644 index 00000000..6a6af111 --- /dev/null +++ b/pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml @@ -0,0 +1,79 @@ +# 台股版 LightGBM + Alpha158 +# 先用 qbot/data/tw_collector.py 建好 ~/.qlib/qlib_data/tw_data,再執行: +# qrun pytrader/strategies/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158_tw.yaml +# 說明見 docs/03-智能策略/台股qlib工作流.md +qlib_init: + provider_uri: "~/.qlib/qlib_data/tw_data" + region: tw # 交易單位 1000 股(一張),需 pyqlib >= 0.8.6 +market: &market top200 # 依過去 60 日成交值每季重選的前 200 檔 +benchmark: &benchmark TW0050 # 0050 還原權值(含息);TWII 是不含息的價格指數 +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2025-12-31 + fit_start_time: 2008-01-01 + fit_end_time: 2017-12-31 + instruments: *market +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy + kwargs: + signal: + topk: 50 + n_drop: 5 + backtest: + # 2015-06-01 起漲跌幅才從 7% 放寬到 10%,回測期間請放在之後 + start_time: 2021-01-01 + # 資料最後一天必須晚於 end_time(qlib 回測需要下一個交易日) + end_time: 2025-12-31 + # 一張要 1000 股,帳戶太小時高價股會買不到;小資金可在 exchange_kwargs 加 trade_unit: 1 模擬零股 + account: 1000000000 + benchmark: *benchmark + exchange_kwargs: + # 漲停價會依升降單位向下取整,實際漲停幅度約 9.5%~10%,因此不用 0.1 + limit_threshold: 0.095 + deal_price: close + open_cost: 0.001425 # 手續費 0.1425%(券商折扣可自行調低) + close_cost: 0.004425 # 手續費 0.1425% + 證交稅 0.3% + min_cost: 20 # 最低手續費 20 元 +task: + model: + class: LGBModel + module_path: qlib.contrib.model.gbdt + kwargs: + loss: mse + colsample_bytree: 0.8879 + learning_rate: 0.2 + subsample: 0.8789 + lambda_l1: 205.6999 + lambda_l2: 580.9768 + max_depth: 8 + num_leaves: 210 + num_threads: 20 + dataset: + class: DatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha158 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2017-12-31] + valid: [2018-01-01, 2020-12-31] + test: [2021-01-01, 2025-12-31] + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: + model: + dataset: + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config diff --git a/qbot/data/dump_bin.py b/qbot/data/dump_bin.py index 3b377f68..70ef7213 100644 --- a/qbot/data/dump_bin.py +++ b/qbot/data/dump_bin.py @@ -149,9 +149,7 @@ def _get_date( def _get_source_data(self, file_path: Path) -> pd.DataFrame: df = pd.read_csv(str(file_path.resolve()), low_memory=False) - df[self.date_field_name] = ( - df[self.date_field_name].astype(str).astype(np.datetime64) - ) + df[self.date_field_name] = pd.to_datetime(df[self.date_field_name].astype(str)) # df.drop_duplicates([self.date_field_name], inplace=True) return df @@ -225,8 +223,8 @@ def data_merge_calendar( ) -> pd.DataFrame: # calendars calendars_df = pd.DataFrame(data=calendars_list, columns=[self.date_field_name]) - calendars_df[self.date_field_name] = calendars_df[self.date_field_name].astype( - np.datetime64 + calendars_df[self.date_field_name] = pd.to_datetime( + calendars_df[self.date_field_name] ) cal_df = calendars_df[ (calendars_df[self.date_field_name] >= df[self.date_field_name].min()) diff --git a/qbot/data/tw_collector.py b/qbot/data/tw_collector.py new file mode 100644 index 00000000..edb23da7 --- /dev/null +++ b/qbot/data/tw_collector.py @@ -0,0 +1,521 @@ +"""台股日線資料下載,並轉成 qlib 格式。 + +流程(可用 `run` 一次跑完): + download 從 Yahoo Finance 下載上市、上櫃股票及基準指數的日線 + normalize 對齊交易日曆、還原權值(qlib 的 factor 慣例)、計算 change / vwap + dump 用 dump_bin.py 轉成 qlib 二進位格式,並產生 twse / tpex 股票池 + universe 依過去成交值每季重選前 N 檔,產生動態股票池(例如 top200) + +範例: + python qbot/data/tw_collector.py run + python qbot/data/tw_collector.py run --symbols_file my_list.txt --top_n 50 + python qbot/data/tw_collector.py download --start 2005-01-01 --skip_exists True + +股票代碼在 qlib 中一律寫成 TW + 代號(TW2330、TW6488), +加權指數為 TWII,元大台灣50 為 TW0050。 +詳細說明見 docs/03-智能策略/台股qlib工作流.md +""" + +import os +import re +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Dict, List, Optional + +import numpy as np +import pandas as pd +from loguru import logger + +DEFAULT_BASE_DIR = "~/.qlib/stock_data/tw" +DEFAULT_QLIB_DIR = "~/.qlib/qlib_data/tw_data" +SOURCE_DIR_NAME = "source" +NORMALIZE_DIR_NAME = "normalize" +STOCK_LIST_FILE = "stock_list.csv" +FAILED_FILE = "failed.txt" + +# qlib 代碼 -> Yahoo 代碼 +BENCHMARKS = {"TWII": "^TWII", "TW0050": "0050.TW"} +CALENDAR_SYMBOL = "TWII" +# 指數沒有可靠的成交量,不套用「量為 0 視為停牌」的規則 +INDEX_SYMBOLS = {"TWII"} + +MARKET_SUFFIX = {"twse": ".TW", "tpex": ".TWO"} +SUFFIX_MARKET = {v: k for k, v in MARKET_SUFFIX.items()} +YAHOO_COLUMNS = { + "Open": "open", + "High": "high", + "Low": "low", + "Close": "close", + "Adj Close": "adjclose", + "Volume": "volume", + "Dividends": "dividends", + "Stock Splits": "stock_splits", +} +PRICE_FIELDS = ["open", "high", "low", "close"] +QLIB_FIELDS = PRICE_FIELDS + ["volume", "vwap", "factor", "change"] + +FINMIND_URL = "https://api.finmindtrade.com/api/v4/data" +STOCK_CODE_PATTERN = re.compile(r"[1-9]\d{3}") + + +def _to_qlib_symbol(code: str) -> str: + return f"TW{code}" + + +def _make_stock_list( + codes: List[str], + markets: List[str], + names: Optional[List[str]] = None, + industries: Optional[List[str]] = None, +) -> pd.DataFrame: + n = len(codes) + df = pd.DataFrame( + { + "code": codes, + "market": markets, + "name": names if names is not None else [""] * n, + "industry": industries if industries is not None else [""] * n, + } + ) + df["symbol"] = df["code"].map(_to_qlib_symbol) + df["yahoo"] = df["code"] + df["market"].map(MARKET_SUFFIX) + return df[["symbol", "code", "market", "yahoo", "name", "industry"]] + + +def filter_stock_info(info: pd.DataFrame) -> pd.DataFrame: + """從 FinMind TaiwanStockInfo 留下上市、上櫃的普通股(4 碼、非 0 開頭)。 + + 0 開頭的是 ETF,興櫃(type=emerging)流動性太低也排除。 + """ + df = info.copy() + df["stock_id"] = df["stock_id"].astype(str).str.strip() + df = df[df["type"].isin(list(MARKET_SUFFIX))] + df = df[df["stock_id"].str.fullmatch(STOCK_CODE_PATTERN.pattern)] + df = df.drop_duplicates("stock_id").sort_values("stock_id") + return _make_stock_list( + df["stock_id"].tolist(), + df["type"].tolist(), + df["stock_name"].tolist(), + df["industry_category"].tolist(), + ) + + +def fetch_finmind_stock_list(token: Optional[str] = None, timeout: int = 30): + """抓 FinMind 的台股清單(TaiwanStockInfo)。token 可省略,但請求額度較低。""" + import requests + + token = token or os.environ.get("FINMIND_TOKEN") + headers = {"Authorization": f"Bearer {token}"} if token else {} + resp = requests.get( + FINMIND_URL, + params={"dataset": "TaiwanStockInfo"}, + headers=headers, + timeout=timeout, + ) + resp.raise_for_status() + payload = resp.json() + if payload.get("status") != 200 or not payload.get("data"): + raise RuntimeError(f"FinMind TaiwanStockInfo failed: {payload.get('msg')}") + return filter_stock_info(pd.DataFrame(payload["data"])) + + +def load_symbols_file(path) -> pd.DataFrame: + """讀取自訂股票清單,每行一檔,# 之後為註解。 + + 可寫 2330(預設上市)、2330.TW、6488.TWO,代號後面可接名稱。 + """ + codes, markets, names = [], [], [] + for line in Path(path).expanduser().read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + parts = line.split(maxsplit=1) + token = parts[0].upper() + code, dot, suffix = token.partition(".") + market = SUFFIX_MARKET.get(f".{suffix}") if dot else "twse" + if market is None or not STOCK_CODE_PATTERN.fullmatch(code): + raise ValueError(f"Unrecognized symbol in {path}: {parts[0]}") + codes.append(code) + markets.append(market) + names.append(parts[1] if len(parts) > 1 else "") + df = _make_stock_list(codes, markets, names) + return df.drop_duplicates("symbol").reset_index(drop=True) + + +def fetch_yahoo(ticker: str, start: str, end: str) -> pd.DataFrame: + """下載 Yahoo 日線(不還原),回傳 date + 小寫欄位;查無資料時回傳空表。""" + import yfinance as yf + + df = yf.Ticker(ticker).history( + start=start, end=end, interval="1d", auto_adjust=False, actions=True + ) + if df is None or df.empty: + return pd.DataFrame() + df = df.rename(columns=YAHOO_COLUMNS) + df.index = pd.to_datetime(df.index) + if df.index.tz is not None: + df.index = df.index.tz_localize(None) + df.index = df.index.normalize() + df.index.name = "date" + columns = [c for c in YAHOO_COLUMNS.values() if c in df.columns] + return df[columns].reset_index() + + +def download( + base_dir: str = DEFAULT_BASE_DIR, + start: str = "2005-01-01", + end: Optional[str] = None, + symbols_file: Optional[str] = None, + finmind_token: Optional[str] = None, + max_workers: int = 4, + delay: float = 0.5, + retries: int = 3, + retry_wait: float = 5.0, + skip_exists: bool = False, +): + """下載股票清單與日線到 base_dir/source。 + + end 含當天,預設為今天。沒給 symbols_file 時從 FinMind 抓全部上市、上櫃股票。 + Yahoo 有流量限制,中斷後可加 --skip_exists True 只補抓缺的檔案。 + """ + base_dir = Path(base_dir).expanduser() + source_dir = base_dir.joinpath(SOURCE_DIR_NAME) + source_dir.mkdir(parents=True, exist_ok=True) + + if symbols_file: + stocks = load_symbols_file(symbols_file) + else: + stocks = fetch_finmind_stock_list(finmind_token) + stocks.to_csv(base_dir.joinpath(STOCK_LIST_FILE), index=False) + logger.info(f"{len(stocks)} stocks in {base_dir.joinpath(STOCK_LIST_FILE)}") + + # yfinance 的 end 不含當天 + end = pd.Timestamp(end) if end else pd.Timestamp.today().normalize() + end = (end + pd.Timedelta(days=1)).strftime("%Y-%m-%d") + jobs = dict(BENCHMARKS) + jobs.update(stocks.set_index("symbol")["yahoo"].to_dict()) + + def _job(symbol: str, ticker: str) -> str: + path = source_dir.joinpath(f"{symbol}.csv") + if skip_exists and path.exists(): + return "skip" + for attempt in range(retries): + try: + df = fetch_yahoo(ticker, start, end) + break + except Exception as e: # yfinance 會丟各種網路、限流例外 + logger.warning(f"{ticker} attempt {attempt + 1} failed: {e}") + if attempt + 1 < retries: + time.sleep(retry_wait * 2**attempt) + else: + return "error" + time.sleep(delay) + if df.empty: + return "empty" + df.to_csv(path, index=False) + return "ok" + + results: Dict[str, str] = {} + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(_job, s, t): s for s, t in jobs.items()} + for i, future in enumerate(as_completed(futures), 1): + results[futures[future]] = future.result() + if i % 100 == 0 or i == len(futures): + logger.info(f"download {i}/{len(futures)}") + + failed = sorted(s for s, r in results.items() if r in ("error", "empty")) + base_dir.joinpath(FAILED_FILE).write_text("\n".join(failed), encoding="utf-8") + counts = pd.Series(results).value_counts().to_dict() + logger.info(f"download finished: {counts}") + if failed: + logger.warning( + f"{len(failed)} symbols failed, see {base_dir.joinpath(FAILED_FILE)}; " + "rerun with --skip_exists True to retry only the missing ones" + ) + if CALENDAR_SYMBOL not in results or results[CALENDAR_SYMBOL] in ("error", "empty"): + logger.warning("TWII download failed; normalize will fall back to stock dates") + + +def _source_files(base_dir: Path) -> Dict[str, Path]: + source_dir = base_dir.joinpath(SOURCE_DIR_NAME) + return {p.stem: p for p in sorted(source_dir.glob("*.csv"))} + + +def _read_source(path: Path) -> pd.DataFrame: + df = pd.read_csv(path) + df["date"] = pd.to_datetime(df["date"]) + return df + + +def load_calendar(base_dir) -> pd.DatetimeIndex: + """以加權指數的交易日為日曆;沒有指數資料時改用所有股票有成交的日期。""" + files = _source_files(Path(base_dir).expanduser()) + if CALENDAR_SYMBOL in files: + df = _read_source(files[CALENDAR_SYMBOL]) + dates = df.loc[df["close"].notna(), "date"] + else: + logger.warning("TWII.csv not found, building calendar from stock dates") + dates = [] + for symbol, path in files.items(): + if symbol in BENCHMARKS: + continue + df = _read_source(path) + dates.extend(df.loc[df["volume"] > 0, "date"]) + return pd.DatetimeIndex(sorted(set(pd.to_datetime(dates))), name="date") + + +def normalize_symbol( + df: pd.DataFrame, + symbol: str, + calendar: Optional[pd.DatetimeIndex] = None, + is_index: bool = False, +) -> pd.DataFrame: + """把 Yahoo 原始日線轉成 qlib 慣例(同 qlib yahoo collector 的 1d normalize)。 + + - 價格為還原權值後再除以首日收盤價,volume 反向調整 + - factor = 還原價 / 原始價,qlib 用 close / factor 換回實際股價來計算整張交易 + - change 為還原收盤價的日報酬,用來判斷漲跌停(除息日不會被當成大跌) + - vwap 用 (high + low + close) / 3 近似(Yahoo 沒有成交金額) + """ + df = df.copy() + df["date"] = pd.to_datetime(df["date"]) + df = df.drop_duplicates("date").set_index("date").sort_index() + if calendar is not None and len(calendar) > 0: + in_range = (calendar >= df.index.min()) & (calendar <= df.index.max()) + df = df.reindex(calendar[in_range]) + df.index.name = "date" + if df.empty: + return pd.DataFrame() + if "adjclose" not in df.columns: + df["adjclose"] = df["close"] + + invalid = df["close"].isna() | (df["close"] <= 0) + if not is_index: + invalid |= df["volume"].isna() | (df["volume"] <= 0) + df.loc[invalid, PRICE_FIELDS + ["adjclose", "volume"]] = np.nan + if df["close"].isna().all(): + return pd.DataFrame() + + df["factor"] = (df["adjclose"] / df["close"]).ffill() + for col in PRICE_FIELDS: + df[col] = df[col] * df["factor"] + df["volume"] = df["volume"] / df["factor"] + + close = df["close"].ffill() + df["change"] = close / close.shift(1) - 1 + df.loc[invalid, "change"] = np.nan + + first_close = df["close"].dropna().iloc[0] + for col in PRICE_FIELDS + ["factor"]: + df[col] = df[col] / first_close + df["volume"] = df["volume"] * first_close + df["vwap"] = (df["high"] + df["low"] + df["close"]) / 3 + + df["symbol"] = symbol + return df.reset_index()[["date", "symbol"] + QLIB_FIELDS] + + +def normalize(base_dir: str = DEFAULT_BASE_DIR): + """把 base_dir/source 的原始資料轉到 base_dir/normalize。""" + base_dir = Path(base_dir).expanduser() + normalize_dir = base_dir.joinpath(NORMALIZE_DIR_NAME) + normalize_dir.mkdir(parents=True, exist_ok=True) + for old in normalize_dir.glob("*.csv"): + old.unlink() + + calendar = load_calendar(base_dir) + logger.info(f"calendar: {len(calendar)} days") + count = 0 + for symbol, path in _source_files(base_dir).items(): + df = normalize_symbol( + _read_source(path), symbol, calendar, is_index=symbol in INDEX_SYMBOLS + ) + if df.empty: + logger.warning(f"{symbol} has no valid rows, skipped") + continue + df.to_csv(normalize_dir.joinpath(f"{symbol}.csv"), index=False) + count += 1 + logger.info(f"normalized {count} symbols into {normalize_dir}") + + +def _instruments_dir(qlib_dir) -> Path: + path = Path(qlib_dir).expanduser().joinpath("instruments") + path.mkdir(parents=True, exist_ok=True) + return path + + +def _read_instruments(path: Path) -> pd.DataFrame: + return pd.read_csv( + path, sep="\t", header=None, names=["symbol", "start", "end"], dtype=str + ) + + +def _write_instruments(df: pd.DataFrame, path: Path): + df[["symbol", "start", "end"]].to_csv(path, sep="\t", header=False, index=False) + + +def write_market_instruments( + base_dir: str = DEFAULT_BASE_DIR, qlib_dir: str = DEFAULT_QLIB_DIR +): + """從 all.txt 移除基準指數,並依股票清單拆出 twse.txt、tpex.txt。""" + inst_dir = _instruments_dir(qlib_dir) + all_path = inst_dir.joinpath("all.txt") + inst = _read_instruments(all_path) + inst = inst[~inst["symbol"].isin(list(BENCHMARKS))] + _write_instruments(inst, all_path) + + stock_list_path = Path(base_dir).expanduser().joinpath(STOCK_LIST_FILE) + if not stock_list_path.exists(): + return + stocks = pd.read_csv(stock_list_path, dtype=str) + for market in MARKET_SUFFIX: + symbols = set(stocks.loc[stocks["market"] == market, "symbol"]) + _write_instruments( + inst[inst["symbol"].isin(symbols)], inst_dir.joinpath(f"{market}.txt") + ) + + +def dump( + base_dir: str = DEFAULT_BASE_DIR, + qlib_dir: str = DEFAULT_QLIB_DIR, + max_workers: int = 8, +): + """用 dump_bin.py 把 base_dir/normalize 轉成 qlib 格式(會覆寫 qlib_dir 的日曆)。""" + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from dump_bin import DumpDataAll + + normalize_dir = Path(base_dir).expanduser().joinpath(NORMALIZE_DIR_NAME) + DumpDataAll( + csv_path=str(normalize_dir), + qlib_dir=qlib_dir, + max_workers=max_workers, + date_field_name="date", + symbol_field_name="symbol", + include_fields=",".join(QLIB_FIELDS), + )() + write_market_instruments(base_dir, qlib_dir) + + +def compute_liquidity_universe( + values: pd.DataFrame, top_n: int = 200, window: int = 60, freq: str = "Q" +) -> pd.DataFrame: + """依過去 window 日平均成交值,在每期最後一個交易日收盤後選出前 top_n 檔。 + + values:index 為交易日,columns 為代碼,上市前或下市後為 NaN,停牌日為 0。 + 選股只用當天(含)以前的資料,成分從下一個交易日生效,到下一次重選日為止, + 所以沒有前視偏差。回傳 symbol / start / end,連續入選的期間會合併成一段。 + """ + dates = values.index + avg = values.rolling(window, min_periods=window).mean() + periods = dates.to_period(freq) + rebalance_dates = dates[~periods.duplicated(keep="last")] + + spans: Dict[str, List[list]] = {} + for i, rebalance in enumerate(rebalance_dates): + pos = dates.get_loc(rebalance) + if pos + 1 >= len(dates): + break + ranked = avg.loc[rebalance].dropna() + ranked = ranked[ranked > 0].nlargest(top_n) + start = dates[pos + 1] + if i + 1 < len(rebalance_dates): + end = rebalance_dates[i + 1] + else: + end = dates[-1] + for symbol in ranked.index: + symbol_spans = spans.setdefault(symbol, []) + # 上一期也有入選(上一段剛好結束在這次重選日)就延長 + if symbol_spans and symbol_spans[-1][1] == rebalance: + symbol_spans[-1][1] = end + else: + symbol_spans.append([start, end]) + + rows = [ + (symbol, start, end) + for symbol, symbol_spans in sorted(spans.items()) + for start, end in symbol_spans + ] + df = pd.DataFrame(rows, columns=["symbol", "start", "end"]) + for col in ("start", "end"): + df[col] = pd.to_datetime(df[col]).dt.strftime("%Y-%m-%d") + return df + + +def _load_trading_values(base_dir: Path, calendar: pd.DatetimeIndex) -> pd.DataFrame: + stock_list_path = base_dir.joinpath(STOCK_LIST_FILE) + stocks = set(pd.read_csv(stock_list_path, dtype=str)["symbol"]) + values = {} + for symbol, path in _source_files(base_dir).items(): + if symbol not in stocks: + continue + df = _read_source(path).drop_duplicates("date").set_index("date").sort_index() + # Yahoo 的 close 只做分割調整、未除息,乘上成交量即為當日成交值 + value = (df["close"] * df["volume"]).clip(lower=0) + in_range = (calendar >= df.index.min()) & (calendar <= df.index.max()) + values[symbol] = value.reindex(calendar[in_range]).fillna(0) + return pd.DataFrame(values).reindex(calendar) + + +def universe( + base_dir: str = DEFAULT_BASE_DIR, + qlib_dir: str = DEFAULT_QLIB_DIR, + top_n: int = 200, + window: int = 60, + freq: str = "Q", + name: Optional[str] = None, +): + """產生動態股票池 instruments/.txt(預設 top200,每季重選)。""" + base_dir = Path(base_dir).expanduser() + name = name or f"top{top_n}" + calendar = load_calendar(base_dir) + values = _load_trading_values(base_dir, calendar) + df = compute_liquidity_universe(values, top_n=top_n, window=window, freq=freq) + path = _instruments_dir(qlib_dir).joinpath(f"{name}.txt") + _write_instruments(df, path) + logger.info(f"universe {name}: {df['symbol'].nunique()} symbols -> {path}") + + +def run( + base_dir: str = DEFAULT_BASE_DIR, + qlib_dir: str = DEFAULT_QLIB_DIR, + start: str = "2005-01-01", + end: Optional[str] = None, + symbols_file: Optional[str] = None, + finmind_token: Optional[str] = None, + max_workers: int = 4, + delay: float = 0.5, + skip_exists: bool = False, + top_n: int = 200, + window: int = 60, + freq: str = "Q", +): + """依序執行 download、normalize、dump、universe。""" + download( + base_dir, + start=start, + end=end, + symbols_file=symbols_file, + finmind_token=finmind_token, + max_workers=max_workers, + delay=delay, + skip_exists=skip_exists, + ) + normalize(base_dir) + dump(base_dir, qlib_dir) + universe(base_dir, qlib_dir, top_n=top_n, window=window, freq=freq) + + +if __name__ == "__main__": + import fire + + fire.Fire( + { + "run": run, + "download": download, + "normalize": normalize, + "dump": dump, + "universe": universe, + } + ) diff --git a/tests/test_tw_collector.py b/tests/test_tw_collector.py new file mode 100644 index 00000000..4b4474a6 --- /dev/null +++ b/tests/test_tw_collector.py @@ -0,0 +1,297 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +import importlib.util +import shutil +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np +import pandas as pd + +sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("qbot", "data"))) +import tw_collector as tw # noqa: E402 + +HAS_QLIB = importlib.util.find_spec("qlib") is not None + + +def make_raw(dates, start_price, seed, ex_div=None, dividend=0.0, suspended=()): + """產生和 tw_collector.fetch_yahoo 同格式的原始日線。""" + rng = np.random.default_rng(seed) + close = start_price * np.cumprod(1 + rng.normal(0, 0.01, len(dates))) + adjclose = close.copy() + if ex_div is not None: + pos = dates.get_loc(pd.Timestamp(ex_div)) + close[pos:] -= dividend + # Yahoo 的還原價:除息日前的價格乘上 (1 - 股息 / 除息前一日收盤) + adjclose = close.copy() + adjclose[:pos] *= 1 - dividend / close[pos - 1] + volume = rng.integers(1_000_000, 5_000_000, len(dates)).astype(float) + df = pd.DataFrame( + { + "date": dates, + "open": close * 0.995, + "high": close * 1.01, + "low": close * 0.99, + "close": close, + "adjclose": adjclose, + "volume": volume, + "dividends": 0.0, + "stock_splits": 0.0, + } + ) + for day in suspended: + df.loc[df["date"] == pd.Timestamp(day), "volume"] = 0 + return df + + +class StockListTest(unittest.TestCase): + def test_filter_stock_info(self): + info = pd.DataFrame( + { + "industry_category": [ + "半導體業", + "ETF", + "電子零組件業", + "半導體業", + "其他", + ], + "stock_id": ["2330", "0050", "6488", "2330", "7777"], + "stock_name": ["台積電", "元大台灣50", "環球晶", "台積電", "興櫃股"], + "type": ["twse", "twse", "tpex", "twse", "emerging"], + "date": ["2026-10-01"] * 5, + } + ) + df = tw.filter_stock_info(info) + self.assertEqual(df["symbol"].tolist(), ["TW2330", "TW6488"]) + self.assertEqual(df["yahoo"].tolist(), ["2330.TW", "6488.TWO"]) + self.assertEqual(df["market"].tolist(), ["twse", "tpex"]) + + def test_load_symbols_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp, "list.txt") + path.write_text( + "# 範例\n2330 台積電\n6488.two\n\n2317.TW # 鴻海\n2330\n", + encoding="utf-8", + ) + df = tw.load_symbols_file(path) + self.assertEqual(df["symbol"].tolist(), ["TW2330", "TW6488", "TW2317"]) + self.assertEqual(df["yahoo"].tolist(), ["2330.TW", "6488.TWO", "2317.TW"]) + self.assertEqual(df["name"].tolist()[0], "台積電") + + path.write_text("0050\n", encoding="utf-8") + with self.assertRaises(ValueError): + tw.load_symbols_file(path) + + +class FetchYahooTest(unittest.TestCase): + def test_columns_and_timezone(self): + index = pd.DatetimeIndex( + ["2024-01-02", "2024-01-03"], tz="Asia/Taipei", name="Date" + ) + history = pd.DataFrame( + { + "Open": [590.0, 584.0], + "High": [593.0, 585.0], + "Low": [589.0, 576.0], + "Close": [593.0, 578.0], + "Adj Close": [570.1, 555.7], + "Volume": [26059058, 37106763], + "Dividends": [0.0, 0.0], + "Stock Splits": [0.0, 0.0], + }, + index=index, + ) + ticker = mock.Mock() + ticker.history.return_value = history + fake_yf = types.SimpleNamespace(Ticker=mock.Mock(return_value=ticker)) + with mock.patch.dict(sys.modules, {"yfinance": fake_yf}): + df = tw.fetch_yahoo("2330.TW", "2024-01-01", "2024-01-04") + fake_yf.Ticker.assert_called_once_with("2330.TW") + self.assertEqual(ticker.history.call_args.kwargs["auto_adjust"], False) + self.assertEqual( + df.columns.tolist(), ["date"] + list(tw.YAHOO_COLUMNS.values()) + ) + self.assertEqual( + df["date"].tolist(), list(pd.to_datetime(["2024-01-02", "2024-01-03"])) + ) + self.assertIsNone(df["date"].dt.tz) + + +class NormalizeTest(unittest.TestCase): + def setUp(self): + self.dates = pd.bdate_range("2024-01-01", periods=30) + self.raw = make_raw( + self.dates, + 100.0, + seed=1, + ex_div=self.dates[10], + dividend=3.0, + suspended=[self.dates[20]], + ) + + def test_adjustment_is_reversible(self): + df = tw.normalize_symbol(self.raw, "TW2330", self.dates).set_index("date") + raw = self.raw.set_index("date") + valid = raw["volume"] > 0 + # qlib 用 close / factor 換回實際股價,用 volume * factor 換回實際股數 + np.testing.assert_allclose( + (df["close"] / df["factor"])[valid], raw["close"][valid] + ) + np.testing.assert_allclose( + (df["volume"] * df["factor"])[valid], raw["volume"][valid] + ) + self.assertAlmostEqual(df["close"].dropna().iloc[0], 1.0) + + def test_change_ignores_dividend_and_suspension(self): + df = tw.normalize_symbol(self.raw, "TW2330", self.dates).set_index("date") + raw = self.raw.set_index("date") + ex_div = self.dates[10] + expected = raw.loc[ex_div, "adjclose"] / raw["adjclose"].shift(1)[ex_div] - 1 + self.assertAlmostEqual(df.loc[ex_div, "change"], expected) + self.assertGreater(df.loc[ex_div, "change"], -0.05) + + suspended = self.dates[20] + self.assertTrue(df.loc[suspended, ["close", "volume", "change"]].isna().all()) + # 復牌日的 change 對比停牌前最後一個收盤價 + resumed = self.dates[21] + expected = df.loc[resumed, "close"] / df.loc[self.dates[19], "close"] - 1 + self.assertAlmostEqual(df.loc[resumed, "change"], expected) + + def test_vwap_and_calendar_alignment(self): + calendar = self.dates.delete(5) + raw = self.raw.drop(index=7) + df = tw.normalize_symbol(raw, "TW2330", calendar).set_index("date") + self.assertNotIn(self.dates[5], df.index) + self.assertTrue(np.isnan(df.loc[self.dates[7], "close"])) + np.testing.assert_allclose( + df["vwap"].dropna(), ((df["high"] + df["low"] + df["close"]) / 3).dropna() + ) + + def test_index_keeps_zero_volume_rows(self): + raw = self.raw.assign(volume=0.0, adjclose=self.raw["close"]) + df = tw.normalize_symbol(raw, "TWII", self.dates, is_index=True) + self.assertEqual(df["close"].notna().sum(), len(self.dates)) + self.assertEqual(df["factor"].nunique(), 1) + + +class UniverseTest(unittest.TestCase): + def test_quarterly_top_n_without_lookahead(self): + dates = pd.bdate_range("2020-01-01", "2020-12-31") + values = pd.DataFrame(index=dates) + values["A"] = 100.0 + values["B"] = np.where(dates <= "2020-06-30", 90.0, 0.0) + values["C"] = np.where(dates < "2020-08-14", 10.0, 1000.0) + values["D"] = 50.0 + df = tw.compute_liquidity_universe(values, top_n=2, window=20, freq="Q") + spans = { + s: g[["start", "end"]].values.tolist() for s, g in df.groupby("symbol") + } + self.assertEqual(spans["A"], [["2020-04-01", "2020-12-31"]]) + self.assertEqual(spans["B"], [["2020-04-01", "2020-09-30"]]) + # C 在 8 月中就放量,但要等 9/30 重選後才入選 + self.assertEqual(spans["C"], [["2020-10-01", "2020-12-31"]]) + self.assertNotIn("D", spans) + + def test_unlisted_stock_waits_for_full_window(self): + dates = pd.bdate_range("2020-01-01", "2020-06-30") + values = pd.DataFrame(index=dates) + values["A"] = 1.0 + values["NEW"] = np.where(dates >= "2020-03-20", 1000.0, np.nan) + df = tw.compute_liquidity_universe(values, top_n=1, window=20, freq="Q") + # 3/31 時 NEW 只有 8 天資料,不夠 20 日不能入選 + self.assertEqual(df.loc[df["symbol"] == "A", "start"].tolist(), ["2020-04-01"]) + self.assertNotIn("NEW", df["symbol"].tolist()) + + +@unittest.skipUnless(HAS_QLIB, "pyqlib is not installed") +class PipelineTest(unittest.TestCase): + STOCKS = { + "2330": "twse", + "2317": "twse", + "2454": "twse", + "2881": "twse", + "6488": "tpex", + "5274": "tpex", + } + + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.base_dir = self.tmp.joinpath("tw") + self.qlib_dir = self.tmp.joinpath("qlib") + self.dates = pd.bdate_range("2022-01-03", "2023-12-29") + self.raw = {"^TWII": make_raw(self.dates, 17000.0, seed=0)} + self.raw["0050.TW"] = make_raw(self.dates, 140.0, seed=1) + for i, (code, market) in enumerate(self.STOCKS.items()): + ticker = code + tw.MARKET_SUFFIX[market] + self.raw[ticker] = make_raw( + self.dates, + 50.0 * (i + 1), + seed=10 + i, + ex_div="2023-07-03", + dividend=2.0, + ) + + def tearDown(self): + shutil.rmtree(self.tmp) + + def _fake_stock_list(self, token=None): + info = pd.DataFrame( + { + "industry_category": ["x"] * len(self.STOCKS), + "stock_id": list(self.STOCKS), + "stock_name": list(self.STOCKS), + "type": list(self.STOCKS.values()), + } + ) + return tw.filter_stock_info(info) + + def _fake_fetch(self, ticker, start, end): + df = self.raw[ticker] + return df[(df["date"] >= start) & (df["date"] < end)].copy() + + def test_run_builds_qlib_data(self): + import qlib + from qlib.config import C + from qlib.data import D + + with mock.patch.object( + tw, "fetch_finmind_stock_list", self._fake_stock_list + ), mock.patch.object(tw, "fetch_yahoo", self._fake_fetch): + tw.run( + base_dir=str(self.base_dir), + qlib_dir=str(self.qlib_dir), + start="2022-01-01", + end="2023-12-31", + delay=0, + top_n=3, + window=20, + ) + + inst_dir = self.qlib_dir.joinpath("instruments") + all_symbols = tw._read_instruments(inst_dir.joinpath("all.txt"))["symbol"] + self.assertEqual(sorted(all_symbols), sorted("TW" + c for c in self.STOCKS)) + tpex = tw._read_instruments(inst_dir.joinpath("tpex.txt"))["symbol"] + self.assertEqual(sorted(tpex), ["TW5274", "TW6488"]) + + qlib.init(provider_uri=str(self.qlib_dir), region="tw") + self.assertEqual(C.trade_unit, 1000) + members = D.list_instruments(D.instruments("top3"), as_list=True) + self.assertTrue(0 < len(members) <= len(self.STOCKS)) + + fields = ["$close", "$factor", "$change", "$vwap"] + feat = D.features(["TW2330", "TW0050"], fields, "2022-01-01", "2023-12-31") + tsmc = feat.loc["TW2330"] + raw = self.raw["2330.TW"].set_index("date") + np.testing.assert_allclose( + (tsmc["$close"] / tsmc["$factor"]).values, raw["close"].values, rtol=1e-5 + ) + self.assertFalse(feat.loc["TW0050", "$close"].isna().any()) + + +if __name__ == "__main__": + unittest.main()