Skip to content

Commit 84c90d5

Browse files
SunsetWolfSunsetWolf
committed
optimize yml
1 parent 680523b commit 84c90d5

5 files changed

Lines changed: 261 additions & 16 deletions

File tree

‎.github/workflows/python-publish.yml‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,9 @@
33

44
name: Upload Python Package
55

6-
# on:
7-
# release:
8-
# types: [published]
9-
106
on:
11-
push:
12-
branches: [ main ]
13-
pull_request:
14-
branches: [ main ]
7+
release:
8+
types: [published]
159

1610
jobs:
1711
deploy_with_bdist_wheel:
@@ -39,12 +33,12 @@ jobs:
3933
run: |
4034
make build
4135
- name: Upload to PyPi
42-
# env:
43-
# TWINE_USERNAME: __token__
44-
# TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
36+
env:
37+
TWINE_USERNAME: __token__
38+
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
4539
run: |
4640
twine check dist/*.whl
47-
# twine upload dist/*.whl --verbose
41+
twine upload dist/*.whl --verbose
4842
4943
deploy_with_manylinux:
5044
runs-on: ubuntu-latest
@@ -63,9 +57,9 @@ jobs:
6357
run: |
6458
python -m pip install twine
6559
- name: Upload to PyPi
66-
# env:
67-
# TWINE_USERNAME: __token__
68-
# TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
60+
env:
61+
TWINE_USERNAME: __token__
62+
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
6963
run: |
7064
twine check dist/pyqlib-*-manylinux*.whl
71-
# twine upload dist/pyqlib-*-manylinux*.whl --verbose
65+
twine upload dist/pyqlib-*-manylinux*.whl --verbose

‎.github/workflows/test_qlib_from_pip.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Test qlib from pip
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
timeout-minutes: 120
12+
13+
runs-on: ${{ matrix.os }}
14+
strategy:
15+
matrix:
16+
os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-14, macos-15]
17+
# In github action, using python 3.7, pip install will not match the latest version of the package.
18+
# Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
19+
# All things considered, we have removed python 3.7.
20+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
21+
22+
steps:
23+
- name: Test qlib from pip
24+
uses: actions/checkout@v3
25+
26+
- name: Set up Python ${{ matrix.python-version }}
27+
uses: actions/setup-python@v4
28+
with:
29+
python-version: ${{ matrix.python-version }}
30+
31+
- name: Update pip to the latest version
32+
run: |
33+
python -m pip install --upgrade pip
34+
35+
# Will cancel this step when the next qlib version is released. The current qlib version is: 0.9.6
36+
- name: Installing pywinpt for windows
37+
if: ${{ matrix.os == 'windows-latest' }}
38+
run: |
39+
python -m pip install pywinpty --only-binary=:all:
40+
41+
# # joblib was released on 2025-05-04 with version 1.5.0, in which _backend_args was removed and replaced by _backend_kwargs.
42+
# This change caused the application to fail, so the version of joblib is restricted here.
43+
# This restriction will be removed in the next release. The current qlib version is: 0.9.6
44+
- name: Qlib installation test
45+
run: |
46+
python -m pip install pyqlib
47+
python -m pip install "joblib<=1.4.2"
48+
49+
- name: Install Lightgbm for MacOS
50+
if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
51+
run: |
52+
brew update
53+
brew install libomp || brew reinstall libomp
54+
python -m pip install --no-binary=:all: lightgbm
55+
56+
# When the new version is released it should be changed to:
57+
# python -m qlib.cli.data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
58+
- name: Downloads dependencies data
59+
run: |
60+
cd ..
61+
python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
62+
cd qlib
63+
64+
- name: Test workflow by config
65+
run: |
66+
qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

‎.github/workflows/test_qlib_from_source.yml‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
name: Test qlib from source
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
timeout-minutes: 180
12+
# we may retry for 3 times for `Unit tests with Pytest`
13+
14+
runs-on: ${{ matrix.os }}
15+
strategy:
16+
matrix:
17+
os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-14, macos-15]
18+
# In github action, using python 3.7, pip install will not match the latest version of the package.
19+
# Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
20+
# All things considered, we have removed python 3.7.
21+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
22+
23+
steps:
24+
- name: Test qlib from source
25+
uses: actions/checkout@v3
26+
27+
- name: Set up Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
32+
- name: Update pip to the latest version
33+
run: |
34+
python -m pip install --upgrade pip
35+
36+
- name: Installing pytorch for macos
37+
if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
38+
run: |
39+
python -m pip install torch torchvision torchaudio
40+
41+
- name: Installing pytorch for ubuntu
42+
if: ${{ matrix.os == 'ubuntu-24.04' || matrix.os == 'ubuntu-22.04' }}
43+
run: |
44+
python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
45+
46+
- name: Installing pytorch for windows
47+
if: ${{ matrix.os == 'windows-latest' }}
48+
run: |
49+
python -m pip install torch torchvision torchaudio
50+
51+
- name: Set up Python tools
52+
run: |
53+
make dev
54+
55+
- name: Lint with Black
56+
run: |
57+
make black
58+
59+
- name: Make html with sphinx
60+
# Since read the docs builds on ubuntu 22.04, we only need to test that the build passes on ubuntu 22.04.
61+
if: ${{ matrix.os == 'ubuntu-22.04' }}
62+
run: |
63+
make docs-gen
64+
65+
- name: Check Qlib with pylint
66+
run: |
67+
make pylint
68+
69+
- name: Check Qlib with flake8
70+
run: |
71+
make flake8
72+
73+
- name: Check Qlib with mypy
74+
run: |
75+
make mypy
76+
77+
- name: Check Qlib ipynb with nbqa
78+
run: |
79+
make nbqa
80+
81+
- name: Test data downloads
82+
run: |
83+
python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
84+
python scripts/get_data.py download_data --file_name rl_data.zip --target_dir tests/.data/rl
85+
86+
- name: Install Lightgbm for MacOS
87+
if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
88+
run: |
89+
brew update
90+
brew install libomp || brew reinstall libomp
91+
python -m pip install --no-binary=:all: lightgbm
92+
93+
- name: Check Qlib ipynb with nbconvert
94+
run: |
95+
make nbconvert
96+
97+
- name: Test workflow by config (install from source)
98+
run: |
99+
python -m pip install numba
100+
python qlib/cli/run.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
101+
102+
- name: Unit tests with Pytest (MacOS)
103+
if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
104+
uses: nick-fields/retry@v2
105+
with:
106+
timeout_minutes: 60
107+
max_attempts: 3
108+
command: |
109+
# Limit the number of threads in various libraries to prevent Segmentation faults caused by OpenMP multithreading conflicts under macOS.
110+
export OMP_NUM_THREADS=1 # Limit the number of OpenMP threads
111+
export MKL_NUM_THREADS=1 # Limit the number of Intel MKL threads
112+
export NUMEXPR_NUM_THREADS=1 # Limit the number of NumExpr threads
113+
export OPENBLAS_NUM_THREADS=1 # Limit the number of OpenBLAS threads
114+
export VECLIB_MAXIMUM_THREADS=1 # Limit the number of macOS Accelerate/vecLib threads
115+
cd tests
116+
python -m pytest . -m "not slow" --durations=0
117+
118+
- name: Unit tests with Pytest (Ubuntu and Windows)
119+
if: ${{ matrix.os != 'macos-13' && matrix.os != 'macos-14' && matrix.os != 'macos-15' }}
120+
uses: nick-fields/retry@v2
121+
with:
122+
timeout_minutes: 60
123+
max_attempts: 3
124+
command: |
125+
cd tests
126+
python -m pytest . -m "not slow" --durations=0

‎.github/workflows/test_qlib_from_source_slow.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: Test qlib from source slow
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
timeout-minutes: 720
12+
# we may retry for 3 times for `Unit tests with Pytest`
13+
14+
runs-on: ${{ matrix.os }}
15+
strategy:
16+
matrix:
17+
os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-14, macos-15]
18+
# In github action, using python 3.7, pip install will not match the latest version of the package.
19+
# Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
20+
# All things considered, we have removed python 3.7.
21+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
22+
23+
steps:
24+
- name: Test qlib from source slow
25+
uses: actions/checkout@v3
26+
27+
- name: Set up Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
32+
- name: Set up Python tools
33+
run: |
34+
make dev
35+
36+
- name: Downloads dependencies data
37+
run: |
38+
python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
39+
40+
# install.sh file contents from: https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh
41+
# brew_install.sh file contents from: https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh
42+
- name: Install Lightgbm for MacOS
43+
if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
44+
run: |
45+
brew update
46+
brew install libomp || brew reinstall libomp
47+
python -m pip install --no-binary=:all: lightgbm
48+
49+
- name: Unit tests with Pytest
50+
uses: nick-fields/retry@v2
51+
with:
52+
timeout_minutes: 240
53+
max_attempts: 3
54+
command: |
55+
cd tests
56+
python -m pytest . -m "slow" --durations=0

‎pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,9 @@ analysis = [
9797
"statsmodels",
9898
]
9999

100+
# In the process of releasing a new version, when checking the manylinux package with twine, an error is reported:
101+
# InvalidDistribution: Invalid distribution metadata: unrecognized or malformed field 'license-file'
102+
# To solve this problem, we added license-files here. Refs: https://github.com/pypa/twine/issues/1216
100103
[tool.setuptools]
101104
packages = [
102105
"qlib",

0 commit comments

Comments
 (0)