qlib/.github/workflows/test_qlib_from_source.yml at main · microsoft/qlib · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
name: Test qlib from source

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  build:
    timeout-minutes: 180
    # we may retry for 3 times for `Unit tests with Pytest`

    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-14, macos-15]
        # In github action, using python 3.7, pip install will not match the latest version of the package.
        # Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
        # All things considered, we have removed python 3.7.
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]

    steps:
    - name: Test qlib from source
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Update pip to the latest version
      run: |
        python -m pip install --upgrade pip

    - name: Installing pytorch for macos
      if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
      run: |
        python -m pip install torch torchvision torchaudio

    - name: Installing pytorch for ubuntu
      if: ${{ matrix.os == 'ubuntu-24.04' || matrix.os == 'ubuntu-22.04' }}
      run: |
        python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu

    - name: Installing pytorch for windows
      if: ${{ matrix.os == 'windows-latest' }}
      run: |
        python -m pip install torch torchvision torchaudio

    - name: Set up Python tools
      run: |
        make dev

    - name: Lint with Black
      run: |
        make black

    - name: Make html with sphinx
      # Since read the docs builds on ubuntu 22.04, we only need to test that the build passes on ubuntu 22.04.
      if: ${{ matrix.os == 'ubuntu-22.04' }}
      run: |
        make docs-gen

    - name: Check Qlib with pylint
      run: |
        make pylint

    - name: Check Qlib with flake8
      run: |
        make flake8

    - name: Check Qlib with mypy
      run: |
        make mypy

    # Due to issues that cannot be automatically fixed when running `nbqa black . -l 120 --check --diff` on Jupyter notebooks,
    # we reverted to a version of `black` earlier than 26.1.0 before performing the checks.
    - name: Check Qlib ipynb with nbqa
      run: |
        python -m pip install "black<26.1"
        make nbqa

    - name: Test data downloads
      run: |
        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
        python scripts/get_data.py download_data --file_name rl_data.zip --target_dir tests/.data/rl

    - name: Install Lightgbm for MacOS
      if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
      run: |
        brew update
        brew install libomp || brew reinstall libomp
        python -m pip install --no-binary=:all: lightgbm

    - name: Check Qlib ipynb with nbconvert
      run: |
        make nbconvert

    - name: Test workflow by config (install from source)
      run: |
        python -m pip install numba
        python qlib/cli/run.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

    - name: Unit tests with Pytest (MacOS)
      if: ${{ matrix.os == 'macos-14' || matrix.os == 'macos-15' }}
      uses: nick-fields/retry@v2
      with:
        timeout_minutes: 60
        max_attempts: 3
        command: |
          # Limit the number of threads in various libraries to prevent Segmentation faults caused by OpenMP multithreading conflicts under macOS.
          export OMP_NUM_THREADS=1  # Limit the number of OpenMP threads
          export MKL_NUM_THREADS=1  # Limit the number of Intel MKL threads
          export NUMEXPR_NUM_THREADS=1  # Limit the number of NumExpr threads
          export OPENBLAS_NUM_THREADS=1  # Limit the number of OpenBLAS threads
          export VECLIB_MAXIMUM_THREADS=1  # Limit the number of macOS Accelerate/vecLib threads
          cd tests
          python -m pytest . -m "not slow" --durations=0

    - name: Unit tests with Pytest (Ubuntu and Windows)
      if: ${{ matrix.os != 'macos-13' && matrix.os != 'macos-14' && matrix.os != 'macos-15' }}
      uses: nick-fields/retry@v2
      with:
        timeout_minutes: 60
        max_attempts: 3
        command: |
          cd tests
          python -m pytest . -m "not slow" --durations=0