fastsafetensors/.github/workflows/test-torch.yaml at main · foundation-model-stack/fastsafetensors · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
name: test-torch

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

permissions:
  contents: read

jobs:
  build-and-test-torch:
    name: Test Python ${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libnuma-dev gcc g++ make
      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          tf_ver=5.0.0
          npy_spec="numpy>=2.2,<2.5"
          pip install torch==2.11.0 --index-url https://download.pytorch.org/whl/cpu
          pip install pytest pytest-cov setuptools_scm safetensors transformers==${tf_ver} "${npy_spec}"
      - name: Build package
        run: |
          pip install .
      - name: Run tests
        run: |
          cd tests/unit
          LIBDIR=`python3 -c "import os; os.chdir('/tmp'); import fastsafetensors; print(os.path.dirname(fastsafetensors.__file__))"`
          mkdir -p /tmp/pytest-log
          export TEST_FASTSAFETENSORS_FRAMEWORK=pytorch
          COVERAGE_FILE=.coverage_0 pytest -s --cov=${LIBDIR} test_fastsafetensors.py > /tmp/pytest-log/0.log 2>&1
          COVERAGE_FILE=.coverage_ep pytest -s --cov=${LIBDIR} test_ep_slice.py > /tmp/pytest-log/ep_slice.log 2>&1
          COVERAGE_FILE=.coverage_rob pytest -s --cov=${LIBDIR} test_robustness.py > /tmp/pytest-log/robustness.log 2>&1
          COVERAGE_FILE=.coverage_config pytest -s --cov=${LIBDIR} test_config.py > /tmp/pytest-log/config.log 2>&1
          COVERAGE_FILE=.coverage_auto pytest -s --cov=${LIBDIR} test_auto_loader.py > /tmp/pytest-log/auto_loader.log 2>&1
          COVERAGE_FILE=.coverage_3fs pytest -s --cov=${LIBDIR} threefs/ > /tmp/pytest-log/threefs.log 2>&1
          COVERAGE_FILE=.coverage_1 torchrun --nnodes=1 --master_addr=0.0.0.0 --master_port=1234 --node_rank=0 test_multi.py --cov=${LIBDIR} -s test_multi.py > /tmp/pytest-log/1.log 2>&1
          COVERAGE_FILE=.coverage_2 torchrun --nnodes=4 --master_addr=0.0.0.0 --master_port=1234 --node_rank=0 test_multi.py --cov=${LIBDIR} -s test_multi.py > /tmp/pytest-log/2.log 2>&1 &
          COVERAGE_FILE=.coverage_3 torchrun --nnodes=4 --master_addr=0.0.0.0 --master_port=1234 --node_rank=1 test_multi.py --cov=${LIBDIR} -s test_multi.py > /tmp/pytest-log/3.log 2>&1 &
          COVERAGE_FILE=.coverage_4 torchrun --nnodes=4 --master_addr=0.0.0.0 --master_port=1234 --node_rank=2 test_multi.py --cov=${LIBDIR} -s test_multi.py > /tmp/pytest-log/4.log 2>&1 &
          COVERAGE_FILE=.coverage_5 torchrun --nnodes=4 --master_addr=0.0.0.0 --master_port=1234 --node_rank=3 test_multi.py --cov=${LIBDIR} -s test_multi.py > /tmp/pytest-log/5.log 2>&1
          coverage combine .coverage_*
          coverage html
          mv htmlcov /tmp/pytest-log
      - name: Upload Pytest log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: pytest-log-${{ matrix.python-version }}
          path: /tmp/pytest-log

  build-and-test-torch-windows-nogds:
    name: Test Windows nogds smoke (Python 3.13)
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.13"
      - name: Install Python dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          pip install torch==2.11.0 --index-url https://download.pytorch.org/whl/cpu
          pip install pytest pytest-cov setuptools_scm safetensors transformers==5.0.0 "numpy>=2.2,<2.5"
          pip install .
      - name: Run tests
        shell: bash
        env:
          TEST_FASTSAFETENSORS_FRAMEWORK: pytorch
        run: |
          LOGDIR="${RUNNER_TEMP}/pytest-log"
          mkdir -p "${LOGDIR}"
          cd tests/unit
          LIBDIR=$(python -c "import os, tempfile; os.chdir(tempfile.gettempdir()); import fastsafetensors; print(os.path.dirname(fastsafetensors.__file__))")
          COVERAGE_FILE=.coverage_0 pytest -s --cov="${LIBDIR}" test_fastsafetensors.py -k "not test_init_gds and not test_close_gds and not test_GdsFileCopier and not test_SafeTensorsFileLoader and not test_SafeTensorsFileLoader_copier_selection" > "${LOGDIR}/0.log" 2>&1
          COVERAGE_FILE=.coverage_3fs pytest -s --cov="${LIBDIR}" threefs > "${LOGDIR}/threefs.log" 2>&1
          coverage combine .coverage_*
          coverage html -d "${LOGDIR}/htmlcov"
      - name: Upload Pytest log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: pytest-log-windows-3.13-nogds
          path: ${{ runner.temp }}/pytest-log