OpenFed/.github/workflows/build.yml at main · FederalLab/OpenFed · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
name: build

on: [push, pull_request]

jobs:
  lint:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - name: Install pre-commit hook
        run: |
          pip install pre-commit
          pre-commit install
      - name: Linting
        run: pre-commit run --all-files

  build_cpu:
    runs-on: ubuntu-18.04
    strategy:
      matrix:
        python-version: [3.6, 3.7, 3.8]
        torch: [1.5.1, 1.6.0, 1.7.0, 1.8.0, 1.9.0]
        include:
          - torch: 1.5.1
            torchvision: 0.6.1
          - torch: 1.6.0
            torchvision: 0.7.0
          - torch: 1.7.0
            torchvision: 0.8.1
          - torch: 1.8.0
            torchvision: 0.9.0
          - torch: 1.9.0
            torchvision: 0.10.0
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install system dependencies
        run: sudo apt-get update
      - name: Install PyTorch
        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
      - name: Upgrade Setuptools
        run: pip install setuptools --upgrade
      - name: Build and install
        run: rm -rf .eggs && pip install -e .
      - name: Validate the installation
        run: python -c "import openfed; print(openfed.__version__)"
      - name: Run unittests and generate coverage report
        run: |
          pip install -r requirements/test.txt
          coverage run --branch --source=openfed -p -m pytest tests/ -k 'not aggregator and not collaborator'
          coverage combine
          coverage xml
          coverage report -m

  build_cu102:
    runs-on: ubuntu-18.04
    env:
      CUDA: 10.2.89-1
      CUDA_SHORT: 10.2
      UBUNTU_VERSION: ubuntu1804
      FORCE_CUDA: 1
    strategy:
      matrix:
        python-version: [3.7]
        torch: [1.9.0+cu102]
        include:
          - torch: 1.9.0+cu102
            torchvision: 0.10.0+cu102

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install CUDA
        run: |
          export INSTALLER=cuda-repo-${UBUNTU_VERSION}_${CUDA}_amd64.deb
          wget http://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/x86_64/${INSTALLER}
          sudo dpkg -i ${INSTALLER}
          wget https://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/x86_64/7fa2af80.pub
          sudo apt-key add 7fa2af80.pub
          sudo apt update -qq
          sudo apt install -y cuda-${CUDA_SHORT/./-} cuda-cufft-dev-${CUDA_SHORT/./-}
          sudo apt clean
          export CUDA_HOME=/usr/local/cuda-${CUDA_SHORT}
          export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${CUDA_HOME}/include:${LD_LIBRARY_PATH}
          export PATH=${CUDA_HOME}/bin:${PATH}
          sudo apt-get install -y ninja-build
      - name: Install PyTorch
        run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
      - name: Install system dependencies
        run: sudo apt-get update
      - name: Upgrade Setuptools
        run: pip install setuptools --upgrade
      - name: Build and install
        run: rm -rf .eggs && pip install -e .
      - name: Validate the installation
        run: python -c "import openfed; print(openfed.__version__)"
      - name: Run unittests and generate coverage report
        run: |
          pip install -r requirements/test.txt
          coverage run --branch --source=openfed -p -m pytest tests/ -k 'not aggregator and not collaborator'
          coverage run --branch --source=openfed -p -m pytest -n 3 tests/test_federated/test_federated.py -k 'federated'
          coverage run --branch --source=openfed -p -m pytest -n 3 tests/test_core/test_maintainer.py -k 'maintainer'
          coverage run --branch --source=openfed -p -m pytest -n 3 tests/test_simulator.py -k 'simulator'
          coverage run --branch --source=openfed -p -m pytest -n 2 tests/test_paillier_crypto.py -k 'paillier_crypto'
          coverage combine
          coverage xml
          coverage report -m
      # Only upload coverage report for python3.7 && pytorch1.6
      - name: Upload coverage to Codecov
        if: ${{matrix.torch == '1.6.0+cu102' && matrix.python-version == '3.7'}}
        uses: codecov/codecov-action@v1.0.14
        with:
          file: ./coverage.xml
          flags: unittests
          env_vars: OS,PYTHON
          name: codecov-umbrella
          fail_ci_if_error: false

  build_macos:
    runs-on: macos-latest
    strategy:
      matrix:
        torch: [1.5.1, 1.6.0, 1.7.0, 1.8.0, 1.9.0]
        include:
          - torch: 1.5.1
            torchvision: 0.6.1
          - torch: 1.6.0
            torchvision: 0.7.0
          - torch: 1.7.0
            torchvision: 0.8.1
          - torch: 1.8.0
            torchvision: 0.9.0
          - torch: 1.9.0
            torchvision: 0.10.0
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - name: Install PyTorch
        run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} --no-cache-dir
      - name: Upgrade Setuptools
        run: pip install setuptools --upgrade
      - name: Build and install
        run: |
          rm -rf .eggs
          CC=clang CXX=clang++ CFLAGS='-stdlib=libc++' pip install -e .
      - name: Validate the installation
        run: python -c "import openfed; print(openfed.__version__)"
      - name: Run unittests
        run: |
          pip install -r requirements/test.txt
          coverage run --branch --source=openfed -p -m pytest tests/ -k 'not aggregator and not collaborator'
          coverage combine
          coverage xml
          coverage report -m