pypsa-de/.github/workflows/test.yaml at main · PyPSA/pypsa-de · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
name: Test

on:
  push:
    branches:
    - main
  pull_request:
    branches:
    - main
  schedule:
  - cron: "0 5 * * 0"
  workflow_dispatch:

# Cancel any in-progress runs when a new run is triggered
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  unit-tests:
    name: Unit
    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash -l {0}

    steps:
    - uses: actions/checkout@v6

    - name: Setup Pixi
      uses: prefix-dev/setup-pixi@v0.9.5
      with:
        pixi-version: v0.59.0
        cache: true
        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}

    - name: Run unit tests
      run: |
        pixi run unit-tests

  integration-tests:
    name: Integration
    runs-on: ${{ matrix.os }}-latest
    strategy:
      fail-fast: false
      matrix:
        # Run windows only on scheduled runs on Sundays, otherwise ignore
        os: ${{ github.event.schedule == '0 5 * * 0' && fromJson('["ubuntu", "macos", "windows"]') || fromJson('["ubuntu", "macos"]') }}

    defaults:
      run:
        shell: bash -l {0}

    steps:
    - uses: actions/checkout@v6

    - uses: dorny/paths-filter@v4
      id: filter
      with:
        filters: |
          src:
            - 'scripts/**'
            - 'rules/**'
            - 'data/**'
            - 'Snakefile'
            - 'config/**'
            - 'pixi.toml'
            - 'pixi.lock'
            - '.github/workflows/test.yaml'

    - name: Free up disk space
      run: |
        echo "Initial disk space"
        df -h
        echo "Free up disk space"
        sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
        sudo docker image prune --all --force
        sudo docker builder prune -a --force
        echo "Final disk space"
        df -h

    - name: Skip - no source changes
      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
      run: echo "Skipping tests because no source code changes detected"

    - name: Setup Pixi
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
      uses: prefix-dev/setup-pixi@v0.9.5
      with:
        pixi-version: v0.59.0
        cache: true
        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}

    - name: Setup cache keys
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
      run: |
        echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data
        echo "MONTH=$(date +'%Y%m')" >> $GITHUB_ENV # cutouts
        echo "VERSIONS_HASH=${{ hashFiles('data/versions.csv') }}" >> $GITHUB_ENV

    - uses: actions/cache@v5
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
      with:
        path: data
        key: data-${{ env.WEEK }}-${{ env.VERSIONS_HASH }}

    - uses: actions/cache@v5
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
      with:
        path: cutouts
        key: cutouts-${{ env.MONTH }}

    - name: Restore git-tracked files in data/
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
      run: git checkout HEAD -- data/

    - name: Run pylint check on scripts
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
      # check for undefined variables to reuse functions across scripts
      run: |
        pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*

    - name: Run snakemake test workflows
      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
      env:
        SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
        SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
      run: |
        pixi run integration-tests

    - name: Upload artifacts
      if: always()
      uses: actions/upload-artifact@v7
      with:
        name: results-${{ matrix.os }}
        path: |
          logs
          .snakemake/log
          results
        retention-days: 3

    - name: Show remaining disk space
      if: always()
      run: df -h