pipeline.yml

name: Pipeline

on:
  push:
    branches: [ main ]
    paths-ignore:
      - 'docs/**'
  pull_request:
    paths-ignore:
      - 'docs/**'

jobs:
  black:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: psf/black@stable
      with:
        options: "--check --diff"
        src: ./src/${{ github.event.repository.name }}

  black_fix:  # in most cases pre-commit is faster
    needs: [black]
    if: failure()
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        token: ${{ secrets.DEPENDABOT_WORKFLOW_TOKEN }}
        ref: ${{ github.event.pull_request.head.ref }} # Check out the head of the actual branch, not the PR
        fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
    - name: format black
      uses: psf/black@stable
      with:
        options: ""
        src: "./src/${{ github.event.repository.name }}"
    - name: commit
      run: |
        git config --local user.email "pyiron@mpie.de"
        git config --local user.name "pyiron-runner"
        git commit -m "Format black" -a
    - name: push
      uses: ad-m/github-push-action@master
      with:
        github_token: ${{ secrets.DEPENDABOT_WORKFLOW_TOKEN }}
        branch: ${{ github.event.pull_request.head.ref }}

  mypy:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"
          architecture: x64
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install mypy
        run: pip install mypy
      - name: Test
        run: mypy --ignore-missing-imports src/${{ github.event.repository.name }}

  minimal:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: "3.13"
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-mini.yml
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        pip install . --no-deps --no-build-isolation
        cd tests
        python -m unittest discover .

  pip_check:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: '3.14'
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-openmpi.yml
    - name: Setup
      shell: bash -l {0}
      run: |
        pip install tomlkit
        python .ci_support/check.py
        cat pyproject.toml
        pip install . --no-deps --no-build-isolation
        pip check

  uv_check:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: astral-sh/setup-uv@v7
    - name: Install the project and pip check
      shell: bash -l {0}
      run: |
        uv sync --all-extras --dev
        pip check

  benchmark:
    needs: [black]
    runs-on: ${{ matrix.operating-system }}
    strategy:
      matrix:
        include:
        - operating-system: ubuntu-latest
          python-version: '3.14'
          environment-file: .ci_support/environment-openmpi.yml

        - operating-system: ubuntu-latest
          python-version: '3.14'
          environment-file: .ci_support/environment-mpich.yml

    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: ${{ matrix.python-version }}
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: ${{ matrix.environment-file }}
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 10
      run: |
        pip install . --no-deps --no-build-isolation
        python tests/benchmark/llh.py static >> timing.log
        python tests/benchmark/llh.py process >> timing.log
        python tests/benchmark/llh.py thread >> timing.log
        mpiexec -n 4 python -m mpi4py.futures tests/benchmark/llh.py mpi4py >> timing.log
        python tests/benchmark/llh.py executorlib >> timing.log
        python tests/benchmark/llh.py block_allocation >> timing.log
        cat timing.log
        python -m unittest tests/benchmark/test_results.py
      env:
        PRTE_MCA_rmaps_default_mapping_policy: ':oversubscribe'

  notebooks:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Merge Notebook environment
        run: |
          cp binder/environment.yml environment.yml
          tail --lines=+4 .ci_support/environment-notebooks.yml >> environment.yml
      - name: Setup Mambaforge
        uses: conda-incubator/setup-miniconda@v3
        with:
          python-version: "3.13"
          miniforge-version: latest
          channels: conda-forge
          conda-remove-defaults: "true"
          environment-file: environment.yml
      - name: Install
        shell: bash -l {0}
        run: pip install . --no-deps --no-build-isolation
      - name: Notebooks
        shell: bash -l {0}
        timeout-minutes: 5
        run: |
          papermill notebooks/1-single-node.ipynb notebooks/1-single-node-out.ipynb -k python3
          flux start papermill notebooks/2-hpc-cluster.ipynb notebooks/2-hpc-cluster-out.ipynb -k python3
          flux start papermill notebooks/3-hpc-job.ipynb notebooks/3-hpc-job-out.ipynb -k python3
          papermill notebooks/5-developer.ipynb notebooks/5-developer-out.ipynb -k python3

  notebooks_integration:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup Mambaforge
        uses: conda-incubator/setup-miniconda@v3
        with:
          python-version: "3.13"
          miniforge-version: latest
          channels: conda-forge
          conda-remove-defaults: "true"
          environment-file: .ci_support/environment-integration.yml
      - name: Install
        shell: bash -l {0}
        run: pip install . --no-deps --no-build-isolation
      - name: Notebooks
        shell: bash -l {0}
        timeout-minutes: 20
        run: |
          flux start papermill notebooks/4-1-gpaw.ipynb notebooks/4-1-gpaw-out.ipynb -k python3
          flux start papermill notebooks/4-2-quantum-espresso.ipynb notebooks/4-2-quantum-espresso-out.ipynb -k python3

  unittest_flux_mpich:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Extend environment
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        echo -e '- coverage\n- flux-core =0.81.0'>> .ci_support/environment-mpich.yml
        cat .ci_support/environment-mpich.yml
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: '3.13'
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-mpich.yml
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        pip install . --no-deps --no-build-isolation
        python -m unittest discover tests
    - name: Test Flux
      shell: bash -l {0}
      timeout-minutes: 5
      run: >
        flux start
        python -m unittest tests/unit/task_scheduler/interactive/test_spawner_flux.py tests/unit/executor/test_flux_job_plot.py tests/unit/executor/test_flux_job.py tests/unit/executor/test_flux_cluster.py;

  unittest_flux_openmpi:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Extend environment
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        echo -e '- coverage\n- flux-core =0.81.0\n- flux-pmix=0.6.0' >> .ci_support/environment-openmpi.yml
        cat .ci_support/environment-openmpi.yml
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: '3.13'
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-openmpi.yml
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        pip install . --no-deps --no-build-isolation
        coverage run -a
    - name: Test Flux with OpenMPI
      shell: bash -l {0}
      timeout-minutes: 5
      run: >
        flux start
        coverage run -a -m unittest tests/unit/task_scheduler/interactive/test_spawner_flux.py tests/unit/executor/test_flux_job_plot.py tests/unit/executor/test_flux_job.py tests/unit/executor/test_flux_cluster.py;
        coverage report;
        coverage xml
      env:
        EXECUTORLIB_PMIX: "pmix"
        TMPDIR: "/tmp"   # required by MacOs https://github.com/open-mpi/ompi/issues/7393
    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@v5
      with:
        token: ${{ secrets.CODECOV_TOKEN }}

  unittest_slurm_mpich:
    needs: [black]
    runs-on: ubuntu-latest
    services:
      mysql:
        image: mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: root
        ports:
          - "8888:3306"
        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
    steps:
    - uses: actions/checkout@v4
    - uses: koesterlab/setup-slurm-action@v1
      timeout-minutes: 10
    - name: ubnuntu install
      run: sudo apt install -y mpich
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: '3.13'
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-mpich.yml
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        pip install . --no-deps --no-build-isolation
        cd tests/unit
        sinfo -o "%n %e %m %a %c %C"
        srun --mpi=list
        python -m unittest executor/test_slurm_job.py
        python -m unittest executor/test_slurm_cluster.py

  unittest_mpich:
    needs: [black]
    runs-on: ${{ matrix.operating-system }}
    strategy:
      matrix:
        include:
        - operating-system: macos-latest
          python-version: '3.14'

        - operating-system: ubuntu-24.04-arm
          python-version: '3.14'

        - operating-system: ubuntu-22.04-arm
          python-version: '3.14'

        - operating-system: ubuntu-latest
          python-version: '3.14'

        - operating-system: ubuntu-latest
          python-version: '3.13'

        - operating-system: ubuntu-latest
          python-version: '3.12'

        - operating-system: ubuntu-latest
          python-version: '3.11'

    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: ${{ matrix.python-version }}
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-mpich.yml
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        pip install . --no-deps --no-build-isolation
        cd tests
        python -m unittest discover .

  unittest_openmpi:
    needs: [black]
    runs-on: ${{ matrix.operating-system }}
    strategy:
      matrix:
        include:
        - operating-system: macos-latest
          python-version: '3.14'

        - operating-system: ubuntu-latest
          python-version: '3.14'

        - operating-system: ubuntu-24.04-arm
          python-version: '3.14'

        - operating-system: ubuntu-22.04-arm
          python-version: '3.14'

        - operating-system: ubuntu-latest
          python-version: '3.13'

        - operating-system: ubuntu-latest
          python-version: '3.12'

        - operating-system: ubuntu-latest
          python-version: '3.11'

    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: ${{ matrix.python-version }}
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-openmpi.yml
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        pip install . --no-deps --no-build-isolation
        cd tests
        python -m unittest discover .

  unittest_win:
    needs: [black]
    runs-on: windows-latest
    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: "3.14"
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-win.yml
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        pip install . --no-deps --no-build-isolation
        cd tests
        python -m unittest discover .

  unittest_old:
    needs: [black]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: '3.10'
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
        environment-file: .ci_support/environment-old.yml
    - name: Test
      shell: bash -l {0}
      timeout-minutes: 5
      run: |
        pip install . --no-deps --no-build-isolation
        cd tests
        python -m unittest discover .
      env:
        OMPI_MCA_plm: 'isolated'
        OMPI_MCA_rmaps_base_oversubscribe: 'yes'
        OMPI_MCA_btl_vader_single_copy_mechanism: 'none'

  autobot:
    needs: [unittest_old, unittest_win, unittest_openmpi, unittest_mpich, unittest_flux_openmpi, unittest_flux_mpich, notebooks, benchmark, minimal, pip_check, mypy]
    permissions:
      contents: write
      pull-requests: write
    runs-on: ubuntu-latest
    if: (github.event.pull_request.user.login == 'dependabot[bot]' || github.event.pull_request.user.login == 'pre-commit-ci[bot]')
    steps:
      - name: Enable auto-merge for bot PRs
        run: gh pr merge --auto --squash "$PR_URL"
        env:
          PR_URL: ${{github.event.pull_request.html_url}}
          GH_TOKEN: ${{secrets.GITHUB_TOKEN}}

  uml:
    needs: [uv_check, unittest_slurm_mpich, unittest_old, unittest_win, unittest_openmpi, unittest_mpich, unittest_flux_openmpi, unittest_flux_mpich, notebooks, benchmark, minimal, pip_check, mypy]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        python-version: "3.13"
        miniforge-version: latest
        channels: conda-forge
        conda-remove-defaults: "true"
    - name: Generate UML diagram
      shell: bash -l {0}
      timeout-minutes: 10
      run: |
        conda install -y pylint graphviz
        pyreverse -o png -p ./src/${{ github.event.repository.name }} ./src/${{ github.event.repository.name }}
        zip -m uml.zip *.png
    - uses: actions/upload-artifact@v4
      with:
        name: uml.zip
        path: uml.zip