pip.yml

---
# After building and testing the pip package,
# this workflow publishes it to Github Release by default
# and to Test PyPI (TEST) or PyPI (PROD) if requested
name: Pip Package
env:
  DEFAULT_SAMPLES_REVISION: 11.0.0
on:
  workflow_dispatch:
    inputs:
      samples-revision:
        default: 11.0.0
        description: khiops-samples repo revision
      image-tag:
        default: latest
        description: Development Docker Image Tag
      pypi-target:
        type: choice
        default: None  # no publishing to any Package Index by default
        options: [None, testpypi, pypi]
        description: Publish to Test PyPI or PyPI
  pull_request:
    paths: [pyproject.toml, LICENSE.md, .github/workflows/pip.yml]
  push:
    tags: ['*']
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true
jobs:
  build:
    runs-on: ubuntu-22.04
    permissions:
      checks: write
      contents: read
      id-token: write
      packages: read
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          # Get Git tags so that versioneer can function correctly
          # See issue https://github.com/actions/checkout/issues/701
          fetch-depth: 0
      - name: Build Pip package
        run: |
          # This is needed so that the Git tag is parsed and the version is retrieved
          git config --global --add safe.directory $(realpath .)

          # Upgrade Pip
          pip install --upgrade pip

          # Install required Python build dependency
          pip install build

          # Build the source package (sdist) only          
          python3 -m build --sdist
      - name: Upload the package as artifact
        uses: actions/upload-artifact@v4
        with:
          name: pip-package
          path: ./dist/khiops*.tar.gz
  test:
    needs: build
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        container: [ubuntu22.04, rocky9, debian13]
    container:
      # 'latest' default image tag cannot be set as an environment variable,
      # because the `env` context is only accessible at the step level;
      # hence, it is hard-coded
      image: |-
        ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || 'latest' }}
    steps:
      - name: Set parameters as env
        run: |
          SAMPLES_REVISION=${{ inputs.samples-revision || env.DEFAULT_SAMPLES_REVISION }}
          echo "SAMPLES_REVISION=$SAMPLES_REVISION" >> "$GITHUB_ENV"
      - name: Checkout Khiops samples
        uses: actions/checkout@v4
        with:
          repository: khiopsml/khiops-samples
          ref: ${{ env.SAMPLES_REVISION }}
          token: ${{ secrets.GITHUB_TOKEN }}
          path: khiops-samples
      - name: Download package artifact
        uses: actions/download-artifact@v4
        with:
          name: pip-package
      - name: Install package
        shell: bash
        run: |
          # Allow Pip to write to its cache
          mkdir -p /github/home/.cache/pip
          chown -R $(whoami) /github/home/.cache/pip
          # Install the Khiops Python library

          # A virtual env is mandatory under debian
          if [[ "${{ matrix.container }}" == "debian13" ]]; then
            python -m venv khiops-debian-venv
            source khiops-debian-venv/bin/activate
          fi
          pip install --upgrade pip
          pip install $(ls khiops*.tar.gz)
          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
            deactivate
          fi
      - name: Run tests
        env:
          KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
          # Force > 2 CPU cores to launch mpiexec
          KHIOPS_PROC_NUMBER: 4
          # Oversubscribe for Open MPI 4.x
          rmaps_base_oversubscribe: true
          OMPI_MCA_rmaps_base_oversubscribe: true
          # Oversubscribe for Open MPI >= 5
          PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
        shell: bash
        run: |-
          # Make sure MPI support is not loaded through env modules
          # Note: As the Docker container's shell is non-interactive, environment
          # modules are currently not initializing the shell anyway
          if [ -n "$MODULESHOME" ]; then module unload mpi; fi

          # A virtual env is mandatory under debian
          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
            source khiops-debian-venv/bin/activate
          fi
          if [[ $RUNNER_OS != 'Windows' ]]; then
            # Set rights to OpenMPI-specific folder
            mkdir -p /github/home/.pmix/components
            chown -R $(whoami) /github/home/.pmix/components
          fi

          # Print khiops installation status
          kh-status

          # Run some simple training tasks
          kh-samples core -i train_predictor -e
          kh-samples core -i train_coclustering -e
          kh-samples sklearn -i khiops_classifier -e

          # Test that the line containing "MPI command" does not contain "<empty>"
          # (as given by kh-status in the absence of MPI support)
          # The MPI command is not always named mpiexec, but can be orterun etc
          # (as given by khiops_env)
          kh-status | grep "MPI command" | grep -vwq "<empty>"
          if [[ "${{ matrix.container }}" == "debian13" ]]; then
            deactivate
          fi
      - name: Test package / Git tag version coherence
        shell: bash
        if: github.ref_type == 'tag'
        run: |
          # Don't exit on first error: print relevant error message
          set +e
          # A virtual env is mandatory under Debian 13
          if [[ "${{ matrix.container }}" == "debian13" ]]; then
            python -m venv khiops-debian-venv
            source khiops-debian-venv/bin/activate
          fi
          # Convert pre-release version specification in the Git tag to the Pip
          # format and check that it matches the Pip package version
          PACKAGE_VERSION=$(pip show khiops | awk 'BEGIN {FS = ": "} $1 ~ /^Version$/ {print $2}')
          if [[ "${{ matrix.container }}" == "debian13" ]]; then
            deactivate
          fi
          echo ${{ github.ref_name }} | tr -d '-' | rev | sed -E 's/\.([^0-9].*)/\1/' | rev | \
            grep -wq $PACKAGE_VERSION
          if [[ $? -ne 0 ]]
          then
            echo "::error::Python package version $PACKAGE_VERSION does not match Git tag ${{ github.ref_name }}"
            false
          fi
  # The implementation of a unique release job with multiple conditional steps
  # for each publishing mode is not chosen
  # because of the required job environment verified for Trusted Publishing
  release-github:
    # only publish on tag pushes
    if: github.ref_type == 'tag' && github.repository_owner == 'KhiopsML'
    needs: [build, test]
    runs-on: ubuntu-22.04
    permissions:
      contents: write
    steps:
      - name: Download package artifact
        uses: actions/download-artifact@v4
        with:
          name: pip-package
      - name: Upload Pip source package to the github release
        uses: ncipollo/release-action@v1.15.0
        with:
          allowUpdates: true
          artifacts: khiops*.tar.gz
          body: '**For testing purposes only**'
          draft: false
          makeLatest: false
          prerelease: true
          updateOnlyUnreleased: true
  release-testpypi:
    # only publish on testpypi if requested and on tag pushes
    if: inputs.pypi-target == 'testpypi' && github.ref_type == 'tag' && github.repository_owner == 'KhiopsML'
    needs: [build, test]
    runs-on: ubuntu-22.04
    permissions:
      # IMPORTANT: OIDC token mandatory for trusted publishing on TestPyPI and PyPI
      id-token: write
    # required job environment verified for Trusted Publishing
    environment:
      name: testpypi
    steps:
      - name: Download package artifact
        uses: actions/download-artifact@v4
        with:
          name: pip-package
          path: dist/
      - name: Publish Python distribution to TestPyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          verbose: true
          repository-url: https://test.pypi.org/legacy/
  release-pypi:
    # only publish on pypi if requested and on tag pushes
    if: inputs.pypi-target == 'pypi' && github.ref_type == 'tag' && github.repository_owner == 'KhiopsML'
    needs: [build, test]
    runs-on: ubuntu-22.04
    permissions:
      # IMPORTANT: OIDC token mandatory for trusted publishing on TestPyPI and PyPI
      id-token: write
    # required job environment verified for Trusted Publishing
    environment:
      name: pypi
    steps:
      - name: Download package artifact
        uses: actions/download-artifact@v4
        with:
          name: pip-package
          path: dist/
      - name: Publish Python distribution to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          verbose: true