tests.yml

---
name: Tests
env:
  DEFAULT_SAMPLES_REVISION: 11.0.0
  DEFAULT_KHIOPS_DESKTOP_REVISION: 11.0.0
on:
  workflow_dispatch:
    inputs:
      samples-revision:
        default: 11.0.0
        description: Git Tag/Branch/Commit for the khiops-samples Repo
      image-tag:
        default: latest
        description: Development Docker Image Tag
      khiops-desktop-revision:
        default: 11.0.0
        description: Khiops Windows Desktop Application Version
      run-expensive-tests:
        type: boolean
        required: false
        default: false
        description: Execute expensive tests
  pull_request:
    paths:
      - khiops/**.py
      - tests/**.py
      - tests/resources/**
      - '!tests/resources/**.md'
      - .github/workflows/tests.yml
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true
jobs:
  run:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
    container:
      # 'latest' default image tag cannot be set as an environment variable,
      # because the `env` context is only accessible at the step level;
      # hence, it is hard-coded
      image: |-
        ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:${{ inputs.image-tag || 'latest' }}
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      id-token: write
      contents: read
      checks: write
      packages: read
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          # Get Git tags so that versioneer can function correctly
          # See issue https://github.com/actions/checkout/issues/701
          fetch-depth: 0
      # We move SAMPLES_REVISION to the environment so that we can use
      # them in both push and workflow_dispatch events
      - name: Set SAMPLES_REVISION on 'pull_request' events
        if: github.event_name == 'pull_request'
        run: echo "SAMPLES_REVISION=${DEFAULT_SAMPLES_REVISION}" >> "$GITHUB_ENV"
      - name: Set SAMPLES_REVISION on 'workflow_dispatch' events
        if: github.event_name == 'workflow_dispatch'
        run: echo "SAMPLES_REVISION=${{ inputs.samples-revision }}" >> "$GITHUB_ENV"
      - name: Checkout Khiops samples
        uses: actions/checkout@v4
        with:
          repository: khiopsml/khiops-samples
          ref: ${{ env.SAMPLES_REVISION }}
          token: ${{ secrets.GITHUB_TOKEN }}
          path: khiops-samples
      - name: Setup and Install Test Requirements
        if: success() || failure()
        run: |
          CONDA="/root/miniforge3/bin/conda"

          # Native Khiops-based Conda environment, and
          # `khiops-core`-based Conda environment
          CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda"
          for CONDA_ENV in $CONDA_ENVS
          do
            mkdir -p -m u+rwx reports/"$CONDA_ENV"

            # install within the conda environments without activating them
            $CONDA install -y -n "$CONDA_ENV" unittest-xml-reporting
            $CONDA install -y -n "$CONDA_ENV" --file test-requirements.txt
          done
      - name: Install khiops-python dependencies
        if: success() || failure()
        run: |
          # The following git command is required,
          # as the Git repository is in a directory the current user does not own,
          # Python versioneer fails to compute the current version correctly otherwise
          git config --global --add safe.directory $(realpath .)
          CONDA="/root/miniforge3/bin/conda"
          # Native Khiops-based Conda environment, and
          # `khiops-core`-based Conda environment
          CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda"
          for CONDA_ENV in $CONDA_ENVS
          do
            # Since Python 3.13, setuptools is not installed automatically anymore
            $CONDA install -y -n "$CONDA_ENV" setuptools

            # Add homogeneous TOML support (Python >= 3.12 has standard tomllib)
            $CONDA install -y -n "$CONDA_ENV" tomli
            $CONDA run --no-capture-output -n "$CONDA_ENV" python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" > requires.txt
            $CONDA install -y -n "$CONDA_ENV" `cat requires.txt`
            rm -f requires.txt
          done
      - name: Configure Expensive Tests Setting
        # Skip expensive tests by default, unless on the `main-v10` or `main` branches
        if: github.ref != 'main-v10' && github.ref != 'main' && ! inputs.run-expensive-tests
        run: echo "SKIP_EXPENSIVE_TESTS=true" >> "$GITHUB_ENV"
      - name: Prepare Integration Tests on remote files
        if: env.SKIP_EXPENSIVE_TESTS != 'true'
        env:
          AWS_ENDPOINT_URL: http://localhost:4569
        shell: bash
        run: |
          # Prepare AWS-S3 credentials and configuration          
          mkdir -p ${GITHUB_WORKSPACE}/.aws/
          cat << EOF > ${GITHUB_WORKSPACE}/.aws/credentials
          [default]
          aws_access_key_id=KEY
          aws_secret_access_key=SECRET           
          EOF
          cat << EOF > ${GITHUB_WORKSPACE}/.aws/configuration
          [default]
          endpoint_url=${AWS_ENDPOINT_URL}
          region=eu-north-1          
          EOF
          echo "Generated AWS credentials..."
          cat  ${GITHUB_WORKSPACE}/.aws/credentials
          echo "Generated AWS configuration..."
          cat  ${GITHUB_WORKSPACE}/.aws/configuration
          /scripts/run_fake_remote_file_servers.sh .  # launch the servers in the background
          # Set environment variables for the tests with GCS
          GCS_BUCKET_NAME=data-test-khiops-driver-gcs/khiops_data
          GCS_DRIVER_LOGLEVEL=info  # set to debug for diagnosis
          # Set environment variables for the tests with S3
          S3_DRIVER_LOGLEVEL=info  # set to debug for diagnosis
          S3_BUCKET_NAME=s3-bucket
          AWS_SHARED_CREDENTIALS_FILE=${{ github.workspace }}/.aws/credentials
          AWS_CONFIG_FILE=${{ github.workspace }}/.aws/configuration
          # Persist environment variables for subsequent steps
          echo "GCS_BUCKET_NAME=${GCS_BUCKET_NAME}" >> "$GITHUB_ENV"
          echo "GCS_DRIVER_LOGLEVEL=${GCS_DRIVER_LOGLEVEL}" >> "$GITHUB_ENV"
          echo "S3_DRIVER_LOGLEVEL=${S3_DRIVER_LOGLEVEL}" >> "$GITHUB_ENV"
          echo "S3_BUCKET_NAME=${S3_BUCKET_NAME}" >> "$GITHUB_ENV"
          echo "AWS_SHARED_CREDENTIALS_FILE=${AWS_SHARED_CREDENTIALS_FILE}" >> "$GITHUB_ENV"
          echo "AWS_CONFIG_FILE=${AWS_CONFIG_FILE}" >> "$GITHUB_ENV"
      - name: Authenticate to GCP using "Workload Identity Federation"
        if: env.SKIP_EXPENSIVE_TESTS != 'true'
        # For integration tests on GCS we use a real Google account
        # Retrieve the Google credentials through "Workload Identity Federation"
        # see https://github.com/google-github-actions/auth?tab=readme-ov-file#workload-identity-federation-through-a-service-account
        uses: google-github-actions/auth@v2
        with:
          service_account: khiops-gcs-driver-test-sa@ino-olr-dak-ideal-sbx.iam.gserviceaccount.com
          workload_identity_provider: projects/322269704080/locations/global/workloadIdentityPools/github/providers/my-repo
          # 'create_credentials_file' is true by default but let's make it explicit
          # After authentication, the required GOOGLE_APPLICATION_CREDENTIALS environment variable is exported
          # https://github.com/google-github-actions/auth?tab=readme-ov-file#inputs-miscellaneous
          create_credentials_file: true
      - name: Run Unit & Integration Tests
        env:
          KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
          KHIOPS_DOCKER_RUNNER_URL: https://localhost:11000
          KHIOPS_DOCKER_RUNNER_SHARED_DIR: /tmp/sandbox
          KHIOPS_RUNNER_SERVICE_PATH: /scripts/run_service.sh
          # Force > 2 CPU cores to launch mpiexec
          KHIOPS_PROC_NUMBER: 4
          # Oversubscribe for Open MPI 4.x
          rmaps_base_oversubscribe: true
          OMPI_MCA_rmaps_base_oversubscribe: true
          # Oversubscribe for Open MPI >= 5
          PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
          # Var for tests with S3
          no_proxy: localhost
        run: |
          # This is needed so that the Git tag is parsed and the khiops-python
          # version is retrieved
          git config --global --add safe.directory $(realpath .)
          CONDA="/root/miniforge3/bin/conda"
          # Native Khiops-based Conda environment, and
          # `khiops-core`-based Conda environment
          CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda"
          for CONDA_ENV in $CONDA_ENVS
          do
            $CONDA run --no-capture-output -n "$CONDA_ENV" coverage run -m xmlrunner -o "reports/$CONDA_ENV" -v
            $CONDA run --no-capture-output -n "$CONDA_ENV" coverage report -m
            $CONDA run --no-capture-output -n "$CONDA_ENV" coverage xml -o "reports/$CONDA_ENV/py-coverage.xml"
          done
      - name: Display Test Reports
        if: success() || failure()
        uses: dorny/test-reporter@v1
        with:
          name: Run Tests ${{ matrix.python-version }}
          path: >-
            reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml,
            reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml
          reporter: java-junit
          path-replace-backslashes: 'true'  # Necessary for windows paths
          fail-on-error: 'false'
      - name: Upload Test Reports as Artifacts
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: test-reports-${{ matrix.python-version }}
          path: |-
            reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml
            reports/py${{ matrix.python-version }}/py-coverage.xml
            reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml
            reports/py${{ matrix.python-version }}_conda/py-coverage.xml
            tests/resources/scenario_generation/*/ref/*._kh
            tests/resources/scenario_generation/*/output/*._kh
            tests/resources/*/output_reports/*.txt
            tests/resources/*/ref_reports/*.txt
            tests/resources/dictionary/ref_kdic/*.kdic
            tests/resources/dictionary/output_kdic/*.kdic
            tests/resources/dictionary/copy_output_kdic/*.kdic
            tests/resources/general_options/general_options/*/*._kh
          retention-days: 7
  check-khiops-integration-on-windows:
    runs-on: windows-2022
    steps:
      - name: Download the Khiops Desktop NSIS Installer
        shell: pwsh
        run: |
          $KHIOPS_DESKTOP_REVISION = '${{ inputs.khiops-desktop-revision || env.DEFAULT_KHIOPS_DESKTOP_REVISION }}'
          $KHIOPS_DOWNLOAD_URL = "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_DESKTOP_REVISION}/khiops-${KHIOPS_DESKTOP_REVISION}-setup.exe"
          Invoke-WebRequest "${KHIOPS_DOWNLOAD_URL}" `
            -OutFile .\khiops-setup.exe `
            -UseBasicParsing
          Unblock-File .\khiops-setup.exe
      - name: Install the Khiops Desktop Application
        shell: pwsh
        run: |
          # Execute the installer
          $ErrorActionPreference = 'Stop'
          $ProgressPreference = 'SilentlyContinue'
          Start-Process `
            -FilePath .\khiops-setup.exe `
            -ArgumentList '/S' `
            -Wait
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          # Get Git tags so that versioneer can function correctly
          # See issue https://github.com/actions/checkout/issues/701
          fetch-depth: 0
      - name: Checkout Khiops samples
        uses: actions/checkout@v4
        with:
          repository: khiopsml/khiops-samples
          ref: ${{ env.SAMPLES_REVISION }}
          token: ${{ secrets.GITHUB_TOKEN }}
          path: khiops-samples
      - name: Setup Python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Install khiops-python dev dependencies
        shell: pwsh
        run: |
          # The following git command is required,
          # as the Git repository is in a directory the current user does not own,
          # Python versioneer fails to compute the current version correctly otherwise
          git config --global --add safe.directory $(Resolve-Path '.' | % {$_.toString()})
          python -m pip install setuptools
          python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" -s "\n"  > requires.txt

          # Install the Python requirements outside a python venv
          Get-Content .\requires.txt `
            | ForEach-Object {python -m pip install $_.toString()}

          # Create and activate a python venv
          python -m venv khiops-windows-venv
          khiops-windows-venv\Scripts\Activate.ps1

          # Install the Python requirements inside a venv
          # The venv python executable is used here
          Get-Content .\requires.txt `
            | ForEach-Object {khiops-windows-venv\Scripts\python -m pip install $_.toString()}

          # Deactivate the python venv
          deactivate
          Remove-Item -force requires.txt
      - name: Setup and Install Test Requirements
        run: python -m pip install -r test-requirements.txt
      - name: Test Khiops Integration
        env:
          # Force > 2 CPU cores to launch MPI
          KHIOPS_PROC_NUMBER: 4
          KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
        shell: pwsh
        run: |
          # This $ErrorActionPreference directive stops the current invoked expression not the whole script          
          $ErrorActionPreference = 'Stop'
          # Set the following $PSNativeCommandUseErrorActionPreference to avoid stopping the whole script in case of error
          # so that the return code can be evaluated if needed
          $PSNativeCommandUseErrorActionPreference = $false
          Set-StrictMode -Version Latest

          # Refresh environment variables by using the Chocolatey tool
          # Otherwise, the env vars set in the registry by the Khiops installer do not get updated
          # See also https://github.com/actions/runner-images/discussions/6065#discussioncomment-3517318
          Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
          refreshenv

          # Invoke the Python version set-up by the `setup-python` step / action
          # Otherwise invoking `python` directly invokes the system-cached Python on the runner
          # This is a side-effect of refreshing the environment
          $Python = "${{ steps.setup-python.outputs.python-path }}"

          # Print status
          Invoke-Expression -Command "$Python -c 'import sys; import khiops.core as kh; return_code = kh.get_runner().print_status(); sys.exit(return_code)'"
          # The installation status check will fail
          # because the library was not installed properly (the source code was only cloned)
          if ($LASTEXITCODE -ne 0) {
            Write-Host "::error::Status error: improper setup, as expected: khiops-python has been cloned, not installed from a package"
          }

          # Run integration tests on Windows
          Invoke-Expression -Command "$Python -m unittest -v tests.test_khiops_integrations"

          # Execute Khiops sample (train and deploy model)
          Invoke-Expression -Command "$Python -m khiops.samples.samples -i deploy_model -e"

          # Execute Khiops Coclustering sample (train and deploy model)
          Invoke-Expression -Command "$Python -m khiops.samples.samples -i deploy_coclustering -e"

          # Install khiops-python in the python venv using the sources          
          # The venv python executable is used here 
          Invoke-Expression -Command "khiops-windows-venv\Scripts\python -m pip install ."
          # Change directory to avoid using the cloned khiops-python
          Invoke-Expression -Command "cd khiops-windows-venv\"

          # Print status
          # The venv python executable is used here
          Invoke-Expression -Command "Scripts\python -c 'import sys; import khiops.core as kh; return_code = kh.get_runner().print_status(); sys.exit(return_code)'"

          # The installation status MUST not fail                    
          if ($LASTEXITCODE -ne 0) {
            Write-Host "::error::Status error: khiops-python installation status check MUST NOT fail"
            exit 1
          }
  check-khiops-integration-on-linux:
    strategy:
      fail-fast: false
      matrix:
        container: [ubuntu22.04, rocky8, rocky9, debian13]
    runs-on: ubuntu-latest
    container:
      # 'latest' default image tag cannot be set as an environment variable,
      # because the `env` context is only accessible at the step level;
      # hence, it is hard-coded
      image: |-
        ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || 'latest' }}
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      id-token: write
      contents: read
      checks: write
      packages: read
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          # Get Git tags so that versioneer can function correctly
          # See issue https://github.com/actions/checkout/issues/701
          fetch-depth: 0
      - name: Checkout Khiops samples
        uses: actions/checkout@v4
        with:
          repository: khiopsml/khiops-samples
          ref: ${{ env.SAMPLES_REVISION }}
          token: ${{ secrets.GITHUB_TOKEN }}
          path: khiops-samples
      - name: Install khiops-python dev dependencies
        shell: bash
        run: |
          # The following git command is required,
          # as the Git repository is in a directory the current user does not own,
          # Python versioneer fails to compute the current version correctly otherwise
          git config --global --add safe.directory $(realpath .)

          # A virtual env is mandatory under debian
          if [[ "${{ matrix.container }}" == "debian13" ]]; then
            python -m venv khiops-debian-venv
            source khiops-debian-venv/bin/activate                      
          fi

          # Install tomli for Python < 3.11
          pip install tomli
          python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" > requires.txt
          pip install `cat requires.txt`
          rm -f requires.txt
          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
            deactivate                      
          fi
      - name: Setup and Install Test Requirements
        shell: bash
        run: |-
          # A virtual env is mandatory under debian
          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
            source khiops-debian-venv/bin/activate                      
          fi
          pip install -r test-requirements.txt
          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
            deactivate                      
          fi
      - name: Test Khiops Integration
        env:
          # Force > 2 CPU cores to launch mpiexec
          KHIOPS_PROC_NUMBER: 4
          KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
          # Oversubscribe for Open MPI 4.x
          rmaps_base_oversubscribe: true
          OMPI_MCA_rmaps_base_oversubscribe: true
          # Oversubscribe for Open MPI >= 5
          PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
        shell: bash
        run: |-
          # Reset the default 'exit-on-error' mode of the bash shell in Github actions
          # so that the return code can be evaluated if needed
          set +e

          # Make sure MPI support is not loaded through env modules
          # Note: As Docker container's shell is non-interactive, environment
          # modules are currently not initializing the shell anyway
          if [ -n "$MODULESHOME" ]; then module unload mpi; fi

          # A virtual env is mandatory under debian
          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
            source khiops-debian-venv/bin/activate                      
          fi

          # Set rights to OpenMPI-specific folder
          mkdir -p /github/home/.pmix/components
          chown -R $(whoami) /github/home/.pmix/components

          # Print status
          PATTERN=$(python -c \
            "import sys; import khiops.core as kh; return_code = kh.get_runner().print_status(); sys.exit(return_code)" 2> >(grep -Ei 'with.*?unknown.*?installer') )                    
          # The installation status check will fail
          # because the library was not installed properly (the source code was only cloned)
          if [ -n "$PATTERN" ]; then
            echo "::error::Status error: improper setup, as expected: khiops-python has been cloned, not installed from a package"
          fi

          # Run the library against an incompatible Khiops (with a different major version)
          # This instance of Khiops is isolated in a dedicated conda environment
          CONDA="/root/miniforge3/bin/conda"                
          # Check an error is raised because of the major version mismatch
          # The khiops-python library from the cloned sources is used here
          PATTERN=$($CONDA run -n py3_khiops10_conda python -c "import khiops.core as kh; print(kh.get_runner().khiops_version)" 2> >(grep -Ei 'major version.*?does not match'))          
          if [ -z "$PATTERN" ]; then
            echo "::error::Status error: khiops-python should fail because of the major version mismatch"          
            if [[ "${{ matrix.container }}" == "debian13" ]]; then            
              deactivate                      
            fi          
            exit 1;
          fi

          # Run the remaining integration tests
          python -m unittest -v tests.test_khiops_integrations

          # Execute Khiops sample (train and deploy model), which uses MPI
          python -m khiops.samples.samples -i deploy_model -e
          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
            deactivate                      
          fi