Merge pull request #592 from KhiopsML/try_fix_windows_without_env

popescu-v · web-flow · commit 53727176d36b · 2026-06-30T18:27:42.000+02:00
fix: use sysconfig to locate khiops_env.cmd on Windows
diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml
@@ -11,9 +11,6 @@ on:
       samples-revision:
         default: 11.0.0
         description: khiops-samples repo revision
-      image-tag:
-        default: latest
-        description: Development Docker Image Tag
       pypi-target:
         type: choice
         default: None  # no publishing to any Package Index by default
@@ -30,10 +27,7 @@ jobs:
   build:
     runs-on: ubuntu-22.04
     permissions:
-      checks: write
       contents: read
-      id-token: write
-      packages: read
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -53,30 +47,63 @@ jobs:
           pip install build
 
           # Build the source package (sdist) only          
-          python3 -m build --sdist
+          python -m build --sdist
       - name: Upload the package as artifact
         uses: actions/upload-artifact@v4
         with:
           name: pip-package
           path: ./dist/khiops*.tar.gz
+  # Test Pip package on brand new environments
   test:
     needs: build
-    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        container: [ubuntu22.04, rocky9, debian13]
-    container:
-      # 'latest' default image tag cannot be set as an environment variable,
-      # because the `env` context is only accessible at the step level;
-      # hence, it is hard-coded
-      image: |-
-        ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || 'latest' }}
+        env:
+          # Test without virtualenv only on platforms that support it out of the
+          # box (i.e. where passing `--break-system-packages` to Pip is not
+          # required)
+          - {os: ubuntu-24.04, json-image: '{"image": null}', use-virtualenv: true}
+          - {os: ubuntu-24.04, json-image: '{"image": null}', use-virtualenv: false}
+          - {os: ubuntu-22.04, json-image: '{"image": "debian:trixie"}', use-virtualenv: true}
+          - {os: ubuntu-22.04, json-image: '{"image": "rockylinux/rockylinux:10"}', use-virtualenv: true}
+          - {os: ubuntu-22.04, json-image: '{"image": "rockylinux/rockylinux:10"}', use-virtualenv: false}
+          - {os: windows-2025, json-image: '{"image": null}', use-virtualenv: true}
+          - {os: windows-2025, json-image: '{"image": null}', use-virtualenv: false}
+          - {os: macos-15, json-image: '{"image": null}', use-virtualenv: true}
+    runs-on: ${{ matrix.env.os }}
+    container: ${{ fromJSON(matrix.env.json-image) }}
     steps:
+      - name: Make sure Python is installed on Linux (might not be in containers)
+        if: fromJSON(matrix.env.json-image).image != null
+        shell: sh
+        run: |
+          if command -v apt-get; then
+            apt-get update -y
+            apt-get install -y python3 python3-pip python3-venv
+          elif command -v dnf; then
+            dnf install -y python3 python3-pip
+            # Install findutils on Rocky Linux when xargs cannot be found
+            if ! command -v xargs; then
+              dnf install -y findutils
+            fi
+          fi
       - name: Set parameters as env
+        shell: bash
         run: |
           SAMPLES_REVISION=${{ inputs.samples-revision || env.DEFAULT_SAMPLES_REVISION }}
           echo "SAMPLES_REVISION=$SAMPLES_REVISION" >> "$GITHUB_ENV"
+          if [[ $RUNNER_OS == 'Windows' ]]; then
+            VENV_ACTIVATE_SCRIPT="khiops-venv/Scripts/activate"
+          else
+            VENV_ACTIVATE_SCRIPT="khiops-venv/bin/activate"
+          fi
+          echo "VENV_ACTIVATE_SCRIPT=$VENV_ACTIVATE_SCRIPT" >> "$GITHUB_ENV"
+          # Setup Python (be portable wrt. python vs python3 in runners and
+          # containers; either python3 or python is present as per the Python
+          # installation step in the Docker container contexts)
+          PYTHON=$(basename $(command -v python3 || command -v python))
+          echo "PYTHON=$PYTHON" >> "$GITHUB_ENV"
       - name: Checkout sources  # Checkout the sources to be able to extract the dependencies list
         uses: actions/checkout@v4
         with:
@@ -97,30 +124,37 @@ jobs:
       - name: Install the package
         shell: bash
         run: |
-          # Allow Pip to write to its cache
-          mkdir -p /github/home/.cache/pip
-          chown -R $(whoami) /github/home/.cache/pip
+          # Allow Pip to write to its cache (only applies to Docker containers)
+          if [[ -n "${{ fromJSON(matrix.env.json-image).image }}" ]]; then
+            mkdir -p /github/home/.cache/pip
+            chown -R $(whoami) /github/home/.cache/pip
+          fi
 
           # Install the Khiops Python library
 
-          # A virtual env is mandatory under debian
-          if [[ "${{ matrix.container }}" == "debian13" ]]; then
-            python -m venv khiops-debian-venv
-            source khiops-debian-venv/bin/activate
+          # Use a virtualenv if required
+          if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then
+            $PYTHON -m venv khiops-venv
+            source "$VENV_ACTIVATE_SCRIPT"
           fi
-          pip install --upgrade pip
+          $PYTHON -m pip install --upgrade pip
           # Add homogeneous TOML support (Python >= 3.12 has standard tomllib)
-          pip install tomli
+          $PYTHON -c "import tomllib" 2>/dev/null || $PYTHON -m pip install tomli
           # First, install all dependencies except khiops-core and khiops-drivers-*
-          python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" --exclude-khiops-family > requires-no-khiops.txt
-          pip install `cat requires-no-khiops.txt`
+          $PYTHON scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" --exclude-khiops-family > requires-no-khiops.txt
+          [ ! -s requires-no-khiops.txt ] || xargs $PYTHON -m pip install < requires-no-khiops.txt
+          # On Windows, the `impi-rt` MPI dependency must be installed
+          # separately, from PyPI, because it is not present on TestPyPI
+          if [[ $RUNNER_OS == 'Windows' ]]; then
+            $PYTHON -m pip install impi-rt
+          fi
           # khiops-core and khiops-drivers-* must always be installed from TestPyPI in order to avoid distorting usage statistics
-          python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" --khiops-family-only > requires-khiops.txt
-          pip install --index-url https://test.pypi.org/simple `cat requires-khiops.txt`
+          $PYTHON scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" --khiops-family-only > requires-khiops.txt
+          [ ! -s requires-khiops.txt ] || xargs $PYTHON -m pip install --index-url https://test.pypi.org/simple < requires-khiops.txt
           rm -f requires-khiops.txt requires-no-khiops.txt
           # Lastly, install khiops-python
-          pip install $(ls khiops*.tar.gz)
-          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+          $PYTHON -m pip install khiops*.tar.gz
+          if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then
             deactivate
           fi
       - name: Run tests
@@ -140,12 +174,13 @@ jobs:
           # modules are currently not initializing the shell anyway
           if [ -n "$MODULESHOME" ]; then module unload mpi; fi
 
-          # A virtual env is mandatory under debian
-          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
-            source khiops-debian-venv/bin/activate
+          # Use a virtualenv if required
+          if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then
+            source "$VENV_ACTIVATE_SCRIPT"
           fi
-          if [[ $RUNNER_OS != 'Windows' ]]; then
-            # Set rights to OpenMPI-specific folder
+          if [[ -n "${{ fromJSON(matrix.env.json-image).image }}" ]]; then
+            # Set rights to OpenMPI-specific folder (only applies to Docker
+            # containers)
             mkdir -p /github/home/.pmix/components
             chown -R $(whoami) /github/home/.pmix/components
           fi
@@ -163,7 +198,7 @@ jobs:
           # The MPI command is not always named mpiexec, but can be orterun etc
           # (as given by khiops_env)
           kh-status | grep "MPI command" | grep -vwq "<empty>"
-          if [[ "${{ matrix.container }}" == "debian13" ]]; then
+          if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then
             deactivate
           fi
       - name: Test package / Git tag version coherence
@@ -172,19 +207,18 @@ jobs:
         run: |
           # Don't exit on first error: print relevant error message
           set +e
-          # A virtual env is mandatory under Debian 13
-          if [[ "${{ matrix.container }}" == "debian13" ]]; then
-            python -m venv khiops-debian-venv
-            source khiops-debian-venv/bin/activate
+          # Use a virtualenv if required
+          if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then
+            source "$VENV_ACTIVATE_SCRIPT"
           fi
           # Convert pre-release version specification in the Git tag to the Pip
           # format and check that it matches the Pip package version
-          PACKAGE_VERSION=$(pip show khiops | awk 'BEGIN {FS = ": "} $1 ~ /^Version$/ {print $2}')
-          if [[ "${{ matrix.container }}" == "debian13" ]]; then
+          PACKAGE_VERSION=$($PYTHON -m pip show khiops | awk 'BEGIN {FS = ": "} $1 ~ /^Version$/ {print $2}')
+          if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then
             deactivate
           fi
-          echo ${{ github.ref_name }} | tr -d '-' | rev | sed -E 's/\.([^0-9].*)/\1/' | rev | \
-            grep -wq $PACKAGE_VERSION
+          echo "${{ github.ref_name }}" | tr -d '-' | rev | sed -E 's/\.([^0-9].*)/\1/' | rev | \
+            grep -wq "$PACKAGE_VERSION"
           if [[ $? -ne 0 ]]
           then
             echo "::error::Python package version $PACKAGE_VERSION does not match Git tag ${{ github.ref_name }}"
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -2,7 +2,6 @@
 name: Tests
 env:
   DEFAULT_SAMPLES_REVISION: 11.0.0
-  DEFAULT_KHIOPS_DESKTOP_REVISION: 11.0.0
 on:
   workflow_dispatch:
     inputs:
@@ -12,9 +11,6 @@ on:
       image-tag:
         default: latest
         description: Development Docker Image Tag
-      khiops-desktop-revision:
-        default: 11.0.0
-        description: Khiops Windows Desktop Application Version
       run-expensive-tests:
         type: boolean
         required: false
@@ -229,25 +225,6 @@ jobs:
   check-khiops-integration-on-windows:
     runs-on: windows-2022
     steps:
-      - name: Download the Khiops Desktop NSIS Installer
-        shell: pwsh
-        run: |
-          $KHIOPS_DESKTOP_REVISION = '${{ inputs.khiops-desktop-revision || env.DEFAULT_KHIOPS_DESKTOP_REVISION }}'
-          $KHIOPS_DOWNLOAD_URL = "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_DESKTOP_REVISION}/khiops-${KHIOPS_DESKTOP_REVISION}-setup.exe"
-          Invoke-WebRequest "${KHIOPS_DOWNLOAD_URL}" `
-            -OutFile .\khiops-setup.exe `
-            -UseBasicParsing
-          Unblock-File .\khiops-setup.exe
-      - name: Install the Khiops Desktop Application
-        shell: pwsh
-        run: |
-          # Execute the installer
-          $ErrorActionPreference = 'Stop'
-          $ProgressPreference = 'SilentlyContinue'
-          Start-Process `
-            -FilePath .\khiops-setup.exe `
-            -ArgumentList '/S' `
-            -Wait
       - name: Checkout sources
         uses: actions/checkout@v4
         with:
diff --git a/khiops/core/internals/runner.py b/khiops/core/internals/runner.py
@@ -20,6 +20,7 @@
 import site
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import uuid
 import warnings
@@ -955,53 +956,59 @@ def __init__(self):
 
     def _initialize_khiops_environment(self):
         installation_method = _infer_khiops_installation_method()
-        # in a 'pip' installation method,
-        # the current Khiops Python library depends on the Khiops binary-only
-        # PyPI package.
-        # Let's ensure this dependency has not disappeared.
-        if installation_method == "pip":
-            try:
-                distribution("khiops-core")
-            except PackageNotFoundError as exc:
-                raise KhiopsEnvironmentError(
-                    f"The Khiops binaries are not installed properly: {exc}. "
-                    "Re-install the Khiops Python library to automatically install "
-                    "Khiops. Go to https://khiops.org for more information.\n"
+        match installation_method:
+            # In conda-based environments, khiops_env is not in PATH;
+            # its location must be inferred from the conda env directory.
+            case "conda-based":
+                khiops_env_path = os.path.join(
+                    _infer_env_bin_dir_for_conda_based_installations(), "khiops_env"
                 )
-        # Search for the `khiops_env` script location
-        if platform.system() == "Windows":
-            sys_executable_direct_parent = Path(sys.executable).parents[0]
-            probable_khiops_env = os.path.join(
-                sys_executable_direct_parent, "khiops_env.cmd"
-            )
-            # The script is found in the current environment
-            if os.path.exists(probable_khiops_env):
-                khiops_env_path = probable_khiops_env
-            # Raise error otherwise
-            else:
-                raise KhiopsEnvironmentError(
-                    "No 'khiops_env.cmd' found in the current environment. "
-                    "Make sure you have installed Khiops properly. "
-                    "Go to https://khiops.org for more information."
-                )
-        # In Conda-based environments, `khiops_env` might not be in the PATH,
-        # hence its path must be inferred
-        elif installation_method == "conda-based":
-            khiops_env_path = os.path.join(
-                _infer_env_bin_dir_for_conda_based_installations(), "khiops_env"
-            )
-            if platform.system() == "Windows":
-                khiops_env_path += ".cmd"
-
-        # On UNIX or Conda, khiops_env is always in path for a proper installation
-        else:
-            khiops_env_path = shutil.which("khiops_env")
-            if khiops_env_path is None:
+                if platform.system() == "Windows":
+                    khiops_env_path += ".cmd"
+            # In an activated conda environment, khiops_env is in PATH.
+            case "conda":
+                khiops_env_path = self._infer_khiops_env_from_path(installation_method)
+            case "pip":
+                # Ensure the binary dependency is still installed.
+                try:
+                    distribution("khiops-core")
+                except PackageNotFoundError as exc:
+                    raise KhiopsEnvironmentError(
+                        f"The Khiops binaries are not installed properly: {exc}. "
+                        "Re-install the Khiops Python library to automatically install "
+                        "Khiops. Go to https://khiops.org for more information.\n"
+                    ) from exc
+                # On Windows, determine the Scripts directory where pip placed
+                # khiops_env.cmd.
+                # If this library is installed under the user site-packages
+                # directory, khiops-core (and its khiops_env.cmd) was also installed
+                # there with `pip install --user`, so use the user Scripts directory.
+                # Otherwise use the standard Scripts directory (venv or system-wide
+                # install). This avoids an ambiguous search and mirrors how pip
+                # resolves scripts.
+                if platform.system() == "Windows":
+                    library_root_dir_path = Path(__file__).parents[2]
+                    user_site_packages_path = Path(site.getusersitepackages())
+                    if library_root_dir_path.is_relative_to(user_site_packages_path):
+                        scripts_dir = sysconfig.get_path("scripts", "nt_user")
+                    else:
+                        scripts_dir = sysconfig.get_path("scripts")
+                    khiops_env_path = os.path.join(scripts_dir, "khiops_env.cmd")
+                    if not os.path.exists(khiops_env_path):
+                        raise KhiopsEnvironmentError(
+                            "No 'khiops_env.cmd' found in the current environment. "
+                            "Make sure you have installed Khiops properly. "
+                            "Go to https://khiops.org for more information."
+                        )
+                # On UNIX, pip places khiops_env in the bin directory,
+                # which is in PATH.
+                else:
+                    khiops_env_path = self._infer_khiops_env_from_path(
+                        installation_method
+                    )
+            case _:
                 raise KhiopsEnvironmentError(
-                    "The 'khiops_env' script not found for the current "
-                    f"'{installation_method}' installation method. Make sure "
-                    "you have installed Khiops properly. "
-                    "Go to https://khiops.org for more information."
+                    f"Unknown installation method '{installation_method}'."
                 )
 
         with subprocess.Popen(
@@ -1063,6 +1070,17 @@ def _initialize_khiops_environment(self):
         # Initialize the default samples dir
         self._initialize_default_samples_dir()
 
+    def _infer_khiops_env_from_path(self, installation_method):
+        khiops_env_path = shutil.which("khiops_env")
+        if khiops_env_path is None:
+            raise KhiopsEnvironmentError(
+                "The 'khiops_env' script not found for the current "
+                f"'{installation_method}' installation method. Make sure "
+                "you have installed Khiops properly. "
+                "Go to https://khiops.org for more information."
+            )
+        return khiops_env_path
+
     def _initialize_default_samples_dir(self):
         """See class docstring"""
         samples_dir = get_default_samples_dir()
@@ -1223,13 +1241,13 @@ def _detect_library_installation_incompatibilities(self, library_root_dir_path):
                         platform.system() == "Windows"
                         and
                         # Under Windows, there are two cases :
-                        (
-                            # for conda-based installations python is inside 'base_dir'
-                            sys_executable_direct_parent != base_dir_path
-                            and
-                            # for 'pip' installations (within a virtual env)
-                            # python is inside 'base_dir'/Scripts
-                            sys_executable_grand_parent != base_dir_path
+                        # for conda-based installations python is inside 'base_dir'
+                        # for 'pip' installations (within a virtual env)
+                        # python is inside 'base_dir'/Scripts
+                        base_dir_path
+                        not in (
+                            sys_executable_direct_parent,
+                            sys_executable_grand_parent,
                         )
                         # Under Linux or MacOS a bin/ folder exists
                         or sys_executable_grand_parent != base_dir_path
