Fix CI: add venv site-packages to PYTHONPATH for msgpack import

When a custom pythonExecutable (e.g. 'python3') is used with a venv,
the venv's site-packages are not on sys.path, causing import msgpack
to fail. This commit adds indVenvSitePackages() which locates
lib/python3.X/site-packages (Unix) or Lib/site-packages (Windows)
inside the venv, and appends it to PYTHONPATH before starting the
Python process. Added 4 unit tests for the new helper method.

Author: howtis

python-embed-runtime/src/main/java/io/github/howtis/pythonembed/PythonProcessManager.java

@@ -6,6 +6,7 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.Map;
@@ -14,6 +15,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Stream;
 
 /**
  * Manages a persistent CPython REPL process that communicates via stdin/stdout
@@ -76,9 +78,21 @@ void start(Path venvPath, Path bridgeScriptPath, Map<String, String> env,
                 + " --max-code-length " + maxCodeLength);
         pb.redirectErrorStream(false);
 
-        // Apply environment variables
+        // Add venv site-packages to PYTHONPATH so msgpack is importable
+        Map<String, String> pbEnv = pb.environment();
+        if (venvPath != null) {
+            Path sitePackages = findVenvSitePackages(venvPath);
+            if (sitePackages != null) {
+                String existing = pbEnv.getOrDefault("PYTHONPATH", "");
+                String newPath = sitePackages.toAbsolutePath().toString();
+                String pythonPath = existing.isEmpty() ? newPath : newPath + java.io.File.pathSeparator + existing;
+                pbEnv.put("PYTHONPATH", pythonPath);
+                logger.info(() -> "Python process PYTHONPATH: " + pythonPath);
+            }
+        }
+
+        // Apply environment variables (may override PYTHONPATH)
         if (env != null && !env.isEmpty()) {
-            Map<String, String> pbEnv = pb.environment();
             pbEnv.putAll(env);
             logger.info(() -> "Python process env vars: " + env.keySet());
         }
@@ -299,6 +313,43 @@ private void shutdownExecutor(ExecutorService executor) {
     }
 
 
+    /**
+     * Finds the site-packages directory inside a venv.
+     *
+     * <p>On Unix: {@code lib/python3.X/site-packages}
+     * On Windows: {@code Lib/site-packages}
+     *
+     * @param venvPath root directory of the venv
+     * @return path to site-packages, or null if not found
+     */
+    static Path findVenvSitePackages(Path venvPath) {
+        boolean isWindows = System.getProperty("os.name", "").toLowerCase().contains("win");
+        if (isWindows) {
+            Path sitePkgs = venvPath.resolve("Lib").resolve("site-packages");
+            if (Files.isDirectory(sitePkgs)) {
+                return sitePkgs;
+            }
+            return null;
+        } else {
+            Path libDir = venvPath.resolve("lib");
+            if (!Files.isDirectory(libDir)) {
+                return null;
+            }
+            try (Stream<Path> entries = Files.list(libDir)) {
+                return entries
+                        .filter(Files::isDirectory)
+                        .filter(p -> p.getFileName().toString().startsWith("python"))
+                        .map(p -> p.resolve("site-packages"))
+                        .filter(Files::isDirectory)
+                        .findFirst()
+                        .orElse(null);
+            } catch (IOException e) {
+                logger.log(Level.WARNING, "Failed to scan venv lib directory", e);
+                return null;
+            }
+        }
+    }
+
     /**
      * Resolves the Python executable within a directory.
      *

python-embed-runtime/src/test/java/io/github/howtis/pythonembed/PythonProcessManagerTest.java

@@ -164,6 +164,70 @@ void resolvePythonExecutable_emptyBasePath() {
         assertFalse(resolved.isEmpty());
     }
 
+    // ------------------------------------------------------------------
+    // findVenvSitePackages()
+    // ------------------------------------------------------------------
+
+    @Test
+    void findVenvSitePackages_unixVenvLayout() throws Exception {
+        tempDir = Files.createTempDirectory("test-venv-sp-");
+        Path sitePkgs = Files.createDirectories(
+                tempDir.resolve("lib").resolve("python3.12").resolve("site-packages"));
+
+        // On Windows the method looks for Lib/site-packages
+        boolean isWindows = System.getProperty("os.name", "").toLowerCase().contains("win");
+        if (isWindows) {
+            // Windows: set up Lib/site-packages
+            Path winSitePkgs = Files.createDirectories(
+                    tempDir.resolve("Lib").resolve("site-packages"));
+            Path found = PythonProcessManager.findVenvSitePackages(tempDir);
+            assertEquals(winSitePkgs, found);
+        } else {
+            Path found = PythonProcessManager.findVenvSitePackages(tempDir);
+            assertEquals(sitePkgs, found);
+        }
+    }
+
+    @Test
+    void findVenvSitePackages_picksFirstMatchingPython() throws Exception {
+        tempDir = Files.createTempDirectory("test-venv-sp-multi-");
+        Path first = Files.createDirectories(
+                tempDir.resolve("lib").resolve("python3.12").resolve("site-packages"));
+        Path second = Files.createDirectories(
+                tempDir.resolve("lib").resolve("python3.11").resolve("site-packages"));
+
+        Path found = PythonProcessManager.findVenvSitePackages(tempDir);
+        boolean isWindows = System.getProperty("os.name", "").toLowerCase().contains("win");
+        if (!isWindows) {
+            assertNotNull(found);
+            // Either is acceptable (order depends on Files.list)
+            assertTrue(found.equals(first) || found.equals(second),
+                    "Expected one of the python site-packages dirs, got: " + found);
+        }
+        // On Windows the unix lib dir won't match, returns null; that's fine
+    }
+
+    @Test
+    void findVenvSitePackages_noLibDir_returnsNull() {
+        tempDir = new java.io.File("nonexistent-" + System.nanoTime()).toPath();
+        Path found = PythonProcessManager.findVenvSitePackages(tempDir);
+        assertNull(found);
+    }
+
+    @Test
+    void findVenvSitePackages_noSitePackages_returnsNull() throws Exception {
+        tempDir = Files.createTempDirectory("test-venv-sp-nosp-");
+        Files.createDirectories(tempDir.resolve("lib").resolve("python3.12"));
+        // No site-packages subdirectory
+
+        Path found = PythonProcessManager.findVenvSitePackages(tempDir);
+        boolean isWindows = System.getProperty("os.name", "").toLowerCase().contains("win");
+        if (!isWindows) {
+            assertNull(found);
+        }
+        // On Windows the unix lib dir won't match, returns null; that's fine
+    }
+
     // ------------------------------------------------------------------
     // isRunning()
     // ------------------------------------------------------------------