Fixes various problems revealed by Windows CI runs (#702)

* applied #685 to async pr

* added missing boost dependency

* ignore root build dirs

* added missing boost packages dependencies

* debug duct tape and more info

* removed msvc-specific instruction

* tests wait for xpython's ready message instead of sleep

* debug duct tape and more info

* fi

* fix

* x6

* one minute

* cleaner fix: lock python's GIL when destroying the debugger

* cleanup python path deduction unicode-friendly (on windows) and absolute paths

* removed debugging logs & cleaned error logs

* imrpoved sub-process tracking output in tests

* removed commented code

* run 20 times on windows

* run tests 20 times on unix

* show test run number in CI logs

* removed changes to python path deduction, improved wrong path detection and report instead

* debugger reference decreased instead of object destroyed

* removed some logs

---------

Co-authored-by: DerThorsten <derthorstenbeier@gmail.com>

Author: Klaim

.github/workflows/main.yml

@@ -69,8 +69,13 @@ jobs:
         run: xpython --version
 
       - name: Test xeus-python C++
-        run: ./test_xeus_python
-        timeout-minutes: 4
+        run: |
+          for i in $(seq 1 10);
+          do
+              echo "tests run $i"
+              ./test_xeus_python
+          done
+        timeout-minutes: 40
         working-directory: build/test
 
       - name: Test xeus-python Python
@@ -139,8 +144,10 @@ jobs:
       - name: Test xeus-python C++
         shell: cmd /C call {0}
         run: |
-          micromamba activate xeus-python
-          test_xeus_python
-        timeout-minutes: 4
+          for /L %%i in (1,1,20) do (
+            echo tests run %%i
+            test_xeus_python
+          )
+        timeout-minutes: 40
         working-directory: build\test
 

.gitignore

@@ -44,6 +44,7 @@ docs/build/
 
 # Build directory
 build/
+/build-*/
 
 # generated kernel specs
 /share/jupyter/kernels/xpython/kernel.json

environment-dev.yml

@@ -25,3 +25,7 @@ dependencies:
   - jupyter_kernel_test<0.8
   - doctest
   - pluggy=1.3
+  - libboost
+  - libboost-devel
+  - libboost-headers
+

include/xeus-python/xdebugger.hpp

@@ -20,6 +20,7 @@
 #include <memory>
 #include <mutex>
 #include <set>
+#include <thread>
 
 #include "nlohmann/json.hpp"
 #include "pybind11/pybind11.h"

src/main.cpp

@@ -77,15 +77,32 @@ int main(int argc, char* argv[])
     PyConfig_InitPythonConfig(&config);
     // config.isolated = 1;
 
+    constexpr std::string_view python_path_help = "PYTHONHOME or PYTHON_EXECUTABLE environment variables can be used to specify the correct path";
+    const auto fail_with_error_message = [](const auto&... message_parts) {
+            std::stringstream message_stream;
+            ((message_stream << message_parts), ...);
+            auto message = message_stream.str();
+            std::cerr << message << std::endl;
+            throw std::runtime_error(std::move(message));
+        };
+
     // Setting Program Name
     static const std::string executable(xpyt::get_python_path());
     static const std::wstring wexecutable(executable.cbegin(), executable.cend());
+    if (!std::filesystem::exists(wexecutable))
+    {
+        fail_with_error_message("cannot find python executable, tried  '", executable, "' - ", python_path_help);
+    }
     config.program_name = const_cast<wchar_t*>(wexecutable.c_str());
 
     // Setting Python Home
     static const std::string pythonhome{ xpyt::get_python_prefix() };
-    static const std::wstring wstr(pythonhome.cbegin(), pythonhome.cend());;
-    config.home = const_cast<wchar_t*>(wstr.c_str());
+    static const std::wstring wpythonhome(pythonhome.cbegin(), pythonhome.cend());
+    if (!std::filesystem::exists(wpythonhome))
+    {
+        fail_with_error_message("cannot find python home directory, tried  '", pythonhome, "' - ", python_path_help);
+    }
+    config.home = const_cast<wchar_t*>(wpythonhome.c_str());
     xpyt::print_pythonhome();
 
     // Implicitly pre-initialize Python

src/xdebugger.cpp

@@ -72,6 +72,9 @@ namespace xpyt
 
     debugger::~debugger()
     {
+        // release/destroy the debugger python object while GIL is acquired
+        pybind11::gil_scoped_acquire gil_lock;
+        m_pydebugger.dec_ref();
     }
 
     nl::json debugger::inspect_variables_request(const nl::json& message)
@@ -286,7 +289,7 @@ namespace xpyt
             py::gil_scoped_acquire acquire;
             py::module xeus_python_shell = py::module::import("xeus_python_shell.debugger");
             m_pydebugger = xeus_python_shell.attr("XDebugger")();
-
+            
             // Get debugpy version
             std::string expression = "debugpy.__version__";
             std::string version = (eval(py::str(expression))).cast<std::string>();

src/xinterpreter.cpp

@@ -342,29 +342,52 @@ namespace xpyt
 
         // Reset traceback
         m_ipython_shell.attr("last_error") = py::none();
-
         try
         {
             exec(py::str(code));
             return xeus::create_successful_reply();
         }
         catch (py::error_already_set& e)
         {
-            // This will grab the latest traceback and set shell.last_error
-            m_ipython_shell.attr("showtraceback")();
-
-            py::list pyerror = m_ipython_shell.attr("last_error");
-
-            xerror error = extract_error(pyerror);
+            try{
+                // This will grab the latest traceback and set shell.last_error
+                m_ipython_shell.attr("showtraceback")();
+                py::list pyerror = m_ipython_shell.attr("last_error");
+                xerror error = extract_error(pyerror);
 
-            publish_execution_error(error.m_ename, error.m_evalue, error.m_traceback);
-            error.m_traceback.resize(1);
-            error.m_traceback[0] = code;
+                publish_execution_error(error.m_ename, error.m_evalue, error.m_traceback);
 
-            return xeus::create_error_reply(error.m_ename, error.m_evalue, error.m_traceback);
+                error.m_traceback.resize(1);
+                error.m_traceback[0] = code;
+                return xeus::create_error_reply(error.m_ename, error.m_evalue, error.m_traceback);
+            }
+            catch (py::error_already_set& e)
+            {
+                std::cerr << "an error occurred during error handling: "<<e.what()<<std::endl;
+                return xeus::create_error_reply("ErrorDuringErrorHandling", e.what(), std::vector<std::string>());
+            }
+            catch(std::exception& e)
+            {
+                std::cerr << "a standard exception occurred during error handling: "<<e.what()<<std::endl;
+                return xeus::create_error_reply("ExceptionDuringErrorHandling", e.what(), std::vector<std::string>());
+            }
+            catch (...)
+            {
+                std::cerr << "an unknown error occurred during error handling"<<std::endl;
+                return xeus::create_error_reply("UnknownErrorDuringErrorHandling", "", std::vector<std::string>());
+            }
+        }
+        catch(std::exception& e)
+        {
+            return xeus::create_error_reply("Exception", e.what(), std::vector<std::string>());
+        }
+        catch (...)
+        {
+            return xeus::create_error_reply("UnknownError", "", std::vector<std::string>());
         }
     }
 
+
     void interpreter::set_request_context(xeus::xrequest_context context)
     {
         py::gil_scoped_acquire acquire;

src/xpaths.cpp

@@ -52,6 +52,13 @@ namespace xpyt
 
     std::string get_python_path()
     {
+        const char* python_exe_environment = std::getenv("PYTHON_EXECUTABLE");
+        if (python_exe_environment != nullptr && std::strlen(python_exe_environment) != 0)
+        {
+            static const std::string python_exe_path = python_exe_environment;
+            return python_exe_path;
+        }
+
         std::string python_prefix = get_python_prefix();
 #ifdef _WIN32
         if (python_prefix.back() != '\\')

test/CMakeLists.txt

@@ -45,6 +45,9 @@ endif()
 find_package(Threads)
 find_package(doctest)
 
+cmake_policy(SET CMP0167 OLD) # use find boost from old cmake
+find_package(Boost REQUIRED COMPONENTS process filesystem)
+
 include_directories(${GTEST_INCLUDE_DIRS} SYSTEM)
 
 set(XEUS_PYTHON_TESTS
@@ -70,7 +73,7 @@ set_target_properties(test_xeus_python PROPERTIES
 )
 
 include_directories(${PYTHON_INCLUDE_DIRS})
-target_link_libraries(test_xeus_python ${PYTHON_LIBRARIES} xeus-zmq doctest::doctest ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(test_xeus_python ${PYTHON_LIBRARIES} xeus-zmq doctest::doctest Boost::headers Boost::filesystem Boost::process ${CMAKE_THREAD_LIBS_INIT})
 target_include_directories(test_xeus_python PRIVATE ${XEUS_PYTHON_INCLUDE_DIR})
 
 add_custom_target(xtest COMMAND test_xeus_python DEPENDS test_xeus_python)