diff --git a/.github/workflows/deploy-github-page.yml b/.github/workflows/deploy-github-page.yml index 08c3fc70..035f25ad 100644 --- a/.github/workflows/deploy-github-page.yml +++ b/.github/workflows/deploy-github-page.yml @@ -47,6 +47,11 @@ jobs: set -eux + + URL=https://github.com/DerThorsten/xeus-python-shell/archive/refs/heads/akernel.zip + curl -L $URL -o xeus-python-shell-akernel.zip + unzip xeus-python-shell-akernel.zip + export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-python-wasm-host echo "PREFIX=$PREFIX" >> $GITHUB_ENV @@ -63,10 +68,12 @@ jobs: - name: Jupyter Lite integration shell: bash -l {0} run: | + THIS_DIR=$(pwd) jupyter lite build \ --XeusAddon.prefix=${{ env.PREFIX }} \ --contents notebooks/ \ - --output-dir dist + --output-dir dist + - name: Upload artifact uses: actions/upload-pages-artifact@v4 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2d11c990..93d8a53a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,6 +32,7 @@ jobs: uses: mamba-org/setup-micromamba@v2 with: environment-file: environment-dev.yml + - name: Make build directory run: mkdir build diff --git a/CMakeLists.txt b/CMakeLists.txt index 83e08b16..50bd7756 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,8 @@ set(XEUS_PYTHON_SRC src/xstream.hpp src/xtraceback.cpp src/xutils.cpp + src/xasync_runner.cpp + src/xaserver.cpp ) set(XEUS_PYTHON_HEADERS @@ -190,6 +192,7 @@ set(XEUS_PYTHON_HEADERS include/xeus-python/xinterpreter_raw.hpp include/xeus-python/xtraceback.hpp include/xeus-python/xutils.hpp + include/xeus-python/xaserver.hpp ) set(XPYTHON_SRC diff --git a/environment-dev.yml b/environment-dev.yml index e1483547..eb273d25 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -15,7 +15,8 @@ dependencies: # The debugger is not available with python 3.13 because # of a spurious bug in debugpy - python <3.13 - - xeus-python-shell>=0.6.3,<0.7 + - pip + - xeus-python-shell>=0.7.0,<0.8 - debugpy>=1.6.5 - ipython # Test dependencies diff --git a/environment-wasm-build.yml b/environment-wasm-build.yml index a9bb8778..69a6ecd4 100644 --- a/environment-wasm-build.yml +++ b/environment-wasm-build.yml @@ -13,3 +13,5 @@ dependencies: - jupyterlite-core >0.6 - jupyter_server - jupyterlite-xeus + - # widgets + - ipywidgets \ No newline at end of file diff --git a/environment-wasm-host.yml b/environment-wasm-host.yml index deaf8b7f..1e230664 100644 --- a/environment-wasm-host.yml +++ b/environment-wasm-host.yml @@ -12,9 +12,11 @@ dependencies: - numpy - xeus-lite - xeus - - xeus-python-shell>=0.6.3 - pyjs >=4,<5 - libpython - zstd - openssl - xz + - ipywidgets + - xeus-python-shell>=0.7.0,<0.8 + diff --git a/include/xeus-python/xaserver.hpp b/include/xeus-python/xaserver.hpp new file mode 100644 index 00000000..edbab446 --- /dev/null +++ b/include/xeus-python/xaserver.hpp @@ -0,0 +1,22 @@ +/*************************************************************************** +* Copyright (c) 2024, Isabel Paredes * +* Copyright (c) 2024, QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include "xeus_python_config.hpp" +#include "xeus/xkernel.hpp" +#include "pybind11/pybind11.h" + +namespace py = pybind11; +namespace nl = nlohmann; + +namespace xpyt +{ + + XEUS_PYTHON_API XPYT_FORCE_PYBIND11_EXPORT xeus::xkernel::server_builder make_async_server_factory(py::dict globals); + +} // namespace xeus diff --git a/include/xeus-python/xdebugger.hpp b/include/xeus-python/xdebugger.hpp index f28678df..47c7c9cc 100644 --- a/include/xeus-python/xdebugger.hpp +++ b/include/xeus-python/xdebugger.hpp @@ -38,7 +38,9 @@ namespace xpyt using base_type = xeus::xdebugger_base; - debugger(xeus::xcontext& context, + debugger( + py::dict globals, + xeus::xcontext& context, const xeus::xkernel_configuration& config, const std::string& user_name, const std::string& session_id, @@ -63,6 +65,9 @@ namespace xpyt xeus::xdebugger_info get_debugger_info() const override; std::string get_cell_temporary_file(const std::string& code) const override; + + py::dict m_global_dict; + std::unique_ptr p_debugpy_client; std::string m_debugpy_host; std::string m_debugpy_port; @@ -72,8 +77,10 @@ namespace xpyt bool m_copy_to_globals_available; }; - XEUS_PYTHON_API - std::unique_ptr make_python_debugger(xeus::xcontext& context, + + XEUS_PYTHON_API XPYT_FORCE_PYBIND11_EXPORT + std::unique_ptr make_python_debugger(py::dict globals, + xeus::xcontext& context, const xeus::xkernel_configuration& config, const std::string& user_name, const std::string& session_id, diff --git a/include/xeus-python/xeus_python_config.hpp b/include/xeus-python/xeus_python_config.hpp index 75481c05..db3baf6b 100644 --- a/include/xeus-python/xeus_python_config.hpp +++ b/include/xeus-python/xeus_python_config.hpp @@ -46,3 +46,9 @@ #define XPYT_FORCE_PYBIND11_EXPORT __attribute__ ((visibility ("default"))) #endif #endif + +#ifdef _MSC_VER + #define XEUS_PYTHON_HIDDEN +#else + #define XEUS_PYTHON_HIDDEN __attribute__ ((visibility ("hidden"))) +#endif \ No newline at end of file diff --git a/include/xeus-python/xinterpreter.hpp b/include/xeus-python/xinterpreter.hpp index 8333331b..83538299 100644 --- a/include/xeus-python/xinterpreter.hpp +++ b/include/xeus-python/xinterpreter.hpp @@ -32,7 +32,7 @@ namespace nl = nlohmann; namespace xpyt { - class XEUS_PYTHON_API interpreter : public xeus::xinterpreter + class XEUS_PYTHON_API XPYT_FORCE_PYBIND11_EXPORT interpreter : public xeus::xinterpreter { public: @@ -44,7 +44,9 @@ namespace xpyt // If redirect_display_enabled is true (default) then this interpreter will // overwrite sys.displayhook and send execution results using publish_execution_result. // Disable this if your interpreter uses custom display hook. - interpreter(bool redirect_output_enabled=true, bool redirect_display_enabled = true); + interpreter( + py::dict globals, + bool redirect_output_enabled=true, bool redirect_display_enabled = true); virtual ~interpreter(); protected: @@ -78,6 +80,7 @@ namespace xpyt void redirect_output(); + py::dict m_global_dict; py::object m_ipython_shell_app; py::object m_ipython_shell; py::object m_displayhook; @@ -94,8 +97,7 @@ namespace xpyt // If an application has already released the GIL by the time the interpreter // is started, m_release_gil_at_startup has to be set to false to prevent // releasing it again in configure_impl(). - // - bool m_release_gil_at_startup = true; + bool m_release_gil_at_startup = false; gil_scoped_release_ptr m_release_gil = nullptr; bool m_redirect_output_enabled; diff --git a/include/xeus-python/xinterpreter_raw.hpp b/include/xeus-python/xinterpreter_raw.hpp index 03532a87..07aedb1c 100644 --- a/include/xeus-python/xinterpreter_raw.hpp +++ b/include/xeus-python/xinterpreter_raw.hpp @@ -30,7 +30,7 @@ namespace nl = nlohmann; namespace xpyt { - class XEUS_PYTHON_API raw_interpreter : public xeus::xinterpreter + class XEUS_PYTHON_API XPYT_FORCE_PYBIND11_EXPORT raw_interpreter : public xeus::xinterpreter { public: @@ -42,7 +42,9 @@ namespace xpyt // If redirect_display_enabled is true (default) then this interpreter will // overwrite sys.displayhook and send execution results using publish_execution_result. // Disable this if your interpreter uses custom display hook. - raw_interpreter(bool redirect_output_enabled=true, bool redirect_display_enabled = true); + raw_interpreter( + py::dict globals, + bool redirect_output_enabled=true, bool redirect_display_enabled = true); virtual ~raw_interpreter(); protected: @@ -90,6 +92,7 @@ namespace xpyt bool m_release_gil_at_startup = true; gil_scoped_release_ptr m_release_gil = nullptr; bool m_redirect_display_enabled; + py::dict m_global_dict; }; } diff --git a/notebooks/xeus-python.ipynb b/notebooks/xeus-python.ipynb index 12a0b75b..3320a866 100644 --- a/notebooks/xeus-python.ipynb +++ b/notebooks/xeus-python.ipynb @@ -49,15 +49,6 @@ "sq(b)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -653,15 +644,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.9 (XPython)", + "display_name": "Python 3.12 (XPython)", "language": "python", "name": "xpython" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", - "version": "3.9.1" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" } }, "nbformat": 4, diff --git a/src/main.cpp b/src/main.cpp index 95af2803..b0c01000 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,4 +1,5 @@ /*************************************************************************** +* Copyright (c) 2026, Thorsten Beier * * Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * * Wolf Vollprecht * * Copyright (c) 2018, QuantStack * @@ -26,7 +27,7 @@ #include "xeus/xinterpreter.hpp" #include "xeus/xhelper.hpp" -#include "xeus-zmq/xserver_zmq_split.hpp" +#include "xeus-zmq/xserver_zmq.hpp" #include "xeus-zmq/xzmq_context.hpp" #include "pybind11/embed.h" @@ -38,10 +39,10 @@ #include "xeus-python/xpaths.hpp" #include "xeus-python/xeus_python_config.hpp" #include "xeus-python/xutils.hpp" +#include "xeus-python/xaserver.hpp" namespace py = pybind11; - int main(int argc, char* argv[]) { if (xeus::should_print_version(argc, argv)) @@ -111,6 +112,10 @@ int main(int argc, char* argv[]) std::cerr << "Error:" << status.err_msg << std::endl; } + // Instantiating the Python interpreter + py::scoped_interpreter guard{}; + py::gil_scoped_acquire acquire; + // Setting argv wchar_t** argw = new wchar_t*[size_t(argc)]; for(auto i = 0; i < argc; ++i) @@ -130,22 +135,23 @@ int main(int argc, char* argv[]) } delete[] argw; - // Instantiating the Python interpreter - py::scoped_interpreter guard; std::unique_ptr context = xeus::make_zmq_context(); + // we want to use **the same global dict everywhere** + py::dict globals = py::globals(); + // Instantiating the xeus xinterpreter bool raw_mode = xpyt::extract_option("-r", "--raw", argc, argv); using interpreter_ptr = std::unique_ptr; interpreter_ptr interpreter; if (raw_mode) { - interpreter = interpreter_ptr(new xpyt::raw_interpreter()); + interpreter = interpreter_ptr(new xpyt::raw_interpreter(globals)); } else { - interpreter = interpreter_ptr(new xpyt::interpreter()); + interpreter = interpreter_ptr(new xpyt::interpreter(globals)); } using history_manager_ptr = std::unique_ptr; @@ -163,6 +169,24 @@ int main(int argc, char* argv[]) nl::json debugger_config; debugger_config["python"] = executable; + + // Factory to create the debugger with the global dict + auto make_the_debugger = [globals]( + xeus::xcontext& context, + const xeus::xkernel_configuration& config, + const std::string& user_name, + const std::string& session_id, + const nl::json& debugger_config) -> std::unique_ptr + { + return xpyt::make_python_debugger( + globals, + context, + config, + user_name, + session_id, + debugger_config); + }; + if (!connection_filename.empty()) { xeus::xconfiguration config = xeus::load_configuration(connection_filename); @@ -171,11 +195,11 @@ int main(int argc, char* argv[]) xeus::get_user_name(), std::move(context), std::move(interpreter), - xeus::make_xserver_shell_main, + xpyt::make_async_server_factory(globals), std::move(hist), xeus::make_console_logger(xeus::xlogger::msg_type, xeus::make_file_logger(xeus::xlogger::content, "xeus.log")), - xpyt::make_python_debugger, + make_the_debugger, debugger_config); std::clog << @@ -183,8 +207,10 @@ int main(int argc, char* argv[]) "If you want to connect to this kernel from an other client, you can use" " the " + connection_filename + " file." << std::endl; - + + std::cout << "Starting kernel..." << std::endl; kernel.start(); + std::cout << "Kernel stopped." << std::endl; } else { @@ -192,10 +218,10 @@ int main(int argc, char* argv[]) xeus::xkernel kernel(xeus::get_user_name(), std::move(context), std::move(interpreter), - xeus::make_xserver_shell_main, + xpyt::make_async_server_factory(globals), std::move(hist), nullptr, - xpyt::make_python_debugger, + make_the_debugger, debugger_config); std::cout << "Getting config" << std::endl; diff --git a/src/xaserver.cpp b/src/xaserver.cpp new file mode 100644 index 00000000..d6d1c120 --- /dev/null +++ b/src/xaserver.cpp @@ -0,0 +1,46 @@ +/*************************************************************************** +* Copyright (c) 2026, Thorsten Beier * +* Copyright (c) 2024, Isabel Paredes * +* Copyright (c) 2024, QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ +#include "xeus-python/xaserver.hpp" +#include "xasync_runner.hpp" +#include "xeus-zmq/xcontrol_default_runner.hpp" +#include "xeus-zmq/xserver_zmq_split.hpp" + +#include "xeus/xkernel.hpp" +#include "xeus/xserver.hpp" +#include "xeus/xeus_context.hpp" +#include "xeus/xkernel_configuration.hpp" + +#include "nlohmann/json.hpp" + + +namespace nl = nlohmann; + +namespace xpyt +{ + xeus::xkernel::server_builder make_async_server_factory(py::dict globals) + { + return [globals]( + xeus::xcontext& context, + const xeus::xconfiguration& config, + nl::json::error_handler_t eh) + { + return xeus::make_xserver_shell + ( + context, + config, + eh, + std::make_unique(), + std::make_unique(globals) + ); + }; + } + + +} // namespace xpyt diff --git a/src/xasync_runner.cpp b/src/xasync_runner.cpp new file mode 100644 index 00000000..f88ed7c8 --- /dev/null +++ b/src/xasync_runner.cpp @@ -0,0 +1,154 @@ +/*************************************************************************** +* Copyright (c) 2026, Thorsten Beier * +* Copyright (c) 2024, Isabel Paredes * +* Copyright (c) 2024, QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include +#include + +#include "xasync_runner.hpp" +#include "pybind11/embed.h" +#include "pybind11/pybind11.h" + +namespace py = pybind11; + +namespace xpyt +{ + + xasync_runner::xasync_runner(py::dict globals) + : xeus::xshell_runner(), + m_global_dict{globals} + { + } + + void xasync_runner::run_impl() + { + + const int fd_shell_int = static_cast(this->get_shell_fd()); + const int fd_controller_int = static_cast(this-> get_shell_controller_fd()); + + // wrap c++ callbacks into python functions + py::cpp_function shell_callback = py::cpp_function([this]() { + this->on_message_doorbell_shell(); + }); + py::cpp_function controller_callback = py::cpp_function([this]() { + this->on_message_doorbell_controller(); + }); + + // ensure gil + py::gil_scoped_acquire acquire; + + // pure python impl of the main loop + exec(R"( + import sys + import asyncio + import traceback + import socket + + is_win = sys.platform.startswith("win") or sys.platform.startswith("cygwin") or sys.platform.startswith("msys") + + if is_win: + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + class ZMQSockReader: + def __init__(self, fd): + self._fd = fd + self._sock = socket.fromfd(self._fd, socket.AF_INET, socket.SOCK_STREAM) + + def fileno(self): + return self._fd + + def __del__(self): + try: + self._sock.detach() + except: + pass + + def make_fd(fd): + if is_win: + return ZMQSockReader(fd) + else: + return fd + + def run_main_non_busy_loop(fd_shell, fd_controller, shell_callback, controller_callback): + # here we create / ensure we have an event loop + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.add_reader(fd_shell, shell_callback) + loop.add_reader(fd_controller, controller_callback) + loop.run_forever() + + def run_main(fd_shell, fd_controller, shell_callback, controller_callback): + try: + fd_controller = make_fd(fd_controller) + fd_shell = make_fd(fd_shell) + run_main_non_busy_loop(fd_shell, fd_controller, shell_callback, controller_callback) + + except Exception as e: + traceback_str = traceback.format_exc() + print(f"Exception in async runner: {e}\n{traceback_str}", file=sys.stderr) + sys.exit(1) + + )", m_global_dict); + + m_global_dict["run_main"](fd_shell_int, fd_controller_int, shell_callback, controller_callback); + + } + + void xasync_runner::on_message_doorbell_shell() + { + int ZMQ_DONTWAIT{ 1 }; // from zmq.h + while (auto msg = read_shell(ZMQ_DONTWAIT)) + { + notify_shell_listener(std::move(msg.value())); + } + } + + void xasync_runner::on_message_doorbell_controller() + { + int ZMQ_DONTWAIT{ 1 }; // from zmq.h + while (auto msg = read_controller(ZMQ_DONTWAIT)) + { + std::string val{ msg.value() }; + + if (val == "stop") + { + send_controller(std::move(val)); + + int fd_shell_int = static_cast(this->get_shell_fd()); + int fd_controller_int = static_cast(this-> get_shell_controller_fd()); + + py::gil_scoped_acquire acquire; + + // Or create via exec if you need a more complex function: + py::exec(R"( + import asyncio + import sys + + def stop_loop(fd_shell, fd_controller): + loop = asyncio.get_event_loop() + loop.remove_reader(make_fd(fd_shell)) + loop.remove_reader(make_fd(fd_controller)) + loop.stop() + + )", m_global_dict); + + py::object stop_func = m_global_dict["stop_loop"]; + stop_func(fd_shell_int, fd_controller_int); + break; + + } + else + { + std::string rep = notify_internal_listener(std::move(val)); + send_controller(std::move(rep)); + } + } + } + +} // namespace xpyt diff --git a/src/xasync_runner.hpp b/src/xasync_runner.hpp new file mode 100644 index 00000000..22857ebc --- /dev/null +++ b/src/xasync_runner.hpp @@ -0,0 +1,49 @@ +/*************************************************************************** +* Copyright (c) 2026, Thorsten Beier * +* Copyright (c) 2024, Isabel Paredes * +* Copyright (c) 2024, QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef XEUS_PYTHON_ASYNC_RUNNER_HPP +#define XEUS_PYTHON_ASYNC_RUNNER_HPP + +#include + +#include "xeus-python/xeus_python_config.hpp" +#include "xeus-zmq/xshell_runner.hpp" +#include "pybind11/pybind11.h" + + +namespace py = pybind11; + +namespace xpyt +{ + + class XEUS_PYTHON_HIDDEN xasync_runner final : public xeus::xshell_runner + { + public: + + xasync_runner(py::dict globals); + xasync_runner(const xasync_runner&) = delete; + xasync_runner& operator=(const xasync_runner&) = delete; + xasync_runner(xasync_runner&&) = delete; + xasync_runner& operator=(xasync_runner&&) = delete; + ~xasync_runner() override = default; + + private: + void on_message_doorbell_shell(); + void on_message_doorbell_controller(); + + void run_impl() override; + + py::dict m_global_dict; + + }; + +} // namespace xeus + +#endif // XEUS_PYTHON_ASYNC_RUNNER_HPP diff --git a/src/xdebugger.cpp b/src/xdebugger.cpp index 41bc09a8..9bbe8f1c 100644 --- a/src/xdebugger.cpp +++ b/src/xdebugger.cpp @@ -20,8 +20,10 @@ #include "pybind11/pybind11.h" #include "pybind11/stl.h" +#include "pybind11/embed.h" #include "xeus/xinterpreter.hpp" +#include "xeus/xeus_context.hpp" #include "xeus/xsystem.hpp" #include "xeus-zmq/xmiddleware.hpp" @@ -38,12 +40,14 @@ using namespace std::placeholders; namespace xpyt { - debugger::debugger(xeus::xcontext& context, + debugger::debugger(py::dict globals, + xeus::xcontext& context, const xeus::xkernel_configuration& config, const std::string& user_name, const std::string& session_id, const nl::json& debugger_config) : xdebugger_base(context) + , m_global_dict{globals} , p_debugpy_client(new xdebugpy_client(context, config, xeus::get_socket_linger(), @@ -144,7 +148,7 @@ namespace xpyt } py::gil_scoped_acquire acquire; - py::object variables = py::globals(); + py::object variables = m_global_dict; py::object repr_data = variables[py::str(var_repr_data)]; py::object repr_metadata = variables[py::str(var_repr_metadata)]; nl::json body = { @@ -254,6 +258,7 @@ namespace xpyt if (std::getenv("XEUS_LOG") != nullptr) { std::ofstream out("xeus.log", std::ios_base::app); + //auto& out = std::cout; out << "===== DEBUGGER CONFIG =====" << std::endl; out << m_debugger_config.dump() << std::endl; } @@ -288,7 +293,7 @@ namespace xpyt // Get debugpy version std::string expression = "debugpy.__version__"; - std::string version = (eval(py::str(expression))).cast(); + std::string version = eval(py::str(expression), m_global_dict).cast(); // Format the version to match [0-9]+(\s[0-9]+)* size_t pos = version.find_first_of("abrc"); @@ -361,13 +366,15 @@ namespace xpyt return get_cell_tmp_file(code); } - std::unique_ptr make_python_debugger(xeus::xcontext& context, + std::unique_ptr make_python_debugger( + py::dict globals, + xeus::xcontext& context, const xeus::xkernel_configuration& config, const std::string& user_name, const std::string& session_id, const nl::json& debugger_config) { - return std::unique_ptr(new debugger(context, + return std::unique_ptr(new debugger(globals,context, config, user_name, session_id, debugger_config)); } diff --git a/src/xdebugpy_client.cpp b/src/xdebugpy_client.cpp index bad415dd..b1d65d0a 100644 --- a/src/xdebugpy_client.cpp +++ b/src/xdebugpy_client.cpp @@ -10,6 +10,7 @@ #include "nlohmann/json.hpp" #include "xeus/xmessage.hpp" +#include "xeus/xeus_context.hpp" #include "xdebugpy_client.hpp" #include diff --git a/src/xdebugpy_client.hpp b/src/xdebugpy_client.hpp index 451226fe..b2d10907 100644 --- a/src/xdebugpy_client.hpp +++ b/src/xdebugpy_client.hpp @@ -12,6 +12,7 @@ #define XPYT_DEBUGPY_CLIENT_HPP #include "xeus-zmq/xdap_tcp_client.hpp" +#include "xeus/xeus_context.hpp" namespace xpyt { diff --git a/src/xinspect.cpp b/src/xinspect.cpp index c820ea9b..1d94635f 100644 --- a/src/xinspect.cpp +++ b/src/xinspect.cpp @@ -22,30 +22,30 @@ using namespace pybind11::literals; namespace xpyt { - py::object static_inspect(const std::string& code) + py::object static_inspect(const std::string& code, py::dict globals) { py::module jedi = py::module::import("jedi"); - return jedi.attr("Interpreter")(code, py::make_tuple(py::globals())); + return jedi.attr("Interpreter")(code, py::make_tuple(globals)); } - py::object static_inspect(const std::string& code, int cursor_pos) + + py::object static_inspect(const std::string& code, int cursor_pos, py::dict globals) { std::string sub_code = code.substr(0, cursor_pos); - return static_inspect(sub_code); + return static_inspect(sub_code, globals); } - py::list get_completions(const std::string& code, int cursor_pos) + + py::list get_completions(const std::string& code, int cursor_pos, py::dict globals) { - return static_inspect(code, cursor_pos).attr("complete")(); + return static_inspect(code, cursor_pos, globals).attr("complete")(); } - py::list get_completions(const std::string& code) + py::list get_completions(const std::string& code, py::dict globals) { - return static_inspect(code).attr("complete")(); + return static_inspect(code, globals).attr("complete")(); } - py::list get_completions(const std::string& code); - std::string formatted_docstring_impl(py::object inter) { py::object definition = py::none(); @@ -121,15 +121,15 @@ namespace xpyt return result; } - std::string formatted_docstring(const std::string& code, int cursor_pos) + std::string formatted_docstring(const std::string& code, int cursor_pos, py::dict globals) { - py::object inter = static_inspect(code, cursor_pos); + py::object inter = static_inspect(code, cursor_pos, globals); return formatted_docstring_impl(inter); } - std::string formatted_docstring(const std::string& code) + std::string formatted_docstring(const std::string& code, py::dict globals) { - py::object inter = static_inspect(code); + py::object inter = static_inspect(code, globals); return formatted_docstring_impl(inter); } } diff --git a/src/xinspect.hpp b/src/xinspect.hpp index edd235da..b3fa6a77 100644 --- a/src/xinspect.hpp +++ b/src/xinspect.hpp @@ -19,11 +19,11 @@ namespace py = pybind11; namespace xpyt { - py::list get_completions(const std::string& code, int cursor_pos); - py::list get_completions(const std::string& code); + py::list get_completions(const std::string& code, int cursor_pos, py::dict globals); + py::list get_completions(const std::string& code, py::dict globals); - std::string formatted_docstring(const std::string& code, int cursor_pos); - std::string formatted_docstring(const std::string& code); + std::string formatted_docstring(const std::string& code, int cursor_pos, py::dict globals); + std::string formatted_docstring(const std::string& code, py::dict globals); } #endif diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index 4ec49c5d..9a1bda89 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -22,6 +22,7 @@ #include "xeus/xhelper.hpp" #include "pybind11/functional.h" +#include "pybind11/embed.h" #include "pybind11_json/pybind11_json.hpp" @@ -44,8 +45,11 @@ using namespace pybind11::literals; namespace xpyt { - interpreter::interpreter(bool redirect_output_enabled /*=true*/, bool redirect_display_enabled /*=true*/) - : m_redirect_output_enabled{redirect_output_enabled}, m_redirect_display_enabled{redirect_display_enabled} + interpreter::interpreter( + py::dict globals, + bool redirect_output_enabled /*=true*/, bool redirect_display_enabled /*=true*/) + : m_global_dict{globals}, + m_redirect_output_enabled{redirect_output_enabled}, m_redirect_display_enabled{redirect_display_enabled} { xeus::register_interpreter(this); } @@ -129,14 +133,41 @@ namespace xpyt // getpass with a function sending input_request messages. auto input_guard = input_redirection(config.allow_stdin); - bool exception_occurred = false; std::string ename; std::string evalue; std::vector traceback; + + auto when_done_callback_lambda = [this, cb, config, user_expressions]() { + py::gil_scoped_acquire acquire; + + // Get payload + nl::json payload = this->m_ipython_shell.attr("payload_manager").attr("read_payload")(); + this->m_ipython_shell.attr("payload_manager").attr("clear_payload")(); + + if (this->m_ipython_shell.attr("last_error").is_none()) + { + nl::json user_exprs = this->m_ipython_shell.attr("user_expressions")(user_expressions); + cb(xeus::create_successful_reply(payload, user_exprs)); + } + else + { + py::list pyerror = this->m_ipython_shell.attr("last_error"); + xerror error = extract_error(pyerror); + + if (!config.silent) + { + publish_execution_error(error.m_ename, error.m_evalue, error.m_traceback); + } + cb(xeus::create_error_reply(error.m_ename, error.m_evalue, error.m_traceback)); + } + }; + std::function when_done_callback = when_done_callback_lambda; + try { - m_ipython_shell.attr("run_cell")(code, "store_history"_a=config.store_history, "silent"_a=config.silent); + + m_ipython_shell.attr("run_cell_async")(code, when_done_callback, "store_history"_a=config.store_history, "silent"_a=config.silent); } catch(std::runtime_error& e) { @@ -145,7 +176,6 @@ namespace xpyt { publish_execution_error("RuntimeError", error_msg, std::vector()); } - exception_occurred = true; } catch (py::error_already_set& e) { @@ -154,11 +184,7 @@ namespace xpyt { publish_execution_error(error.m_ename, error.m_evalue, error.m_traceback); } - - ename = error.m_ename; - evalue = error.m_evalue; - traceback = error.m_traceback; - exception_occurred = true; + cb(xeus::create_error_reply(error.m_ename, error.m_evalue, error.m_traceback)); } catch(...) { @@ -166,41 +192,16 @@ namespace xpyt { publish_execution_error("unknown_error", "", std::vector()); } - ename = "UnknownError"; - evalue = ""; - exception_occurred = true; - } - - // Get payload - nl::json payload = m_ipython_shell.attr("payload_manager").attr("read_payload")(); - m_ipython_shell.attr("payload_manager").attr("clear_payload")(); - - if(exception_occurred) - { - cb(xeus::create_error_reply(ename, evalue, traceback)); - return; + cb(xeus::create_error_reply("UnknownError", "", std::vector())); } + } - if (m_ipython_shell.attr("last_error").is_none()) - { - nl::json user_exprs = m_ipython_shell.attr("user_expressions")(user_expressions); - cb(xeus::create_successful_reply(payload, user_exprs)); - } - else - { - py::list pyerror = m_ipython_shell.attr("last_error"); - - xerror error = extract_error(pyerror); - - if (!config.silent) - { - publish_execution_error(error.m_ename, error.m_evalue, error.m_traceback); - } - - cb(xeus::create_error_reply(error.m_ename, error.m_evalue, error.m_traceback)); - } + nl::json interpreter::shutdown_request_impl(bool /*restart*/) + { + return xeus::create_shutdown_reply(false); } + nl::json interpreter::complete_request_impl( const std::string& code, int cursor_pos) @@ -325,11 +326,6 @@ namespace xpyt return rep; } - nl::json interpreter::shutdown_request_impl(bool /*restart*/) - { - return xeus::create_shutdown_reply(false); - } - nl::json interpreter::interrupt_request_impl() { return xeus::create_interrupt_reply(); @@ -344,7 +340,7 @@ namespace xpyt m_ipython_shell.attr("last_error") = py::none(); try { - exec(py::str(code)); + exec(py::str(code), m_global_dict); return xeus::create_successful_reply(); } catch (py::error_already_set& e) @@ -385,8 +381,8 @@ namespace xpyt { return xeus::create_error_reply("UnknownError", "", std::vector()); } - } + } void interpreter::set_request_context(xeus::xrequest_context context) { diff --git a/src/xinterpreter_raw.cpp b/src/xinterpreter_raw.cpp index 3ab30cec..7115224c 100644 --- a/src/xinterpreter_raw.cpp +++ b/src/xinterpreter_raw.cpp @@ -45,14 +45,19 @@ using namespace pybind11::literals; namespace xpyt { - raw_interpreter::raw_interpreter(bool redirect_output_enabled /*=true*/, bool redirect_display_enabled /*=true*/) - :m_redirect_display_enabled{ redirect_display_enabled } + raw_interpreter::raw_interpreter( + py::dict globals, + bool redirect_output_enabled /*=true*/, bool redirect_display_enabled /*=true*/) + + : m_redirect_display_enabled{ redirect_display_enabled }, + m_global_dict{globals} { xeus::register_interpreter(this); if (redirect_output_enabled) { redirect_output(); } + m_release_gil_at_startup = false; } raw_interpreter::~raw_interpreter() @@ -85,8 +90,8 @@ namespace xpyt } // Expose display functions to Python - py::globals()["display"] = display_module.attr("display"); - py::globals()["update_display"] = display_module.attr("update_display"); + m_global_dict["display"] = display_module.attr("display"); + m_global_dict["update_display"] = display_module.attr("update_display"); // Monkey patching "import IPython.core.display" sys.attr("modules")["IPython.core.display"] = display_module; @@ -98,13 +103,12 @@ namespace xpyt sys.attr("modules")["IPython.core.getipython"] = kernel_module; // Add get_ipython to global namespace - py::globals()["get_ipython"] = kernel_module.attr("get_ipython"); + m_global_dict["get_ipython"] = kernel_module.attr("get_ipython"); kernel_module.attr("get_ipython")(); - py::globals()["_i"] = ""; - py::globals()["_ii"] = ""; - py::globals()["_iii"] = ""; - + m_global_dict["_i"] = ""; + m_global_dict["_ii"] = ""; + m_global_dict["_iii"] = ""; py::module context_module = get_request_context_module(); } @@ -146,6 +150,7 @@ namespace xpyt xeus::execute_request_config config, nl::json /*user_expressions*/) { + std::cout<<"execute_request_impl()"< matches; int cursor_start = cursor_pos; - py::list completions = get_completions(code, cursor_pos); + py::list completions = get_completions(code, cursor_pos, m_global_dict); if (py::len(completions) != 0) { @@ -258,7 +263,7 @@ namespace xpyt nl::json kernel_res; nl::json pub_data; - std::string docstring = formatted_docstring(code, cursor_pos); + std::string docstring = formatted_docstring(code, cursor_pos, m_global_dict); bool found = false; if (!docstring.empty()) @@ -277,7 +282,6 @@ namespace xpyt nl::json raw_interpreter::kernel_info_request_impl() { - /* The jupyter-console banner for xeus-python is the following: __ _____ _ _ ___ \ \/ / _ \ | | / __| @@ -342,14 +346,32 @@ namespace xpyt context_module.attr("set_request_context")(context); } + namespace + { + xeus::xrequest_context empty_request_context{}; + } + const xeus::xrequest_context& raw_interpreter::get_request_context() const noexcept { py::gil_scoped_acquire acquire; py::module context_module = get_request_context_module(); - py::object res = context_module.attr("get_request_context")(); - return *(res.cast()); + // When the debugger is started, it send some python code to execute, that triggers + // a call to publish_stream, and ultimately to this function. However: + // - we are out of the handling of an execute_request, therefore set_request_context + // has not been called + // - we cannot set it from another thread (the context of the context variable would + // be different) + // Therefore, we have to catch the exception thrown when the context variable is empty. + try + { + py::object res = context_module.attr("get_request_context")(); + return *(res.cast()); + } + catch (py::error_already_set& e) + { + return empty_request_context; + } } - void raw_interpreter::redirect_output() { py::module sys = py::module::import("sys"); diff --git a/src/xinterpreter_wasm.cpp b/src/xinterpreter_wasm.cpp index 38c3e205..333a56e1 100644 --- a/src/xinterpreter_wasm.cpp +++ b/src/xinterpreter_wasm.cpp @@ -22,7 +22,7 @@ namespace xpyt { wasm_interpreter::wasm_interpreter() - : interpreter(true, true) + : interpreter(py::globals(), true, true) { m_release_gil_at_startup = false; } diff --git a/src/xpaths.cpp b/src/xpaths.cpp index 3fbd84a2..3093195b 100644 --- a/src/xpaths.cpp +++ b/src/xpaths.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "pybind11/pybind11.h" @@ -44,6 +45,7 @@ namespace xpyt #elif defined(XEUS_PYTHONHOME_ABSPATH) static const std::string pythonhome = XPYT_STRINGIFY(XEUS_PYTHONHOME_ABSPATH); #else + using namespace std::filesystem; static const std::string pythonhome = xeus::prefix_path(); #endif return pythonhome; diff --git a/src/xpython_extension.cpp b/src/xpython_extension.cpp index c27618b0..511de1af 100644 --- a/src/xpython_extension.cpp +++ b/src/xpython_extension.cpp @@ -35,6 +35,8 @@ #include "xeus-python/xdebugger.hpp" #include "xeus-python/xutils.hpp" +#include "xeus-python/xaserver.hpp" + namespace py = pybind11; @@ -70,17 +72,36 @@ void launch(const py::list args_list) std::unique_ptr context = xeus::make_zmq_context(); + + py::dict globals = py::globals(); + // Instantiating the xeus xinterpreter using interpreter_ptr = std::unique_ptr; interpreter_ptr interpreter; if (raw_mode) { - interpreter = interpreter_ptr(new xpyt::raw_interpreter()); + interpreter = interpreter_ptr(new xpyt::raw_interpreter(globals)); } else { - interpreter = interpreter_ptr(new xpyt::interpreter()); + interpreter = interpreter_ptr(new xpyt::interpreter(globals)); } + + auto make_the_debugger = [&globals]( + xeus::xcontext& context, + const xeus::xkernel_configuration& config, + const std::string& user_name, + const std::string& session_id, + const nl::json& debugger_config) -> std::unique_ptr + { + return xpyt::make_python_debugger( + globals, + context, + config, + user_name, + session_id, + debugger_config); + }; using history_manager_ptr = std::unique_ptr; history_manager_ptr hist = xeus::make_in_memory_history_manager(); @@ -100,11 +121,11 @@ void launch(const py::list args_list) xeus::get_user_name(), std::move(context), std::move(interpreter), - xeus::make_xserver_shell_main, + xpyt::make_async_server_factory(globals), std::move(hist), xeus::make_console_logger(xeus::xlogger::msg_type, xeus::make_file_logger(xeus::xlogger::content, "xeus.log")), - xpyt::make_python_debugger); + make_the_debugger); std::clog << "Starting xeus-python kernel...\n\n" @@ -119,10 +140,10 @@ void launch(const py::list args_list) xeus::xkernel kernel(xeus::get_user_name(), std::move(context), std::move(interpreter), - xeus::make_xserver_shell_main, + xpyt::make_async_server_factory(globals), std::move(hist), nullptr, - xpyt::make_python_debugger); + make_the_debugger); const auto& config = kernel.get_config(); std::clog << diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5aceaf62..c6b110c6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -12,7 +12,6 @@ # ========== cmake_minimum_required(VERSION 3.20) - if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) project(xeus-python-test) diff --git a/test/test_xeus_python_kernel.py b/test/test_xeus_python_kernel.py index d4f28514..726b62f6 100644 --- a/test/test_xeus_python_kernel.py +++ b/test/test_xeus_python_kernel.py @@ -10,8 +10,8 @@ import unittest import jupyter_kernel_test - from jupyter_client.manager import start_new_kernel +import textwrap class XeusPythonTests(jupyter_kernel_test.KernelTests): @@ -48,6 +48,32 @@ def test_xeus_python_stdout(self): def test_xeus_python_stderr(self): reply, output_msgs = self.execute_helper(code='a = []; a.push_back(3)') self.assertEqual(output_msgs[0]['msg_type'], 'error') + + def test_toplevel_await(self): + code = textwrap.dedent(R""" + import asyncio + async def f(): + await asyncio.sleep(0.25) + print("World") + print("Hello") + await f() + print("!") + """) + reply, output_msgs = self.execute_helper(code=code) + self.assertEqual(reply['content']['status'], 'ok') + import json + print(json.dumps(output_msgs, indent=2, default=str)) + + def checkMsg(msg, text): + self.assertEqual(msg['msg_type'], 'stream') + self.assertEqual(msg['content']['name'], 'stdout') + self.assertEqual(msg['content']['text'], text) + + checkMsg(output_msgs[0], 'Hello') + checkMsg(output_msgs[1], '\n') # The newline after "Hello" + checkMsg(output_msgs[2], 'World') + checkMsg(output_msgs[3], '\n') # The newline after "World" + checkMsg(output_msgs[4], '!') if __name__ == '__main__':