feat(pj_base): add pj.data_processors.v1 data-only SDK service for plugin-created transforms (#125)

Adds the `pj.data_processors.v1` host service: a DATA-ONLY C ABI that lets a
toolbox plugin create catalog-resident "transform" nodes in the host. Only
strings cross the boundary (script source + input/output names + params JSON +
an id); the host owns all execution, so a transform survives plugin unload and
session reload — the originating plugin is needed only for re-editing.

- PJ_data_processors_host_vtable_t (create/remove/list/config), protocol_version 1,
  struct_size-gated and ABI-appendable; all slots [main-thread], PJ_NOEXCEPT.
- sdk::DataProcessorsHostView C++ sugar + DataProcessorsHostService trait.
- Payload (script/params) is binary-safe (PJ_string_view_t {data,size}), so the
  native "door" is WASM bytes through this same surface — a future host-owned
  WASM/Python backend is purely additive and survives unload, deliberately NOT a
  C++ kernel vtable that would dangle on DSO unload.
- The view now rejects an empty output list (transforms create named topics, so
  >=1 output is required) before it ever reaches the vtable.
- Fake-host gtest suite (8 cases) incl. a binary-safe payload round-trip and the
  empty-outputs guard.

Additive new service -> Conan MINOR (0.8.0 -> 0.9.0); abi/baseline.abi untouched.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: facontidavide

CMakeLists.txt

@@ -111,7 +111,7 @@ endif()
 if(PJ_INSTALL_SDK)
   include(CMakePackageConfigHelpers)
 
-  set(PJ_PACKAGE_VERSION "0.8.1")
+  set(PJ_PACKAGE_VERSION "0.9.0")
   set(PJ_PACKAGE_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/plotjuggler_sdk)
 
   install(EXPORT plotjuggler_sdkTargets

conanfile.py

@@ -6,7 +6,7 @@
   plugin_sdk   — umbrella for plugin authors (base + dialog SDK + parser SDK)
   plugin_host  — umbrella for host loaders (data_source/parser/toolbox/dialog)
 
-A consuming Conan recipe declares e.g. `plotjuggler_sdk/0.8.1` and then:
+A consuming Conan recipe declares e.g. `plotjuggler_sdk/0.9.0` and then:
 
     find_package(plotjuggler_sdk REQUIRED COMPONENTS plugin_sdk)
     target_link_libraries(my_plugin PRIVATE plotjuggler_sdk::plugin_sdk)
@@ -30,7 +30,7 @@
 
 class PlotjugglerSdkConan(ConanFile):
     name = "plotjuggler_sdk"
-    version = "0.8.1"
+    version = "0.9.0"
     # Apache-2.0 covers the whole SDK (pj_base + pj_plugins). See LICENSE.
     license = "Apache-2.0"
     url = "https://github.com/PlotJuggler/plotjuggler_sdk"

pj_base/CMakeLists.txt

@@ -72,6 +72,7 @@ if(PJ_BUILD_TESTS)
     tests/expected_test.cpp
     tests/number_parse_test.cpp
     tests/plugin_data_api_test.cpp
+    tests/data_processors_api_test.cpp
     tests/settings_store_host_test.cpp
     tests/data_source_protocol_test.cpp
     # TODO: data_source_plugin_base_test.cpp and

pj_base/include/pj_base/plugin_data_api.h

@@ -754,6 +754,85 @@ typedef struct {
   const PJ_colormap_registry_vtable_t* vtable;
 } PJ_colormap_registry_t;
 
+/**
+ * Data-processor host service ("pj.data_processors.v1", protocol_version 1).
+ *
+ * Lets a toolbox plugin create catalog-resident "transform" nodes in the host
+ * by DATA only. Nothing executable crosses the boundary — only strings:
+ *
+ *   - id          : per-call stable key. The host namespaces it under the
+ *                   calling plugin. create() UPSERTS by id (idempotent Save).
+ *                   ids survive a session reload, so a plugin can re-edit its
+ *                   nodes after the host has replayed them from persisted
+ *                   recipes (see data_processor_config).
+ *   - inputs      : topic OR topic-field names ("pose/orientation" or
+ *                   "pose/orientation/x"); the host resolves them and exact-joins
+ *                   co-timestamped inputs. Field-level resolution is a host
+ *                   concern and needs no ABI change.
+ *   - outputs     : REQUIRED non-empty. This service creates named catalog
+ *                   topics (transforms). View-bound hidden-output "filters" are
+ *                   host-internal and are NOT created through this ABI.
+ *   - script      : the processor class source. Its first-line directive
+ *                   ("-- pj-script: <lang>") selects the backend; an unsupported
+ *                   one is rejected by the host at create() time. BINARY-SAFE:
+ *                   script is a PJ_string_view_t {data,size}, NOT a C string, so
+ *                   it MAY carry a non-text blob — e.g. a future WASM module
+ *                   detected by the leading "\0asm" magic. Nothing on the path
+ *                   may treat it as NUL-terminated (no strlen).
+ *   - params_json : forwarded VERBATIM to the backend's create(params); the host
+ *                   does not interpret its keys (translucent pass-through). Also
+ *                   a binary-safe PJ_string_view_t {data,size}.
+ *
+ * Per-plugin isolation: the host namespaces every id under the calling plugin, so
+ * list/remove/config only ever see or affect THAT plugin's nodes; one plugin can
+ * neither enumerate nor remove another's.
+ *
+ * FORWARD-COMPAT — the native door is WASM, not a C++ kernel. Because the script
+ * slot is a binary-safe blob the host owns and runs (today Luau; tomorrow a
+ * host-owned WASM/Python backend), a native processor needs NO new ABI: it ships
+ * as bytes through this same data-only surface, survives plugin unload (the host
+ * owns the payload + runtime), and is sandboxed. Adding a backend is purely
+ * additive. This is deliberately the OPPOSITE of crossing a C++ vtable across the
+ * DSO boundary (which dangles on unload).
+ *
+ * ABI-APPENDABLE: new slots may be added at the tail; struct_size gates read.
+ */
+typedef struct PJ_data_processors_host_vtable_t {
+  uint32_t protocol_version;  // = 1
+  uint32_t struct_size;       // = sizeof(PJ_data_processors_host_vtable_t)
+
+  /* [main-thread] Create or replace (upsert by id) a transform node. outputs
+   * MUST be non-empty. Transactional: on failure no partial node/topic is left
+   * behind. All string arguments are borrowed for the duration of the call. */
+  bool (*create_data_processor)(
+      void* ctx, PJ_string_view_t id, const PJ_string_view_t* inputs, uint64_t input_count,
+      const PJ_string_view_t* outputs, uint64_t output_count, PJ_string_view_t script, PJ_string_view_t params_json,
+      PJ_error_t* out_error) PJ_NOEXCEPT;
+
+  /* [main-thread] Remove a node by id. An unknown id is an error. */
+  bool (*remove_data_processor)(void* ctx, PJ_string_view_t id, PJ_error_t* out_error) PJ_NOEXCEPT;
+
+  /* [main-thread] Enumerate the ids of THIS plugin's live processors.
+   * Count-then-fill: pass capacity 0 to read *out_count, then call again with a
+   * buffer of that size. On success the first min(capacity, *out_count) entries
+   * of out_ids are filled and point into host storage valid only until the next
+   * call on this vtable. */
+  bool (*list_data_processor_ids)(
+      void* ctx, PJ_string_view_t* out_ids, uint64_t capacity, uint64_t* out_count, PJ_error_t* out_error) PJ_NOEXCEPT;
+
+  /* [main-thread] Read a node's full recipe as JSON
+   * {"inputs":[...],"outputs":[...],"params":{...}} for re-edit (e.g. after a
+   * session reload). *out_recipe_json is borrowed, valid only until the next
+   * call on this vtable. An unknown id is an error. */
+  bool (*data_processor_config)(
+      void* ctx, PJ_string_view_t id, PJ_string_view_t* out_recipe_json, PJ_error_t* out_error) PJ_NOEXCEPT;
+} PJ_data_processors_host_vtable_t;
+
+typedef struct {
+  void* ctx;
+  const PJ_data_processors_host_vtable_t* vtable;
+} PJ_data_processors_host_t;
+
 /**
  * Settings store service ("pj.settings.v1", protocol_version 1).
  *

pj_base/include/pj_base/sdk/plugin_data_api.hpp

@@ -1286,6 +1286,109 @@ class ColorMapRegistryView {
   PJ_colormap_registry_t registry_{};
 };
 
+// ---------------------------------------------------------------------------
+// DataProcessorsHostView — typed C++ view over PJ_data_processors_host_t
+// ---------------------------------------------------------------------------
+
+/// C++ wrapper around PJ_data_processors_host_t for plugins that create
+/// catalog-resident transform nodes in the host (see the C ABI doc-comment on
+/// PJ_data_processors_host_vtable_t). Empty-constructible; `valid()` tells
+/// whether the host exposed the service. Strings returned by `list()`/
+/// `recipeOf()` are copied into owned values, so they stay valid past the next
+/// vtable call.
+class DataProcessorsHostView {
+ public:
+  DataProcessorsHostView() = default;
+  explicit DataProcessorsHostView(PJ_data_processors_host_t host) : host_(host) {}
+
+  [[nodiscard]] bool valid() const noexcept {
+    return host_.vtable != nullptr && host_.ctx != nullptr;
+  }
+
+  /// Create or replace (upsert by id) a transform node. `outputs` must be
+  /// non-empty (the host rejects empty output lists). `params_json` is
+  /// forwarded verbatim to the script's create(params).
+  [[nodiscard]] Status createTransform(
+      std::string_view id, Span<const std::string_view> inputs, Span<const std::string_view> outputs,
+      std::string_view script, std::string_view params_json) const {
+    if (!valid() || host_.vtable->create_data_processor == nullptr) {
+      return unexpected("data processors host is not bound");
+    }
+    if (outputs.empty()) {
+      return unexpected("data processors transform requires at least one output topic");
+    }
+    std::vector<PJ_string_view_t> in_abi;
+    in_abi.reserve(inputs.size());
+    for (const auto& name : inputs) {
+      in_abi.push_back(toAbiString(name));
+    }
+    std::vector<PJ_string_view_t> out_abi;
+    out_abi.reserve(outputs.size());
+    for (const auto& name : outputs) {
+      out_abi.push_back(toAbiString(name));
+    }
+    PJ_error_t err{};
+    if (!host_.vtable->create_data_processor(
+            host_.ctx, toAbiString(id), in_abi.data(), in_abi.size(), out_abi.data(), out_abi.size(),
+            toAbiString(script), toAbiString(params_json), &err)) {
+      return unexpected(errorToString(err));
+    }
+    return okStatus();
+  }
+
+  /// Remove a previously created node by id.
+  [[nodiscard]] Status remove(std::string_view id) const {
+    if (!valid() || host_.vtable->remove_data_processor == nullptr) {
+      return unexpected("data processors host is not bound");
+    }
+    PJ_error_t err{};
+    if (!host_.vtable->remove_data_processor(host_.ctx, toAbiString(id), &err)) {
+      return unexpected(errorToString(err));
+    }
+    return okStatus();
+  }
+
+  /// Enumerate the ids of this plugin's live processors (owned copies).
+  [[nodiscard]] Expected<std::vector<std::string>> list() const {
+    if (!valid() || host_.vtable->list_data_processor_ids == nullptr) {
+      return unexpected("data processors host is not bound");
+    }
+    PJ_error_t err{};
+    uint64_t count = 0;
+    if (!host_.vtable->list_data_processor_ids(host_.ctx, nullptr, 0, &count, &err)) {
+      return unexpected(errorToString(err));
+    }
+    std::vector<PJ_string_view_t> borrowed(count);
+    uint64_t filled = 0;
+    if (count != 0 &&
+        !host_.vtable->list_data_processor_ids(host_.ctx, borrowed.data(), borrowed.size(), &filled, &err)) {
+      return unexpected(errorToString(err));
+    }
+    std::vector<std::string> ids;
+    ids.reserve(filled);
+    for (uint64_t i = 0; i < filled; ++i) {
+      ids.emplace_back(toStringView(borrowed[i]));
+    }
+    return ids;
+  }
+
+  /// Read a node's full recipe JSON (owned copy) for re-edit.
+  [[nodiscard]] Expected<std::string> recipeOf(std::string_view id) const {
+    if (!valid() || host_.vtable->data_processor_config == nullptr) {
+      return unexpected("data processors host is not bound");
+    }
+    PJ_error_t err{};
+    PJ_string_view_t recipe{};
+    if (!host_.vtable->data_processor_config(host_.ctx, toAbiString(id), &recipe, &err)) {
+      return unexpected(errorToString(err));
+    }
+    return std::string(toStringView(recipe));
+  }
+
+ private:
+  PJ_data_processors_host_t host_{};
+};
+
 // ---------------------------------------------------------------------------
 // SettingsView — typed C++ view over PJ_settings_store_t
 // ---------------------------------------------------------------------------

pj_base/include/pj_base/sdk/service_traits.hpp

@@ -158,6 +158,19 @@ struct ColorMapRegistryService {
   static_assert(detail::isValidServiceName(kName), "kName must match the pj naming rule");
 };
 
+/// Optional service for plugins that create catalog-resident transform nodes in
+/// the host by data (script + input/output names + params JSON). Nothing
+/// executable crosses the boundary; the host owns execution. See the C ABI
+/// doc-comment on PJ_data_processors_host_vtable_t.
+struct DataProcessorsHostService {
+  static constexpr const char* kName = "pj.data_processors.v1";
+  static constexpr uint32_t kMinVersion = 1;
+  using Raw = PJ_data_processors_host_t;
+  using Vtable = PJ_data_processors_host_vtable_t;
+  using View = DataProcessorsHostView;
+  static_assert(detail::isValidServiceName(kName), "kName must match the pj naming rule");
+};
+
 /// Optional QSettings-like key/value persistence exposed to any plugin family.
 /// Host-backed (QSettings in the GUI app, JSON in a headless host); keys are
 /// namespaced per plugin by the host.