Implement an LWTNN neural network node

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "xxhash"]
 	path = xxhash
 	url = https://github.com/Cyan4973/xxHash.git
+[submodule "lwtnn"]
+	path = lwtnn
+	url = https://github.com/lwtnn/lwtnn.git

CMakeLists.txt

@@ -1,13 +1,16 @@
-cmake_minimum_required(VERSION 3.11)
+cmake_minimum_required(VERSION 3.15...4.1)
 
 if(NOT SKBUILD_PROJECT_VERSION)
   set(SKBUILD_PROJECT_VERSION "0.0.0")  # provided by scikit-build-core
 endif()
 string(REPLACE "." ";" VERSION_SPLIT ${SKBUILD_PROJECT_VERSION})
+set(CORRECTIONLIB_SKBUILD_PROJECT_VERSION "${SKBUILD_PROJECT_VERSION}")
 list(GET VERSION_SPLIT 0 SPLIT_VERSION_MAJOR)
 list(GET VERSION_SPLIT 1 SPLIT_VERSION_MINOR)
 
 project(correctionlib VERSION ${SPLIT_VERSION_MAJOR}.${SPLIT_VERSION_MINOR} LANGUAGES CXX)
+set(CORRECTIONLIB_VERSION_MAJOR "${correctionlib_VERSION_MAJOR}")
+set(CORRECTIONLIB_VERSION_MINOR "${correctionlib_VERSION_MINOR}")
 
 option(BUILD_DEMO "Build demo program" ON)
 
@@ -24,12 +27,25 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads)
 find_package(ZLIB)
 
+# Configure lwtnn to build static library only and download dependencies
+set(BUILD_STATIC_LIBRARY ON)
+set(BUILD_TESTING OFF)
+set(BUILTIN_BOOST ON)
+set(BUILTIN_EIGEN ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+add_subdirectory(lwtnn EXCLUDE_FROM_ALL)
+
 
 configure_file(include/version.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/correctionlib_version.h)
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/include/correctionlib_version.h DESTINATION ${PKG_INSTALL}/include)
 
 
-add_library(correctionlib SHARED src/correction.cc src/formula_ast.cc)
+add_library(correctionlib SHARED
+  src/correction.cc
+  src/formula_ast.cc
+  src/detail_impl.cc
+  src/lwtnn.cc
+  )
 set_target_properties(correctionlib PROPERTIES PUBLIC_HEADER include/correction.h WINDOWS_EXPORT_ALL_SYMBOLS ON)
 target_include_directories(correctionlib
   PUBLIC
@@ -41,13 +57,15 @@ target_include_directories(correctionlib
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-peglib>
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/xxhash>
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/pcg-cpp/include>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lwtnn/include>
   )
 target_compile_features(correctionlib PUBLIC cxx_std_17)
 if(MSVC)
   target_compile_options(correctionlib PRIVATE /Zc:__cplusplus /utf-8)
 else()
   target_compile_options(correctionlib PRIVATE -Wall -Wextra -Wpedantic -Werror)
 endif()
+target_link_libraries(correctionlib PRIVATE lwtnn-stat)
 if(ZLIB_FOUND)
   target_link_libraries(correctionlib PRIVATE ZLIB::ZLIB)
 endif()
@@ -86,6 +104,12 @@ if(NOT BUILD_DEMO)
   set_target_properties(demo PROPERTIES EXCLUDE_FROM_ALL ON)
 endif()
 
+add_executable(lwtnn_demo src/lwtnn_demo.cc)
+target_link_libraries(lwtnn_demo PRIVATE correctionlib)
+target_compile_features(lwtnn_demo PUBLIC cxx_std_17)
+if(NOT BUILD_DEMO)
+  set_target_properties(lwtnn_demo PROPERTIES EXCLUDE_FROM_ALL ON)
+endif()
 
 install(EXPORT correctionlib-targets FILE correctionlib-targets.cmake DESTINATION ${PKG_INSTALL}/cmake)
 include(CMakePackageConfigHelpers)

README.md

@@ -85,6 +85,30 @@ using the
 [parse_obj](https://pydantic-docs.helpmanual.io/usage/models/#helper-functions)
 class method or by directly constructing them using keyword arguments.
 
+## Compatibility
+
+The evaluator keys on the `schema_version` field in the JSON data and only
+operates with the same major schema version. For minor versions, the evaluator
+MUST remain backwards-compatible with all older minor versions of the same major
+series, but MAY introduce new features that make it forwards-incompatible with
+older readers. For patch versions, the library MUST be fully
+backwards-compatible and MUST NOT introduce new features or schema changes.
+
+In practice we have not always lived up to these goals. The table below lists
+known deviations. Forwards-incompatible changes (marked "Forwards") are allowed
+under the versioning policy and are included for informational purposes only;
+backwards-incompatible changes (marked "Backwards") represent unintended breaks.
+
+| Version       | Break type                      | Change                                                                                                                                      |
+| ------------- | ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
+| v2.2.0        | Forwards (new node)             | `HashPRNG` node added; files using it require v2.2.0+                                                                                       |
+| v2.3.0        | Forwards (new edge format)      | `UniformBinning` object allowed for `edges`; requires v2.3.0+                                                                               |
+| v2.6.0        | Forwards (new edge values)      | String infinity sentinels (`"inf"`, `"-inf"`) allowed in edges; requires v2.6.0+                                                            |
+| v2.6.0–v2.6.3 | Backwards (patch rule violated) | Float `±inf` in edges rejected with no escape hatch; files using float infinities broke. `IGNORE_FLOAT_INF=True` workaround added in v2.6.4 |
+| v2.7.0        | Forwards (new flow value)       | `flow: "wrap"` added to `Binning`/`MultiBinning`; requires v2.7.0+                                                                          |
+| v2.8.0        | Forwards (`$schema` field)      | `"$schema"` key added to JSON, rejected by older Python parsers (`extra="forbid"`)                                                          |
+| v2.9.0        | Forwards (new node)             | `LWTNN` node added; files using it require v2.9.0+                                                                                          |
+
 ## Developing
 
 See [CONTRIBUTING.md](./CONTRIBUTING.md) for details on setting up a development

include/correction.h

@@ -20,6 +20,8 @@ class Variable {
     typedef std::variant<int64_t, double, std::string> Type;
 
     Variable(const JSONObject& json);
+    Variable(const std::string& name, const std::string& description, VarType type) :
+      name_(name), description_(description), type_(type) {};
     std::string name() const { return name_; };
     std::string description() const { return description_; };
     VarType type() const { return type_; };
@@ -38,10 +40,11 @@ class Formula;
 class FormulaRef;
 class Transform;
 class HashPRNG;
+class LWTNN;
 class Binning;
 class MultiBinning;
 class Category;
-typedef std::variant<double, Formula, FormulaRef, Transform, HashPRNG, Binning, MultiBinning, Category> Content;
+typedef std::variant<double, Formula, FormulaRef, Transform, HashPRNG, LWTNN, Binning, MultiBinning, Category> Content;
 class Correction;
 
 class FormulaAst {
@@ -176,36 +179,62 @@ class HashPRNG {
     Distribution dist_;
 };
 
-// common internal for Binning and MultiBinning
-enum class _FlowBehavior {value, clamp, error, wrap};
+// Forward declaration (opaque outside correctionlib source)
+namespace detail {
+  class LWTNNEvaluationContext;
+}
 
-using _NonUniformBins = std::vector<double>;
+class LWTNN {
+  public:
+    LWTNN(const JSONObject& json, const Correction& context);
+    double evaluate(const std::vector<Variable::Type>& values) const;
+
+    // this variant is in a separate source file, so move/delete needs to be explicit
+    // TODO: eventually break all the Content variants into separate source files
+    ~LWTNN();
+    LWTNN(LWTNN&&);
+    LWTNN& operator=(LWTNN&&);
 
-struct _UniformBins {
-   std::size_t n; // number of bins
-   double low; // lower edge of first bin
-   double high; // upper edge of last bin
+  private:
+    std::unique_ptr<const detail::LWTNNEvaluationContext> model_;
 };
 
+namespace detail {
+  // common internal for Binning and MultiBinning
+  enum class FlowBehavior {value, clamp, error, wrap};
+
+  using NonUniformBins = std::vector<double>;
+
+  struct UniformBins {
+    std::size_t n; // number of bins
+    double low; // lower edge of first bin
+    double high; // upper edge of last bin
+  };
+
+  using EdgesType = std::variant<UniformBins, NonUniformBins>;
+}
+
 class Binning {
   public:
     Binning(const JSONObject& json, const Correction& context);
     double evaluate(const std::vector<Variable::Type>& values) const;
 
   private:
-    std::variant<_UniformBins, _NonUniformBins> bins_; // bin edges
+    detail::EdgesType bins_; // bin edges
     // bin contents: contents_[i] is the value corresponding to bins_[i+1].
     // the default value is at contents_[0]
     std::vector<Content> contents_;
     size_t variableIdx_;
-    _FlowBehavior flow_;
+    detail::FlowBehavior flow_;
 };
 
-struct _MultiBinningAxis {
-  size_t variableIdx;
-  size_t stride;
-  std::variant<_UniformBins, _NonUniformBins> bins;
-};
+namespace detail {
+  struct MultiBinningAxis {
+    size_t variableIdx;
+    size_t stride;
+    detail::EdgesType bins;
+  };
+}
 
 class MultiBinning {
   public:
@@ -216,9 +245,9 @@ class MultiBinning {
   private:
     size_t nbins(size_t dimension) const;
 
-    std::vector<_MultiBinningAxis> axes_;
+    std::vector<detail::MultiBinningAxis> axes_;
     std::vector<Content> content_;
-    _FlowBehavior flow_;
+    detail::FlowBehavior flow_;
 };
 
 class Category {

include/version.h.in

@@ -4,9 +4,9 @@
 #include <string_view>
 
 namespace correction {
-  constexpr std::string_view correctionlib_version{"@SKBUILD_PROJECT_VERSION@"};
-  constexpr int correctionlib_major_version{@correctionlib_VERSION_MAJOR@};
-  constexpr int correctionlib_minor_version{@correctionlib_VERSION_MINOR@};
+	constexpr std::string_view correctionlib_version{"@CORRECTIONLIB_SKBUILD_PROJECT_VERSION@"};
+	constexpr int correctionlib_major_version{@CORRECTIONLIB_VERSION_MAJOR@};
+	constexpr int correctionlib_minor_version{@CORRECTIONLIB_VERSION_MINOR@};
 }
 
 #endif // CORRECTIONLIB_VERSION_H