Merge branch 'main' into fix-compatible-mode

Author: orisgarno

.github/workflows/ci.yml

@@ -218,7 +218,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.12]
+        python-version: [3.8, 3.12]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/release.yaml

@@ -32,7 +32,7 @@ jobs:
       url: https://pypi.org/project/pyfury
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, 3.10.12, 3.11, 3.12]
+        python-version: [3.8, 3.9, 3.10.12, 3.11, 3.12]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

BUILD

@@ -62,6 +62,7 @@ pyx_library(
     ),
     deps = [
         "//cpp/fury/util:fury_util",
+        "//cpp/fury/type:fury_type",
         "@com_google_absl//absl/container:flat_hash_map",
     ],
 )

README.md

@@ -278,7 +278,7 @@ class SomeClass:
     f3: Dict[str, str]
 
 fury = pyfury.Fury(ref_tracking=True)
-fury.register_class(SomeClass, type_tag="example.SomeClass")
+fury.register_type(SomeClass, typename="example.SomeClass")
 obj = SomeClass()
 obj.f2 = {"k1": "v1", "k2": "v2"}
 obj.f1, obj.f3 = obj, obj.f2

ci/run_ci.py

@@ -17,6 +17,7 @@
 
 
 import argparse
+import shutil
 import subprocess
 import platform
 import urllib.request as ulib
@@ -37,6 +38,11 @@
 )
 
 
+def _bazel(cmd: str):
+    bazel_cmd = "bazel" if _is_windows() else "~/bin/bazel"
+    return _exec_cmd(f"{bazel_cmd} {cmd}")
+
+
 def _exec_cmd(cmd: str):
     logging.info(f"running command: {cmd}")
     try:
@@ -81,10 +87,8 @@ def _cd_project_subdir(subdir):
 def _run_cpp():
     _install_cpp_deps()
     # run test
-    query_result = _exec_cmd("bazel query //...")
-    _exec_cmd(
-        "bazel test {}".format(query_result.replace("\n", " ").replace("\r", " "))
-    )
+    query_result = _bazel("query //...")
+    _bazel("test {}".format(query_result.replace("\n", " ").replace("\r", " ")))
 
 
 def _run_rust():
@@ -132,11 +136,14 @@ def _install_bazel():
         bazel_path = os.path.join(os.getcwd(), local_name)
         _exec_cmd(f'setx path "%PATH%;{bazel_path}"')
     else:
+        if shutil.which("bazel"):
+            os.remove(shutil.which("bazel"))
         _exec_cmd(f"./{local_name} --user")
+        _update_shell_profile()
         os.remove(local_name)
 
     # bazel install status check
-    _exec_cmd("bazel --version")
+    _bazel("--version")
 
     # default is byte
     psutil = importlib.import_module("psutil")
@@ -146,6 +153,21 @@ def _install_bazel():
         file.write(f"\nbuild --jobs={limit_jobs}")
 
 
+def _update_shell_profile():
+    home = os.path.expanduser("~")
+    profiles = [".bashrc", ".bash_profile", ".zshrc"]
+    path_export = 'export PATH="$PATH:$HOME/bin" # Add Bazel to PATH\n'
+    for profile in profiles:
+        profile_path = os.path.join(home, profile)
+        if os.path.exists(profile_path):
+            with open(profile_path, "a") as f:
+                f.write(path_export)
+            logging.info(f"Updated {profile} to include Bazel PATH.")
+            break
+    else:
+        logging.info("No shell profile found. Please add Bazel to PATH manually.")
+
+
 def _parse_args():
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter

cpp/fury/type/BUILD

@@ -0,0 +1,15 @@
+load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
+
+cc_library(
+    name = "fury_type",
+    srcs = glob(["*.cc"], exclude=["*test.cc"]),
+    hdrs = glob(["*.h"]),
+    copts = ["-mavx2"],  # Enable AVX2 support
+    linkopts = ["-mavx2"],  # Ensure linker also knows about AVX2
+    strip_include_prefix = "/cpp",
+    alwayslink=True,
+    linkstatic=True,
+    deps = [
+    ],
+    visibility = ["//visibility:public"],
+)

cpp/fury/type/type.h

@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <cstdint> // For fixed-width integer types
+
+namespace fury {
+enum class TypeId : int32_t {
+  // a boolean value (true or false).
+  BOOL = 1,
+  // a 8-bit signed integer.
+  INT8 = 2,
+  // a 16-bit signed integer.
+  INT16 = 3,
+  // a 32-bit signed integer.
+  INT32 = 4,
+  // a 32-bit signed integer which use fury var_int32 encoding.
+  VAR_INT32 = 5,
+  // a 64-bit signed integer.
+  INT64 = 6,
+  // a 64-bit signed integer which use fury PVL encoding.
+  VAR_INT64 = 7,
+  // a 64-bit signed integer which use fury SLI encoding.
+  SLI_INT64 = 8,
+  // a 16-bit floating point number.
+  FLOAT16 = 9,
+  // a 32-bit floating point number.
+  FLOAT32 = 10,
+  // a 64-bit floating point number including NaN and Infinity.
+  FLOAT64 = 11,
+  // a text string encoded using Latin1/UTF16/UTF-8 encoding.
+  STRING = 12,
+  // a data type consisting of a set of named values. Rust enum with
+  // non-predefined field values are not supported as an enum
+  ENUM = 13,
+  // an enum whose value will be serialized as the registered name.
+  NAMED_ENUM = 14,
+  // a morphic(final) type serialized by Fury Struct serializer. i.e. it doesn't
+  // have subclasses. Suppose we're
+  // deserializing `List<SomeClass>`, we can save dynamic serializer dispatch
+  // since `SomeClass` is morphic(final).
+  STRUCT = 15,
+  // a type which is not morphic(not final). i.e. it has subclasses. Suppose
+  // we're deserializing
+  // `List<SomeClass>`, we must dispatch serializer dynamically since
+  // `SomeClass` is polymorphic(non-final).
+  POLYMORPHIC_STRUCT = 16,
+  // a morphic(final) type serialized by Fury compatible Struct serializer.
+  COMPATIBLE_STRUCT = 17,
+  // a non-morphic(non-final) type serialized by Fury compatible Struct
+  // serializer.
+  POLYMORPHIC_COMPATIBLE_STRUCT = 18,
+  // a `struct` whose type mapping will be encoded as a name.
+  NAMED_STRUCT = 19,
+  // a `polymorphic_struct` whose type mapping will be encoded as a name.
+  NAMED_POLYMORPHIC_STRUCT = 20,
+  // a `compatible_struct` whose type mapping will be encoded as a name.
+  NAMED_COMPATIBLE_STRUCT = 21,
+  // a `polymorphic_compatible_struct` whose type mapping will be encoded as a
+  // name.
+  NAMED_POLYMORPHIC_COMPATIBLE_STRUCT = 22,
+  // a type which will be serialized by a customized serializer.
+  EXT = 23,
+  // an `ext` type which is not morphic(not final).
+  POLYMORPHIC_EXT = 24,
+  // an `ext` type whose type mapping will be encoded as a name.
+  NAMED_EXT = 25,
+  // an `polymorphic_ext` type whose type mapping will be encoded as a name.
+  NAMED_POLYMORPHIC_EXT = 26,
+  // a sequence of objects.
+  LIST = 27,
+  // an unordered set of unique elements.
+  SET = 28,
+  // a map of key-value pairs. Mutable types such as
+  // `list/map/set/array/tensor/arrow` are not allowed as key of map.
+  MAP = 29,
+  // an absolute length of time, independent of any calendar/timezone, as a
+  // count of nanoseconds.
+  DURATION = 30,
+  // a point in time, independent of any calendar/timezone, as a count of
+  // nanoseconds. The count is relative
+  // to an epoch at UTC midnight on January 1, 1970.
+  TIMESTAMP = 31,
+  // a naive date without timezone. The count is days relative to an epoch at
+  // UTC midnight on Jan 1, 1970.
+  LOCAL_DATE = 32,
+  // exact decimal value represented as an integer value in two's complement.
+  DECIMAL = 33,
+  // an variable-length array of bytes.
+  BINARY = 34,
+  // a multidimensional array which every sub-array can have different sizes but
+  // all have same type.
+  // only allow numeric components. Other arrays will be taken as List. The
+  // implementation should support the
+  // interoperability between array and list.
+  ARRAY = 35,
+  // one dimensional bool array.
+  BOOL_ARRAY = 36,
+  // one dimensional int16 array.
+  INT8_ARRAY = 37,
+  // one dimensional int16 array.
+  INT16_ARRAY = 38,
+  // one dimensional int32 array.
+  INT32_ARRAY = 39,
+  // one dimensional int64 array.
+  INT64_ARRAY = 40,
+  // one dimensional half_float_16 array.
+  FLOAT16_ARRAY = 41,
+  // one dimensional float32 array.
+  FLOAT32_ARRAY = 42,
+  // one dimensional float64 array.
+  FLOAT64_ARRAY = 43,
+  // an arrow [record
+  // batch](https://arrow.apache.org/docs/cpp/tables.html#record-batches)
+  // object.
+  ARROW_RECORD_BATCH = 44,
+  // an arrow [table](https://arrow.apache.org/docs/cpp/tables.html#tables)
+  // object.
+  ARROW_TABLE = 45,
+  BOUND = 64
+};
+
+inline bool IsNamespacedType(int32_t type_id) {
+  switch (static_cast<TypeId>(type_id)) {
+  case TypeId::NAMED_ENUM:
+  case TypeId::NAMED_STRUCT:
+  case TypeId::NAMED_POLYMORPHIC_STRUCT:
+  case TypeId::NAMED_COMPATIBLE_STRUCT:
+  case TypeId::NAMED_POLYMORPHIC_COMPATIBLE_STRUCT:
+  case TypeId::NAMED_EXT:
+  case TypeId::NAMED_POLYMORPHIC_EXT:
+    return true;
+  default:
+    return false;
+  }
+}
+
+} // namespace fury

cpp/fury/util/buffer.h

@@ -27,6 +27,7 @@
 
 #include "fury/util/bit_util.h"
 #include "fury/util/logging.h"
+#include "fury/util/status.h"
 
 namespace fury {
 
@@ -133,6 +134,29 @@ class Buffer {
 
   inline double GetDouble(uint32_t offset) { return Get<double>(offset); }
 
+  inline Status GetBytesAsInt64(uint32_t offset, uint32_t length,
+                                int64_t *target) {
+    if (length == 0) {
+      *target = 0;
+      return Status::OK();
+    }
+    if (size_ - (offset + 8) > 0) {
+      uint64_t mask = 0xffffffffffffffff;
+      uint64_t x = (mask >> (8 - length) * 8);
+      *target = GetInt64(offset) & x;
+    } else {
+      if (size_ - (offset + length) < 0) {
+        return Status::OutOfBound("buffer out of bound");
+      }
+      int64_t result = 0;
+      for (size_t i = 0; i < length; i++) {
+        result = result | ((int64_t)(data_[offset + i])) << (i * 8);
+      }
+      *target = result;
+    }
+    return Status::OK();
+  }
+
   inline uint32_t PutVarUint32(uint32_t offset, int32_t value) {
     if (value >> 7 == 0) {
       data_[offset] = (int8_t)value;

cpp/fury/util/buffer_test.cc

@@ -65,6 +65,16 @@ TEST(Buffer, TestVarUint) {
   }
 }
 
+TEST(Buffer, TestGetBytesAsInt64) {
+  std::shared_ptr<Buffer> buffer;
+  AllocateBuffer(64, &buffer);
+  buffer->UnsafePut<int32_t>(0, 100);
+  int64_t result = -1;
+  EXPECT_TRUE(buffer->GetBytesAsInt64(0, 0, &result).ok());
+  EXPECT_EQ(result, 0);
+  EXPECT_TRUE(buffer->GetBytesAsInt64(0, 1, &result).ok());
+  EXPECT_EQ(result, 100);
+}
 } // namespace fury
 
 int main(int argc, char **argv) {

cpp/fury/util/logging.cc

@@ -111,4 +111,4 @@ bool FuryLog::IsLevelEnabled(FuryLogLevel log_level) {
   return log_level >= fury_severity_threshold;
 }
 
-} // namespace fury
+} // namespace fury