Support uninitialized tensors in file IO reads

docs_input/api/io/index.rst

@@ -3,8 +3,67 @@
 Input/Output
 ############
 
+MatX file IO helpers read and write common array file formats through the
+optional ``MATX_ENABLE_FILEIO`` support. Include ``matx.h`` and use the
+``matx::io`` namespace functions shown below.
+
+Read functions can write into an already-sized tensor, or into a
+default-constructed tensor with the desired rank and value type. When the tensor
+has no storage yet, MatX allocates it after discovering the shape from the file.
+
+CSV
+===
+
+CSV files support rank-1 and rank-2 tensors. The delimiter is passed explicitly,
+and ``read_csv`` skips the first row by default.
+
+.. code-block:: cpp
+
+   tensor_t<float, 2> samples;
+   io::read_csv(samples, "samples.csv", ",");
+
+   io::write_csv(samples, "samples_out.csv", ",");
+
+To read a file without skipping the first row, pass ``false`` for the final
+argument.
+
+.. code-block:: cpp
+
+   io::read_csv(samples, "samples_out.csv", ",", false);
+
+MAT
+===
+
+MAT files can contain multiple named variables. ``read_mat`` and ``write_mat``
+therefore take a variable name in addition to the file name.
+
+.. code-block:: cpp
+
+   tensor_t<float, 2> A;
+   io::read_mat(A, "arrays.mat", "A");
+
+   auto B = io::read_mat<tensor_t<float, 2>>("arrays.mat", "B");
+
+   io::write_mat(A, "arrays_out.mat", "A");
+
+MATLAB v7.3 MAT files are HDF5-based, but the MatX MAT helpers are variable
+oriented and use SciPy's MAT-file routines. Treat them as MAT-file readers and
+writers rather than as a general HDF5 interface.
+
+NPY
+===
+
+NPY files store a single NumPy array per file.
+
+.. code-block:: cpp
+
+   tensor_t<float, 2> x;
+   io::read_npy(x, "x.npy");
+
+   io::write_npy(x, "x_out.npy");
+
 .. toctree::
    :maxdepth: 1
    :glob:
 
-   *
+   *

docs_input/api/io/read_mat.rst

@@ -3,11 +3,15 @@
 read_mat
 ========
 
-Read a CSV file into a tensor
+Read a variable from a MAT file into a tensor
 
 .. note::
    This function requires the optional ``MATX_ENABLE_FILEIO`` compile flag
 
+MAT files can contain multiple named variables. Pass the variable name in
+``var`` to select the tensor to read. MATLAB v7.3 MAT files are HDF5-based, but
+these helpers use SciPy's MAT-file routines and are not a general HDF5
+interface.
 
 
 .. versionadded:: 0.3.0

docs_input/api/io/write_mat.rst

@@ -3,11 +3,13 @@
 write_mat
 =========
 
-Write an operator to a MAT file
+Write a tensor to a MAT file variable
 
 .. note::
    This function requires the optional ``MATX_ENABLE_FILEIO`` compile flag
 
+MAT files can contain multiple named variables. ``write_mat`` writes the tensor
+under the variable name passed in ``var``.
 
 
 .. versionadded:: 0.3.0

include/matx/core/dynamic_tensor.h

@@ -218,6 +218,11 @@ class dynamic_tensor_t {
 
   __MATX_INLINE__ T *Data() const { return ldata_; }
 
+  __MATX_INLINE__ bool IsInitialized() const noexcept
+  {
+    return Data() != nullptr;
+  }
+
   __MATX_INLINE__ index_t TotalSize() const {
     index_t total = 1;
     for (int i = 0; i < rank_; ++i) {

include/matx/core/pybind.h

@@ -40,6 +40,7 @@ MATX_IGNORE_WARNING_PUSH_GCC("-Wnull-dereference")
 #include <pybind11/embed.h>
 #include <pybind11/numpy.h>
 MATX_IGNORE_WARNING_POP_GCC
+#include <algorithm>
 #include <optional>
 #include <filesystem>
 
@@ -378,23 +379,54 @@ class MATX_PYBIND_VISIBILITY MatXPybind {
     return indices;
   }
 
+  template <typename TensorType>
+  static constexpr bool has_is_initialized_v = requires(const TensorType &ten) {
+    ten.IsInitialized();
+  };
+
   template <typename TensorType>
   void NumpyToTensorView(TensorType &ten,
-                         const std::string fname)
+                         const std::string fname,
+                         bool exact_shape = false)
   {
     auto resobj = res_dict[fname.c_str()];
-    NumpyToTensorView(ten, resobj);
+    NumpyToTensorView(ten, resobj, exact_shape);
   }
 
   template <typename TensorType>
-  void NumpyToTensorView(TensorType ten,
-                         const pybind11::object &np_ten)
+  void NumpyToTensorView(TensorType &&ten,
+                         const pybind11::object &np_ten,
+                         bool exact_shape = false)
   {
-    using T = typename TensorType::value_type;
-    constexpr int RANK = TensorType::Rank();
+    using Tensor = remove_cvref_t<TensorType>;
+    using T = typename Tensor::value_type;
+    constexpr int RANK = Tensor::Rank();
 
     using ntype = matx_convert_complex_type<T>;
     auto ften = pybind11::array_t<ntype>(np_ten);
+    auto info = ften.request();
+
+    MATX_ASSERT_STR(info.ndim == RANK, matxInvalidDim,
+                    "Numpy array rank does not match tensor rank");
+
+    if constexpr (has_is_initialized_v<Tensor>) {
+      if (!ten.IsInitialized()) {
+        cuda::std::array<matx::index_t, RANK> shape;
+        std::copy_n(info.shape.begin(), RANK, std::begin(shape));
+        // The copy below writes from host code, so use the default
+        // host-accessible allocation path.
+        make_tensor(ten, shape);
+      }
+    }
+
+    // File IO requires exact shapes. Some generated test vectors intentionally
+    // copy a smaller tensor from a larger NumPy array, so keep that path bounded.
+    for (int d = 0; d < RANK; d++) {
+      const auto np_size = static_cast<index_t>(info.shape[d]);
+      const bool valid_size = exact_shape ? ten.Size(d) == np_size : ten.Size(d) <= np_size;
+      MATX_ASSERT_STR(valid_size, matxInvalidSize,
+                      "Numpy array dimension size is incompatible with tensor size");
+    }
 
     if constexpr (RANK == 0) {
       ten() = ConvertComplex(ften.at());

include/matx/core/tensor_impl.h

@@ -1557,6 +1557,16 @@ MATX_IGNORE_WARNING_POP_GCC
       return data_.ldata_;
     }
 
+    /**
+     * @brief Check whether this tensor has an assigned data pointer.
+     *
+     * @return true if the tensor has storage or a non-owning data pointer
+     */
+    __MATX_INLINE__ __MATX_HOST__ __MATX_DEVICE__ bool IsInitialized() const noexcept
+    {
+      return Data() != nullptr;
+    }
+
     /**
      * @brief Set data pointer
      *

include/matx/file_io/file_io.h

@@ -158,7 +158,7 @@ void read_csv(TensorType &t, const std::string fname,
   auto obj = np.attr("genfromtxt")("fname"_a = fname.c_str(), "delimiter"_a = delimiter,
                                    "skip_header"_a = skip_header,
                                    "dtype"_a = detail::MatXPybind::GetNumpyDtype<typename TensorType::value_type>());
-  pb->NumpyToTensorView(t, obj);
+  pb->NumpyToTensorView(t, obj, true);
 }
 
 /**
@@ -200,9 +200,9 @@ void write_csv(const TensorType &t, const std::string fname,
 /**
  * @brief Read a MAT file into a tensor view
  *
- * MAT files use SciPy's loadmat() function to read various MATLAB file
- * types in. MAT files are supersets of HDF5 files, and are allowed to
- * have multiple fields in them.
+ * MAT files use SciPy's loadmat() function to read MATLAB variables. MATLAB
+ * v7.3 MAT files are HDF5-based, but this helper is intended for MAT-file
+ * variables rather than as a general HDF5 interface.
  *
  * @tparam TensorType
  *   Data type of tensor
@@ -235,15 +235,15 @@ void read_mat(TensorType &t, const std::string fname,
   auto obj = (pybind11::dict)sp.attr("loadmat")("file_name"_a = fname);
   auto v = obj[var.c_str()];
 
-  pb->NumpyToTensorView(t, v);
+  pb->NumpyToTensorView(t, v, true);
 }
 
 /**
  * @brief Read a MAT file and return a tensor view
  *
- * MAT files use SciPy's loadmat() function to read various MATLAB file
- * types in. MAT files are supersets of HDF5 files, and are allowed to
- * have multiple fields in them.
+ * MAT files use SciPy's loadmat() function to read MATLAB variables. MATLAB
+ * v7.3 MAT files are HDF5-based, but this helper is intended for MAT-file
+ * variables rather than as a general HDF5 interface.
  *
  * @tparam TensorType
  *   Data type of tensor
@@ -333,7 +333,7 @@ void read_npy(TensorType &t, const std::string& fname)
   auto np = pybind11::module_::import("numpy");
   auto obj = np.attr("load")("file"_a = fname);
 
-  pb->NumpyToTensorView(t, obj);
+  pb->NumpyToTensorView(t, obj, true);
 }
 
 /**

test/00_io/FileIOTests.cu

@@ -75,6 +75,21 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, SmallCSVRead)
   MATX_EXIT_HANDLER();
 }
 
+TYPED_TEST(FileIoTestsNonComplexFloatTypes, SmallCSVReadUninitialized)
+{
+  MATX_ENTER_HANDLER();
+  using TestType = cuda::std::tuple_element_t<0, TypeParam>;
+  tensor_t<TestType, 2> t;
+
+  io::read_csv(t, this->small_csv, ",");
+
+  ASSERT_EQ(t.Size(0), 10);
+  ASSERT_EQ(t.Size(1), 2);
+  MATX_TEST_ASSERT_COMPARE(this->pb, t, this->small_csv.c_str(), 0.01);
+
+  MATX_EXIT_HANDLER();
+}
+
 TYPED_TEST(FileIoTestsNonComplexFloatTypes, CSVReadFileNotFound)
 {
   MATX_ENTER_HANDLER();
@@ -146,6 +161,21 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATRead)
   MATX_EXIT_HANDLER();
 }
 
+TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATReadUninitialized)
+{
+  MATX_ENTER_HANDLER();
+  using TestType = cuda::std::tuple_element_t<0, TypeParam>;
+  tensor_t<TestType, 2> t;
+
+  io::read_mat(t, "../test/00_io/test.mat", "myvar");
+
+  ASSERT_EQ(t.Size(0), 1);
+  ASSERT_EQ(t.Size(1), 10);
+  ASSERT_NEAR(t(0,0), 1.456, 0.001);
+
+  MATX_EXIT_HANDLER();
+}
+
 TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATReadFileNotFound)
 {
   MATX_ENTER_HANDLER();
@@ -338,6 +368,37 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYRead)
   MATX_EXIT_HANDLER();
 }
 
+TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadUninitialized)
+{
+  MATX_ENTER_HANDLER();
+  using TestType = cuda::std::tuple_element_t<0, TypeParam>;
+
+  tensor_t<TestType, 2> t;
+
+  io::read_npy(t, "../test/00_io/test.npy");
+
+  ASSERT_EQ(t.Size(0), 2);
+  ASSERT_EQ(t.Size(1), 3);
+  ASSERT_NEAR(t(0, 0), 1.5, 0.001);
+  ASSERT_NEAR(t(1, 2), 6.5, 0.001);
+
+  MATX_EXIT_HANDLER();
+}
+
+TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadInitializedShapeMismatch)
+{
+  MATX_ENTER_HANDLER();
+  using TestType = cuda::std::tuple_element_t<0, TypeParam>;
+
+  auto t = make_tensor<TestType>({1, 3});
+
+  ASSERT_THROW({
+    io::read_npy(t, "../test/00_io/test.npy");
+  }, matx::detail::matxException);
+
+  MATX_EXIT_HANDLER();
+}
+
 TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadFileNotFound)
 {
   MATX_ENTER_HANDLER();
@@ -375,4 +436,4 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYWrite)
   }
 
   MATX_EXIT_HANDLER();
-}
+}

test/00_tensor/TensorCreationTests.cu

@@ -210,11 +210,17 @@ TYPED_TEST(TensorCreationTestsAll, StaticTensorDataPointer)
 {
   using TestType = cuda::std::tuple_element_t<0, TypeParam>;
 
+  tensor_t<TestType, 2> uninitialized;
+  ASSERT_EQ(uninitialized.Data(), nullptr);
+  ASSERT_FALSE(uninitialized.IsInitialized());
+
   auto mt1 = make_tensor<TestType, 10>();
   ASSERT_NE(mt1.Data(), nullptr);
+  ASSERT_TRUE(mt1.IsInitialized());
 
   auto mt2 = make_tensor<TestType, 4, 5>();
   ASSERT_NE(mt2.Data(), nullptr);
+  ASSERT_TRUE(mt2.IsInitialized());
 }
 
 TYPED_TEST(TensorCreationTestsAll, StaticTensorAssignOnes)