Support uninitialized tensors in file IO reads

Add IsInitialized helpers to tensor types so callers do not inspect Data() directly.

Allocate destination tensors during NumPy-backed file reads when the tensor has no data pointer, using the shape discovered from the file before copying data.

Document CSV, MAT, and NPY IO examples, and clarify that MAT helpers operate on MAT-file variables rather than providing a general HDF5 interface.

Add CSV, MAT, NPY, and tensor initialization regression coverage.

Author: cliffburdick

docs_input/api/io/index.rst

@@ -3,8 +3,67 @@
 Input/Output
 ############
 
+MatX file IO helpers read and write common array file formats through the
+optional ``MATX_ENABLE_FILEIO`` support. Include ``matx.h`` and use the
+``matx::io`` namespace functions shown below.
+
+Read functions can write into an already-sized tensor, or into a
+default-constructed tensor with the desired rank and value type. When the tensor
+has no storage yet, MatX allocates it after discovering the shape from the file.
+
+CSV
+===
+
+CSV files support rank-1 and rank-2 tensors. The delimiter is passed explicitly,
+and ``read_csv`` skips the first row by default.
+
+.. code-block:: cpp
+
+   tensor_t<float, 2> samples;
+   io::read_csv(samples, "samples.csv", ",");
+
+   io::write_csv(samples, "samples_out.csv", ",");
+
+To read a file without skipping the first row, pass ``false`` for the final
+argument.
+
+.. code-block:: cpp
+
+   io::read_csv(samples, "samples_out.csv", ",", false);
+
+MAT
+===
+
+MAT files can contain multiple named variables. ``read_mat`` and ``write_mat``
+therefore take a variable name in addition to the file name.
+
+.. code-block:: cpp
+
+   tensor_t<float, 2> A;
+   io::read_mat(A, "arrays.mat", "A");
+
+   auto B = io::read_mat<tensor_t<float, 2>>("arrays.mat", "B");
+
+   io::write_mat(A, "arrays_out.mat", "A");
+
+MATLAB v7.3 MAT files are HDF5-based, but the MatX MAT helpers are variable
+oriented and use SciPy's MAT-file routines. Treat them as MAT-file readers and
+writers rather than as a general HDF5 interface.
+
+NPY
+===
+
+NPY files store a single NumPy array per file.
+
+.. code-block:: cpp
+
+   tensor_t<float, 2> x;
+   io::read_npy(x, "x.npy");
+
+   io::write_npy(x, "x_out.npy");
+
 .. toctree::
    :maxdepth: 1
    :glob:
 
-   *
+   *

docs_input/api/io/read_mat.rst

@@ -3,11 +3,15 @@
 read_mat
 ========
 
-Read a CSV file into a tensor
+Read a variable from a MAT file into a tensor
 
 .. note::
    This function requires the optional ``MATX_ENABLE_FILEIO`` compile flag
 
+MAT files can contain multiple named variables. Pass the variable name in
+``var`` to select the tensor to read. MATLAB v7.3 MAT files are HDF5-based, but
+these helpers use SciPy's MAT-file routines and are not a general HDF5
+interface.
 
 
 .. versionadded:: 0.3.0

docs_input/api/io/write_mat.rst

@@ -3,11 +3,13 @@
 write_mat
 =========
 
-Write an operator to a MAT file
+Write a tensor to a MAT file variable
 
 .. note::
    This function requires the optional ``MATX_ENABLE_FILEIO`` compile flag
 
+MAT files can contain multiple named variables. ``write_mat`` writes the tensor
+under the variable name passed in ``var``.
 
 
 .. versionadded:: 0.3.0

include/matx/core/dynamic_tensor.h

@@ -218,6 +218,11 @@ class dynamic_tensor_t {
 
   __MATX_INLINE__ T *Data() const { return ldata_; }
 
+  __MATX_INLINE__ bool IsInitialized() const noexcept
+  {
+    return Data() != nullptr;
+  }
+
   __MATX_INLINE__ index_t TotalSize() const {
     index_t total = 1;
     for (int i = 0; i < rank_; ++i) {

include/matx/core/pybind.h

@@ -40,6 +40,7 @@ MATX_IGNORE_WARNING_PUSH_GCC("-Wnull-dereference")
 #include <pybind11/embed.h>
 #include <pybind11/numpy.h>
 MATX_IGNORE_WARNING_POP_GCC
+#include <algorithm>
 #include <optional>
 #include <filesystem>
 
@@ -378,6 +379,11 @@ class MATX_PYBIND_VISIBILITY MatXPybind {
     return indices;
   }
 
+  template <typename TensorType>
+  static constexpr bool has_is_initialized_v = requires(const TensorType &ten) {
+    ten.IsInitialized();
+  };
+
   template <typename TensorType>
   void NumpyToTensorView(TensorType &ten,
                          const std::string fname)
@@ -387,14 +393,35 @@ class MATX_PYBIND_VISIBILITY MatXPybind {
   }
 
   template <typename TensorType>
-  void NumpyToTensorView(TensorType ten,
+  void NumpyToTensorView(TensorType &&ten,
                          const pybind11::object &np_ten)
   {
-    using T = typename TensorType::value_type;
-    constexpr int RANK = TensorType::Rank();
+    using Tensor = remove_cvref_t<TensorType>;
+    using T = typename Tensor::value_type;
+    constexpr int RANK = Tensor::Rank();
 
     using ntype = matx_convert_complex_type<T>;
     auto ften = pybind11::array_t<ntype>(np_ten);
+    auto info = ften.request();
+
+    MATX_ASSERT_STR(info.ndim == RANK, matxInvalidDim,
+                    "Numpy array rank does not match tensor rank");
+
+    if constexpr (has_is_initialized_v<Tensor>) {
+      if (!ten.IsInitialized()) {
+        cuda::std::array<matx::index_t, RANK> shape;
+        std::copy_n(info.shape.begin(), RANK, std::begin(shape));
+        // The copy below writes from host code, so use the default
+        // host-accessible allocation path.
+        make_tensor(ten, shape);
+      }
+    }
+
+    for (int d = 0; d < RANK; d++) {
+      MATX_ASSERT_STR(ten.Size(d) == static_cast<index_t>(info.shape[d]),
+                      matxInvalidSize,
+                      "Numpy array dimension size does not match tensor size");
+    }
 
     if constexpr (RANK == 0) {
       ten() = ConvertComplex(ften.at());

include/matx/core/tensor_impl.h

@@ -1557,6 +1557,16 @@ MATX_IGNORE_WARNING_POP_GCC
       return data_.ldata_;
     }
 
+    /**
+     * @brief Check whether this tensor has an assigned data pointer.
+     *
+     * @return true if the tensor has storage or a non-owning data pointer
+     */
+    __MATX_INLINE__ __MATX_HOST__ __MATX_DEVICE__ bool IsInitialized() const noexcept
+    {
+      return Data() != nullptr;
+    }
+
     /**
      * @brief Set data pointer
      *

include/matx/file_io/file_io.h

@@ -200,9 +200,9 @@ void write_csv(const TensorType &t, const std::string fname,
 /**
  * @brief Read a MAT file into a tensor view
  *
- * MAT files use SciPy's loadmat() function to read various MATLAB file
- * types in. MAT files are supersets of HDF5 files, and are allowed to
- * have multiple fields in them.
+ * MAT files use SciPy's loadmat() function to read MATLAB variables. MATLAB
+ * v7.3 MAT files are HDF5-based, but this helper is intended for MAT-file
+ * variables rather than as a general HDF5 interface.
  *
  * @tparam TensorType
  *   Data type of tensor
@@ -241,9 +241,9 @@ void read_mat(TensorType &t, const std::string fname,
 /**
  * @brief Read a MAT file and return a tensor view
  *
- * MAT files use SciPy's loadmat() function to read various MATLAB file
- * types in. MAT files are supersets of HDF5 files, and are allowed to
- * have multiple fields in them.
+ * MAT files use SciPy's loadmat() function to read MATLAB variables. MATLAB
+ * v7.3 MAT files are HDF5-based, but this helper is intended for MAT-file
+ * variables rather than as a general HDF5 interface.
  *
  * @tparam TensorType
  *   Data type of tensor

test/00_io/FileIOTests.cu

@@ -75,6 +75,21 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, SmallCSVRead)
   MATX_EXIT_HANDLER();
 }
 
+TYPED_TEST(FileIoTestsNonComplexFloatTypes, SmallCSVReadUninitialized)
+{
+  MATX_ENTER_HANDLER();
+  using TestType = cuda::std::tuple_element_t<0, TypeParam>;
+  tensor_t<TestType, 2> t;
+
+  io::read_csv(t, this->small_csv, ",");
+
+  ASSERT_EQ(t.Size(0), 10);
+  ASSERT_EQ(t.Size(1), 2);
+  MATX_TEST_ASSERT_COMPARE(this->pb, t, this->small_csv.c_str(), 0.01);
+
+  MATX_EXIT_HANDLER();
+}
+
 TYPED_TEST(FileIoTestsNonComplexFloatTypes, CSVReadFileNotFound)
 {
   MATX_ENTER_HANDLER();
@@ -146,6 +161,21 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATRead)
   MATX_EXIT_HANDLER();
 }
 
+TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATReadUninitialized)
+{
+  MATX_ENTER_HANDLER();
+  using TestType = cuda::std::tuple_element_t<0, TypeParam>;
+  tensor_t<TestType, 2> t;
+
+  io::read_mat(t, "../test/00_io/test.mat", "myvar");
+
+  ASSERT_EQ(t.Size(0), 1);
+  ASSERT_EQ(t.Size(1), 10);
+  ASSERT_NEAR(t(0,0), 1.456, 0.001);
+
+  MATX_EXIT_HANDLER();
+}
+
 TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATReadFileNotFound)
 {
   MATX_ENTER_HANDLER();
@@ -338,6 +368,37 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYRead)
   MATX_EXIT_HANDLER();
 }
 
+TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadUninitialized)
+{
+  MATX_ENTER_HANDLER();
+  using TestType = cuda::std::tuple_element_t<0, TypeParam>;
+
+  tensor_t<TestType, 2> t;
+
+  io::read_npy(t, "../test/00_io/test.npy");
+
+  ASSERT_EQ(t.Size(0), 2);
+  ASSERT_EQ(t.Size(1), 3);
+  ASSERT_NEAR(t(0, 0), 1.5, 0.001);
+  ASSERT_NEAR(t(1, 2), 6.5, 0.001);
+
+  MATX_EXIT_HANDLER();
+}
+
+TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadInitializedShapeMismatch)
+{
+  MATX_ENTER_HANDLER();
+  using TestType = cuda::std::tuple_element_t<0, TypeParam>;
+
+  auto t = make_tensor<TestType>({1, 3});
+
+  ASSERT_THROW({
+    io::read_npy(t, "../test/00_io/test.npy");
+  }, matx::detail::matxException);
+
+  MATX_EXIT_HANDLER();
+}
+
 TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadFileNotFound)
 {
   MATX_ENTER_HANDLER();
@@ -375,4 +436,4 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYWrite)
   }
 
   MATX_EXIT_HANDLER();
-}
+}

test/00_tensor/TensorCreationTests.cu

@@ -210,11 +210,17 @@ TYPED_TEST(TensorCreationTestsAll, StaticTensorDataPointer)
 {
   using TestType = cuda::std::tuple_element_t<0, TypeParam>;
 
+  tensor_t<TestType, 2> uninitialized;
+  ASSERT_EQ(uninitialized.Data(), nullptr);
+  ASSERT_FALSE(uninitialized.IsInitialized());
+
   auto mt1 = make_tensor<TestType, 10>();
   ASSERT_NE(mt1.Data(), nullptr);
+  ASSERT_TRUE(mt1.IsInitialized());
 
   auto mt2 = make_tensor<TestType, 4, 5>();
   ASSERT_NE(mt2.Data(), nullptr);
+  ASSERT_TRUE(mt2.IsInitialized());
 }
 
 TYPED_TEST(TensorCreationTestsAll, StaticTensorAssignOnes)