From c30d2e3d87a7e20e8424bb6d06cc3356fcf4f6bb Mon Sep 17 00:00:00 2001 From: Cliff Burdick Date: Tue, 26 May 2026 13:35:11 -0700 Subject: [PATCH] Support uninitialized tensors in file IO reads Add IsInitialized helpers to tensor types so callers do not inspect Data() directly. Allocate destination tensors during NumPy-backed file reads when the tensor has no data pointer, using the shape discovered from the file before copying data. Document CSV, MAT, and NPY IO examples, and clarify that MAT helpers operate on MAT-file variables rather than providing a general HDF5 interface. Add CSV, MAT, NPY, and tensor initialization regression coverage. --- docs_input/api/io/index.rst | 61 +++++++++++++++++++++++++- docs_input/api/io/read_mat.rst | 6 ++- docs_input/api/io/write_mat.rst | 4 +- include/matx/core/dynamic_tensor.h | 5 +++ include/matx/core/pybind.h | 44 ++++++++++++++++--- include/matx/core/tensor_impl.h | 10 +++++ include/matx/file_io/file_io.h | 18 ++++---- test/00_io/FileIOTests.cu | 63 ++++++++++++++++++++++++++- test/00_tensor/TensorCreationTests.cu | 6 +++ 9 files changed, 198 insertions(+), 19 deletions(-) diff --git a/docs_input/api/io/index.rst b/docs_input/api/io/index.rst index 5cacb5f28..a023a7201 100644 --- a/docs_input/api/io/index.rst +++ b/docs_input/api/io/index.rst @@ -3,8 +3,67 @@ Input/Output ############ +MatX file IO helpers read and write common array file formats through the +optional ``MATX_ENABLE_FILEIO`` support. Include ``matx.h`` and use the +``matx::io`` namespace functions shown below. + +Read functions can write into an already-sized tensor, or into a +default-constructed tensor with the desired rank and value type. When the tensor +has no storage yet, MatX allocates it after discovering the shape from the file. + +CSV +=== + +CSV files support rank-1 and rank-2 tensors. The delimiter is passed explicitly, +and ``read_csv`` skips the first row by default. + +.. code-block:: cpp + + tensor_t samples; + io::read_csv(samples, "samples.csv", ","); + + io::write_csv(samples, "samples_out.csv", ","); + +To read a file without skipping the first row, pass ``false`` for the final +argument. + +.. code-block:: cpp + + io::read_csv(samples, "samples_out.csv", ",", false); + +MAT +=== + +MAT files can contain multiple named variables. ``read_mat`` and ``write_mat`` +therefore take a variable name in addition to the file name. + +.. code-block:: cpp + + tensor_t A; + io::read_mat(A, "arrays.mat", "A"); + + auto B = io::read_mat>("arrays.mat", "B"); + + io::write_mat(A, "arrays_out.mat", "A"); + +MATLAB v7.3 MAT files are HDF5-based, but the MatX MAT helpers are variable +oriented and use SciPy's MAT-file routines. Treat them as MAT-file readers and +writers rather than as a general HDF5 interface. + +NPY +=== + +NPY files store a single NumPy array per file. + +.. code-block:: cpp + + tensor_t x; + io::read_npy(x, "x.npy"); + + io::write_npy(x, "x_out.npy"); + .. toctree:: :maxdepth: 1 :glob: - * \ No newline at end of file + * diff --git a/docs_input/api/io/read_mat.rst b/docs_input/api/io/read_mat.rst index 2cbe276a7..dfb9c205d 100644 --- a/docs_input/api/io/read_mat.rst +++ b/docs_input/api/io/read_mat.rst @@ -3,11 +3,15 @@ read_mat ======== -Read a CSV file into a tensor +Read a variable from a MAT file into a tensor .. note:: This function requires the optional ``MATX_ENABLE_FILEIO`` compile flag +MAT files can contain multiple named variables. Pass the variable name in +``var`` to select the tensor to read. MATLAB v7.3 MAT files are HDF5-based, but +these helpers use SciPy's MAT-file routines and are not a general HDF5 +interface. .. versionadded:: 0.3.0 diff --git a/docs_input/api/io/write_mat.rst b/docs_input/api/io/write_mat.rst index c70d0dd2f..46744b08d 100644 --- a/docs_input/api/io/write_mat.rst +++ b/docs_input/api/io/write_mat.rst @@ -3,11 +3,13 @@ write_mat ========= -Write an operator to a MAT file +Write a tensor to a MAT file variable .. note:: This function requires the optional ``MATX_ENABLE_FILEIO`` compile flag +MAT files can contain multiple named variables. ``write_mat`` writes the tensor +under the variable name passed in ``var``. .. versionadded:: 0.3.0 diff --git a/include/matx/core/dynamic_tensor.h b/include/matx/core/dynamic_tensor.h index e57b6e651..d683a5fc0 100644 --- a/include/matx/core/dynamic_tensor.h +++ b/include/matx/core/dynamic_tensor.h @@ -218,6 +218,11 @@ class dynamic_tensor_t { __MATX_INLINE__ T *Data() const { return ldata_; } + __MATX_INLINE__ bool IsInitialized() const noexcept + { + return Data() != nullptr; + } + __MATX_INLINE__ index_t TotalSize() const { index_t total = 1; for (int i = 0; i < rank_; ++i) { diff --git a/include/matx/core/pybind.h b/include/matx/core/pybind.h index 05d1210eb..607978e92 100644 --- a/include/matx/core/pybind.h +++ b/include/matx/core/pybind.h @@ -40,6 +40,7 @@ MATX_IGNORE_WARNING_PUSH_GCC("-Wnull-dereference") #include #include MATX_IGNORE_WARNING_POP_GCC +#include #include #include @@ -378,23 +379,54 @@ class MATX_PYBIND_VISIBILITY MatXPybind { return indices; } + template + static constexpr bool has_is_initialized_v = requires(const TensorType &ten) { + ten.IsInitialized(); + }; + template void NumpyToTensorView(TensorType &ten, - const std::string fname) + const std::string fname, + bool exact_shape = false) { auto resobj = res_dict[fname.c_str()]; - NumpyToTensorView(ten, resobj); + NumpyToTensorView(ten, resobj, exact_shape); } template - void NumpyToTensorView(TensorType ten, - const pybind11::object &np_ten) + void NumpyToTensorView(TensorType &&ten, + const pybind11::object &np_ten, + bool exact_shape = false) { - using T = typename TensorType::value_type; - constexpr int RANK = TensorType::Rank(); + using Tensor = remove_cvref_t; + using T = typename Tensor::value_type; + constexpr int RANK = Tensor::Rank(); using ntype = matx_convert_complex_type; auto ften = pybind11::array_t(np_ten); + auto info = ften.request(); + + MATX_ASSERT_STR(info.ndim == RANK, matxInvalidDim, + "Numpy array rank does not match tensor rank"); + + if constexpr (has_is_initialized_v) { + if (!ten.IsInitialized()) { + cuda::std::array shape; + std::copy_n(info.shape.begin(), RANK, std::begin(shape)); + // The copy below writes from host code, so use the default + // host-accessible allocation path. + make_tensor(ten, shape); + } + } + + // File IO requires exact shapes. Some generated test vectors intentionally + // copy a smaller tensor from a larger NumPy array, so keep that path bounded. + for (int d = 0; d < RANK; d++) { + const auto np_size = static_cast(info.shape[d]); + const bool valid_size = exact_shape ? ten.Size(d) == np_size : ten.Size(d) <= np_size; + MATX_ASSERT_STR(valid_size, matxInvalidSize, + "Numpy array dimension size is incompatible with tensor size"); + } if constexpr (RANK == 0) { ten() = ConvertComplex(ften.at()); diff --git a/include/matx/core/tensor_impl.h b/include/matx/core/tensor_impl.h index 95d3e4e41..d9d1b2084 100644 --- a/include/matx/core/tensor_impl.h +++ b/include/matx/core/tensor_impl.h @@ -1557,6 +1557,16 @@ MATX_IGNORE_WARNING_POP_GCC return data_.ldata_; } + /** + * @brief Check whether this tensor has an assigned data pointer. + * + * @return true if the tensor has storage or a non-owning data pointer + */ + __MATX_INLINE__ __MATX_HOST__ __MATX_DEVICE__ bool IsInitialized() const noexcept + { + return Data() != nullptr; + } + /** * @brief Set data pointer * diff --git a/include/matx/file_io/file_io.h b/include/matx/file_io/file_io.h index c5385f4ce..38f0c694d 100644 --- a/include/matx/file_io/file_io.h +++ b/include/matx/file_io/file_io.h @@ -158,7 +158,7 @@ void read_csv(TensorType &t, const std::string fname, auto obj = np.attr("genfromtxt")("fname"_a = fname.c_str(), "delimiter"_a = delimiter, "skip_header"_a = skip_header, "dtype"_a = detail::MatXPybind::GetNumpyDtype()); - pb->NumpyToTensorView(t, obj); + pb->NumpyToTensorView(t, obj, true); } /** @@ -200,9 +200,9 @@ void write_csv(const TensorType &t, const std::string fname, /** * @brief Read a MAT file into a tensor view * - * MAT files use SciPy's loadmat() function to read various MATLAB file - * types in. MAT files are supersets of HDF5 files, and are allowed to - * have multiple fields in them. + * MAT files use SciPy's loadmat() function to read MATLAB variables. MATLAB + * v7.3 MAT files are HDF5-based, but this helper is intended for MAT-file + * variables rather than as a general HDF5 interface. * * @tparam TensorType * Data type of tensor @@ -235,15 +235,15 @@ void read_mat(TensorType &t, const std::string fname, auto obj = (pybind11::dict)sp.attr("loadmat")("file_name"_a = fname); auto v = obj[var.c_str()]; - pb->NumpyToTensorView(t, v); + pb->NumpyToTensorView(t, v, true); } /** * @brief Read a MAT file and return a tensor view * - * MAT files use SciPy's loadmat() function to read various MATLAB file - * types in. MAT files are supersets of HDF5 files, and are allowed to - * have multiple fields in them. + * MAT files use SciPy's loadmat() function to read MATLAB variables. MATLAB + * v7.3 MAT files are HDF5-based, but this helper is intended for MAT-file + * variables rather than as a general HDF5 interface. * * @tparam TensorType * Data type of tensor @@ -333,7 +333,7 @@ void read_npy(TensorType &t, const std::string& fname) auto np = pybind11::module_::import("numpy"); auto obj = np.attr("load")("file"_a = fname); - pb->NumpyToTensorView(t, obj); + pb->NumpyToTensorView(t, obj, true); } /** diff --git a/test/00_io/FileIOTests.cu b/test/00_io/FileIOTests.cu index c0527ee4b..be06c27ca 100644 --- a/test/00_io/FileIOTests.cu +++ b/test/00_io/FileIOTests.cu @@ -75,6 +75,21 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, SmallCSVRead) MATX_EXIT_HANDLER(); } +TYPED_TEST(FileIoTestsNonComplexFloatTypes, SmallCSVReadUninitialized) +{ + MATX_ENTER_HANDLER(); + using TestType = cuda::std::tuple_element_t<0, TypeParam>; + tensor_t t; + + io::read_csv(t, this->small_csv, ","); + + ASSERT_EQ(t.Size(0), 10); + ASSERT_EQ(t.Size(1), 2); + MATX_TEST_ASSERT_COMPARE(this->pb, t, this->small_csv.c_str(), 0.01); + + MATX_EXIT_HANDLER(); +} + TYPED_TEST(FileIoTestsNonComplexFloatTypes, CSVReadFileNotFound) { MATX_ENTER_HANDLER(); @@ -146,6 +161,21 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATRead) MATX_EXIT_HANDLER(); } +TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATReadUninitialized) +{ + MATX_ENTER_HANDLER(); + using TestType = cuda::std::tuple_element_t<0, TypeParam>; + tensor_t t; + + io::read_mat(t, "../test/00_io/test.mat", "myvar"); + + ASSERT_EQ(t.Size(0), 1); + ASSERT_EQ(t.Size(1), 10); + ASSERT_NEAR(t(0,0), 1.456, 0.001); + + MATX_EXIT_HANDLER(); +} + TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATReadFileNotFound) { MATX_ENTER_HANDLER(); @@ -338,6 +368,37 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYRead) MATX_EXIT_HANDLER(); } +TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadUninitialized) +{ + MATX_ENTER_HANDLER(); + using TestType = cuda::std::tuple_element_t<0, TypeParam>; + + tensor_t t; + + io::read_npy(t, "../test/00_io/test.npy"); + + ASSERT_EQ(t.Size(0), 2); + ASSERT_EQ(t.Size(1), 3); + ASSERT_NEAR(t(0, 0), 1.5, 0.001); + ASSERT_NEAR(t(1, 2), 6.5, 0.001); + + MATX_EXIT_HANDLER(); +} + +TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadInitializedShapeMismatch) +{ + MATX_ENTER_HANDLER(); + using TestType = cuda::std::tuple_element_t<0, TypeParam>; + + auto t = make_tensor({1, 3}); + + ASSERT_THROW({ + io::read_npy(t, "../test/00_io/test.npy"); + }, matx::detail::matxException); + + MATX_EXIT_HANDLER(); +} + TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadFileNotFound) { MATX_ENTER_HANDLER(); @@ -375,4 +436,4 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYWrite) } MATX_EXIT_HANDLER(); -} \ No newline at end of file +} diff --git a/test/00_tensor/TensorCreationTests.cu b/test/00_tensor/TensorCreationTests.cu index 0174fd85a..829a99fd0 100644 --- a/test/00_tensor/TensorCreationTests.cu +++ b/test/00_tensor/TensorCreationTests.cu @@ -210,11 +210,17 @@ TYPED_TEST(TensorCreationTestsAll, StaticTensorDataPointer) { using TestType = cuda::std::tuple_element_t<0, TypeParam>; + tensor_t uninitialized; + ASSERT_EQ(uninitialized.Data(), nullptr); + ASSERT_FALSE(uninitialized.IsInitialized()); + auto mt1 = make_tensor(); ASSERT_NE(mt1.Data(), nullptr); + ASSERT_TRUE(mt1.IsInitialized()); auto mt2 = make_tensor(); ASSERT_NE(mt2.Data(), nullptr); + ASSERT_TRUE(mt2.IsInitialized()); } TYPED_TEST(TensorCreationTestsAll, StaticTensorAssignOnes)