diff --git a/docs_input/api/io/index.rst b/docs_input/api/io/index.rst index 5cacb5f28..a023a7201 100644 --- a/docs_input/api/io/index.rst +++ b/docs_input/api/io/index.rst @@ -3,8 +3,67 @@ Input/Output ############ +MatX file IO helpers read and write common array file formats through the +optional ``MATX_ENABLE_FILEIO`` support. Include ``matx.h`` and use the +``matx::io`` namespace functions shown below. + +Read functions can write into an already-sized tensor, or into a +default-constructed tensor with the desired rank and value type. When the tensor +has no storage yet, MatX allocates it after discovering the shape from the file. + +CSV +=== + +CSV files support rank-1 and rank-2 tensors. The delimiter is passed explicitly, +and ``read_csv`` skips the first row by default. + +.. code-block:: cpp + + tensor_t samples; + io::read_csv(samples, "samples.csv", ","); + + io::write_csv(samples, "samples_out.csv", ","); + +To read a file without skipping the first row, pass ``false`` for the final +argument. + +.. code-block:: cpp + + io::read_csv(samples, "samples_out.csv", ",", false); + +MAT +=== + +MAT files can contain multiple named variables. ``read_mat`` and ``write_mat`` +therefore take a variable name in addition to the file name. + +.. code-block:: cpp + + tensor_t A; + io::read_mat(A, "arrays.mat", "A"); + + auto B = io::read_mat>("arrays.mat", "B"); + + io::write_mat(A, "arrays_out.mat", "A"); + +MATLAB v7.3 MAT files are HDF5-based, but the MatX MAT helpers are variable +oriented and use SciPy's MAT-file routines. Treat them as MAT-file readers and +writers rather than as a general HDF5 interface. + +NPY +=== + +NPY files store a single NumPy array per file. + +.. code-block:: cpp + + tensor_t x; + io::read_npy(x, "x.npy"); + + io::write_npy(x, "x_out.npy"); + .. toctree:: :maxdepth: 1 :glob: - * \ No newline at end of file + * diff --git a/docs_input/api/io/read_mat.rst b/docs_input/api/io/read_mat.rst index 2cbe276a7..dfb9c205d 100644 --- a/docs_input/api/io/read_mat.rst +++ b/docs_input/api/io/read_mat.rst @@ -3,11 +3,15 @@ read_mat ======== -Read a CSV file into a tensor +Read a variable from a MAT file into a tensor .. note:: This function requires the optional ``MATX_ENABLE_FILEIO`` compile flag +MAT files can contain multiple named variables. Pass the variable name in +``var`` to select the tensor to read. MATLAB v7.3 MAT files are HDF5-based, but +these helpers use SciPy's MAT-file routines and are not a general HDF5 +interface. .. versionadded:: 0.3.0 diff --git a/docs_input/api/io/write_mat.rst b/docs_input/api/io/write_mat.rst index c70d0dd2f..46744b08d 100644 --- a/docs_input/api/io/write_mat.rst +++ b/docs_input/api/io/write_mat.rst @@ -3,11 +3,13 @@ write_mat ========= -Write an operator to a MAT file +Write a tensor to a MAT file variable .. note:: This function requires the optional ``MATX_ENABLE_FILEIO`` compile flag +MAT files can contain multiple named variables. ``write_mat`` writes the tensor +under the variable name passed in ``var``. .. versionadded:: 0.3.0 diff --git a/include/matx/core/dynamic_tensor.h b/include/matx/core/dynamic_tensor.h index e57b6e651..d683a5fc0 100644 --- a/include/matx/core/dynamic_tensor.h +++ b/include/matx/core/dynamic_tensor.h @@ -218,6 +218,11 @@ class dynamic_tensor_t { __MATX_INLINE__ T *Data() const { return ldata_; } + __MATX_INLINE__ bool IsInitialized() const noexcept + { + return Data() != nullptr; + } + __MATX_INLINE__ index_t TotalSize() const { index_t total = 1; for (int i = 0; i < rank_; ++i) { diff --git a/include/matx/core/pybind.h b/include/matx/core/pybind.h index 05d1210eb..607978e92 100644 --- a/include/matx/core/pybind.h +++ b/include/matx/core/pybind.h @@ -40,6 +40,7 @@ MATX_IGNORE_WARNING_PUSH_GCC("-Wnull-dereference") #include #include MATX_IGNORE_WARNING_POP_GCC +#include #include #include @@ -378,23 +379,54 @@ class MATX_PYBIND_VISIBILITY MatXPybind { return indices; } + template + static constexpr bool has_is_initialized_v = requires(const TensorType &ten) { + ten.IsInitialized(); + }; + template void NumpyToTensorView(TensorType &ten, - const std::string fname) + const std::string fname, + bool exact_shape = false) { auto resobj = res_dict[fname.c_str()]; - NumpyToTensorView(ten, resobj); + NumpyToTensorView(ten, resobj, exact_shape); } template - void NumpyToTensorView(TensorType ten, - const pybind11::object &np_ten) + void NumpyToTensorView(TensorType &&ten, + const pybind11::object &np_ten, + bool exact_shape = false) { - using T = typename TensorType::value_type; - constexpr int RANK = TensorType::Rank(); + using Tensor = remove_cvref_t; + using T = typename Tensor::value_type; + constexpr int RANK = Tensor::Rank(); using ntype = matx_convert_complex_type; auto ften = pybind11::array_t(np_ten); + auto info = ften.request(); + + MATX_ASSERT_STR(info.ndim == RANK, matxInvalidDim, + "Numpy array rank does not match tensor rank"); + + if constexpr (has_is_initialized_v) { + if (!ten.IsInitialized()) { + cuda::std::array shape; + std::copy_n(info.shape.begin(), RANK, std::begin(shape)); + // The copy below writes from host code, so use the default + // host-accessible allocation path. + make_tensor(ten, shape); + } + } + + // File IO requires exact shapes. Some generated test vectors intentionally + // copy a smaller tensor from a larger NumPy array, so keep that path bounded. + for (int d = 0; d < RANK; d++) { + const auto np_size = static_cast(info.shape[d]); + const bool valid_size = exact_shape ? ten.Size(d) == np_size : ten.Size(d) <= np_size; + MATX_ASSERT_STR(valid_size, matxInvalidSize, + "Numpy array dimension size is incompatible with tensor size"); + } if constexpr (RANK == 0) { ten() = ConvertComplex(ften.at()); diff --git a/include/matx/core/tensor_impl.h b/include/matx/core/tensor_impl.h index 95d3e4e41..d9d1b2084 100644 --- a/include/matx/core/tensor_impl.h +++ b/include/matx/core/tensor_impl.h @@ -1557,6 +1557,16 @@ MATX_IGNORE_WARNING_POP_GCC return data_.ldata_; } + /** + * @brief Check whether this tensor has an assigned data pointer. + * + * @return true if the tensor has storage or a non-owning data pointer + */ + __MATX_INLINE__ __MATX_HOST__ __MATX_DEVICE__ bool IsInitialized() const noexcept + { + return Data() != nullptr; + } + /** * @brief Set data pointer * diff --git a/include/matx/file_io/file_io.h b/include/matx/file_io/file_io.h index c5385f4ce..38f0c694d 100644 --- a/include/matx/file_io/file_io.h +++ b/include/matx/file_io/file_io.h @@ -158,7 +158,7 @@ void read_csv(TensorType &t, const std::string fname, auto obj = np.attr("genfromtxt")("fname"_a = fname.c_str(), "delimiter"_a = delimiter, "skip_header"_a = skip_header, "dtype"_a = detail::MatXPybind::GetNumpyDtype()); - pb->NumpyToTensorView(t, obj); + pb->NumpyToTensorView(t, obj, true); } /** @@ -200,9 +200,9 @@ void write_csv(const TensorType &t, const std::string fname, /** * @brief Read a MAT file into a tensor view * - * MAT files use SciPy's loadmat() function to read various MATLAB file - * types in. MAT files are supersets of HDF5 files, and are allowed to - * have multiple fields in them. + * MAT files use SciPy's loadmat() function to read MATLAB variables. MATLAB + * v7.3 MAT files are HDF5-based, but this helper is intended for MAT-file + * variables rather than as a general HDF5 interface. * * @tparam TensorType * Data type of tensor @@ -235,15 +235,15 @@ void read_mat(TensorType &t, const std::string fname, auto obj = (pybind11::dict)sp.attr("loadmat")("file_name"_a = fname); auto v = obj[var.c_str()]; - pb->NumpyToTensorView(t, v); + pb->NumpyToTensorView(t, v, true); } /** * @brief Read a MAT file and return a tensor view * - * MAT files use SciPy's loadmat() function to read various MATLAB file - * types in. MAT files are supersets of HDF5 files, and are allowed to - * have multiple fields in them. + * MAT files use SciPy's loadmat() function to read MATLAB variables. MATLAB + * v7.3 MAT files are HDF5-based, but this helper is intended for MAT-file + * variables rather than as a general HDF5 interface. * * @tparam TensorType * Data type of tensor @@ -333,7 +333,7 @@ void read_npy(TensorType &t, const std::string& fname) auto np = pybind11::module_::import("numpy"); auto obj = np.attr("load")("file"_a = fname); - pb->NumpyToTensorView(t, obj); + pb->NumpyToTensorView(t, obj, true); } /** diff --git a/test/00_io/FileIOTests.cu b/test/00_io/FileIOTests.cu index c0527ee4b..be06c27ca 100644 --- a/test/00_io/FileIOTests.cu +++ b/test/00_io/FileIOTests.cu @@ -75,6 +75,21 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, SmallCSVRead) MATX_EXIT_HANDLER(); } +TYPED_TEST(FileIoTestsNonComplexFloatTypes, SmallCSVReadUninitialized) +{ + MATX_ENTER_HANDLER(); + using TestType = cuda::std::tuple_element_t<0, TypeParam>; + tensor_t t; + + io::read_csv(t, this->small_csv, ","); + + ASSERT_EQ(t.Size(0), 10); + ASSERT_EQ(t.Size(1), 2); + MATX_TEST_ASSERT_COMPARE(this->pb, t, this->small_csv.c_str(), 0.01); + + MATX_EXIT_HANDLER(); +} + TYPED_TEST(FileIoTestsNonComplexFloatTypes, CSVReadFileNotFound) { MATX_ENTER_HANDLER(); @@ -146,6 +161,21 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATRead) MATX_EXIT_HANDLER(); } +TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATReadUninitialized) +{ + MATX_ENTER_HANDLER(); + using TestType = cuda::std::tuple_element_t<0, TypeParam>; + tensor_t t; + + io::read_mat(t, "../test/00_io/test.mat", "myvar"); + + ASSERT_EQ(t.Size(0), 1); + ASSERT_EQ(t.Size(1), 10); + ASSERT_NEAR(t(0,0), 1.456, 0.001); + + MATX_EXIT_HANDLER(); +} + TYPED_TEST(FileIoTestsNonComplexFloatTypes, MATReadFileNotFound) { MATX_ENTER_HANDLER(); @@ -338,6 +368,37 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYRead) MATX_EXIT_HANDLER(); } +TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadUninitialized) +{ + MATX_ENTER_HANDLER(); + using TestType = cuda::std::tuple_element_t<0, TypeParam>; + + tensor_t t; + + io::read_npy(t, "../test/00_io/test.npy"); + + ASSERT_EQ(t.Size(0), 2); + ASSERT_EQ(t.Size(1), 3); + ASSERT_NEAR(t(0, 0), 1.5, 0.001); + ASSERT_NEAR(t(1, 2), 6.5, 0.001); + + MATX_EXIT_HANDLER(); +} + +TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadInitializedShapeMismatch) +{ + MATX_ENTER_HANDLER(); + using TestType = cuda::std::tuple_element_t<0, TypeParam>; + + auto t = make_tensor({1, 3}); + + ASSERT_THROW({ + io::read_npy(t, "../test/00_io/test.npy"); + }, matx::detail::matxException); + + MATX_EXIT_HANDLER(); +} + TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYReadFileNotFound) { MATX_ENTER_HANDLER(); @@ -375,4 +436,4 @@ TYPED_TEST(FileIoTestsNonComplexFloatTypes, NPYWrite) } MATX_EXIT_HANDLER(); -} \ No newline at end of file +} diff --git a/test/00_tensor/TensorCreationTests.cu b/test/00_tensor/TensorCreationTests.cu index 0174fd85a..829a99fd0 100644 --- a/test/00_tensor/TensorCreationTests.cu +++ b/test/00_tensor/TensorCreationTests.cu @@ -210,11 +210,17 @@ TYPED_TEST(TensorCreationTestsAll, StaticTensorDataPointer) { using TestType = cuda::std::tuple_element_t<0, TypeParam>; + tensor_t uninitialized; + ASSERT_EQ(uninitialized.Data(), nullptr); + ASSERT_FALSE(uninitialized.IsInitialized()); + auto mt1 = make_tensor(); ASSERT_NE(mt1.Data(), nullptr); + ASSERT_TRUE(mt1.IsInitialized()); auto mt2 = make_tensor(); ASSERT_NE(mt2.Data(), nullptr); + ASSERT_TRUE(mt2.IsInitialized()); } TYPED_TEST(TensorCreationTestsAll, StaticTensorAssignOnes)