Merge pull request #68 from GPUEngineering/hf/bt-data-storage

Proper interoperability in data formatting (C++ and Python)

Author: alphaville

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+<!-- ---------------------
+      v1.7.1
+     --------------------- -->
+## v1.7.1 - 4-12-2024
+
+### Fixed
+
+- Compatibility between Python and C++ in how the data is stored in bt files
+
+
 <!-- ---------------------
       v1.7.0
      --------------------- -->

python/VERSION

@@ -1 +1 @@
-1.7.0
+1.7.1

python/gputils_api/gputils_api.py

@@ -14,7 +14,11 @@ def read_array_from_gputils_binary_file(path, dt=np.dtype('d')):
         nc = int.from_bytes(f.read(8), byteorder='little', signed=False)  # read number of columns
         nm = int.from_bytes(f.read(8), byteorder='little', signed=False)  # read number of matrices
         dat = np.fromfile(f, dtype=np.dtype(dt))  # read data
-        dat = dat.reshape((nr, nc, nm))  # reshape
+
+        if nm >= 2:  # if we actually have a 3D tensor (not a matrix or a vector)
+            dat = dat.reshape((nm, nc, nr)).swapaxes(0, 2)  # I'll explain this to you when you grow up
+        else:
+            dat = dat.reshape((nr, nc, nm))  # reshape
     return dat
 
 
@@ -27,6 +31,7 @@ def write_array_to_gputils_binary_file(x, path):
     :raises ValueError: if `x` has more than 3 dimensions
     :raises ValueError: if the file name specified `path` does not have the .bt extension
     """
+
     if not path.endswith(".bt"):
         raise ValueError("The file must have the .bt extension")
     x_shape = x.shape
@@ -36,8 +41,12 @@ def write_array_to_gputils_binary_file(x, path):
     nr = x_shape[0]
     nc = x_shape[1] if x_dims >= 2 else 1
     nm = x_shape[2] if x_dims == 3 else 1
+    if x_dims == 3:
+        x = x.swapaxes(0, 2).reshape(-1)  # column-major storage; axis 2 last
+    else:
+        x = x.T.reshape(-1)  # column-major storage
     with open(path, 'wb') as f:
         f.write(nr.to_bytes(8, 'little'))  # write number of rows
         f.write(nc.to_bytes(8, 'little'))  # write number of columns
         f.write(nm.to_bytes(8, 'little'))  # write number of matrices
-        x.reshape(nr*nc*nm, 1).tofile(f)  # write data
+        x.tofile(f)  # write data

python/test/test.py

@@ -37,6 +37,7 @@ def setUpClass(cls):
 
         a = np.linspace(-100, 100, 4 * 5).reshape((4, 5)).astype('d')
         gpuapi.write_array_to_gputils_binary_file(a, os.path.join(base_dir, 'a_d.bt'))
+
         gpuapi.write_array_to_gputils_binary_file(cls._B, os.path.join(base_dir, 'b_d.bt'))
 
     def __test_read_eye(self, dt):

test/testTensor.cu

@@ -183,9 +183,24 @@ TEST_F(TensorTest, parseTensorFromFileBinary) {
 TEST_F(TensorTest, parseTensorFromBinaryPython) {
     std::string fName = "../../python/b_d.bt";
     DTensor<double> b = DTensor<double>::parseFromFile(fName);
-    std::vector<double> vb(12);
-    b.download(vb);
-    for (size_t i = 0; i < 12; i++) EXPECT_NEAR(i + 1., vb[i], PRECISION_HIGH);
+    for (size_t i=0; i<3; i++) {
+        for (size_t j=0; j<3; j++) {
+            EXPECT_NEAR(1 + 2*j + 6*i, b(i, j, 0), PRECISION_HIGH);
+            EXPECT_NEAR(2 + 2*j + 6*i, b(i, j, 1), PRECISION_HIGH);
+        }
+    }
+}
+
+
+/* ---------------------------------------
+ * Parse not existing file
+ * --------------------------------------- */
+
+TEST_F(TensorTest, parseTensorFromNonexistentFile) {
+    std::string fName = "../../python/whatever.bt";
+    EXPECT_THROW(DTensor<double> b = DTensor<double>::parseFromFile(fName, rowMajor), std::invalid_argument);
+    std::string fName2 = "../../python/whatever.txt";
+    EXPECT_THROW(DTensor<double> b = DTensor<double>::parseFromFile(fName2, rowMajor), std::invalid_argument);
 }