bench deser dataset only

Author: gauteh

benches/index.rs

@@ -8,3 +8,9 @@ use hidefix::idx::Index;
 fn chunked_1d(b: &mut Bencher) {
     b.iter(|| Index::index("tests/data/dmrpp/chunked_oneD.h5").unwrap())
 }
+
+#[ignore]
+#[bench]
+fn meps(b: &mut Bencher) {
+    b.iter(|| test::black_box(Index::index("tests/data/meps_det_vc_2_5km_latest.nc")).unwrap())
+}

benches/serde_db.rs

@@ -3,7 +3,7 @@ extern crate test;
 use tempfile::{NamedTempFile, TempDir};
 use test::Bencher;
 
-use hidefix::idx::Index;
+use hidefix::idx::{DatasetD, Index};
 
 mod serde_db_sled {
     use super::*;
@@ -107,6 +107,60 @@ mod serde_db_sqlite {
         })
     }
 
+    #[ignore]
+    #[bench]
+    fn deserialize_meps_bincode_only_read_x_wind_ml(b: &mut Bencher) {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let i = Index::index("tests/data/meps_det_vc_2_5km_latest.nc").unwrap();
+        let d = i.dataset("x_wind_ml").unwrap();
+
+        let bts = bincode::serialize(d).unwrap();
+
+        println!("serialized size: {}", bts.len());
+
+        let db = NamedTempFile::new().unwrap();
+
+        let pool = rt.block_on(async {
+            let pool = SqlitePool::connect(&format!("sqlite:{}", db.path().to_str().unwrap()))
+                .await
+                .unwrap();
+            let mut c = pool.acquire().await.unwrap();
+
+            sqlx::query("CREATE TABLE hdf5 (dataset TEXT, idx BLOB)")
+                .execute(&mut c)
+                .await
+                .unwrap();
+            sqlx::query("CREATE INDEX dataset_idx ON hdf5 (dataset)")
+                .execute(&mut c)
+                .await
+                .unwrap();
+
+            // Insert index into db
+            sqlx::query("INSERT INTO hdf5 (dataset, idx) VALUES (?1, ?2)")
+                .bind("meps")
+                .bind(&bts)
+                .execute(&mut c)
+                .await
+                .unwrap();
+
+            pool
+        });
+
+        b.iter(|| {
+            let (_nbts,): (Vec<u8>,) = test::black_box(
+                rt.block_on(
+                    sqlx::query_as("SELECT idx FROM hdf5 WHERE dataset = ?1")
+                        .bind("meps")
+                        .fetch_one(&pool),
+                )
+                .unwrap(),
+            );
+            // assert_eq!(&bts, &nbts);
+
+            // let bts = rt.block_on(async { test::black_box(Vec::<u8>::with_capacity(8_000_000)) });
+        })
+    }
+
     #[ignore]
     #[bench]
     fn deserialize_meps_bincode(b: &mut Bencher) {
@@ -228,6 +282,7 @@ mod serde_db_redis {
         let i = Index::index("tests/data/meps_det_vc_2_5km_latest.nc").unwrap();
 
         let bts = bincode::serialize(&i).unwrap();
+        println!("bytes: {}", bts.len());
 
         let mut db = redis::Client::open("redis://:@127.1:6379")
             .unwrap()
@@ -237,11 +292,33 @@ mod serde_db_redis {
         db.set::<_, _, ()>("meps", bts.as_slice()).unwrap();
 
         b.iter(|| {
-            let nbts: Vec<u8> = db.get("meps").unwrap();
+            let nbts: Vec<u8> = test::black_box(db.get("meps").unwrap());
             test::black_box(bincode::deserialize::<Index>(&nbts).unwrap());
         })
     }
 
+    #[ignore]
+    #[bench]
+    fn deserialize_meps_bincode_x_wind_ml(b: &mut Bencher) {
+        let i = Index::index("tests/data/meps_det_vc_2_5km_latest.nc").unwrap();
+        let d = i.dataset("x_wind_ml").unwrap();
+
+        let bts = bincode::serialize(d).unwrap();
+        println!("bytes: {}", bts.len());
+
+        let mut db = redis::Client::open("redis://:@127.1:6379")
+            .unwrap()
+            .get_connection()
+            .unwrap();
+
+        db.set::<_, _, ()>("meps", bts.as_slice()).unwrap();
+
+        b.iter(|| {
+            let nbts: Vec<u8> = test::black_box(db.get("meps").unwrap());
+            test::black_box(bincode::deserialize::<DatasetD>(&nbts).unwrap());
+        })
+    }
+
     #[ignore]
     #[bench]
     fn deserialize_meps_bincode_only_read(b: &mut Bencher) {