Skip to content

Commit 534e555

Browse files
Lizhe Jifacebook-github-bot
authored andcommitted
Add MetaHeader parity test addressing D108793458 review comment (#5935)
Summary: X-link: facebookresearch/FBGEMM#2853 This diff addresses the review comment in D108793458 inline 1189-1193 about hardcoded magic number 16 in values[j].size() >= 16 and sizeof(int64_t) offset baking in MetaHeader layout assumptions. Adds SSDTableBatchedEmbeddingsTest.MetaHeaderParityWithDRAM to verify SSD hard-coded 16-byte header parsing produces identical output to DRAM FixedBlockPool::get_metaheader_raw for same MetaHeader bytes. Keeps production ssd_table_batched_embeddings.h unchanged per follow-up discussion — hard-coded 16 / 8 remains with comment, test is source of truth for cross-backend parity. Updates test BUCK dep to include dram_kv_embedding_inference for FixedBlockPool header access. No production behavior change. Reviewed By: vinares Differential Revision: D109084819
1 parent defd174 commit 534e555

1 file changed

Lines changed: 50 additions & 0 deletions

File tree

fbgemm_gpu/src/ssd_split_embeddings_cache/test/ssd_table_batched_embeddings_test.cpp

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <gtest/gtest.h>
1111
#include <cstring>
1212
#include <filesystem>
13+
#include "deeplearning/fbgemm/fbgemm_gpu/src/dram_kv_embedding_cache/fixed_block_pool.h"
1314
#include "deeplearning/fbgemm/fbgemm_gpu/src/ssd_split_embeddings_cache/ssd_table_batched_embeddings.h"
1415

1516
using namespace ::testing;
@@ -641,3 +642,52 @@ TEST(
641642
EXPECT_TRUE(mock->is_metadata_cf_initialized(0));
642643
}
643644
}
645+
646+
TEST(SSDTableBatchedEmbeddingsTest, MetaHeaderParityWithDRAM) {
647+
// Verify the SSD read path (get_kv_zch_eviction_metadata_by_snapshot)
648+
// produces the same packed eviction-metadata word as the DRAM
649+
// FixedBlockPool helper for identical MetaHeader bytes. Ensures cross-backend
650+
// compatibility.
651+
kv_mem::FixedBlockPool::MetaHeader dram_hdr{};
652+
dram_hdr.key = 0x1122334455667788LL;
653+
dram_hdr.timestamp = 0xAABBCCDD;
654+
dram_hdr.count = 0x1234567;
655+
dram_hdr.used = true;
656+
657+
uint64_t dram_out = kv_mem::FixedBlockPool::get_metaheader_raw(&dram_hdr);
658+
659+
// SSD side: store the identical MetaHeader bytes into the metadata column
660+
// family, then read them back through the production accessor.
661+
auto mock_embedding_rocks = getMockEmbeddingRocksDB(
662+
/*num_shards=*/1,
663+
"metaHeaderParity",
664+
/*enable_raw_embedding_streaming=*/false,
665+
/*table_names=*/{},
666+
/*table_offsets=*/{},
667+
/*table_sizes=*/{},
668+
/*enable_metadata_cf=*/true,
669+
/*metadata_dim=*/kMetadataDimFp32);
670+
671+
auto indices =
672+
at::tensor({dram_hdr.key}, at::TensorOptions().dtype(at::kLong));
673+
auto metadata =
674+
at::zeros({1, kMetadataDimFp32}, at::TensorOptions().dtype(at::kFloat));
675+
std::memcpy(metadata.data_ptr<float>(), &dram_hdr, sizeof(dram_hdr));
676+
auto count = at::tensor({1L}, at::TensorOptions().dtype(at::kLong));
677+
mock_embedding_rocks->set_kv_metadata_async(indices, metadata, count).wait();
678+
679+
auto metadata_out =
680+
mock_embedding_rocks->get_kv_zch_eviction_metadata_by_snapshot(
681+
indices, count, /*snapshot_handle=*/nullptr);
682+
ASSERT_EQ(metadata_out.numel(), 1);
683+
uint64_t ssd_out = static_cast<uint64_t>(metadata_out.data_ptr<int64_t>()[0]);
684+
685+
EXPECT_EQ(dram_out, ssd_out);
686+
687+
// unpack matches Python test expectation
688+
uint32_t ts = static_cast<uint32_t>(dram_out & 0xFFFFFFFFu);
689+
uint32_t count_used = static_cast<uint32_t>(dram_out >> 32);
690+
EXPECT_EQ(ts, 0xAABBCCDDu);
691+
EXPECT_EQ(count_used & 0x7FFFFFFFu, 0x1234567u);
692+
EXPECT_EQ(count_used >> 31, 1u);
693+
}

0 commit comments

Comments
 (0)