Add an option to sst_dump to list all metadata blocks (facebook#13838)

Summary:
Add the --list_meta_blocks option to sst_dump. This PR also refactors some of the test code in sst_dump_test.

Pull Request resolved: facebook#13838

Reviewed By: cbi42

Differential Revision: D80320812

Pulled By: anand1976

fbshipit-source-id: 921b6560fbd756f5f8b364893700d240d3b7ad00

Author: anand1976

table/sst_file_dumper.cc

@@ -158,6 +158,14 @@ Status SstFileDumper::GetTableReader(const std::string& file_path) {
       s = SetOldTableOptions();
     }
     options_.comparator = internal_comparator_.user_comparator();
+
+    {
+      Status status = ReadMetaIndexBlockInFile(
+          file_.get(), file_size, magic_number, ImmutableOptions(options_),
+          ReadOptions(), &meta_index_contents_);
+      // Ignore any errors since this is required for a specific CLI option
+      status.PermitUncheckedError();
+    }
   }
 
   if (s.ok()) {

table/sst_file_dumper.h

@@ -51,6 +51,8 @@ class SstFileDumper {
   Status ShowCompressionSize(size_t block_size, CompressionType compress_type,
                              const CompressionOptions& compress_opt);
 
+  BlockContents& GetMetaIndexContents() { return meta_index_contents_; }
+
  private:
   // Get the TableReader implementation for the sst file
   Status GetTableReader(const std::string& file_path);
@@ -96,6 +98,7 @@ class SstFileDumper {
   ReadOptions read_options_;
   InternalKeyComparator internal_comparator_;
   std::unique_ptr<TableProperties> table_properties_;
+  BlockContents meta_index_contents_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

tools/sst_dump_test.cc

@@ -175,6 +175,30 @@ class SSTDumpToolTest : public testing::Test {
 
  protected:
   constexpr static int kNumKey = 1024;
+
+  void SSTDumpToolTestCase(Options& opts, bool filter, int wide_column_one_in,
+                           const char* cmd_arg) {
+    opts.env = env();
+    BlockBasedTableOptions table_opts;
+    if (filter) {
+      table_opts.filter_policy.reset(
+          ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10));
+    }
+    opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
+    std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
+    createSST(opts, file_path, wide_column_one_in);
+
+    char* usage[3];
+    PopulateCommandArgs(file_path, cmd_arg, usage);
+
+    ROCKSDB_NAMESPACE::SSTDumpTool tool;
+    ASSERT_TRUE(!tool.Run(3, usage, opts));
+
+    cleanup(opts, file_path);
+    for (int i = 0; i < 3; i++) {
+      delete[] usage[i];
+    }
+  }
 };
 
 
@@ -194,156 +218,52 @@ TEST_F(SSTDumpToolTest, HelpAndVersion) {
 
 TEST_F(SSTDumpToolTest, EmptyFilter) {
   Options opts;
-  opts.env = env();
-  std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
-  createSST(opts, file_path, 10);
-
-  char* usage[3];
-  PopulateCommandArgs(file_path, "--command=raw", usage);
-
-  ROCKSDB_NAMESPACE::SSTDumpTool tool;
-  ASSERT_TRUE(!tool.Run(3, usage, opts));
-
-  cleanup(opts, file_path);
-  for (int i = 0; i < 3; i++) {
-    delete[] usage[i];
-  }
+  SSTDumpToolTestCase(opts, /*filter=*/false, /*wide_column_one_in=*/10,
+                      "--command=raw");
 }
 
 TEST_F(SSTDumpToolTest, SstDumpReverseBytewiseComparator) {
   Options opts;
-  opts.env = env();
   opts.comparator = ReverseBytewiseComparator();
-  BlockBasedTableOptions table_opts;
-  table_opts.filter_policy.reset(
-      ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
-  opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
-  std::string file_path =
-      MakeFilePath("rocksdb_sst_reverse_bytewise_comparator.sst");
-  createSST(opts, file_path);
-
-  char* usage[3];
-  PopulateCommandArgs(file_path, "--command=raw", usage);
-
-  ROCKSDB_NAMESPACE::SSTDumpTool tool;
-  ASSERT_TRUE(!tool.Run(3, usage, opts));
-
-  cleanup(opts, file_path);
-  for (int i = 0; i < 3; i++) {
-    delete[] usage[i];
-  }
+  SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/10,
+                      "--command=raw");
 }
 
 TEST_F(SSTDumpToolTest, SstDumpComparatorWithU64Ts) {
   Options opts;
-  opts.env = env();
   opts.comparator = test::BytewiseComparatorWithU64TsWrapper();
-  BlockBasedTableOptions table_opts;
-  table_opts.filter_policy.reset(
-      ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
-  opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
-  std::string file_path =
-      MakeFilePath("rocksdb_sst_comparator_with_u64_ts.sst");
-  createSST(opts, file_path, 10);
-
-  char* usage[3];
-  PopulateCommandArgs(file_path, "--command=raw", usage);
-
-  ROCKSDB_NAMESPACE::SSTDumpTool tool;
-  ASSERT_TRUE(!tool.Run(3, usage, opts));
-
-  cleanup(opts, file_path);
-  for (int i = 0; i < 3; i++) {
-    delete[] usage[i];
-  }
+  SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/10,
+                      "--command=raw");
 }
 
-TEST_F(SSTDumpToolTest, FilterBlock) {
+TEST_F(SSTDumpToolTest, FilterBlockWideColumn) {
   Options opts;
-  opts.env = env();
-  BlockBasedTableOptions table_opts;
-  table_opts.filter_policy.reset(
-      ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, true));
-  opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
-  std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
-  createSST(opts, file_path, 10);
-
-  char* usage[3];
-  PopulateCommandArgs(file_path, "--command=raw", usage);
-
-  ROCKSDB_NAMESPACE::SSTDumpTool tool;
-  ASSERT_TRUE(!tool.Run(3, usage, opts));
-
-  cleanup(opts, file_path);
-  for (int i = 0; i < 3; i++) {
-    delete[] usage[i];
-  }
+  SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/10,
+                      "--command=raw");
 }
 
-TEST_F(SSTDumpToolTest, FullFilterBlock) {
+TEST_F(SSTDumpToolTest, FilterBlock) {
   Options opts;
-  opts.env = env();
-  BlockBasedTableOptions table_opts;
-  table_opts.filter_policy.reset(
-      ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
-  opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
-  std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
-  createSST(opts, file_path);
-
-  char* usage[3];
-  PopulateCommandArgs(file_path, "--command=raw", usage);
-
-  ROCKSDB_NAMESPACE::SSTDumpTool tool;
-  ASSERT_TRUE(!tool.Run(3, usage, opts));
-
-  cleanup(opts, file_path);
-  for (int i = 0; i < 3; i++) {
-    delete[] usage[i];
-  }
+  SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/0,
+                      "--command=raw");
 }
 
 TEST_F(SSTDumpToolTest, GetProperties) {
   Options opts;
-  opts.env = env();
-  BlockBasedTableOptions table_opts;
-  table_opts.filter_policy.reset(
-      ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
-  opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
-  std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
-  createSST(opts, file_path);
-
-  char* usage[3];
-  PopulateCommandArgs(file_path, "--show_properties", usage);
-
-  ROCKSDB_NAMESPACE::SSTDumpTool tool;
-  ASSERT_TRUE(!tool.Run(3, usage, opts));
-
-  cleanup(opts, file_path);
-  for (int i = 0; i < 3; i++) {
-    delete[] usage[i];
-  }
+  SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/0,
+                      "--show_properties");
 }
 
 TEST_F(SSTDumpToolTest, CompressedSizes) {
   Options opts;
-  opts.env = env();
-  BlockBasedTableOptions table_opts;
-  table_opts.filter_policy.reset(
-      ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
-  opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
-  std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
-  createSST(opts, file_path, 10);
-
-  char* usage[3];
-  PopulateCommandArgs(file_path, "--command=recompress", usage);
-
-  ROCKSDB_NAMESPACE::SSTDumpTool tool;
-  ASSERT_TRUE(!tool.Run(3, usage, opts));
+  SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/10,
+                      "--command=recompress");
+}
 
-  cleanup(opts, file_path);
-  for (int i = 0; i < 3; i++) {
-    delete[] usage[i];
-  }
+TEST_F(SSTDumpToolTest, ListMetaBlocks) {
+  Options opts;
+  SSTDumpToolTestCase(opts, /*filter=*/true, /*wide_column_one_in=*/0,
+                      "--list_meta_blocks");
 }
 
 namespace {
@@ -455,8 +375,8 @@ TEST_F(SSTDumpToolTest, ReadaheadSize) {
 
   // The file is approximately 10MB. Readahead is 4MB.
   // We usually need 3 reads + one metadata read.
-  // One extra read is needed before opening the file for metadata.
-  ASSERT_EQ(5, num_reads);
+  // Three extra read is needed before opening the file for metadata.
+  ASSERT_EQ(7, num_reads);
 
   SyncPoint::GetInstance()->ClearAllCallBacks();
   SyncPoint::GetInstance()->DisableProcessing();

tools/sst_dump_tool.cc

@@ -14,6 +14,7 @@
 #include "port/port.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/utilities/ldb_cmd.h"
+#include "table/block_based/block.h"
 #include "table/sst_file_dumper.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -121,6 +122,9 @@ void print_help(bool to_stderr) {
 
     --compression_use_zstd_finalize_dict
       Use zstd's finalizeDictionary() API instead of zstd's dictionary trainer to generate dictionary.
+
+    --list_meta_blocks
+      Print the list of all meta blocks in the file
 )",
       supported_compressions.c_str());
 }
@@ -162,6 +166,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
   bool use_from_as_prefix = false;
   bool show_properties = false;
   bool show_summary = false;
+  bool list_meta_blocks = false;
   bool set_block_size = false;
   bool has_compression_level_from = false;
   bool has_compression_level_to = false;
@@ -335,6 +340,8 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
       compression_max_dict_buffer_bytes = static_cast<uint64_t>(tmp_val);
     } else if (strcmp(argv[i], "--compression_use_zstd_finalize_dict") == 0) {
       compression_use_zstd_finalize_dict = true;
+    } else if (strcmp(argv[i], "--list_meta_blocks") == 0) {
+      list_meta_blocks = true;
     } else if (strcmp(argv[i], "--help") == 0) {
       print_help(/*to_stderr*/ false);
       return 0;
@@ -561,7 +568,35 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
         fprintf(stderr, "Reader unexpectedly returned null properties\n");
       }
     }
+
+    BlockContents& meta_index_contents = dumper.GetMetaIndexContents();
+    if (list_meta_blocks && meta_index_contents.data.size() > 0) {
+      Block meta_index_block(std::move(meta_index_contents));
+      std::unique_ptr<MetaBlockIter> meta_index_iter;
+      meta_index_iter.reset(meta_index_block.NewMetaIterator());
+      meta_index_iter->SeekToFirst();
+      fprintf(stdout,
+              "Meta Blocks:\n"
+              "------------------------------\n");
+      while (meta_index_iter->status().ok() && meta_index_iter->Valid()) {
+        Slice v = meta_index_iter->value();
+        BlockHandle handle;
+        st = handle.DecodeFrom(&v);
+        if (!st.ok()) {
+          fprintf(stderr, "%s: Could not decode block handle - %s\n",
+                  filename.c_str(), st.ToString().c_str());
+        } else {
+          fprintf(stdout, "  %s: %" PRIu64 " %" PRIu64 "\n",
+                  meta_index_iter->key().ToString().c_str(), handle.offset(),
+                  handle.size());
+        }
+        meta_index_iter->Next();
+      }
+    } else if (list_meta_blocks) {
+      fprintf(stderr, "Could not read the meta index block\n");
+    }
   }
+
   if (show_summary) {
     fprintf(stdout, "total number of files: %" PRIu64 "\n", total_num_files);
     fprintf(stdout, "total number of data blocks: %" PRIu64 "\n",