Add a c++ implementation for podio-dump (#620)

* Move sortAlphabetically to public header

* Implement podio-dump-tool in c++ and wrap it in thin script

Drop-in replacement of podio-dump python implementation which gets moved
to podio-dump.py because it supports dumping the pre-release legacy files

* Fix test env and paths for roundtrip tests

* Keep podio-dump.py in set of installed programs for now

* Fix minor differences in output

* Use ranges where possible

* Remove unnecessary flushing of stdout

* Switch to println where possible

* Rename legacy tool to podio-dump-legacy

* Make more things done at compile time

* Enable more tests for sanitizers

* Fix pre-commit issues

* Simplify parse function

Co-authored-by: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>

* Use ranges and views in some places

* Disable some tests still for TSan and UBSan

* Range-ify more of the implementation

Co-authored-by: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>

* Introduce alias namespace for better readability

* Improve error message

---------

Co-authored-by: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com>

Author: tmadlener

CMakeLists.txt

@@ -190,6 +190,9 @@ if(BUILD_TESTING)
   include(cmake/podioTest.cmake)
   add_subdirectory(tests)
 endif()
+
+find_package(fmt 9 REQUIRED)
+
 add_subdirectory(tools)
 add_subdirectory(python)
 

cmake/podioTest.cmake

@@ -12,6 +12,7 @@ function(PODIO_SET_TEST_ENV test)
       IO_HANDLERS=${IO_HANDLERS}
       PODIO_USE_CLANG_FORMAT=${PODIO_USE_CLANG_FORMAT}
       PODIO_BASE=${PROJECT_SOURCE_DIR}
+      PODIO_BUILD_BASE=${PROJECT_BINARY_DIR}
       ENABLE_SIO=${ENABLE_SIO}
       PODIO_BUILD_BASE=${PROJECT_BINARY_DIR}
       LSAN_OPTIONS=suppressions=${PROJECT_SOURCE_DIR}/tests/root_io/leak_sanitizer_suppressions.txt

include/podio/utilities/MiscHelpers.h

@@ -0,0 +1,32 @@
+#ifndef PODIO_UTILITIES_MISCHELPERS_H
+#define PODIO_UTILITIES_MISCHELPERS_H
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+namespace podio::utils {
+
+///  Sort the input vector of strings alphabetically, case insensitive.
+///
+///  @param strings The strings that should be sorted alphabetically
+///
+///  @returns A vector of strings sorted alphabetically, case insensitive
+inline std::vector<std::string> sortAlphabeticaly(std::vector<std::string> strings) {
+  // Obviously there is no tolower(std::string) in c++, so this is slightly more
+  // involved and we make use of the fact that lexicographical_compare works on
+  // ranges and the fact that we can feed it a dedicated comparison function,
+  // where we convert the strings to lower case char-by-char. The alternative is
+  // to make string copies inside the first lambda, transform them to lowercase
+  // and then use operator< of std::string, which would be effectively
+  // hand-writing what is happening below.
+  std::ranges::sort(strings, [](const auto& lhs, const auto& rhs) {
+    return std::lexicographical_compare(
+        lhs.begin(), lhs.end(), rhs.begin(), rhs.end(),
+        [](const auto& cl, const auto& cr) { return std::tolower(cl) < std::tolower(cr); });
+  });
+  return strings;
+}
+} // namespace podio::utils
+
+#endif // PODIO_UTILITIES_MISCHELPERS_H

src/RNTupleWriter.cc

@@ -58,7 +58,7 @@ void RNTupleWriter::writeFrame(const podio::Frame& frame, const std::string& cat
   const bool new_category = (catInfo.writer == nullptr);
   if (new_category) {
     // This is the minimal information that we need for now
-    catInfo.names = root_utils::sortAlphabeticaly(collsToWrite);
+    catInfo.names = podio::utils::sortAlphabeticaly(collsToWrite);
   }
 
   std::vector<root_utils::StoreCollection> collections;

src/ROOTWriter.cc

@@ -33,7 +33,7 @@ void ROOTWriter::writeFrame(const podio::Frame& frame, const std::string& catego
   // been initialized
   if (catInfo.tree == nullptr) {
     catInfo.idTable = frame.getCollectionIDTableForWrite();
-    catInfo.collsToWrite = root_utils::sortAlphabeticaly(collsToWrite);
+    catInfo.collsToWrite = podio::utils::sortAlphabeticaly(collsToWrite);
     catInfo.tree = new TTree(category.c_str(), (category + " data tree").c_str());
     catInfo.tree->SetDirectory(m_file.get());
   }

src/rootUtils.h

@@ -2,6 +2,7 @@
 #define PODIO_ROOT_UTILS_H // NOLINT(llvm-header-guard): internal headers confuse clang-tidy
 
 #include "podio/CollectionIDTable.h"
+#include "podio/utilities/MiscHelpers.h"
 #include "podio/utilities/RootHelpers.h"
 #include "podio/utilities/TypeHelpers.h"
 
@@ -280,25 +281,6 @@ inline auto reconstructCollectionInfo(TTree* eventTree, podio::CollectionIDTable
   return collInfo;
 }
 
-/**
- * Sort the input vector of strings alphabetically, case insensitive.
- */
-inline std::vector<std::string> sortAlphabeticaly(std::vector<std::string> strings) {
-  // Obviously there is no tolower(std::string) in c++, so this is slightly more
-  // involved and we make use of the fact that lexicographical_compare works on
-  // ranges and the fact that we can feed it a dedicated comparison function,
-  // where we convert the strings to lower case char-by-char. The alternative is
-  // to make string copies inside the first lambda, transform them to lowercase
-  // and then use operator< of std::string, which would be effectively
-  // hand-writing what is happening below.
-  std::ranges::sort(strings, [](const auto& lhs, const auto& rhs) {
-    return std::lexicographical_compare(
-        lhs.begin(), lhs.end(), rhs.begin(), rhs.end(),
-        [](const auto& cl, const auto& cr) { return std::tolower(cl) < std::tolower(cr); });
-  });
-  return strings;
-}
-
 /**
  * Check whether existingColls and candidateColls both contain the same
  * collection names. Returns false if the two vectors differ in content. Inputs

tests/CTestCustom.cmake

@@ -40,27 +40,16 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ
 
     pyunittest
 
-    podio-dump-help
     podio-dump-root
     podio-dump-detailed-root
     podio-dump-legacy_root_v00-16-06
     podio-dump-legacy_root-detailed_v00-16-06
 
-    podio-dump-sio
-    podio-dump-detailed-sio
     podio-dump-legacy_sio_v00-16-06
     podio-dump-legacy_sio-detailed_v00-16-06
 
-    podio-dump-rntuple
-    podio-dump-detailed-rntuple
-
     datamodel_def_store_roundtrip_root
     datamodel_def_store_roundtrip_root_extension
-    datamodel_def_store_roundtrip_sio
-    datamodel_def_store_roundtrip_sio_extension
-    datamodel_def_store_roundtrip_rntuple
-    datamodel_def_store_roundtrip_rntuple_extension
-
 
     write_old_data_root
     read_new_data_root
@@ -100,6 +89,12 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ
       read_rntuple
       read_interface_rntuple
       selected_colls_roundtrip_rntuple
+
+      podio-dump-rntuple
+      podio-dump-detailed-rntuple
+
+      datamodel_def_store_roundtrip_rntuple
+      datamodel_def_store_roundtrip_rntuple_extension
     )
   endif()
 
@@ -112,6 +107,12 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ
       write_interface_rntuple
       read_interface_rntuple
       selected_colls_roundtrip_rntuple
+
+      podio-dump-rntuple
+      podio-dump-detailed-rntuple
+
+      datamodel_def_store_roundtrip_rntuple
+      datamodel_def_store_roundtrip_rntuple_extension
     )
 
   endif()

tests/scripts/dumpModelRoundTrip.sh

@@ -21,7 +21,7 @@ OUTPUT_FOLDER=${INPUT_FILE}.dumped_${EDM_NAME}
 mkdir -p ${OUTPUT_FOLDER}
 
 # Dump the model to a yaml file
-${PODIO_BASE}/tools/podio-dump --dump-edm ${EDM_NAME} ${INPUT_FILE} > ${DUMPED_MODEL}
+${PODIO_BUILD_BASE}/tools/podio-dump --dump-edm ${EDM_NAME} ${INPUT_FILE} > ${DUMPED_MODEL}
 
 # Regenerate the code via the class generator and the freshly dumped model
 ${PODIO_BASE}/python/podio_class_generator.py \

tools/CMakeLists.txt

@@ -1,4 +1,11 @@
+add_executable(podio-dump-tool src/podio-dump-tool.cpp)
+target_link_libraries(podio-dump-tool PRIVATE podio::podio podio::podioIO fmt::fmt)
+
+install(TARGETS podio-dump-tool EXPORT podioTargets DESTINATION ${CMAKE_INSTALL_BINDIR})
+
 install(PROGRAMS ${CMAKE_CURRENT_LIST_DIR}/podio-dump DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(PROGRAMS ${CMAKE_CURRENT_LIST_DIR}/podio-dump-legacy DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(PROGRAMS ${CMAKE_CURRENT_LIST_DIR}/json-to-yaml DESTINATION ${CMAKE_INSTALL_BINDIR})
 install(PROGRAMS ${CMAKE_CURRENT_LIST_DIR}/podio-vis DESTINATION ${CMAKE_INSTALL_BINDIR})
 if(ENABLE_RNTUPLE)
   install(PROGRAMS ${CMAKE_CURRENT_LIST_DIR}/podio-ttree-to-rntuple DESTINATION ${CMAKE_INSTALL_BINDIR})
@@ -37,18 +44,22 @@ endif()
 
 # Add a very basic tests here to make sure that podio-dump at least runs
 if(BUILD_TESTING)
+  # Copy these two files into the build tree to be able to test things
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/json-to-yaml ${CMAKE_CURRENT_BINARY_DIR}/json-to-yaml COPYONLY)
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/podio-dump ${CMAKE_CURRENT_BINARY_DIR}/podio-dump COPYONLY)
+
   # Helper function for easily creating "tests" that simply execute podio-dump
   # with different arguments. Not crashing is considered success.
   #
   # Args:
   #     name        the name of the test
   #     depends_on  the target name of the test that produces the required input file
   function(CREATE_DUMP_TEST name depends_on)
-    add_test(NAME ${name} COMMAND ./podio-dump ${ARGN})
+    add_test(NAME ${name} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/podio-dump ${ARGN})
     PODIO_SET_TEST_ENV(${name})
 
     set_tests_properties(${name} PROPERTIES
-       WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
        )
     if (depends_on)
       set_tests_properties(${name} PROPERTIES
@@ -69,7 +80,7 @@ if(BUILD_TESTING)
     set(_name podio-dump-legacy_${name}_${version})
     ExternalData_Add_Test(legacy_test_cases
       NAME ${_name}
-      COMMAND ./podio-dump ${ARGN} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}}
+      COMMAND ./podio-dump-legacy ${ARGN} DATA{${PROJECT_SOURCE_DIR}/tests/input_files/${input_file}}
     )
     PODIO_SET_TEST_ENV(${_name})
 

tools/json-to-yaml

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+"""Tiny script to ingest a json string and dump it as a yaml string"""
+
+import sys
+import json
+import yaml
+
+
+def main():
+    """Main, read json from stdin and dump yaml to stdout"""
+    input_data = sys.stdin.read()
+    model_def = json.loads(input_data)
+    print(yaml.dump(model_def, sort_keys=False, default_flow_style=False))
+
+
+if __name__ == "__main__":
+    main()