Optimize include time, push definitions to object files (#1774)

* WIP: Remove std::variant from public headers

* Fix compile time and size

* Fix tests

* Fixes

* Add loadChunkVariant again

* Move load/storeChunk implementations to object file

* Properly deactivate test

* Fixes for old compilers

* Move Memory.hpp implementations to .cpp

* Move UniquePtr implementation to .cpp file

* Move BaseRecord helpers to .cpp implementation file

* Fixes

* Move the rest of BaseRecord to .cpp

* Fix UniquePtr stuff

* Begin moving Container methods to .cpp

* Continue with that

* Finish moving Container implementation to cpp

* CI fixes, cleanup

* Implement Variant::index()

* Cleanup

* Doxygen

* @doxygen stop it

* CI fixes

Author: franzpoeschel

CMakeLists.txt

@@ -409,9 +409,15 @@ set(CORE_SOURCE
         src/auxiliary/Filesystem.cpp
         src/auxiliary/JSON.cpp
         src/auxiliary/JSONMatcher.cpp
+        src/auxiliary/Memory.cpp
         src/auxiliary/Mpi.cpp
+        src/auxiliary/UniquePtr.cpp
+        src/auxiliary/Variant.cpp
         src/backend/Attributable.cpp
+        src/backend/Attribute.cpp
+        src/backend/BaseRecord.cpp
         src/backend/BaseRecordComponent.cpp
+        src/backend/Container.cpp
         src/backend/MeshRecordComponent.cpp
         src/backend/PatchRecord.cpp
         src/backend/PatchRecordComponent.cpp

include/openPMD/Datatype.hpp

@@ -99,33 +99,6 @@ enum class Datatype : int
  */
 std::vector<Datatype> openPMD_Datatypes();
 
-namespace detail
-{
-    struct bottom
-    {};
-
-    // std::variant, but ignore first template parameter
-    // little trick to avoid trailing commas in the macro expansions below
-    template <typename Arg, typename... Args>
-    using variant_tail_t = std::variant<Args...>;
-} // namespace detail
-
-#define OPENPMD_ENUMERATE_TYPES(type) , type
-
-using dataset_types =
-    detail::variant_tail_t<detail::bottom OPENPMD_FOREACH_DATASET_DATATYPE(
-        OPENPMD_ENUMERATE_TYPES)>;
-
-using non_vector_types =
-    detail::variant_tail_t<detail::bottom OPENPMD_FOREACH_NONVECTOR_DATATYPE(
-        OPENPMD_ENUMERATE_TYPES)>;
-
-using attribute_types =
-    detail::variant_tail_t<detail::bottom OPENPMD_FOREACH_DATATYPE(
-        OPENPMD_ENUMERATE_TYPES)>;
-
-#undef OPENPMD_ENUMERATE_TYPES
-
 /** @brief Fundamental equivalence check for two given types T and U.
  *
  * This checks whether the fundamental datatype (i.e. that of a single value
@@ -451,7 +424,7 @@ inline size_t toBits(Datatype d)
  * @param d Datatype to test
  * @return true if vector type, else false
  */
-inline bool isVector(Datatype d)
+constexpr inline bool isVector(Datatype d)
 {
     using DT = Datatype;
 
@@ -777,6 +750,12 @@ void warnWrongDtype(std::string const &key, Datatype store, Datatype request);
 
 std::ostream &operator<<(std::ostream &, openPMD::Datatype const &);
 
+template <typename T>
+constexpr auto datatypeIndex() -> size_t
+{
+    return static_cast<size_t>(static_cast<int>(determineDatatype<T>()));
+}
+
 /**
  * Generalizes switching over an openPMD datatype.
  *

include/openPMD/DatatypeMacros.hpp

@@ -95,6 +95,27 @@ using openpmd_array_double_7 = std::array<double, 7>;
     MACRO(std::string)                                                         \
     MACRO(bool)
 
+#define OPENPMD_FOREACH_VECTOR_DATATYPE(MACRO)                                 \
+    MACRO(std::vector<char>)                                                   \
+    MACRO(std::vector<short>)                                                  \
+    MACRO(std::vector<int>)                                                    \
+    MACRO(std::vector<long>)                                                   \
+    MACRO(std::vector<long long>)                                              \
+    MACRO(std::vector<unsigned char>)                                          \
+    MACRO(std::vector<unsigned short>)                                         \
+    MACRO(std::vector<unsigned int>)                                           \
+    MACRO(std::vector<unsigned long>)                                          \
+    MACRO(std::vector<unsigned long long>)                                     \
+    MACRO(std::vector<float>)                                                  \
+    MACRO(std::vector<double>)                                                 \
+    MACRO(std::vector<long double>)                                            \
+    MACRO(std::vector<std::complex<float>>)                                    \
+    MACRO(std::vector<std::complex<double>>)                                   \
+    MACRO(std::vector<std::complex<long double>>)                              \
+    MACRO(std::vector<signed char>)                                            \
+    MACRO(std::vector<std::string>)                                            \
+    MACRO(openpmd_array_double_7)
+
 #define OPENPMD_FOREACH_DATASET_DATATYPE(MACRO)                                \
     MACRO(char)                                                                \
     MACRO(unsigned char)                                                       \

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -36,6 +36,7 @@
 #include "openPMD/ThrowError.hpp"
 #include "openPMD/auxiliary/JSON_internal.hpp"
 #include "openPMD/auxiliary/StringManip.hpp"
+#include "openPMD/backend/Variant_internal.hpp"
 #include "openPMD/backend/Writable.hpp"
 #include "openPMD/config.hpp"
 #include <stdexcept>
@@ -659,7 +660,7 @@ namespace detail
             size_t step,
             adios2::IO &IO,
             std::string name,
-            Attribute::resource &resource,
+            Parameter<Operation::READ_ATT> &,
             detail::AdiosAttributes const &);
 
         template <int n, typename... Params>

include/openPMD/IO/IOTask.hpp

@@ -26,11 +26,11 @@
 #include "openPMD/Streaming.hpp"
 #include "openPMD/auxiliary/Export.hpp"
 #include "openPMD/auxiliary/Memory.hpp"
-#include "openPMD/auxiliary/TypeTraits.hpp"
 #include "openPMD/auxiliary/Variant.hpp"
 #include "openPMD/backend/Attribute.hpp"
 #include "openPMD/backend/ParsePreference.hpp"
 
+#include <any>
 #include <cstddef>
 #include <memory>
 #include <optional>
@@ -621,7 +621,12 @@ struct OPENPMDAPI_EXPORT
         IfPossible
     };
     ChangesOverSteps changesOverSteps = ChangesOverSteps::No;
-    Attribute::resource resource;
+    // attribute_types
+    std::any m_resource;
+    template <typename T>
+    void setResource(T val);
+    template <typename variant_t>
+    variant_t const &resource() const;
 };
 
 template <>
@@ -642,8 +647,13 @@ struct OPENPMDAPI_EXPORT
 
     std::string name = "";
     std::shared_ptr<Datatype> dtype = std::make_shared<Datatype>();
-    std::shared_ptr<Attribute::resource> resource =
-        std::make_shared<Attribute::resource>();
+
+    // attribute_types
+    std::shared_ptr<std::any> m_resource = std::make_shared<std::any>();
+    template <typename variant_t>
+    variant_t const &resource() const;
+    template <typename T>
+    void setResource(T val);
 };
 
 template <>
@@ -665,17 +675,14 @@ struct OPENPMDAPI_EXPORT
     std::string name = "";
     std::shared_ptr<Datatype> dtype = std::make_shared<Datatype>();
 
-    struct to_vector_type
-    {
-        template <typename T>
-        using type = std::vector<T>;
-    };
-    // std::variant<std::vector<T_1>, std::vector<T_2>, ...>
-    // for all T_i in openPMD::Datatype.
-    using result_type = typename auxiliary::detail::
-        map_variant<to_vector_type, Attribute::resource>::type;
-
-    std::shared_ptr<result_type> resource = std::make_shared<result_type>();
+    // vector_of_attributes_type
+    std::shared_ptr<std::any> m_resource = std::make_shared<std::any>();
+    template <typename variant_t>
+    variant_t const &resource() const;
+    template <typename variant_t>
+    variant_t &resource();
+    template <typename T>
+    void setResource(std::vector<T> val);
 };
 
 template <>

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -27,6 +27,7 @@
 #include "openPMD/IO/JSON/JSONFilePosition.hpp"
 #include "openPMD/auxiliary/Filesystem.hpp"
 #include "openPMD/auxiliary/JSON_internal.hpp"
+#include "openPMD/backend/Variant_internal.hpp"
 #include "openPMD/config.hpp"
 
 #include <istream>
@@ -473,7 +474,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     struct AttributeWriter
     {
         template <typename T>
-        static void call(nlohmann::json &, Attribute::resource const &);
+        static void call(nlohmann::json &, attribute_types const &);
 
         static constexpr char const *errorMsg = "JSON: writeAttribute";
     };

include/openPMD/RecordComponent.hpp

@@ -28,6 +28,9 @@
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/BaseRecordComponent.hpp"
 
+// comment to prevent this include from being moved by clang-format
+#include "openPMD/DatatypeMacros.hpp"
+
 #include <array>
 #include <cmath>
 #include <limits>
@@ -38,6 +41,7 @@
 #include <string>
 #include <type_traits>
 #include <utility>
+#include <variant>
 #include <vector>
 
 // expose private and protected members for invasive testing
@@ -229,8 +233,11 @@ class RecordComponent : public BaseRecordComponent
     template <typename T>
     std::shared_ptr<T> loadChunk(Offset = {0u}, Extent = {-1u});
 
-    using shared_ptr_dataset_types = auxiliary::detail::
-        map_variant<auxiliary::detail::as_shared_pointer, dataset_types>::type;
+#define OPENPMD_ENUMERATE_TYPES(type) , std::shared_ptr<type>
+    using shared_ptr_dataset_types = auxiliary::detail::variant_tail_t<
+        auxiliary::detail::bottom OPENPMD_FOREACH_DATASET_DATATYPE(
+            OPENPMD_ENUMERATE_TYPES)>;
+#undef OPENPMD_ENUMERATE_TYPES
 
     /** std::variant-based version of allocating loadChunk<T>(Offset, Extent)
      *
@@ -546,4 +553,6 @@ OPENPMD_protected
 
 } // namespace openPMD
 
+#include "openPMD/UndefDatatypeMacros.hpp"
+// comment to prevent these includes from being moved by clang-format
 #include "RecordComponent.tpp"