Merge ae63253 into sapling-pr-archive-ktf

Author: ktf

Framework/AnalysisSupport/src/AODWriterHelpers.cxx

@@ -21,7 +21,6 @@
 #include "Framework/TableConsumer.h"
 #include "Framework/DataOutputDirector.h"
 #include "Framework/TableTreeHelpers.h"
-#include "Framework/Monitoring.h"
 #include "Framework/Signpost.h"
 
 #include <Monitoring/Monitoring.h>
@@ -32,8 +31,6 @@
 #include <TMap.h>
 #include <TObjString.h>
 #include <arrow/table.h>
-#include <chrono>
-#include <ios>
 
 O2_DECLARE_DYNAMIC_LOG(histogram_registry);
 
@@ -157,7 +154,7 @@ AlgorithmSpec AODWriterHelpers::getOutputTTreeWriter(ConfigContext const& ctx)
         }
 
         // skip non-AOD refs
-        if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) {
+        if (!DataSpecUtils::partialMatch(*ref.spec, AODOrigins)) {
           continue;
         }
         startTime = DataRefUtils::getHeader<DataProcessingHeader*>(ref)->startTime;

Framework/Core/include/Framework/ASoA.h

@@ -51,6 +51,8 @@ DECLARE_SOA_TABLE(BCFlags, "AOD", "BCFLAG", //! flag for tagging UPCs, joinable
                   bc::Flags);
 
 using BCs = BCs_001; // current version
+template <aod::is_origin_hash O>
+using BCsFrom = BCs_001From<O>;
 using BC = BCs::iterator;
 
 namespace timestamp
@@ -66,7 +68,7 @@ using BCsWithTimestamps = soa::Join<aod::BCs, aod::Timestamps>;
 
 namespace soa
 {
-extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::BCs, aod::Timestamps>;
+extern template struct Join<aod::BCs, aod::Timestamps>;
 }
 namespace aod
 {
@@ -514,11 +516,11 @@ DECLARE_SOA_TABLE_FULL(StoredTracksIU, "Tracks_IU", "AOD", "TRACK_IU", //! On di
                        track::Sign<track::Signed1Pt>,
                        o2::soa::Marker<2>);
 
-DECLARE_SOA_EXTENDED_TABLE(TracksIU, StoredTracksIU, "EXTRACK_IU", 0, //! Track parameters at inner most update (e.g. ITS) as it comes from the tracking
-                           aod::track::Pt,
-                           aod::track::P,
-                           aod::track::Eta,
-                           aod::track::Phi);
+DECLARE_SOA_EXTENDED_TABLE_NG(TracksIU, StoredTracksIU, "EXTRACK_IU", 0, //! Track parameters at inner most update (e.g. ITS) as it comes from the tracking
+                              aod::track::Pt,
+                              aod::track::P,
+                              aod::track::Eta,
+                              aod::track::Phi);
 
 DECLARE_SOA_TABLE_FULL(StoredTracksCov, "TracksCov", "AOD", "TRACKCOV", //! On disk version of the TracksCov table at collision vertex
                        track::SigmaY, track::SigmaZ, track::SigmaSnp, track::SigmaTgl, track::Sigma1Pt,
@@ -680,9 +682,9 @@ using Run2TrackExtra = Run2TrackExtras::iterator;
 } // namespace aod
 namespace soa
 {
-extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Tracks, aod::TracksExtra>;
-extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Tracks, aod::TracksCov, aod::TracksExtra>;
-extern template struct soa::JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::TracksExtension, aod::StoredTracks>;
+extern template struct soa::Join<aod::Tracks, aod::TracksExtra>;
+extern template struct soa::Join<aod::Tracks, aod::TracksCov, aod::TracksExtra>;
+extern template struct soa::Join<aod::TracksExtension, aod::StoredTracks>;
 } // namespace soa
 namespace aod
 {
@@ -926,6 +928,8 @@ using MFTTracks = MFTTracks_001;
 using StoredMFTTracks = StoredMFTTracks_001;
 
 using MFTTrack = MFTTracks::iterator;
+template <aod::is_origin_hash O>
+using MFTTracksFrom = MFTTracks_001From<O>;
 
 namespace fwdtrack // Index to MFTtrack column must be defined after table definition.
 {
@@ -1005,7 +1009,7 @@ using MFTTrackCovFwd = MFTTracksCov::iterator;
 } // namespace aod
 namespace soa
 {
-extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::FwdTracks, aod::FwdTracksCov>;
+extern template struct Join<aod::FwdTracks, aod::FwdTracksCov>;
 }
 namespace aod
 {
@@ -2026,6 +2030,8 @@ DECLARE_SOA_EXTENDED_TABLE(McParticles_001, StoredMcParticles_001, "EXMCPARTICLE
 using StoredMcParticles = StoredMcParticles_001;
 using McParticles = McParticles_001;
 using McParticle = McParticles::iterator;
+template <aod::is_origin_hash O>
+using McParticlesFrom = McParticles_001From<O>;
 } // namespace aod
 namespace soa
 {
@@ -2191,11 +2197,11 @@ DECLARE_SOA_INDEX_COLUMN(FDD, fdd);                    //!
 // First entry: Collision
 #define INDEX_LIST_RUN2 indices::CollisionId, indices::ZdcId, indices::BCId, indices::FT0Id, indices::FV0AId, indices::FV0CId, indices::FDDId
 DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(Run2MatchedExclusive, BCs, "MA_RN2_EX", INDEX_LIST_RUN2); //!
-DECLARE_SOA_INDEX_TABLE(Run2MatchedSparse, BCs, "MA_RN2_SP", INDEX_LIST_RUN2);              //!
+DECLARE_SOA_INDEX_TABLE(Run2MatchedSparse, BCs_001, "MA_RN2_SP", INDEX_LIST_RUN2);          //!
 
 #define INDEX_LIST_RUN3 indices::CollisionId, indices::ZdcId, indices::BCId, indices::FT0Id, indices::FV0AId, indices::FDDId
 DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(Run3MatchedExclusive, BCs, "MA_RN3_EX", INDEX_LIST_RUN3); //!
-DECLARE_SOA_INDEX_TABLE(Run3MatchedSparse, BCs, "MA_RN3_SP", INDEX_LIST_RUN3);              //!
+DECLARE_SOA_INDEX_TABLE(Run3MatchedSparse, BCs_001, "MA_RN3_SP", INDEX_LIST_RUN3);          //!
 
 // First entry: BC
 DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(MatchedBCCollisionsExclusive, BCs, "MA_BCCOL_EX", //!
@@ -2225,8 +2231,8 @@ DECLARE_EQUIVALENT_FOR_INDEX(aod::StoredTracksIU, aod::McTrackLabels);
 DECLARE_EQUIVALENT_FOR_INDEX(aod::Collisions, aod::McCollisionLabels);
 // Joins with collisions (only for sparse ones)
 // NOTE: index table needs to be always last argument
-extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Collisions, aod::Run2MatchedSparse>;
-extern template struct JoinFull<o2::aod::Hash<"JOIN/0"_h>, aod::Collisions, aod::Run3MatchedSparse>;
+extern template struct Join<aod::Collisions, aod::Run2MatchedSparse>;
+extern template struct Join<aod::Collisions, aod::Run3MatchedSparse>;
 } // namespace soa
 namespace aod
 {

Framework/Core/include/Framework/AnalysisDataModel.h

@@ -79,6 +79,7 @@ auto makeEmptyTable(const char* name)
 }
 
 template <soa::TableRef R>
+  requires(soa::not_void<typename aod::MetadataTrait<aod::Hash<R.desc_hash>>::metadata>)
 auto makeEmptyTable()
 {
   auto schema = std::make_shared<arrow::Schema>(soa::createFieldsFromColumns(typename aod::MetadataTrait<aod::Hash<R.desc_hash>>::metadata::persistent_columns_t{}));
@@ -93,6 +94,7 @@ auto makeEmptyTable(const char* name, framework::pack<Cs...> p)
 }
 
 template <aod::is_aod_hash D>
+  requires(soa::not_void<typename aod::MetadataTrait<D>::metadata>)
 auto makeEmptyTable(const char* name)
 {
   auto schema = std::make_shared<arrow::Schema>(soa::createFieldsFromColumns(typename aod::MetadataTrait<D>::metadata::persistent_columns_t{}));
@@ -216,6 +218,26 @@ inline constexpr auto getSourceSchemas()
   }.template operator()<T::sources.size(), T::sources>();
 }
 
+template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
+inline constexpr auto getSources()
+{
+  return []<size_t N, std::array<soa::TableRef, N> refs>() {
+    return []<size_t... Is>(std::index_sequence<Is...>) {
+      return std::vector{soa::tableRef2ConfigParamSpec<refs[Is]>()...};
+    }(std::make_index_sequence<N>());
+  }.template operator()<T::N, T::template generateSources<O>()>();
+}
+
+template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
+inline constexpr auto getSourceSchemas()
+{
+  return []<size_t N, std::array<soa::TableRef, N> refs>() {
+    return []<size_t... Is>(std::index_sequence<Is...>) {
+      return std::vector{soa::tableRef2Schema<refs[Is]>()...};
+    }(std::make_index_sequence<N>());
+  }.template operator()<T::N, T::template generateSources<O>()>();
+}
+
 template <soa::with_ccdb_urls T>
 inline constexpr auto getCCDBUrls()
 {
@@ -257,7 +279,7 @@ inline constexpr auto getIndexMapping()
   using indices = T::index_pack_t;
   using Key = T::Key;
   [&idx]<size_t... Is>(std::index_sequence<Is...>) mutable {
-    constexpr auto refs = T::sources;
+    constexpr auto refs = T::generateSources();
     ([&idx]<TableRef ref, typename C>() mutable {
       constexpr auto pos = o2::aod::MetadataTrait<o2::aod::Hash<ref.desc_hash>>::metadata::template getIndexPosToKey<Key>();
       if constexpr (pos == -1) {
@@ -272,6 +294,26 @@ inline constexpr auto getIndexMapping()
   return idx;
 }
 
+template <soa::with_sources_generator T, aod::is_origin_hash O = o2::aod::Hash<"AOD"_h>>
+constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
+{
+  std::vector<framework::ConfigParamSpec> inputMetadata;
+
+  auto inputSources = getSources<T, O>();
+  std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
+  auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
+  inputSources.erase(last, inputSources.end());
+  inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end());
+
+  auto inputSchemas = getSourceSchemas<T, O>();
+  std::sort(inputSchemas.begin(), inputSchemas.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; });
+  last = std::unique(inputSchemas.begin(), inputSchemas.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; });
+  inputSchemas.erase(last, inputSchemas.end());
+  inputMetadata.insert(inputMetadata.end(), inputSchemas.begin(), inputSchemas.end());
+
+  return inputMetadata;
+}
+
 template <soa::with_sources T>
 constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
 {
@@ -293,7 +335,7 @@ constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
 }
 
 template <typename T>
-  requires(!soa::with_sources<T>)
+  requires(!(soa::with_sources<T> || soa::with_sources_generator<T>))
 constexpr auto getInputMetadata() -> std::vector<framework::ConfigParamSpec>
 {
   return {};
@@ -358,14 +400,19 @@ template <TableRef R>
 constexpr auto tableRef2InputSpec()
 {
   std::vector<framework::ConfigParamSpec> metadata;
-  auto m = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
-  metadata.insert(metadata.end(), m.begin(), m.end());
-  auto ccdbMetadata = getCCDBMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
-  metadata.insert(metadata.end(), ccdbMetadata.begin(), ccdbMetadata.end());
-  auto p = getExpressionMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
-  metadata.insert(metadata.end(), p.begin(), p.end());
-  auto idx = getIndexMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
-  metadata.insert(metadata.end(), idx.begin(), idx.end());
+  std::vector<framework::ConfigParamSpec> sources;
+  if constexpr (soa::with_sources<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>) {
+    sources = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
+  } else if constexpr (soa::with_sources_generator<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>) {
+    sources = getInputMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata, o2::aod::Hash<R.origin_hash>>();
+  }
+  metadata.insert(metadata.end(), sources.begin(), sources.end());
+  auto ccdbURLs = getCCDBMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
+  metadata.insert(metadata.end(), ccdbURLs.begin(), ccdbURLs.end());
+  auto expressions = getExpressionMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
+  metadata.insert(metadata.end(), expressions.begin(), expressions.end());
+  auto indices = getIndexMetadata<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>();
+  metadata.insert(metadata.end(), indices.begin(), indices.end());
   if constexpr (!soa::with_ccdb_urls<typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata>) {
     metadata.emplace_back(framework::ConfigParamSpec{"schema", framework::VariantType::String, framework::serializeSchema(o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata::getSchema()), {"\"\""}});
   }
@@ -382,11 +429,22 @@ constexpr auto tableRef2InputSpec()
 template <TableRef R>
 constexpr auto tableRef2OutputSpec()
 {
+  std::vector<framework::ConfigParamSpec> metadata;
+  using md = typename o2::aod::MetadataTrait<o2::aod::Hash<R.desc_hash>>::metadata;
+  if constexpr (soa::with_ccdb_urls<md>) {
+    metadata.emplace_back("ccdb:", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""});
+  } else if constexpr (soa::with_expression_pack<md>) {
+    metadata.emplace_back("projectors", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""});
+  } else if constexpr (soa::with_index_pack<md>) {
+    metadata.emplace_back("index-records", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""});
+  }
   return framework::OutputSpec{
     framework::OutputLabel{o2::aod::label<R>()},
     o2::aod::origin<R>(),
     o2::aod::description(o2::aod::signature<R>()),
-    R.version};
+    R.version,
+    framework::Lifetime::Timeframe,
+    metadata};
 }
 
 template <TableRef R>
@@ -504,14 +562,14 @@ struct OutputForTable {
   using table_t = decltype(typeWithRef<T>());
   using metadata = aod::MetadataTrait<o2::aod::Hash<table_t::ref.desc_hash>>::metadata;
 
-  static OutputSpec const spec()
+  static constexpr auto spec()
   {
-    return OutputSpec{OutputLabel{aod::label<table_t::ref>()}, o2::aod::origin<table_t::ref>(), o2::aod::description(o2::aod::signature<table_t::ref>()), table_t::ref.version};
+    return soa::tableRef2OutputSpec<table_t::ref>();
   }
 
-  static OutputRef ref()
+  static constexpr auto ref()
   {
-    return OutputRef{aod::label<table_t::ref>(), table_t::ref.version};
+    return soa::tableRef2OutputRef<table_t::ref>();
   }
 };
 
@@ -544,7 +602,7 @@ concept is_produces_group = std::derived_from<T, ProducesGroup>;
 template <soa::is_metadata M, soa::TableRef Ref>
 struct TableTransform {
   using metadata = M;
-  constexpr static auto sources = M::sources;
+  constexpr static auto sources = M::template generateSources<o2::aod::Hash<Ref.origin_hash>>();
 
   template <soa::TableRef R>
   static auto base_spec()
@@ -578,23 +636,23 @@ struct TableTransform {
 /// This helper struct allows you to declare extended tables which should be
 /// created by the task (as opposed to those pre-defined by data model)
 template <typename T>
-concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
+concept is_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>> && soa::has_extension<typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata>;
 
 template <typename T>
-concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata>;
+concept is_dynamically_spawnable = soa::has_metadata<aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>> && soa::has_configurable_extension<typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata>;
 
 template <is_spawnable T>
 constexpr auto transformBase()
 {
-  using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::ref.desc_hash>>::metadata;
-  return TableTransform<metadata, metadata::extension_table_t::ref>{};
+  using metadata = typename aod::MetadataTrait<o2::aod::Hash<T::originals[T::originals.size() - 1].desc_hash>>::metadata;
+  return TableTransform<metadata, metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>::ref>{};
 }
 
 template <is_spawnable T>
 struct Spawns : decltype(transformBase<T>()) {
   using spawnable_t = T;
   using metadata = decltype(transformBase<T>())::metadata;
-  using extension_t = typename metadata::extension_table_t;
+  using extension_t = typename metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>;
   using expression_pack_t = typename metadata::expression_pack_t;
   static constexpr size_t N = framework::pack_size(expression_pack_t{});
 
@@ -644,7 +702,7 @@ struct Defines : decltype(transformBase<T>()) {
   static constexpr bool delayed = DELAYED;
   using spawnable_t = T;
   using metadata = decltype(transformBase<T>())::metadata;
-  using extension_t = typename metadata::extension_table_t;
+  using extension_t = typename metadata::template extension_table_t_from<o2::aod::Hash<T::originals[T::originals.size() - 1].origin_hash>>;
   using placeholders_pack_t = typename metadata::placeholders_pack_t;
   static constexpr size_t N = framework::pack_size(placeholders_pack_t{});