Streamlining definition and reading of tables (#4369)

Author: pbuehler

Framework/Core/CMakeLists.txt

@@ -28,6 +28,7 @@ o2_add_library(Framework
                        src/ASoA.cxx
                        ${GUI_SOURCES}
                        src/AnalysisHelpers.cxx
+                       src/AnalysisDataModelHelpers.cxx
                        src/BoostOptionsRetriever.cxx
                        src/ChannelConfigurationPolicy.cxx
                        src/ChannelMatching.cxx

Framework/Core/include/Framework/AnalysisDataModel.h

@@ -205,7 +205,7 @@ DECLARE_SOA_DYNAMIC_COLUMN(TPCFractionSharedCls, tpcFractionSharedCls, [](uint8_
 });
 } // namespace track
 
-DECLARE_SOA_TABLE_FULL(StoredTracks, "Tracks", "AOD", "TRACKPAR",
+DECLARE_SOA_TABLE_FULL(StoredTracks, "Tracks", "AOD", "TRACK:PAR",
                        o2::soa::Index<>, track::CollisionId, track::TrackType,
                        track::X, track::Alpha,
                        track::Y, track::Z, track::Snp, track::Tgl,
@@ -216,18 +216,18 @@ DECLARE_SOA_TABLE_FULL(StoredTracks, "Tracks", "AOD", "TRACKPAR",
                        track::Pz<track::Signed1Pt, track::Tgl>,
                        track::Charge<track::Signed1Pt>);
 
-DECLARE_SOA_EXTENDED_TABLE(Tracks, StoredTracks, "TRACKPAR",
+DECLARE_SOA_EXTENDED_TABLE(Tracks, StoredTracks, "TRACK:PAR",
                            aod::track::Pt,
                            aod::track::P,
                            aod::track::Eta,
                            aod::track::RawPhi);
 
-DECLARE_SOA_TABLE_FULL(StoredTracksCov, "TracksCov", "AOD", "TRACKPARCOV",
+DECLARE_SOA_TABLE_FULL(StoredTracksCov, "TracksCov", "AOD", "TRACK:PARCOV",
                        track::SigmaY, track::SigmaZ, track::SigmaSnp, track::SigmaTgl, track::Sigma1Pt,
                        track::RhoZY, track::RhoSnpY, track::RhoSnpZ, track::RhoTglY, track::RhoTglZ,
                        track::RhoTglSnp, track::Rho1PtY, track::Rho1PtZ, track::Rho1PtSnp, track::Rho1PtTgl);
 
-DECLARE_SOA_EXTENDED_TABLE(TracksCov, StoredTracksCov, "TRACKPARCOV",
+DECLARE_SOA_EXTENDED_TABLE(TracksCov, StoredTracksCov, "TRACK:PARCOV",
                            aod::track::CYY,
                            aod::track::CZY,
                            aod::track::CZZ,
@@ -244,7 +244,7 @@ DECLARE_SOA_EXTENDED_TABLE(TracksCov, StoredTracksCov, "TRACKPARCOV",
                            aod::track::C1PtTgl,
                            aod::track::C1Pt21Pt2);
 
-DECLARE_SOA_TABLE(TracksExtra, "AOD", "TRACKEXTRA",
+DECLARE_SOA_TABLE(TracksExtra, "AOD", "TRACK:EXTRA",
                   track::TPCInnerParam, track::Flags, track::ITSClusterMap,
                   track::TPCNClsFindable, track::TPCNClsFindableMinusFound, track::TPCNClsFindableMinusCrossedRows,
                   track::TPCNClsShared, track::TRDPattern, track::ITSChi2NCl,
@@ -468,7 +468,7 @@ DECLARE_SOA_INDEX_COLUMN_FULL(NegTrack, negTrack, int, FullTracks, "fNegTrackID"
 DECLARE_SOA_INDEX_COLUMN(Collision, collision);
 } // namespace v0
 
-DECLARE_SOA_TABLE(StoredV0s, "AOD", "O2v0", o2::soa::Index<>, v0::PosTrackId, v0::NegTrackId);
+DECLARE_SOA_TABLE(StoredV0s, "AOD", "V0", o2::soa::Index<>, v0::PosTrackId, v0::NegTrackId);
 DECLARE_SOA_TABLE(TransientV0s, "AOD", "V0INDEX", v0::CollisionId);
 
 using V0s = soa::Join<TransientV0s, StoredV0s>;
@@ -481,7 +481,7 @@ DECLARE_SOA_INDEX_COLUMN_FULL(Bachelor, bachelor, int, FullTracks, "fTracksID");
 DECLARE_SOA_INDEX_COLUMN(Collision, collision);
 } // namespace cascade
 
-DECLARE_SOA_TABLE(StoredCascades, "AOD", "O2cascade", o2::soa::Index<>, cascade::V0Id, cascade::BachelorId);
+DECLARE_SOA_TABLE(StoredCascades, "AOD", "CASCADE", o2::soa::Index<>, cascade::V0Id, cascade::BachelorId);
 DECLARE_SOA_TABLE(TransientCascades, "AOD", "CASCADEINDEX", cascade::CollisionId);
 
 using Cascades = soa::Join<TransientCascades, StoredCascades>;
@@ -503,7 +503,7 @@ DECLARE_SOA_COLUMN(BBFlag, bbFlag, uint64_t);
 DECLARE_SOA_COLUMN(BGFlag, bgFlag, uint64_t);
 } // namespace run2v0
 
-DECLARE_SOA_TABLE(Run2V0s, "AOD", "RUN2V0", o2::soa::Index<>, run2v0::BCId,
+DECLARE_SOA_TABLE(Run2V0s, "RN2", "V0", o2::soa::Index<>, run2v0::BCId,
                   run2v0::Adc, run2v0::Time, run2v0::Width,
                   run2v0::MultA, run2v0::MultC,
                   run2v0::TimeA, run2v0::TimeC,

Framework/Core/include/Framework/DataAllocator.h

@@ -133,7 +133,6 @@ class DataAllocator
       void* t2tr = nullptr;
       call_if_defined<struct TreeToTable>([&](auto* p) {
         auto t2t = new std::decay_t<decltype(*p)>(args...);
-        t2t->addAllColumns();
         adopt(spec, t2t);
         t2tr = t2t;
       });

Framework/Core/src/AODReaderHelpers.cxx

@@ -10,7 +10,7 @@
 
 #include "Framework/TableTreeHelpers.h"
 #include "Framework/AODReaderHelpers.h"
-#include "Framework/AnalysisDataModel.h"
+#include "AnalysisDataModelHelpers.h"
 #include "DataProcessingHelpers.h"
 #include "ExpressionHelpers.h"
 #include "Framework/RootTableBuilderHelpers.h"
@@ -41,105 +41,6 @@
 
 namespace o2::framework::readers
 {
-enum AODTypeMask : uint64_t {
-  None = 0,
-  Track = 1 << 0,
-  TrackCov = 1 << 1,
-  TrackExtra = 1 << 2,
-  Calo = 1 << 3,
-  CaloTrigger = 1 << 4,
-  Muon = 1 << 5,
-  MuonCluster = 1 << 6,
-  Zdc = 1 << 7,
-  BC = 1 << 8,
-  Collision = 1 << 9,
-  FT0 = 1 << 10,
-  FV0 = 1 << 11,
-  FDD = 1 << 12,
-  UnassignedTrack = 1 << 13,
-  Run2V0 = 1 << 14,
-  McCollision = 1 << 15,
-  McTrackLabel = 1 << 16,
-  McCaloLabel = 1 << 17,
-  McCollisionLabel = 1 << 18,
-  McParticle = 1 << 19,
-  Unknown = 1 << 20
-};
-
-uint64_t getMask(header::DataDescription description)
-{
-
-  if (description == header::DataDescription{"TRACKPAR"}) {
-    return AODTypeMask::Track;
-  } else if (description == header::DataDescription{"TRACKPARCOV"}) {
-    return AODTypeMask::TrackCov;
-  } else if (description == header::DataDescription{"TRACKEXTRA"}) {
-    return AODTypeMask::TrackExtra;
-  } else if (description == header::DataDescription{"CALO"}) {
-    return AODTypeMask::Calo;
-  } else if (description == header::DataDescription{"CALOTRIGGER"}) {
-    return AODTypeMask::CaloTrigger;
-  } else if (description == header::DataDescription{"MUON"}) {
-    return AODTypeMask::Muon;
-  } else if (description == header::DataDescription{"MUONCLUSTER"}) {
-    return AODTypeMask::MuonCluster;
-  } else if (description == header::DataDescription{"ZDC"}) {
-    return AODTypeMask::Zdc;
-  } else if (description == header::DataDescription{"BC"}) {
-    return AODTypeMask::BC;
-  } else if (description == header::DataDescription{"COLLISION"}) {
-    return AODTypeMask::Collision;
-  } else if (description == header::DataDescription{"FT0"}) {
-    return AODTypeMask::FT0;
-  } else if (description == header::DataDescription{"FV0"}) {
-    return AODTypeMask::FV0;
-  } else if (description == header::DataDescription{"FDD"}) {
-    return AODTypeMask::FDD;
-  } else if (description == header::DataDescription{"UNASSIGNEDTRACK"}) {
-    return AODTypeMask::UnassignedTrack;
-  } else if (description == header::DataDescription{"RUN2V0"}) {
-    return AODTypeMask::Run2V0;
-  } else if (description == header::DataDescription{"MCCOLLISION"}) {
-    return AODTypeMask::McCollision;
-  } else if (description == header::DataDescription{"MCTRACKLABEL"}) {
-    return AODTypeMask::McTrackLabel;
-  } else if (description == header::DataDescription{"MCCALOLABEL"}) {
-    return AODTypeMask::McCaloLabel;
-  } else if (description == header::DataDescription{"MCCOLLISLABEL"}) {
-    return AODTypeMask::McCollisionLabel;
-  } else if (description == header::DataDescription{"MCPARTICLE"}) {
-    return AODTypeMask::McParticle;
-  } else {
-    LOG(DEBUG) << "This is a tree of unknown type! " << description.str;
-    return AODTypeMask::Unknown;
-  }
-}
-
-uint64_t calculateReadMask(std::vector<OutputRoute> const& routes, header::DataOrigin const&)
-{
-  uint64_t readMask = None;
-  for (auto& route : routes) {
-    auto concrete = DataSpecUtils::asConcreteDataTypeMatcher(route.matcher);
-    auto description = concrete.description;
-
-    readMask |= getMask(description);
-  }
-  return readMask;
-}
-
-std::vector<OutputRoute> getListOfUnknown(std::vector<OutputRoute> const& routes)
-{
-
-  std::vector<OutputRoute> unknows;
-  for (auto& route : routes) {
-    auto concrete = DataSpecUtils::asConcreteDataTypeMatcher(route.matcher);
-
-    if (getMask(concrete.description) == AODTypeMask::Unknown)
-      unknows.push_back(route);
-  }
-  return unknows;
-}
-
 AlgorithmSpec AODReaderHelpers::aodSpawnerCallback(std::vector<InputSpec> requested)
 {
   return AlgorithmSpec::InitCallback{[requested](InitContext& ic) {
@@ -174,9 +75,9 @@ AlgorithmSpec AODReaderHelpers::aodSpawnerCallback(std::vector<InputSpec> reques
           return o2::soa::spawner(expressions{}, original_table.get());
         };
 
-        if (description == header::DataDescription{"TRACKPAR"}) {
+        if (description == header::DataDescription{"TRACK:PAR"}) {
           outputs.adopt(Output{origin, description}, maker(o2::aod::TracksExtensionMetadata{}));
-        } else if (description == header::DataDescription{"TRACKPARCOV"}) {
+        } else if (description == header::DataDescription{"TRACK:PARCOV"}) {
           outputs.adopt(Output{origin, description}, maker(o2::aod::TracksCovExtensionMetadata{}));
         } else if (description == header::DataDescription{"MUON"}) {
           outputs.adopt(Output{origin, description}, maker(o2::aod::MuonsExtensionMetadata{}));
@@ -206,16 +107,11 @@ AlgorithmSpec AODReaderHelpers::rootFileReaderCallback()
     // get the run time watchdog
     auto* watchdog = new RuntimeWatchdog(options.get<int64_t>("time-limit"));
 
-    // analyze type of requested tables
-    uint64_t readMask = calculateReadMask(spec.outputs, header::DataOrigin{"AOD"});
-    std::vector<OutputRoute> unknowns;
-    if (readMask & AODTypeMask::Unknown) {
-      unknowns = getListOfUnknown(spec.outputs);
-    }
+    // create list of requested tables
+    std::vector<OutputRoute> requestedTables(spec.outputs);
 
     auto counter = std::make_shared<int>(0);
-    return adaptStateless([readMask,
-                           unknowns,
+    return adaptStateless([requestedTables,
                            watchdog,
                            didir](DataAllocator& outputs, ControlService& control, DeviceSpec const& device) {
       // check if RuntimeLimit is reached
@@ -242,66 +138,36 @@ AlgorithmSpec AODReaderHelpers::rootFileReaderCallback()
         return;
       }
 
-      auto tableMaker = [&readMask, &outputs, fi, didir](auto metadata, AODTypeMask mask, char const* treeName) {
-        if (readMask & mask) {
+      // loop over requested tables
+      for (auto route : requestedTables) {
 
-          auto dh = header::DataHeader(decltype(metadata)::description(), decltype(metadata)::origin(), 0);
-          auto reader = didir->getTreeReader(dh, fi, treeName);
+        // create a TreeToTable object
+        auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
+        auto dh = header::DataHeader(concrete.description, concrete.origin, concrete.subSpec);
 
-          using table_t = typename decltype(metadata)::table_t;
-          if (!reader || (reader->IsInvalid())) {
-            LOGP(ERROR, "Requested \"{}\" tree not found in input file \"{}\"", treeName, didir->getInputFilename(dh, fi));
-          } else {
-            auto& builder = outputs.make<TableBuilder>(Output{decltype(metadata)::origin(), decltype(metadata)::description()});
-            RootTableBuilderHelpers::convertASoA<table_t>(builder, *reader);
-          }
+        auto tr = didir->getDataTree(dh, fi);
+        if (!tr) {
+          char* table;
+          sprintf(table, "%s/%s/%" PRIu32, concrete.origin.str, concrete.description.str, concrete.subSpec);
+          LOGP(ERROR, "Error while retrieving the tree for \"{}\"!", table);
+          return;
         }
-      };
-      tableMaker(o2::aod::CollisionsMetadata{}, AODTypeMask::Collision, "O2collision");
-      tableMaker(o2::aod::StoredTracksMetadata{}, AODTypeMask::Track, "O2track");
-      tableMaker(o2::aod::StoredTracksCovMetadata{}, AODTypeMask::TrackCov, "O2track");
-      tableMaker(o2::aod::TracksExtraMetadata{}, AODTypeMask::TrackExtra, "O2track");
-      tableMaker(o2::aod::CalosMetadata{}, AODTypeMask::Calo, "O2calo");
-      tableMaker(o2::aod::CaloTriggersMetadata{}, AODTypeMask::CaloTrigger, "O2calotrigger");
-      tableMaker(o2::aod::StoredMuonsMetadata{}, AODTypeMask::Muon, "O2muon");
-      tableMaker(o2::aod::MuonClustersMetadata{}, AODTypeMask::Muon, "O2muoncluster");
-      tableMaker(o2::aod::ZdcsMetadata{}, AODTypeMask::Zdc, "O2zdc");
-      tableMaker(o2::aod::BCsMetadata{}, AODTypeMask::BC, "O2bc");
-      tableMaker(o2::aod::FT0sMetadata{}, AODTypeMask::FT0, "O2ft0");
-      tableMaker(o2::aod::FV0sMetadata{}, AODTypeMask::FV0, "O2fv0");
-      tableMaker(o2::aod::FDDsMetadata{}, AODTypeMask::FDD, "O2fdd");
-      tableMaker(o2::aod::UnassignedTracksMetadata{}, AODTypeMask::UnassignedTrack, "O2unassignedtrack");
-      tableMaker(o2::aod::Run2V0sMetadata{}, AODTypeMask::Run2V0, "Run2v0");
-      tableMaker(o2::aod::McCollisionsMetadata{}, AODTypeMask::McCollision, "O2mccollision");
-      tableMaker(o2::aod::McTrackLabelsMetadata{}, AODTypeMask::McTrackLabel, "O2mctracklabel");
-      tableMaker(o2::aod::McCaloLabelsMetadata{}, AODTypeMask::McCaloLabel, "O2mccalolabel");
-      tableMaker(o2::aod::McCollisionLabelsMetadata{}, AODTypeMask::McCollisionLabel, "O2mccollisionlabel");
-      tableMaker(o2::aod::McParticlesMetadata{}, AODTypeMask::McParticle, "O2mcparticle");
-
-      // tables not included in the DataModel
-      if (readMask & AODTypeMask::Unknown) {
 
-        // loop over unknowns
-        for (auto route : unknowns) {
+        auto o = Output(dh);
+        auto& t2t = outputs.make<TreeToTable>(o, tr);
 
-          // create a TreeToTable object
-          auto concrete = DataSpecUtils::asConcreteDataMatcher(route.matcher);
-          auto dh = header::DataHeader(concrete.description, concrete.origin, concrete.subSpec);
-
-          auto tr = didir->getDataTree(dh, fi);
-          if (!tr) {
-            char* table;
-            sprintf(table, "%s/%s/%" PRIu32, concrete.origin.str, concrete.description.str, concrete.subSpec);
-            LOGP(ERROR, "Error while retrieving the tree for \"{}\"!", table);
-            return;
+        // add branches to read
+        auto colnames = aod::datamodel::getColumnNames(dh);
+        if (colnames.size() == 0) {
+          t2t.addAllColumns();
+        } else {
+          for (auto colname : colnames) {
+            t2t.addColumn(colname.c_str());
           }
-
-          auto o = Output(dh);
-          auto& t2t = outputs.make<TreeToTable>(o, tr);
-
-          // fill the table
-          t2t.fill();
         }
+
+        // fill the table
+        t2t.fill();
       }
     });
   })};

Framework/Core/src/AnalysisDataModelHelpers.cxx

@@ -0,0 +1,70 @@
+// Copyright CERN and copyright holders of ALICE O2. This software is
+// distributed under the terms of the GNU General Public License v3 (GPL
+// Version 3), copied verbatim in the file "COPYING".
+//
+// See http://alice-o2.web.cern.ch/license for full licensing information.
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "AnalysisDataModelHelpers.h"
+#include "Framework/AnalysisDataModel.h"
+#include "Framework/StringHelpers.h"
+#include "Framework/Logger.h"
+#include <boost/algorithm/string.hpp>
+
+namespace o2::aod::datamodel
+{
+std::string getTreeName(header::DataHeader dh)
+{
+  std::string description = std::string(dh.dataDescription.str);
+  std::string origin = std::string(dh.dataOrigin.str);
+
+  // lower case of first part of description
+  auto found = description.find_first_of(":");
+  std::string treeName = boost::algorithm::to_lower_copy(description).substr(0, found);
+
+  // add prefix according to origin
+  if (origin == "AOD") {
+    treeName = "O2" + treeName;
+  } else if (origin == "RN2") {
+    treeName = "Run2" + treeName;
+  }
+
+  // exceptions from this
+  if (origin == "AOD" && description == "MCCOLLISLABEL") {
+    treeName = "O2mccollisionlabel";
+  }
+
+  return treeName;
+}
+
+template <typename... C>
+static constexpr auto columnNamesTrait(framework::pack<C...>)
+{
+  return std::vector<std::string>{C::columnLabel()...};
+}
+
+std::vector<std::string> getColumnNames(header::DataHeader dh)
+{
+  auto description = std::string(dh.dataDescription.str);
+  auto origin = std::string(dh.dataOrigin.str);
+
+  // get column names
+  // AOD / RN2
+  if (origin == "AOD") {
+    if (description == "TRACK:PAR") {
+      return columnNamesTrait(typename StoredTracksMetadata::table_t::persistent_columns_t{});
+    } else if (description == "TRACK:PARCOV") {
+      return columnNamesTrait(typename StoredTracksCovMetadata::table_t::persistent_columns_t{});
+    } else if (description == "TRACK:EXTRA") {
+      return columnNamesTrait(typename TracksExtraMetadata::table_t::persistent_columns_t{});
+    }
+  }
+
+  // default: column names = {}
+  return std::vector<std::string>({});
+}
+
+} // namespace o2::aod::datamodel