DPL Analysis: make table origin a dynamic property in preparation to …

…handle multiple sources (#13358)

Detectors/AOD/include/AODProducerWorkflow/AODProducerHelpers.h

@@ -50,7 +50,7 @@ struct TripletEqualTo {
 typedef boost::unordered_map<Triplet_t, int, TripletHash, TripletEqualTo> TripletsMap_t;
 
 template <typename T>
-framework::Produces<T> createTableCursor(framework::ProcessingContext& pc)
+auto createTableCursor(framework::ProcessingContext& pc)
 {
   framework::Produces<T> c;
   c.resetCursor(pc.outputs()

Detectors/AOD/src/AODProducerWorkflowSpec.cxx

@@ -1814,7 +1814,7 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
   auto tracksCursor = createTableCursor<o2::aod::StoredTracksIU>(pc);
   auto tracksCovCursor = createTableCursor<o2::aod::StoredTracksCovIU>(pc);
   auto tracksExtraCursor = createTableCursor<o2::aod::StoredTracksExtra>(pc);
-  auto tracksQACursor = createTableCursor<o2::aod::TrackQA>(pc);
+  auto tracksQACursor = createTableCursor<o2::aod::TracksQA>(pc);
   auto ambigTracksCursor = createTableCursor<o2::aod::AmbiguousTracks>(pc);
   auto ambigMFTTracksCursor = createTableCursor<o2::aod::AmbiguousMFTTracks>(pc);
   auto ambigFwdTracksCursor = createTableCursor<o2::aod::AmbiguousFwdTracks>(pc);
@@ -2986,7 +2986,7 @@ DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, boo
     OutputForTable<Calos>::spec(),
     OutputForTable<CaloTriggers>::spec(),
     OutputForTable<CPVClusters>::spec(),
-    OutputForTable<Origin>::spec(),
+    OutputForTable<Origins>::spec(),
     OutputSpec{"TFN", "TFNumber"},
     OutputSpec{"TFF", "TFFilename"},
     OutputSpec{"AMD", "AODMetadataKeys"},

Framework/Core/include/Framework/AnalysisDataModel.h

@@ -562,9 +562,9 @@ DECLARE_SOA_COLUMN(TPCdEdxTot3R, tpcdEdxTot3R, uint8_t);             //! TPC dEd
 
 DECLARE_SOA_TABLE(TracksQA, "AOD", "TRACKQA", //! trackQA information - sampled QA information currently for the TPC
                   o2::soa::Index<>, trackqa::TrackId, trackqa::TPCTime0, trackqa::TPCDCAR, trackqa::TPCDCAZ, trackqa::TPCClusterByteMask,
-                  //                  o2::soa::Index<>, trackqa::TrackId, trackqa::TPCDCAR, trackqa::TPCDCAZ, trackqa::TPCClusterByteMask,
                   trackqa::TPCdEdxMax0R, trackqa::TPCdEdxMax1R, trackqa::TPCdEdxMax2R, trackqa::TPCdEdxMax3R,
                   trackqa::TPCdEdxTot0R, trackqa::TPCdEdxTot1R, trackqa::TPCdEdxTot2R, trackqa::TPCdEdxTot3R);
+//                  o2::soa::Index<>, trackqa::TrackId, trackqa::TPCDCAR, trackqa::TPCDCAZ, trackqa::TPCClusterByteMask,
 
 using TrackQA = TracksQA::iterator;
 

Framework/Core/include/Framework/AnalysisHelpers.h

@@ -37,9 +37,9 @@ struct WritingCursor {
 /// Helper class actually implementing the cursor which can write to
 /// a table. The provided template arguments are if type Column and
 /// therefore refer only to the persisted columns.
-template <typename... PC>
-struct WritingCursor<soa::Table<PC...>> {
-  using persistent_table_t = soa::Table<PC...>;
+template <o2::header::DataOrigin ORIGIN, typename... PC>
+struct WritingCursor<soa::Table<ORIGIN, PC...>> {
+  using persistent_table_t = soa::Table<ORIGIN, PC...>;
   using cursor_t = decltype(std::declval<TableBuilder>().cursor<persistent_table_t>());
 
   template <typename... T>
@@ -124,11 +124,11 @@ struct OutputForTable {
 /// means of the WritingCursor helper class, from which produces actually
 /// derives.
 template <typename T>
-struct Produces : WritingCursor<typename soa::PackToTable<typename T::table_t::persistent_columns_t>::table> {
+requires(!std::is_same_v<void, typename aod::MetadataTrait<T>::metadata>) struct Produces : WritingCursor<typename soa::PackToTable<aod::MetadataTrait<T>::metadata::origin(), typename T::table_t::persistent_columns_t>::table> {
 };
 
-template <template <typename...> class T, typename... C>
-struct Produces<T<C...>> : WritingCursor<typename soa::PackToTable<typename T<C...>::table_t::persistent_columns_t>::table> {
+template <template <o2::header::DataOrigin, typename...> class T, o2::header::DataOrigin ORIGIN, typename... C>
+struct Produces<T<ORIGIN, C...>> : WritingCursor<typename soa::PackToTable<ORIGIN, typename T<ORIGIN, C...>::table_t::persistent_columns_t>::table> {
 };
 
 /// Use this to group together produces. Useful to separate them logically
@@ -615,8 +615,8 @@ template <typename T, typename... Cs>
 auto Extend(T const& table)
 {
   static_assert((soa::is_type_spawnable_v<Cs> && ...), "You can only extend a table with expression columns");
-  using output_t = Join<T, soa::Table<Cs...>>;
-  return output_t{{o2::framework::spawner(framework::pack<Cs...>{}, {table.asArrowTable()}, "dynamicExtension"), table.asArrowTable()}, 0};
+  using output_t = Join<T, soa::Table<o2::header::DataOrigin{"JOIN"}, Cs...>>;
+  return output_t{{o2::framework::spawner<o2::header::DataOrigin{"JOIN"}>(framework::pack<Cs...>{}, {table.asArrowTable()}, "dynamicExtension"), table.asArrowTable()}, 0};
 }
 
 /// Template function to attach dynamic columns on-the-fly (e.g. inside
@@ -625,7 +625,7 @@ template <typename T, typename... Cs>
 auto Attach(T const& table)
 {
   static_assert((framework::is_base_of_template_v<o2::soa::DynamicColumn, Cs> && ...), "You can only attach dynamic columns");
-  using output_t = Join<T, o2::soa::Table<Cs...>>;
+  using output_t = Join<T, o2::soa::Table<o2::header::DataOrigin{"JOIN"}, Cs...>>;
   return output_t{{table.asArrowTable()}, table.offset()};
 }
 } // namespace o2::soa

Framework/Core/include/Framework/AnalysisManagers.h

@@ -351,7 +351,7 @@ struct OutputManager<Spawns<T>> {
       originalTable = makeEmptyTable<base_table_t>(aod::MetadataTrait<typename Spawns<T>::extension_t>::metadata::tableLabel());
     }
 
-    what.extension = std::make_shared<typename Spawns<T>::extension_t>(o2::framework::spawner(what.pack(), extractOriginals(what.sources_pack(), pc), aod::MetadataTrait<typename Spawns<T>::extension_t>::metadata::tableLabel()));
+    what.extension = std::make_shared<typename Spawns<T>::extension_t>(o2::framework::spawner<aod::MetadataTrait<typename Spawns<T>::extension_t>::metadata::origin()>(what.pack(), extractOriginals(what.sources_pack(), pc), aod::MetadataTrait<typename Spawns<T>::extension_t>::metadata::tableLabel()));
     what.table = std::make_shared<typename T::table_t>(soa::ArrowHelpers::joinTables({what.extension->asArrowTable(), originalTable}));
     return true;
   }

Framework/Core/include/Framework/TableBuilder.h

@@ -843,12 +843,12 @@ std::shared_ptr<arrow::Table> spawnerHelper(std::shared_ptr<arrow::Table>& fullT
                                             expressions::Projector* projectors, std::vector<std::shared_ptr<arrow::Field>> const& fields, const char* name);
 
 /// Expression-based column generator to materialize columns
-template <typename... C>
+template <o2::header::DataOrigin ORIGIN, typename... C>
 auto spawner(framework::pack<C...> columns, std::vector<std::shared_ptr<arrow::Table>>&& tables, const char* name)
 {
   auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables));
   if (fullTable->num_rows() == 0) {
-    return makeEmptyTable<soa::Table<C...>>(name);
+    return makeEmptyTable<soa::Table<ORIGIN, C...>>(name);
   }
   static auto fields = o2::soa::createFieldsFromColumns(columns);
   static auto new_schema = std::make_shared<arrow::Schema>(fields);

Framework/Core/src/AODReaderHelpers.cxx

@@ -164,7 +164,7 @@ AlgorithmSpec AODReaderHelpers::aodSpawnerCallback(std::vector<InputSpec>& reque
               originalTables.push_back(pc.inputs().get<TableConsumer>(spec.binding)->asArrowTable());
             }
           }
-          return o2::framework::spawner(expressions{}, std::move(originalTables), input.binding.c_str());
+          return o2::framework::spawner<metadata_t::origin()>(expressions{}, std::move(originalTables), input.binding.c_str());
         };
 
         if (description == header::DataDescription{"TRACK"}) {

Framework/Core/test/benchmark_ASoA.cxx

@@ -217,7 +217,7 @@ static void BM_ASoASimpleForLoop(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
 
   for (auto _ : state) {
     float sum = 0;
@@ -245,7 +245,7 @@ static void BM_ASoASimpleForLoopWithOp(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X, test::Y>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X, test::Y>;
 
   for (auto _ : state) {
     Test tests{table};
@@ -273,7 +273,7 @@ static void BM_ASoADynamicColumnPresent(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X, test::Y, test::Z, test::Sum<test::X, test::Y>>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X, test::Y, test::Z, test::Sum<test::X, test::Y>>;
 
   for (auto _ : state) {
     Test tests{table};
@@ -301,7 +301,7 @@ static void BM_ASoADynamicColumnCall(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X, test::Y, test::Sum<test::X, test::Y>>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X, test::Y, test::Sum<test::X, test::Y>>;
 
   Test tests{table};
   for (auto _ : state) {

Framework/Core/test/benchmark_ASoAHelpers.cxx

@@ -97,7 +97,7 @@ static void BM_ASoAHelpersNaiveSimplePairs(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
   Test tests{table};
   int64_t count = 0;
 
@@ -131,7 +131,7 @@ static void BM_ASoAHelpersNaiveSimpleFives(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
   Test tests{table};
   int64_t count = 0;
 
@@ -323,7 +323,7 @@ static void BM_ASoAHelpersCombGenSimplePairs(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
   Test tests{table};
 
   int64_t count = 0;
@@ -354,7 +354,7 @@ static void BM_ASoAHelpersCombGenSimpleFives(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
   Test tests{table};
 
   int64_t count = 0;
@@ -520,8 +520,8 @@ static void BM_ASoAHelpersCombGenSimpleFivesMultipleChunks(benchmark::State& sta
   }
   auto tableB = builderB.finalize();
 
-  using TestA = o2::soa::Table<o2::soa::Index<>, test::X, test::Y>;
-  using TestB = o2::soa::Table<o2::soa::Index<>, test::X>;
+  using TestA = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::soa::Index<>, test::X, test::Y>;
+  using TestB = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::soa::Index<>, test::X>;
   using ConcatTest = Concat<TestA, TestB>;
 
   ConcatTest tests{tableA, tableB};
@@ -596,7 +596,7 @@ static void BM_ASoAHelpersCombGenSimplePairsSameCategories(benchmark::State& sta
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
   Test tests{table};
   NoBinningPolicy<test::X> noBinning;
 
@@ -629,7 +629,7 @@ static void BM_ASoAHelpersCombGenSimpleFivesSameCategories(benchmark::State& sta
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
   Test tests{table};
   NoBinningPolicy<test::X> noBinning;
 
@@ -662,7 +662,7 @@ static void BM_ASoAHelpersCombGenSimplePairsCategories(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
   Test tests{table};
   NoBinningPolicy<test::X> noBinning;
 
@@ -695,7 +695,7 @@ static void BM_ASoAHelpersCombGenSimpleFivesCategories(benchmark::State& state)
   }
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<test::X>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X>;
   Test tests{table};
   NoBinningPolicy<test::X> noBinning;
 

Framework/Core/test/benchmark_TableBuilder.cxx

@@ -132,7 +132,7 @@ DECLARE_SOA_COLUMN(Y, y, float);
 DECLARE_SOA_COLUMN(Z, z, float);
 } // namespace test
 
-using TestVectors = o2::soa::Table<test::X, test::Y, test::Z>;
+using TestVectors = o2::soa::Table<o2::header::DataOrigin{"AOD"}, test::X, test::Y, test::Z>;
 
 static void BM_TableBuilderSoA(benchmark::State& state)
 {

Framework/Core/test/test_ASoA.cxx

@@ -130,7 +130,7 @@ TEST_CASE("TestTableIteration")
   ++tests;
   REQUIRE(tests.x() == 0);
   REQUIRE(tests.y() == 1);
-  using Test = o2::soa::Table<o2::aod::test::X, o2::aod::test::Y>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::aod::test::X, o2::aod::test::Y>;
   Test tests2{table};
   size_t value = 0;
   auto b = tests2.begin();
@@ -180,14 +180,14 @@ TEST_CASE("TestDynamicColumns")
   rowWriter(0, 1, 7);
   auto table = builder.finalize();
 
-  using Test = o2::soa::Table<o2::aod::test::X, o2::aod::test::Y, o2::aod::test::Sum<o2::aod::test::X, o2::aod::test::Y>>;
+  using Test = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::aod::test::X, o2::aod::test::Y, o2::aod::test::Sum<o2::aod::test::X, o2::aod::test::Y>>;
 
   Test tests{table};
   for (auto& test : tests) {
     REQUIRE(test.sum() == test.x() + test.y());
   }
 
-  using Test2 = o2::soa::Table<o2::aod::test::X, o2::aod::test::Y, o2::aod::test::Sum<o2::aod::test::Y, o2::aod::test::Y>>;
+  using Test2 = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::aod::test::X, o2::aod::test::Y, o2::aod::test::Sum<o2::aod::test::Y, o2::aod::test::Y>>;
 
   Test2 tests2{table};
   for (auto& test : tests2) {
@@ -267,9 +267,9 @@ TEST_CASE("TestJoinedTables")
   rowWriterZ(0, 8);
   auto tableZ = builderZ.finalize();
 
-  using TestX = o2::soa::Table<o2::aod::test::X>;
-  using TestY = o2::soa::Table<o2::aod::test::Y>;
-  using TestZ = o2::soa::Table<o2::aod::test::Z>;
+  using TestX = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::aod::test::X>;
+  using TestY = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::aod::test::Y>;
+  using TestZ = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::aod::test::Z>;
   using Test = Join<TestX, TestY>;
 
   REQUIRE(Test::contains<TestX>());
@@ -286,14 +286,14 @@ TEST_CASE("TestJoinedTables")
     REQUIRE(7 == test.x() + test.y());
   }
 
-  auto tests2 = join(TestX{tableX}, TestY{tableY});
+  auto tests2 = join<o2::header::DataOrigin{"JOIN"}>(TestX{tableX}, TestY{tableY});
   static_assert(std::is_same_v<Test::table_t, decltype(tests2)>,
                 "Joined tables should have the same type, regardless how we construct them");
   for (auto& test : tests2) {
     REQUIRE(7 == test.x() + test.y());
   }
 
-  auto tests3 = join(TestX{tableX}, TestY{tableY}, TestZ{tableZ});
+  auto tests3 = join<o2::header::DataOrigin{"JOIN"}>(TestX{tableX}, TestY{tableY}, TestZ{tableZ});
 
   for (auto& test : tests3) {
     REQUIRE(15 == test.x() + test.y() + test.z());
@@ -356,25 +356,25 @@ TEST_CASE("TestConcatTables")
   rowWriterD(0, 23, 15);
   auto tableD = builderD.finalize();
 
-  using TestA = o2::soa::Table<o2::soa::Index<>, o2::aod::test::X, o2::aod::test::Y>;
-  using TestB = o2::soa::Table<o2::soa::Index<>, o2::aod::test::X>;
-  using TestC = o2::soa::Table<o2::aod::test::Z>;
-  using TestD = o2::soa::Table<o2::aod::test::X, o2::aod::test::Z>;
+  using TestA = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::soa::Index<>, o2::aod::test::X, o2::aod::test::Y>;
+  using TestB = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::soa::Index<>, o2::aod::test::X>;
+  using TestC = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::aod::test::Z>;
+  using TestD = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::aod::test::X, o2::aod::test::Z>;
   using ConcatTest = Concat<TestA, TestB>;
   using JoinedTest = Join<TestA, TestC>;
   using NestedJoinTest = Join<JoinedTest, TestD>;
   using NestedConcatTest = Concat<Join<TestA, TestB>, TestD>;
 
-  static_assert(std::is_same_v<NestedJoinTest::table_t, o2::soa::Table<o2::soa::Index<>, o2::aod::test::Y, o2::aod::test::X, o2::aod::test::Z>>, "Bad nested join");
+  static_assert(std::is_same_v<NestedJoinTest::table_t, o2::soa::Table<o2::header::DataOrigin{"JOIN"}, o2::soa::Index<>, o2::aod::test::Y, o2::aod::test::X, o2::aod::test::Z>>, "Bad nested join");
 
-  static_assert(std::is_same_v<ConcatTest::table_t, o2::soa::Table<o2::soa::Index<>, o2::aod::test::X>>, "Bad intersection of columns");
+  static_assert(std::is_same_v<ConcatTest::table_t, o2::soa::Table<o2::header::DataOrigin{"CONC"}, o2::soa::Index<>, o2::aod::test::X>>, "Bad intersection of columns");
   ConcatTest tests{tableA, tableB};
   REQUIRE(16 == tests.size());
   for (auto& test : tests) {
     REQUIRE(test.index() == test.x());
   }
 
-  static_assert(std::is_same_v<NestedConcatTest::table_t, o2::soa::Table<o2::aod::test::X>>, "Bad nested concat");
+  static_assert(std::is_same_v<NestedConcatTest::table_t, o2::soa::Table<o2::header::DataOrigin{"CONC"}, o2::aod::test::X>>, "Bad nested concat");
 
   // Hardcode a selection for the first 5 odd numbers
   using FilteredTest = Filtered<TestA>;
@@ -539,7 +539,7 @@ TEST_CASE("TestDereference")
   REQUIRE(j.pointB().x() == 3);
   REQUIRE(j.pointB().y() == 4);
 
-  auto joined = join(segments, segmentsExtras);
+  auto joined = join<o2::header::DataOrigin{"JOIN"}>(segments, segmentsExtras);
   joined.bindExternalIndices(&points, &infos);
   auto se = joined.begin();
   REQUIRE(se.n() == 10);
@@ -574,7 +574,7 @@ TEST_CASE("TestFilteredOperators")
   auto tableA = builderA.finalize();
   REQUIRE(tableA->num_rows() == 8);
 
-  using TestA = o2::soa::Table<o2::soa::Index<>, o2::aod::test::X, o2::aod::test::Y>;
+  using TestA = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::soa::Index<>, o2::aod::test::X, o2::aod::test::Y>;
   using FilteredTest = Filtered<TestA>;
   using NestedFilteredTest = Filtered<Filtered<TestA>>;
   using namespace o2::framework;
@@ -611,7 +611,7 @@ TEST_CASE("TestFilteredOperators")
   REQUIRE(0 == filteredIntersection.size());
 
   i = 0;
-  for (auto& f : filteredIntersection) {
+  for (auto const& _ : filteredIntersection) {
     i++;
   }
   REQUIRE(i == 0);
@@ -650,7 +650,7 @@ TEST_CASE("TestNestedFiltering")
   auto tableA = builderA.finalize();
   REQUIRE(tableA->num_rows() == 8);
 
-  using TestA = o2::soa::Table<o2::soa::Index<>, o2::aod::test::X, o2::aod::test::Y>;
+  using TestA = o2::soa::Table<o2::header::DataOrigin{"AOD"}, o2::soa::Index<>, o2::aod::test::X, o2::aod::test::Y>;
   using FilteredTest = Filtered<TestA>;
   using NestedFilteredTest = Filtered<Filtered<TestA>>;
   using TripleNestedFilteredTest = Filtered<Filtered<Filtered<TestA>>>;