Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[hadd] re-add RNTupleMerger change compression integration test #1204

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion root/io/hadd/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
if(ROOT_pyroot_FOUND)
ROOTTEST_ADD_TEST(compression_settings
MACRO test_compression_settings.py)

ROOTTEST_ADD_TEST(input_validation
MACRO input_validation.py)
endif()

ROOTTEST_ADD_TEST(test_MergeChangeComp
COPY_TO_BUILDDIR merge_gen_input_tuples.C merge_changeComp_check_output.C
PRECMD ${ROOT_root_CMD} -q -b -l "merge_gen_input_tuples.C(\"test_comp_in1.root\", \"test_comp_in2.root\")"
COMMAND ${ROOT_hadd_CMD} -f404 test_comp_out.root test_comp_in1.root test_comp_in2.root
POSTCMD ${ROOT_root_CMD} -q -b -l "merge_changeComp_check_output.C(404, \"test_comp_out.root\", \"test_comp_in1.root\", \"test_comp_in2.root\")"
)

ROOTTEST_ADD_TEST(test_TreeChangeComp
COPY_TO_BUILDDIR hadd_gen_input_tree.C hadd_check_comp_tree.C
PRECMD ${ROOT_root_CMD} -q -b -l "hadd_gen_input_tree.C(\"test_comp_tree_in.root\")"
Expand Down
1 change: 1 addition & 0 deletions root/io/hadd/Testing/Temporary/CTestCostData.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
---
Binary file added root/io/hadd/foo.root
Binary file not shown.
63 changes: 63 additions & 0 deletions root/io/hadd/merge_changeComp_check_output.C
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#include <ROOT/RLogger.hxx>
#include <ROOT/RNTupleModel.hxx>
#include <ROOT/RNTupleReader.hxx>
#include <ROOT/RNTupleWriter.hxx>
#include <ROOT/RPageStorageFile.hxx>
#include <Compression.h>
#include <RZip.h>

#include <TSystem.h>

int merge_changeComp_check_output(int expectedCompression, const char *fnameOut, const char *fnameIn1,
const char *fnameIn2)
{
using namespace ROOT::Experimental;

auto noPrereleaseWarning = RLogScopedVerbosity(NTupleLog(), ROOT::Experimental::ELogLevel::kError);

Internal::RPageSourceFile source("ntpl", fnameOut, RNTupleReadOptions());
source.Attach();

Internal::RClusterPool pool{source};

const auto expCompAlgo = ROOT::RCompressionSetting::AlgorithmFromCompressionSettings(expectedCompression);
const auto &desc = source.GetSharedDescriptorGuard();
auto clusterIter = desc->GetClusterIterable();
for (const auto &clusterDesc : clusterIter) {
// check advertised compression
int advertisedCompression = clusterDesc.GetColumnRange(0).fCompressionSettings;
if (advertisedCompression != expectedCompression) {
std::cerr << "Expected advertised compression to be " << expectedCompression << " but it is "
<< advertisedCompression << "\n";
return 1;
}

// check actual compression
for (const auto &column : desc->GetColumnIterable()) {
const auto &pages = clusterDesc.GetPageRange(column.GetLogicalId());
std::uint64_t pageIdx = 0;
for (const auto &pageInfo : pages.fPageInfos) {
std::vector<Internal::RCluster::RKey> keys{clusterDesc.GetId(), {column.GetPhysicalId()}};
auto cluster = pool.GetCluster(clusterDesc.GetId(), {column.GetPhysicalId()});
Internal::ROnDiskPage::Key key{column.GetPhysicalId(), pageIdx};
auto onDiskPage = cluster->GetOnDiskPage(key);
R__ASSERT(onDiskPage);
const auto actualCompAlgo =
R__getCompressionAlgorithm((const unsigned char *)onDiskPage->GetAddress(), onDiskPage->GetSize());
if (actualCompAlgo != expCompAlgo) {
std::cerr << "Expected actual compression to be "
<< ROOT::RCompressionSetting::AlgorithmToString(expCompAlgo) << " but it is "
<< ROOT::RCompressionSetting::AlgorithmToString(actualCompAlgo) << "\n";
return 1;
}
++pageIdx;
}
}
}

gSystem->Unlink(fnameOut);
gSystem->Unlink(fnameIn1);
gSystem->Unlink(fnameIn2);

return 0;
}
45 changes: 45 additions & 0 deletions root/io/hadd/merge_gen_input_tuples.C
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#include <ROOT/RLogger.hxx>
#include <ROOT/RNTupleModel.hxx>
#include <ROOT/RNTupleReader.hxx>
#include <ROOT/RNTupleWriter.hxx>
#include <ROOT/RPageStorageFile.hxx>
#include <Compression.h>

#include <TSystem.h>

#include <string>
#include <utility>

void merge_gen_input_tuples(const char *fname1 = "test_rntuple_input1.root", const char *fname2 = "test_rntuple_input2.root")
{
using namespace ROOT::Experimental;

auto noPrereleaseWarning = RLogScopedVerbosity(NTupleLog(), ROOT::Experimental::ELogLevel::kError);

{
auto model = RNTupleModel::Create();
auto fi = model->MakeField<int>("I", 1337);
auto fl = model->MakeField<long>("L", 666);
auto opts = RNTupleWriteOptions{};
opts.SetCompression(ROOT::CompressionSettings(ROOT::RCompressionSetting::EAlgorithm::kZSTD, 5));
auto writer = RNTupleWriter::Recreate(std::move(model), "ntpl", fname1, opts);
for (int i = 0; i < 1000; ++i) {
*fi = i;
*fl = i;
writer->Fill();
}
}
{
auto model = RNTupleModel::Create();
auto fi = model->MakeField<int>("I", 123);
auto fl = model->MakeField<long>("L", 420);
auto opts = RNTupleWriteOptions{};
opts.SetCompression(ROOT::CompressionSettings(ROOT::RCompressionSetting::EAlgorithm::kZLIB, 1));
auto writer = RNTupleWriter::Recreate(std::move(model), "ntpl", fname2, opts);
for (int i = 0; i < 100; ++i) {
*fi = i;
*fl = i;
writer->Fill();
}
}
}