Skip to content

Added parameter optimization and evaluation functionality for iotools benchmarks #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
Open

Added parameter optimization and evaluation functionality for iotools benchmarks #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8e7918b
test push
DanteNiewenhuis Apr 17, 2023
8a2bdee
Added flags for page and cluster size to gen files
DanteNiewenhuis Apr 17, 2023
e4d3cb2
Added all files used for parameter optimization
DanteNiewenhuis Jun 19, 2023
88653f5
changed default clustersize for generators
DanteNiewenhuis Jun 19, 2023
b3ff00e
updates for the PR
DanteNiewenhuis Jul 25, 2023
537c9f4
Merge remote-tracking branch 'upstream/master'
DanteNiewenhuis Jul 25, 2023
0dfc990
Added flags for page and cluster size to gen files
DanteNiewenhuis Apr 17, 2023
9af1614
changed default clustersize for generators
DanteNiewenhuis Jun 19, 2023
3a700b8
removing merge conflicts
DanteNiewenhuis Jul 25, 2023
d4fc1a5
updated the convertToByte function
DanteNiewenhuis Jul 25, 2023
8deaaa3
Updates based on the comments in the PR
DanteNiewenhuis Jul 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,14 @@ graph_*
AutoDict_*
_make_*
compare/h5hep

atlas
cms
h1
lhcb
gen_atlas
gen_cms
gen_h1
gen_lhcb

*.pyc
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ all: lhcb cms h1 gen_lhcb prepare_cms gen_cms gen_h1 ntuple_info tree_info \
fuse_forward clock check-uring

benchmarks: lhcb h1 cms atlas

generators: gen_lhcb gen_h1 gen_cms gen_atlas

### DATA #######################################################################

Expand Down
37 changes: 32 additions & 5 deletions gen_atlas.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ using RNTupleImporter = ROOT::Experimental::RNTupleImporter;

void Usage(char *progname)
{
std::cout << "Usage: " << progname << " -o <ntuple-path> -c <compression> [-m(t)] <H1 root file>"
std::cout << "Usage: " << progname << " -o <ntuple-path> -c <compression> -p <page-size> -x <cluster-size> [-m(t)] <ATLAS root file>"
<< std::endl;
}

Expand All @@ -25,10 +25,14 @@ int main(int argc, char **argv)
int compressionSettings = 0;
std::string compressionShorthand = "none";
std::string treeName = "mini";
int pagesize = -1;
int clustersize = -1;

int c;
while ((c = getopt(argc, argv, "hvi:o:c:m")) != -1) {
switch (c) {
while ((c = getopt(argc, argv, "hvi:o:c:p:x:m")) != -1)
{
switch (c)
{
case 'h':
case 'v':
Usage(argv[0]);
Expand All @@ -46,19 +50,42 @@ int main(int argc, char **argv)
case 'm':
ROOT::EnableImplicitMT();
break;
case 'p':
pagesize = atoi(optarg);
break;
case 'x':
clustersize = atoi(optarg);
break;
default:
fprintf(stderr, "Unknown option: -%c\n", c);
Usage(argv[0]);
return 1;
}
}
std::string dsName = "gg_data";
std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand + ".ntuple";

std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand;
unlink(outputFile.c_str());
auto importer = RNTupleImporter::Create(inputFile, treeName, outputFile);
auto options = importer->GetWriteOptions();
options.SetCompression(compressionSettings);

// Change pagesize and add pagesize to outputfile if pagesize was given
if (pagesize >= 0)
{
options.SetApproxUnzippedPageSize(pagesize);
options.SetApproxUnzippedPageSize(pagesize);
outputFile += "_pagesize=" std::to_string(pagesize);
}

// Change clustersize and add clustersize to outputfile if clustersize was given
if (clustersize >= 0)
{
options.SetApproxZippedClusterSize(clustersize);
outputFile += "_clustersize=" std::to_string(clustersize);
}

outputFile += ".ntuple";

importer->SetWriteOptions(options);
importer->Import();

Expand Down
44 changes: 36 additions & 8 deletions gen_cms.cxx
View file
Open in desktop
Copy link
Contributor

@jalopezg-git jalopezg-git Jul 13, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comments in gen_atlas.cxx apply here.

Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,27 @@

using RNTupleImporter = ROOT::Experimental::RNTupleImporter;

static void Usage(char *progname)
void Usage(char *progname)
{
std::cout << "Usage: " << progname << " -i <ttjet_13tev_june2019.root> -o <ntuple-path> -c <compression> [-m(t)]"
std::cout << "Usage: " << progname << " -o <ntuple-path> -c <compression> -p <page-size> -x <cluster-size> [-m(t)] <CMS root file>"
<< std::endl;
}

int main(int argc, char **argv)
{
std::string inputFile = "ttjet_13tev_june2019.root";
std::string inputFile = "ttjet.root";
std::string outputPath = ".";
int compressionSettings = 0;
std::string compressionShorthand = "none";
std::string treeName = "Events";
int pagesize = -1;
int clustersize = -1;

int c;
while ((c = getopt(argc, argv, "hvi:o:c:m")) != -1) {
switch (c) {
while ((c = getopt(argc, argv, "hvi:o:c:p:x:m")) != -1)
{
switch (c)
{
case 'h':
case 'v':
Usage(argv[0]);
Expand All @@ -46,19 +50,43 @@ int main(int argc, char **argv)
case 'm':
ROOT::EnableImplicitMT();
break;
case 'p':
pagesize = atoi(optarg);
break;
case 'x':
clustersize = atoi(optarg);
break;

default:
fprintf(stderr, "Unknown option: -%c\n", c);
Usage(argv[0]);
return 1;
}
}
std::string dsName = "ttjet_13tev_june2019";
std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand + ".ntuple";

std::string dsName = "ttjet";
std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand;
unlink(outputFile.c_str());
auto importer = RNTupleImporter::Create(inputFile, treeName, outputFile);
auto options = importer->GetWriteOptions();
options.SetCompression(compressionSettings);

// Change pagesize and add pagesize to outputfile if pagesize was given
if (pagesize >= 0)
{
options.SetApproxUnzippedPageSize(pagesize);
options.SetApproxUnzippedPageSize(pagesize);
outputFile += "_pagesize=" std::to_string(pagesize);
}

// Change clustersize and add clustersize to outputfile if clustersize was given
if (clustersize >= 0)
{
options.SetApproxZippedClusterSize(clustersize);
outputFile += "_clustersize=" std::to_string(clustersize);
}

outputFile += ".ntuple";

importer->SetWriteOptions(options);
importer->Import();

Expand Down
37 changes: 32 additions & 5 deletions gen_h1.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ using RNTupleImporter = ROOT::Experimental::RNTupleImporter;

void Usage(char *progname)
{
std::cout << "Usage: " << progname << " -o <ntuple-path> -c <compression> [-m(t)] <H1 root file>"
std::cout << "Usage: " << progname << " -o <ntuple-path> -c <compression> -p <page-size> -x <cluster-size> [-m(t)] <H1 root file>"
<< std::endl;
}

Expand All @@ -25,10 +25,14 @@ int main(int argc, char **argv)
int compressionSettings = 0;
std::string compressionShorthand = "none";
std::string treeName = "h42";
int pagesize = -1;
int clustersize = -1;

int c;
while ((c = getopt(argc, argv, "hvi:o:c:m")) != -1) {
switch (c) {
while ((c = getopt(argc, argv, "hvi:o:c:p:x:m")) != -1)
{
switch (c)
{
case 'h':
case 'v':
Usage(argv[0]);
Expand All @@ -46,19 +50,42 @@ int main(int argc, char **argv)
case 'm':
ROOT::EnableImplicitMT();
break;
case 'p':
pagesize = atoi(optarg);
break;
case 'x':
clustersize = atoi(optarg);
break;
default:
fprintf(stderr, "Unknown option: -%c\n", c);
Usage(argv[0]);
return 1;
}
}
std::string dsName = "h1dstX10";
std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand + ".ntuple";

std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand;
unlink(outputFile.c_str());
auto importer = RNTupleImporter::Create(inputFile, treeName, outputFile);
auto options = importer->GetWriteOptions();
options.SetCompression(compressionSettings);

// Change pagesize and add pagesize to outputfile if pagesize was given
if (pagesize >= 0)
{
options.SetApproxUnzippedPageSize(pagesize);
options.SetApproxUnzippedPageSize(pagesize);
outputFile += "_pagesize=" std::to_string(pagesize);
}

// Change clustersize and add clustersize to outputfile if clustersize was given
if (clustersize >= 0)
{
options.SetApproxZippedClusterSize(clustersize);
outputFile += "_clustersize=" std::to_string(clustersize);
}

outputFile += ".ntuple";

importer->SetWriteOptions(options);
importer->Import();

Expand Down
39 changes: 34 additions & 5 deletions gen_lhcb.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ using RNTupleImporter = ROOT::Experimental::RNTupleImporter;

void Usage(char *progname)
{
std::cout << "Usage: " << progname << " -o <ntuple-path> -c <compression> [-m(t)] <B2HHH.root>"
std::cout << "Usage: " << progname << " -o <ntuple-path> -c <compression> -p <page-size> -x <cluster-size> [-m(t)] <LHCb root file>"
<< std::endl;
}

Expand All @@ -25,10 +25,14 @@ int main(int argc, char **argv)
int compressionSettings = 0;
std::string compressionShorthand = "none";
std::string treeName = "DecayTree";
int pagesize = -1;
int clustersize = -1;

int c;
while ((c = getopt(argc, argv, "hvi:o:c:m")) != -1) {
switch (c) {
while ((c = getopt(argc, argv, "hvi:o:c:p:x:m")) != -1)
{
switch (c)
{
case 'h':
case 'v':
Usage(argv[0]);
Expand All @@ -46,19 +50,44 @@ int main(int argc, char **argv)
case 'm':
ROOT::EnableImplicitMT();
break;
case 'p':
pagesize = atoi(optarg);
break;
case 'x':
clustersize = atoi(optarg);
break;
default:
fprintf(stderr, "Unknown option: -%c\n", c);
Usage(argv[0]);
return 1;
}
}
std::string dsName = "B2HHH";
std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand + ".ntuple";
std::cout << "clustersize: " << clustersize << std::endl;

std::string dsName = "B2HHH";
std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand;
unlink(outputFile.c_str());
auto importer = RNTupleImporter::Create(inputFile, treeName, outputFile);
auto options = importer->GetWriteOptions();
options.SetCompression(compressionSettings);

// Change pagesize and add pagesize to outputfile if pagesize was given
if (pagesize >= 0)
{
options.SetApproxUnzippedPageSize(pagesize);
options.SetApproxUnzippedPageSize(pagesize);
outputFile += "_pagesize=" std::to_string(pagesize);
}

// Change clustersize and add clustersize to outputfile if clustersize was given
if (clustersize >= 0)
{
options.SetApproxZippedClusterSize(clustersize);
outputFile += "_clustersize=" std::to_string(clustersize);
}

outputFile += ".ntuple";

importer->SetWriteOptions(options);
importer->Import();

Expand Down
Loading