diff --git a/.gitignore b/.gitignore index 519de8c..ad69a18 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,14 @@ graph_* AutoDict_* _make_* compare/h5hep + +atlas +cms +h1 +lhcb +gen_atlas +gen_cms +gen_h1 +gen_lhcb + +*.pyc diff --git a/Makefile b/Makefile index 8c01742..3009113 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ all: lhcb cms h1 gen_lhcb prepare_cms gen_cms gen_h1 ntuple_info tree_info \ fuse_forward clock check-uring benchmarks: lhcb h1 cms atlas - +generators: gen_lhcb gen_h1 gen_cms gen_atlas ### DATA ####################################################################### diff --git a/gen_atlas.cxx b/gen_atlas.cxx index 6c698d0..5006897 100644 --- a/gen_atlas.cxx +++ b/gen_atlas.cxx @@ -14,7 +14,7 @@ using RNTupleImporter = ROOT::Experimental::RNTupleImporter; void Usage(char *progname) { - std::cout << "Usage: " << progname << " -o -c [-m(t)]

" + std::cout << "Usage: " << progname << " -o -c -p -x [-m(t)] " << std::endl; } @@ -25,10 +25,14 @@ int main(int argc, char **argv) int compressionSettings = 0; std::string compressionShorthand = "none"; std::string treeName = "mini"; + int pagesize = -1; + int clustersize = -1; int c; - while ((c = getopt(argc, argv, "hvi:o:c:m")) != -1) { - switch (c) { + while ((c = getopt(argc, argv, "hvi:o:c:p:x:m")) != -1) + { + switch (c) + { case 'h': case 'v': Usage(argv[0]); @@ -46,6 +50,12 @@ int main(int argc, char **argv) case 'm': ROOT::EnableImplicitMT(); break; + case 'p': + pagesize = atoi(optarg); + break; + case 'x': + clustersize = atoi(optarg); + break; default: fprintf(stderr, "Unknown option: -%c\n", c); Usage(argv[0]); @@ -53,12 +63,29 @@ int main(int argc, char **argv) } } std::string dsName = "gg_data"; - std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand + ".ntuple"; - + std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand; unlink(outputFile.c_str()); auto importer = RNTupleImporter::Create(inputFile, treeName, outputFile); auto options = importer->GetWriteOptions(); options.SetCompression(compressionSettings); + + // Change pagesize and add pagesize to outputfile if pagesize was given + if (pagesize >= 0) + { + options.SetApproxUnzippedPageSize(pagesize); + options.SetApproxUnzippedPageSize(pagesize); + outputFile += "_pagesize=" std::to_string(pagesize); + } + + // Change clustersize and add clustersize to outputfile if clustersize was given + if (clustersize >= 0) + { + options.SetApproxZippedClusterSize(clustersize); + outputFile += "_clustersize=" std::to_string(clustersize); + } + + outputFile += ".ntuple"; + importer->SetWriteOptions(options); importer->Import(); diff --git a/gen_cms.cxx b/gen_cms.cxx index 2f8fc96..392a412 100644 --- a/gen_cms.cxx +++ b/gen_cms.cxx @@ -12,23 +12,27 @@ using RNTupleImporter = ROOT::Experimental::RNTupleImporter; -static void Usage(char *progname) +void Usage(char *progname) { - std::cout << "Usage: " << progname << " -i -o -c [-m(t)]" + std::cout << "Usage: " << progname << " -o -c -p -x [-m(t)] " << std::endl; } int main(int argc, char **argv) { - std::string inputFile = "ttjet_13tev_june2019.root"; + std::string inputFile = "ttjet.root"; std::string outputPath = "."; int compressionSettings = 0; std::string compressionShorthand = "none"; std::string treeName = "Events"; + int pagesize = -1; + int clustersize = -1; int c; - while ((c = getopt(argc, argv, "hvi:o:c:m")) != -1) { - switch (c) { + while ((c = getopt(argc, argv, "hvi:o:c:p:x:m")) != -1) + { + switch (c) + { case 'h': case 'v': Usage(argv[0]); @@ -46,19 +50,43 @@ int main(int argc, char **argv) case 'm': ROOT::EnableImplicitMT(); break; + case 'p': + pagesize = atoi(optarg); + break; + case 'x': + clustersize = atoi(optarg); + break; + default: fprintf(stderr, "Unknown option: -%c\n", c); Usage(argv[0]); return 1; } } - std::string dsName = "ttjet_13tev_june2019"; - std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand + ".ntuple"; - + std::string dsName = "ttjet"; + std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand; unlink(outputFile.c_str()); auto importer = RNTupleImporter::Create(inputFile, treeName, outputFile); auto options = importer->GetWriteOptions(); options.SetCompression(compressionSettings); + + // Change pagesize and add pagesize to outputfile if pagesize was given + if (pagesize >= 0) + { + options.SetApproxUnzippedPageSize(pagesize); + options.SetApproxUnzippedPageSize(pagesize); + outputFile += "_pagesize=" std::to_string(pagesize); + } + + // Change clustersize and add clustersize to outputfile if clustersize was given + if (clustersize >= 0) + { + options.SetApproxZippedClusterSize(clustersize); + outputFile += "_clustersize=" std::to_string(clustersize); + } + + outputFile += ".ntuple"; + importer->SetWriteOptions(options); importer->Import(); diff --git a/gen_h1.cxx b/gen_h1.cxx index 973fc19..e573dd2 100644 --- a/gen_h1.cxx +++ b/gen_h1.cxx @@ -14,7 +14,7 @@ using RNTupleImporter = ROOT::Experimental::RNTupleImporter; void Usage(char *progname) { - std::cout << "Usage: " << progname << " -o -c [-m(t)]

" + std::cout << "Usage: " << progname << " -o -c -p -x [-m(t)]

" << std::endl; } @@ -25,10 +25,14 @@ int main(int argc, char **argv) int compressionSettings = 0; std::string compressionShorthand = "none"; std::string treeName = "h42"; + int pagesize = -1; + int clustersize = -1; int c; - while ((c = getopt(argc, argv, "hvi:o:c:m")) != -1) { - switch (c) { + while ((c = getopt(argc, argv, "hvi:o:c:p:x:m")) != -1) + { + switch (c) + { case 'h': case 'v': Usage(argv[0]); @@ -46,6 +50,12 @@ int main(int argc, char **argv) case 'm': ROOT::EnableImplicitMT(); break; + case 'p': + pagesize = atoi(optarg); + break; + case 'x': + clustersize = atoi(optarg); + break; default: fprintf(stderr, "Unknown option: -%c\n", c); Usage(argv[0]); @@ -53,12 +63,29 @@ int main(int argc, char **argv) } } std::string dsName = "h1dstX10"; - std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand + ".ntuple"; - + std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand; unlink(outputFile.c_str()); auto importer = RNTupleImporter::Create(inputFile, treeName, outputFile); auto options = importer->GetWriteOptions(); options.SetCompression(compressionSettings); + + // Change pagesize and add pagesize to outputfile if pagesize was given + if (pagesize >= 0) + { + options.SetApproxUnzippedPageSize(pagesize); + options.SetApproxUnzippedPageSize(pagesize); + outputFile += "_pagesize=" std::to_string(pagesize); + } + + // Change clustersize and add clustersize to outputfile if clustersize was given + if (clustersize >= 0) + { + options.SetApproxZippedClusterSize(clustersize); + outputFile += "_clustersize=" std::to_string(clustersize); + } + + outputFile += ".ntuple"; + importer->SetWriteOptions(options); importer->Import(); diff --git a/gen_lhcb.cxx b/gen_lhcb.cxx index fd699ab..2dabcfc 100644 --- a/gen_lhcb.cxx +++ b/gen_lhcb.cxx @@ -14,7 +14,7 @@ using RNTupleImporter = ROOT::Experimental::RNTupleImporter; void Usage(char *progname) { - std::cout << "Usage: " << progname << " -o -c [-m(t)] " + std::cout << "Usage: " << progname << " -o -c -p -x [-m(t)] " << std::endl; } @@ -25,10 +25,14 @@ int main(int argc, char **argv) int compressionSettings = 0; std::string compressionShorthand = "none"; std::string treeName = "DecayTree"; + int pagesize = -1; + int clustersize = -1; int c; - while ((c = getopt(argc, argv, "hvi:o:c:m")) != -1) { - switch (c) { + while ((c = getopt(argc, argv, "hvi:o:c:p:x:m")) != -1) + { + switch (c) + { case 'h': case 'v': Usage(argv[0]); @@ -46,19 +50,44 @@ int main(int argc, char **argv) case 'm': ROOT::EnableImplicitMT(); break; + case 'p': + pagesize = atoi(optarg); + break; + case 'x': + clustersize = atoi(optarg); + break; default: fprintf(stderr, "Unknown option: -%c\n", c); Usage(argv[0]); return 1; } } - std::string dsName = "B2HHH"; - std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand + ".ntuple"; + std::cout << "clustersize: " << clustersize << std::endl; + std::string dsName = "B2HHH"; + std::string outputFile = outputPath + "/" + dsName + "~" + compressionShorthand; unlink(outputFile.c_str()); auto importer = RNTupleImporter::Create(inputFile, treeName, outputFile); auto options = importer->GetWriteOptions(); options.SetCompression(compressionSettings); + + // Change pagesize and add pagesize to outputfile if pagesize was given + if (pagesize >= 0) + { + options.SetApproxUnzippedPageSize(pagesize); + options.SetApproxUnzippedPageSize(pagesize); + outputFile += "_pagesize=" std::to_string(pagesize); + } + + // Change clustersize and add clustersize to outputfile if clustersize was given + if (clustersize >= 0) + { + options.SetApproxZippedClusterSize(clustersize); + outputFile += "_clustersize=" std::to_string(clustersize); + } + + outputFile += ".ntuple"; + importer->SetWriteOptions(options); importer->Import(); diff --git a/optimization_tools/Benchmarking/Algorithms.py b/optimization_tools/Benchmarking/Algorithms.py new file mode 100644 index 0000000..f5c806b --- /dev/null +++ b/optimization_tools/Benchmarking/Algorithms.py @@ -0,0 +1,327 @@ +from dataclasses import dataclass +from datetime import datetime +import numpy as np +from pathlib import Path +from random import random + +from Benchmarking.DataStructures.Configuration import Configuration +from Benchmarking.benchmark_utils import ( + get_size_decrease, + get_throughput_increase, + get_memory_usage_decrease, + get_performance, + evaluate_default_parameters, +) +from Benchmarking.variables import default_variable_values, path_to_results + + +@dataclass +class Walker: + configuration: Configuration = None + + # benchmark definitions + benchmark: str = "lhcb" + + # Performance parameters + default_file_size: int = None + default_throughput: float = None + default_memory_usage: float = None + performance: float = None + + # TODO: add weights + weights: list[float] = None + + write_file: Path = path_to_results + + multi_change: bool = False + + def __post_init__(self): + if self.configuration == None: + self.configuration = Configuration() + + class_name = self.get_class_name() + + folder_path = self.write_file / f"{self.benchmark}/{class_name}" + folder_path.mkdir(parents=True, exist_ok=True) + + self.write_file = folder_path / f'{datetime.now().strftime("%y-%m-%d_%H:%M:%S")}.csv' + + def get_class_name(self) -> str: + raise NotImplementedError + + def is_accepted(self, performance: float, iteration: int) -> bool: + raise NotImplementedError + + change_probabilities: list[list[float]] = None + + def create_probability_list(self, iterations: int): + x = np.arange(len(self.configuration.mutatable_parameters)) + 1 + prop = x[::-1] / np.sum(x) + last_prop = np.zeros(len(self.configuration.mutatable_parameters)) + last_prop[0] = 1 + + self.change_probabilities = np.linspace(prop, last_prop, iterations) + + def get_num_changes(self, iteration: int) -> list[float]: + if iteration > len(self.change_probabilities): + return self.change_probabilities[-1] + + return self.change_probabilities[iteration] + + def aggregate_performance(self, throughputs, memory_usages): + mean_throughput = np.mean(throughputs) + mean_memory_usage = np.mean(memory_usages) + return mean_throughput, mean_memory_usage + + def normalize_performance( + self, file_size: int, mean_throughput: float, mean_memory_usage: float + ) -> tuple[float, float, float, float]: + size_decrease = get_size_decrease(file_size, self.default_file_size) + throughput_increase = get_throughput_increase(mean_throughput, self.default_throughput) + memory_usage_decrease = get_memory_usage_decrease(mean_memory_usage, self.default_memory_usage) + + performance = get_performance([size_decrease, throughput_increase, memory_usage_decrease], self.weights) + + return size_decrease, throughput_increase, memory_usage_decrease, performance + + def log_step( + self, + generated_file_size: int, + throughputs: list[float], + memory_usages: list[float], + mean_throughput: float, + mean_memory_usage: float, + size_decrease: float, + throughput_increase: float, + memory_usage_decrease: float, + performance: float, + accepted: bool = True, + is_default: bool = False, + ): + """Log a taken step + + Args: + results (list[float]) + throughput (float) + throughput_increase (float) + size (int) + size_decrease (float) + performance (float, optional): Defaults to 1.0. + accepted (bool, optional): Defaults to False. + """ + with open(self.write_file, "a") as wf: + wf.write(f'{datetime.now().strftime("%y-%m-%d_%H:%M:%S")},') + + if is_default: + wf.write(f"{default_variable_values['compression_type']},") + wf.write(f"{default_variable_values['cluster_size']},") + wf.write(f"{default_variable_values['page_size']},") + wf.write(f"{default_variable_values['cluster_bunch']},") + else: + for value in self.configuration.values: + wf.write(f"{value},") + + wf.write( + f"{accepted},{performance:.2f},{size_decrease:.2f},{throughput_increase:.3f},{memory_usage_decrease:.3f},{generated_file_size},{mean_throughput:.3f},{mean_memory_usage:.1f}" + ) + + for throughput in throughputs: + wf.write(f",{throughput:.1f}") + for memory_usage in memory_usages: + wf.write(f",{memory_usage:.1f}") + + wf.write("\n") + + def step(self, iteration: int, evaluations: int = 10): + """Change the current configuration. + Determine the performance of the new configuration. + Determine if you want to keep the new configuration. + + Args: + iteration (int) + evaluations (int, optional). Defaults to 10. + """ + if iteration > 0: + if self.multi_change: + self.configuration.step_multi(self.get_num_changes(iteration)) + else: + self.configuration.step() + + generated_file_size, throughputs, memory_usages = self.configuration.evaluate( + self.benchmark, evaluations=evaluations + ) + + mean_throughput, mean_memory_usage = self.aggregate_performance(throughputs, memory_usages) + + size_decrease, throughput_increase, memory_usage_decrease, performance = self.normalize_performance( + generated_file_size, mean_throughput, mean_memory_usage + ) + + accepted = self.is_accepted(performance, iteration) + + # Logging + if accepted: + self.log_step( + generated_file_size, + throughputs, + memory_usages, + mean_throughput, + mean_memory_usage, + size_decrease, + throughput_increase, + memory_usage_decrease, + performance, + accepted=True, + ) + self.performance = performance + + else: + self.log_step( + generated_file_size, + throughputs, + memory_usages, + mean_throughput, + mean_memory_usage, + size_decrease, + throughput_increase, + memory_usage_decrease, + performance, + accepted=False, + ) + self.configuration.revert() + + def get_default_performance(self, evaluations: int = 10): + """Get the performance of the "base" configuration. + This performance will be used to normalize all other results + + Args: + evaluations (int, optional) Defaults to 10. + """ + generated_file_size, throughputs, memory_usages = evaluate_default_parameters(self.benchmark, evaluations) + + mean_throughput, mean_memory_usage = self.aggregate_performance(throughputs, memory_usages) + + self.performance = 0 + self.default_file_size = generated_file_size + self.default_throughput = mean_throughput + self.default_memory_usage = mean_memory_usage + + self.log_step( + generated_file_size, + throughputs, + memory_usages, + mean_throughput, + mean_memory_usage, + 0, + 0, + 0, + -999, + accepted=False, + is_default=True, + ) + + def evolve(self, steps: int = 100, evaluations: int = 10): + """Evolve the current configuration using the simmulated annealing algorithm + + Args: + steps (int, optional): Number of steps to evolve for. Defaults to 100. + evaluations (int, optional): Number of times to run a benchmark with a configuration. Defaults to 10. + """ + + with open(self.write_file, "w") as wf: + wf.write("time,") + + for name in self.configuration.names: + wf.write(f"{name},") + + wf.write( + f"accepted,performance(%),size_decrease(%),throughput_increase(%),memory_usage_decrease(%),size(MB),mean_throughput(MB/s),mean_memory_usage" + ) + + for i in range(evaluations): + wf.write(f",throughput_{i}") + for i in range(evaluations): + wf.write(f",memory_usage_{i}") + + wf.write("\n") + + # Log initial configuration + print(f"Calculating Initial Performance") + self.get_default_performance(evaluations) + + # # evolve the configuration for the given number of steps + print(f"Starting evolution") + for i in range(steps): + print(f"Step: {i} => Throughput: {self.performance:.3f}") + self.step(i, evaluations) + + +###################################################################################################### +# Random Walker +###################################################################################################### + + +class RandomWalker(Walker): + def is_accepted(self, performance: float, iteration: int) -> bool: + return True + + def get_class_name(self) -> str: + return "RandomWalker" + + +###################################################################################################### +# HillClimber +###################################################################################################### + + +class HillClimber(Walker): + def is_accepted(self, performance: float, iteration: int) -> bool: + return performance > self.performance + + def get_class_name(self) -> str: + return "RandomWalker" + + +###################################################################################################### +# Simmulated annealer +###################################################################################################### + + +@dataclass +class Annealer(Walker): + # Annealer parameters + temperature_const: float = 2.5 + iteration: int = 0 + + def get_class_name(self) -> str: + return "Annealer" + + def get_temperature(self, iteration: int) -> float: + """Get temperature based on the current iteration + + Args: + iteration (int) + + Returns: + float + """ + return self.temperature_const / np.log(iteration + 2) + + def get_probability(self, iteration: int, c: float) -> float: + """Get the probability of accepting a change based on + the current iteration and the difference in performance + + Args: + iteration (int) + c (float): The difference between the performance of + the new and old configuration + + Returns: + float + """ + return np.exp(c / self.get_temperature(iteration)) + + def is_accepted(self, performance: float, iteration: int) -> bool: + # Determine if new configuration will be accepted + c = performance - self.performance + return (c > 0) or (self.get_probability(iteration, c) > random()) diff --git a/optimization_tools/Benchmarking/DataStructures/Configuration.py b/optimization_tools/Benchmarking/DataStructures/Configuration.py new file mode 100644 index 0000000..98b5760 --- /dev/null +++ b/optimization_tools/Benchmarking/DataStructures/Configuration.py @@ -0,0 +1,212 @@ +from dataclasses import dataclass +from random import choice +import os +import numpy as np +from pathlib import Path + + +from Benchmarking.benchmark_utils import ( + convertToByteList, + convertByteToStr, + evaluate_parameters, +) +from Benchmarking.DataStructures.Parameters import ( + Parameter, + CategoricalParameter, + DiscreteParameter, + getCompressionParameter, + getClusterSizeParameter, + getClusterBunchParameter, + getPageSizeParameter, +) + +from Benchmarking.variables import path_to_generated_files, default_variable_values + + +############################################################################################################################### +# Configuration +############################################################################################################################### + + +@dataclass +class Configuration: + compression_type: CategoricalParameter = None + cluster_size: DiscreteParameter = None + page_size: DiscreteParameter = None + cluster_bunch: DiscreteParameter = None + + parameters: list[Parameter] = None + mutatable_parameters: list[Parameter] = None + mutated_variable: Parameter = None + + mutated_parameters: list[Parameter] = None + + def __post_init__(self): + if self.compression_type == None: + self.createBaseConfig() + + self.parameters = [ + self.compression_type, + self.cluster_size, + self.page_size, + self.cluster_bunch, + ] + + self.mutatable_parameters = [ + parameter for parameter in self.parameters if parameter.can_mutate + ] + + @property + def names(self) -> list[str]: + return [parameter.parameter_name for parameter in self.parameters] + + @property + def values(self) -> list[str]: + return [parameter.value for parameter in self.parameters] + + def createBaseConfig(self): + """Creates the configuration that is the default configuration defined by ROOT""" + self.compression_type = CategoricalParameter( + "Compression Type", ["none", "zlib", "lz4", "lzma", "zstd"], current_idx=2 + ) + + page_sizes = [ + (16, "KB"), + (32, "KB"), + (64, "KB"), + (128, "KB"), + (256, "KB"), + (512, "KB"), + (1, "MB"), + (2, "MB"), + (4, "MB"), + (8, "MB"), + (16, "MB"), + ] + self.page_size = DiscreteParameter( + "Page Size", + convertToByteList(page_sizes, base=2), + value_names=convertByteToStr(page_sizes), + current_idx=2, + ) + + cluster_sizes = [ + (20, "MB"), + (30, "MB"), + (40, "MB"), + (50, "MB"), + (100, "MB"), + (200, "MB"), + (300, "MB"), + (400, "MB"), + (500, "MB"), + ] + self.cluster_size = DiscreteParameter( + "Cluster Size", + convertToByteList(cluster_sizes, base=10), + value_names=convertByteToStr(cluster_sizes), + current_idx=3, + ) + + self.cluster_bunch = DiscreteParameter( + "Cluster Bunch", [1, 2, 3, 4, 5], current_idx=0 + ) + + def randomize(self): + """Change all parameters to a random value""" + for var in self.parameters: + var.initialize() + + def step(self): + """Make a random variable set a step""" + self.mutated_variable = self.mutatable_parameters[ + choice(range(len(self.mutatable_parameters))) + ] + self.mutated_variable.step() + + def step_multi(self, prop): + x = np.arange(len(self.mutatable_parameters)) + 1 + number_of_changes = np.random.choice(x, p=prop) + + idxs_to_change = np.random.choice(x - 1, number_of_changes, replace=False) + + self.mutated_mutatable_parameters = [] + for i in idxs_to_change: + self.mutatable_parameters[i].step() + self.mutated_mutatable_parameters.append(self.mutatable_parameters[i]) + + return number_of_changes + + def revert(self): + """Revert the previous step""" + self.mutated_variable.revert() + + def revert_multi(self): + for var in self.mutated_parameters: + var.revert() + + self.mutated_parameters = [] + + def __str__(self) -> str: + s = f"Current configuration:\n" + + for var in self.parameters: + s += f"{var.__str__()}\n" + + return s + + def evaluate( + self, + benchmark: str, + evaluations: int = 10, + path_to_output_folder: Path = path_to_generated_files, + remove_generated_file: bool = True, + ) -> list[float]: + """Evaluate the current configuration using a specific benchmark. + The benchmark is generated, and run multiple times. + + Args: + benchmark_file (str): The type of benchmark to generate + data_file (str): The data file that is used for generation + evaluations (int, optional): number of times the benchmark should be run. Defaults to 10. + + Returns: + list[float]: The throughput of each run + """ + + return evaluate_parameters( + benchmark, + self.compression_type.value, + self.cluster_size.value, + self.page_size.value, + self.cluster_bunch.value, + path_to_output_folder=path_to_output_folder, + evaluations=evaluations, + remove_generated_file=remove_generated_file, + ) + + +def getConfiguration( + compression_type: str = default_variable_values["compression_type"], + cluster_size: int = default_variable_values["cluster_size"], + page_size: int = default_variable_values["page_size"], + cluster_bunch: int = default_variable_values["cluster_bunch"], + compression_types: list[str] = None, +): + if compression_types: + compression_type = getCompressionParameter( + compression_type, compression_types=compression_types + ) + else: + compression_type = getCompressionParameter(compression_type) + + page_size = getPageSizeParameter(int(page_size)) + cluster_size = getClusterSizeParameter(int(cluster_size)) + cluster_bunch = getClusterBunchParameter(int(cluster_bunch)) + + return Configuration( + compression_type=compression_type, + cluster_size=cluster_size, + page_size=page_size, + cluster_bunch=cluster_bunch, + ) diff --git a/optimization_tools/Benchmarking/DataStructures/Parameters.py b/optimization_tools/Benchmarking/DataStructures/Parameters.py new file mode 100644 index 0000000..01d93f8 --- /dev/null +++ b/optimization_tools/Benchmarking/DataStructures/Parameters.py @@ -0,0 +1,249 @@ +from dataclasses import dataclass, field +from random import randint, choice, uniform + +from Benchmarking.benchmark_utils import ( + convertToByte, + convertToByteList, + convertByteToStr, +) + + +@dataclass +class Parameter: + parameter_name: str + + def initialize(self): + raise NotImplementedError + + def step(self): + raise NotImplementedError + + def revert(self): + raise NotImplementedError + + @property + def value(self): + raise NotImplementedError + + @property + def can_mutate(self): + raise NotImplementedError + + def __str__(self) -> str: + raise NotImplementedError + + +@dataclass +class ListBasedParameter(Parameter): + allowed_values: list + value_names: list[str] = None + current_idx: int = None + previous_idx: int = None + + def __post_init__(self): + if self.value_names == None: + self.value_names = [str(x) for x in self.allowed_values] + + if self.current_idx == None: + self.randomize() + + @property + def can_mutate(self) -> bool: + return len(self.allowed_values) > 1 + + def randomize(self): + """Set the value to a random value""" + self.current_idx = randint(0, len(self.allowed_values) - 1) + + def revert(self): + """Revert Parameter to the previous state + + Raises: + ValueError: No previous state is available + """ + if self.previous_idx == None: + raise ValueError("No previous idx available") + self.current_idx = self.previous_idx + + @property + def value(self) -> list: + return self.allowed_values[self.current_idx] + + def __str__(self) -> str: + if self.current_idx == None: + return f"Valiable {self.parameter_name} is not yet initialized" + + return f"{self.parameter_name} \t=> {self.value_names[self.current_idx]}" + + +@dataclass +class DiscreteParameter(ListBasedParameter): + def step(self): + """Make a step by either increasing, or decreasing the current index""" + + self.previous_idx = self.current_idx + if self.current_idx == 0: + self.current_idx = 1 + return + + if self.current_idx == len(self.allowed_values) - 1: + self.current_idx = len(self.allowed_values) - 2 + return + + self.current_idx = choice([self.current_idx - 1, self.current_idx + 1]) + + +@dataclass +class CategoricalParameter(ListBasedParameter): + def step(self): + """Make a step by chosing a random new index""" + self.previous_idx = self.current_idx + self.current_idx = choice( + [x for x in range(len(self.allowed_values)) if x != self.current_idx] + ) + + +@dataclass +class ContinuousParameter(Parameter): + lower_bound: float + upper_bound: float + value: float = None + previous_value: float = None + + @property + def value(self) -> float: + return self._value + + @value.setter + def value(self, value: float): + # Handle default value + if isinstance(value, property): + self.randomize_value() + return + + # Bound the value between upper and lower bound + value = self.upper_bound if value > self.upper_bound else value + value = self.lower_bound if value < self.lower_bound else value + + self._value = value + + def randomize_value(self): + """Set the value to a random value within the bounds""" + self.value = uniform(self.lower_bound, self.upper_bound) + + @property + def max_step_size(self) -> float: + return (self.upper_bound - self.lower_bound) / 10 + + @property + def can_mutate(self) -> bool: + return True + + def step(self): + """Make a step by taking a step of a random size between -max_step and max_step. + max_step is 10% of the allowed range of the Parameter. + With be capped to the bounds if it exceeds it + """ + self.previous_value = self.value + + step_size = self.max_step_size + direction = uniform(-step_size, step_size) + + self.value += direction + + def revert(self): + """Revert Parameter to the previous state + + Raises: + ValueError: No previous state is available + """ + if self.previous_value is None: + raise ValueError("No previous value available") + + self.value = self.previous_value + + +################################################################################################### +# Utils +################################################################################################### + + +def getCompressionParameter( + current_value: str, + compression_types: list[str] = ["none", "zlib", "lz4", "lzma", "zstd"], +) -> CategoricalParameter: + if current_value not in compression_types: + raise ValueError( + f"{current_value = } is not a valid value out {compression_types = }" + ) + current_idx = compression_types.index(current_value) + + return CategoricalParameter( + "Compression Type", compression_types, current_idx=current_idx + ) + + +def getPageSizeParameter(current_value) -> DiscreteParameter: + page_sizes = [ + (16, "KB"), + (32, "KB"), + (64, "KB"), + (128, "KB"), + (256, "KB"), + (512, "KB"), + (1, "MB"), + (2, "MB"), + (4, "MB"), + (8, "MB"), + (16, "MB"), + ] + + value_names = convertByteToStr(page_sizes) + page_sizes = convertToByteList(page_sizes, base=2) + + if current_value not in page_sizes: + raise ValueError(f"{current_value = } is not a valid value out {page_sizes = }") + current_idx = page_sizes.index(current_value) + + return DiscreteParameter( + "Page Size", page_sizes, value_names=value_names, current_idx=current_idx + ) + + +def getClusterSizeParameter(current_value) -> DiscreteParameter: + cluster_sizes = [ + (20, "MB"), + (30, "MB"), + (40, "MB"), + (50, "MB"), + (100, "MB"), + (200, "MB"), + (300, "MB"), + (400, "MB"), + (500, "MB"), + ] + + value_names = convertByteToStr(cluster_sizes) + + cluster_sizes = convertToByteList(cluster_sizes, base=10) + + if current_value not in cluster_sizes: + raise ValueError( + f"{current_value = } is not a valid value out {cluster_sizes = }" + ) + current_idx = cluster_sizes.index(current_value) + + return DiscreteParameter( + "Cluster Size", cluster_sizes, value_names=value_names, current_idx=current_idx + ) + + +def getClusterBunchParameter(current_value: int) -> DiscreteParameter: + cluster_bunches = [1, 2, 3, 4, 5] + if current_value not in cluster_bunches: + raise ValueError( + f"{current_value = } is not a valid value out {cluster_bunches = }" + ) + current_idx = cluster_bunches.index(current_value) + + return DiscreteParameter("Cluster Bunch", cluster_bunches, current_idx=current_idx) diff --git a/optimization_tools/Benchmarking/benchmark_utils.py b/optimization_tools/Benchmarking/benchmark_utils.py new file mode 100644 index 0000000..b3acc2f --- /dev/null +++ b/optimization_tools/Benchmarking/benchmark_utils.py @@ -0,0 +1,404 @@ +""" +File consisting of all the benchmarking functions +""" + +from pathlib import Path +import numpy as np +from tqdm import tqdm +import re +import os +import subprocess +from Benchmarking.variables import ( + path_to_iotools, + path_to_generated_files, + path_to_reference_files, + default_variable_values, + benchmark_datafile_dict, + compression_types, +) + +############################################################################################################################################## +# Utils +############################################################################################################################################## + + +def convertToByte(value: int, metric: str = "b", base=2) -> int: + """Convert value to byte, kilobyte, or megabyte + + Args: + value (int) + metric (str, optional): The desired output. Defaults to "b". + + Returns: + int + """ + if base == 2: + if metric == "b" or metric == "B": + return value + + if metric == "kb" or metric == "KB": + return value * 1024 + + if metric == "mb" or metric == "MB": + return value * 1024 * 1024 + + if metric == "gb" or metric == "GB": + return value * 1024 * 1024 * 1024 + + if base == 10: + if metric == "b" or metric == "B": + return value + + if metric == "kb" or metric == "KB": + return value * 1000 + + if metric == "mb" or metric == "MB": + return value * 1000 * 1000 + + if metric == "gb" or metric == "GB": + return value * 1000 * 1000 * 1000 + + +def convertToByteList(inp_list: list[tuple[int, str]], base=2) -> list[int]: + """Convert a list of values using the convertToByte function + + Args: + inp_list (list[tuple[int, str]]): list of value, form pairs + + Returns: + list[int] + """ + return [convertToByte(x, y, base) for x, y in inp_list] + + +def convertByteToStr(inp_list: list[tuple[int, str]]) -> list[str]: + """Convert a list of values into a list of strings by concatinating them + Args: + inp_list (list[tuple[int, str]]): list of value, form pairs + + Returns: + list[str] + """ + return [str(f"{x} {y}") for x, y in inp_list] + + +def is_valid_compression(compression_type: str) -> bool: + if compression_type in compression_types: + return True + return False + + +def get_throughput_increase(throughput: float, base_throughput: float) -> float: + """Gets the increase in throughput based on a base throughput. + The resulting value is a percentage increase. + This means the result is 0.0 when the throughput is equal to the base + + Args: + throughput (float): throughput acheived on a benchmark + base_throughput (float): base throughput to compare to + + Returns: + float: The new throughput in percentage + """ + return ((throughput - base_throughput) / base_throughput) * 100 + + +def get_size_decrease(size: float, base_size: float) -> float: + """Gets the decrease in size based on a base size. + The resulting value is a percentage decrease. + This means the result is 0.0 when the size is equal to the base + + Args: + size (float): size acheived on a benchmark + base_size (float): base size to compare to + + Returns: + float: The new size in percentage + """ + return ((base_size - size) / base_size) * 100 + + +def get_memory_usage_decrease(memory_usage: float, base_memory_usage: float) -> float: + """Gets the decrease in memory_usage based on a base memory_usage. + The resulting value is a percentage decrease. + This means the result is 0.0 when the memory_usage is equal to the base + + Args: + memory_usage (float): memory_usage acheived on a benchmark + base_memory_usage (float): base memory_usage to compare to + + Returns: + float: The new memory_usage in percentage + """ + return ((base_memory_usage - memory_usage) / base_memory_usage) * 100 + + +def get_performance(performance_values: list[float], weights: list[float] = None) -> float: + """Get the aggragated performance by taking the mean of the results. + If weights are given it becomes a weighted avarage + + Args: + performance_values (list[float]) + weights (list[float], optional): All metrics are weighted equally, if no weights are given + + Returns: + float: A single performance measure + """ + + if weights is None: + return np.mean(performance_values) + + return np.average(performance_values, weights) + + +def get_runtime(benchmark_output: str, target: str = "Runtime-Main:") -> int: + """Get the runtime of a benchmark based on the benchmark_outputut + + Args: + benchmark_output (str): benchmark benchmark_outputut + target (str, optional): what to read out. Defaults to "Runtime-Main:". + + Returns: + int: the runtime + """ + for line in benchmark_output.split("\n"): + if target in line: + return int(line.split(target)[1].strip()[:-2]) + + +def get_metric(benchmark_output: str, target: str) -> float: + """Get a metric from an benchmark benchmark_outputut string + + Args: + benchmark_output (str) + target (str) + + Returns: + float: value of the given metric + """ + for line in benchmark_output.split("\n"): + if target in line: + return float(line.split("|")[-1].strip()) + + +def get_throughput(benchmark_output: str) -> float: + """Calculate the throughput of a benchmark given its benchmark_outputut + Throughput is defined in MB/s based on the unzipped size, and total processing time + + Args: + benchmark_output (str) + + Returns: + float + """ + volume = get_metric(benchmark_output, "RNTupleReader.RPageSourceFile.szUnzip") + volume_MB = volume / 1_000_000 + + upzip_time = get_metric(benchmark_output, "RNTupleReader.RPageSourceFile.timeWallUnzip") + read_time = get_metric(benchmark_output, "RNTupleReader.RPageSourceFile.timeWallRead") + total_time = upzip_time + read_time + + total_time_s = total_time / 1_000_000_000 + + return volume_MB / total_time_s + + +def get_memory_usage(benchmark_output: str) -> float: + """Calculate the throughput of a benchmark given its benchmark_outputut + Throughput is defined in MB/s based on the unzipped size, and total processing time + + Args: + benchmark_output (str) + + Returns: + float + """ + return int(re.findall(" (\d+)maxresident", benchmark_output)[0]) + + +def get_size(path_to_file: Path) -> int: + """Get the size of a file from its Path + + Args: + path_to_file (Path) + + Returns: + int + """ + return os.stat(path_to_file).st_size + + +############################################################################################################################################## +# Benchmarking +############################################################################################################################################## + + +def generate_file( + path_to_generator: Path, + path_to_reference_file: Path, + compression_type: str, + cluster_size: int, + page_size: int, + path_to_output_folder: Path = path_to_generated_files, +) -> Path: + """Generate a new RNTuple based on a generator and reference data file + + Args: + path_to_generator (Path): path to an executable that generates an RNTuple + path_to_reference_file (Path): path to the reference root file used to generate the new RNTuple + compression_type (str) + cluster_size (int) + page_size (int) + path_to_output_folder (Path, optional) + + Returns: + Path: path to the generated RNTuple file + """ + + file_name = path_to_reference_file.stem + if "~" in file_name: + file_name = file_name.split("~")[0] + path_to_output = path_to_output_folder / f"{file_name}~{compression_type}_{page_size}_{cluster_size}.ntuple" + + if path_to_output.exists(): + print(f"output file already available => {path_to_output.resolve()}") + return path_to_output + + print(f"Start generating file: {path_to_output.name}") + status, result_str = subprocess.getstatusoutput( + f"{path_to_generator.resolve()} -i {path_to_reference_file.resolve()} -o {path_to_output_folder.resolve()} -c {compression_type} -p {page_size} -x {cluster_size}" + ) + + if status != 0: + raise ValueError( + f"Error {status} raised when generating file: {path_to_output.resolve()}\n error message provided: {result_str}" + ) + + return path_to_output + + +def run_benchmark( + path_to_benchmark: Path, + path_to_datafile: Path, + cluster_bunch: int, + use_rdf: bool = False, + evaluations: int = 10, +) -> tuple[list[float], list[float]]: + """Run the benchmark multiple times with the given parameters, and return the results + + Args: + path_to_benchmark (Path) + path_to_datafile (Path) + cluster_bunch (int) + use_rdf (bool, optional): Defaults to False. + evaluations (int, optional): number of times the benchmark is executed. Default is 10 + + Returns: + tuple[list, list]: list of throughput, and memory usage during the benchmark runs + """ + + # Get flags + rdf_flag = "-r" if use_rdf else "" + + throughputs = [] + memory_usages = [] + + for _ in tqdm(range(evaluations)): + # Reset cache + os.system(f"{path_to_iotools}/clear_page_cache") + + # Run benchmark + status, result_str = subprocess.getstatusoutput( + f"/usr/bin/time {path_to_benchmark.resolve()} -i {path_to_datafile.resolve()} -x {cluster_bunch} {rdf_flag} -p" + ) + + if status != 0: + raise ValueError(result_str) + + throughputs.append(get_throughput(result_str)) + memory_usages.append(get_memory_usage(result_str)) + + return throughputs, memory_usages + + +def evaluate_parameters( + benchmark: str, + compression_type: str, + cluster_size: int, + page_size: int, + cluster_bunch: int, + use_rdf: bool = False, + evaluations: int = 10, + path_to_output_folder: Path = path_to_generated_files, + remove_generated_file: bool = True, +) -> tuple[int, list[float], list[float]]: + """Evaluate parameters on a given benchmark. + First, a new RNTuple is created. + After, the benchmark is executed using the new RNTuple multiple times. + + Args: + benchmark (str) + compression_type (str) + cluster_size (int) + page_size (int) + cluster_bunch (int) + use_rdf (bool, optional): Defaults to False. + evaluations (int, optional): number of times the benchmark is executed. Default is 10 + path_to_output_folder (Path, optional) + remove_generated_file (bool, optional): remove the generated file at the end of the evaluation, Default is True + + Returns: + tuple[int,list[float],list[float]]: file_size, list of throughputs, and list of memory usage + """ + + print(f"Evaluating parameters: {compression_type = }, {cluster_size = }, {page_size = }, {cluster_bunch = }") + print(f"{benchmark = }") + + # Get Paths + path_to_generator = path_to_iotools / f"gen_{benchmark}" + path_to_benchmark = path_to_iotools / f"{benchmark}" + path_to_reference_file = path_to_reference_files / f"{benchmark_datafile_dict[benchmark]}" + + # Generate RNTuple and get the path to it + path_to_generated_file = generate_file( + path_to_generator, + path_to_reference_file, + compression_type, + cluster_size, + page_size, + path_to_output_folder, + ) + + generated_file_size = get_size(path_to_generated_file) + + # Run the benchmark on the generated file + throughputs, memory_usages = run_benchmark( + path_to_benchmark, path_to_generated_file, cluster_bunch, use_rdf, evaluations + ) + + if remove_generated_file: + os.remove(path_to_generated_file) + + return generated_file_size, throughputs, memory_usages + + +def evaluate_default_parameters(benchmark: str, evaluations: int = 10) -> tuple[int, list[float], list[float]]: + """Get the performance of the default parameters usingf the basic evaluate_parameter function + + Args: + benchmark (str): + evaluations (int, optional): Defaults to 10. + + Returns: + tuple[int,list[float],list[float]]: file_size, list of throughputs, and list of memory usage + """ + + return evaluate_parameters( + benchmark, + default_variable_values["compression_type"], + default_variable_values["cluster_size"], + default_variable_values["page_size"], + default_variable_values["cluster_bunch"], + default_variable_values["use_rdf"], + evaluations, + ) diff --git a/optimization_tools/Benchmarking/variables.py b/optimization_tools/Benchmarking/variables.py new file mode 100644 index 0000000..e96ee47 --- /dev/null +++ b/optimization_tools/Benchmarking/variables.py @@ -0,0 +1,51 @@ +from pathlib import Path + +#################################################################################################### +# PATHS +#################################################################################################### + +# path to the folder that holds the iotools generators and benchmarks +# path_to_iotools: Path = Path("/your/path/to/iotools") +path_to_iotools: Path = Path("/home/dante/Documents/iotools") + +if not path_to_iotools.exists(): + raise ValueError(f"Cannot find the iotools at path: {path_to_iotools.absolute()}") + +# path to the folder that holds the python files +path_to_optimization_tools: Path = path_to_iotools / "optimization_tools" +if not path_to_iotools.exists(): + raise ValueError(f"Cannot find the optimization_tools at path: {path_to_optimization_tools.absolute()}") + +# path to the folder when results can be placed +path_to_results: Path = Path("/home/dante/Documents/CERN-parameter-optimization/results") +if not path_to_iotools.exists(): + raise ValueError(f"Cannot find the results folder at path: {path_to_optimization_tools.absolute()}") + +# path to the folder that holds the reference files needed for the generators +path_to_reference_files: Path = Path("/home/dante/Documents/CERN-parameter-optimization/reference_files") +if not path_to_iotools.exists(): + raise ValueError(f"Cannot find the reference files at path: {path_to_optimization_tools.absolute()}") + +# path to the folder that will hold the temporary generated files during benchmarking +path_to_generated_files: Path = Path("/home/dante/Documents/CERN-parameter-optimization/generated_files") +if not path_to_iotools.exists(): + raise ValueError(f"Cannot find the folder for generated files at path: {path_to_optimization_tools.absolute()}") + +#################################################################################################### +# VARIABLES +#################################################################################################### +compression_types = ["none", "zstd", "zlib", "lz4", "lzma"] +benchmark_datafile_dict = { + "atlas": "gg_data~zstd.root", + "cms": "ttjet~zstd.root", + "h1": "h1dstX10~zstd.root", + "lhcb": "B2HHH~zstd.root", +} + +default_variable_values = { + "compression_type": "lz4", + "cluster_size": (50 * 1000 * 1000), + "page_size": (64 * 1024), + "cluster_bunch": 1, + "use_rdf": False, +} diff --git a/optimization_tools/README.md b/optimization_tools/README.md new file mode 100644 index 0000000..ff07103 --- /dev/null +++ b/optimization_tools/README.md @@ -0,0 +1,15 @@ +This folder contains python files used to benchmark and optimize parameters based on the iotools benchmarks. + +To run the files, first the paths in Benchmarking/variables.py need to be updated. +To run the full optimization, three folders need to be available: +1. reference_files holds the reference files which are used in the generation step of the benchmarking + These files can be found at https://root.cern/files/RNTuple/treeref/ +2. generated_files holds the files that are generated during the benchmarking +3. results holds the results of the optimization. + +Two examples are provided in the tutorials folder. +"evaluate_parameters.py" benchmarks a parameter configuration, and compares it to the default parameters. +"run_annealer.py" optimizes the parameters for a single benchmark + +To run the examples, the user needs to change the path inserted in the add_path.py file to the path +of the optimization_tools. diff --git a/optimization_tools/tutorials/add_path.py b/optimization_tools/tutorials/add_path.py new file mode 100644 index 0000000..13f1305 --- /dev/null +++ b/optimization_tools/tutorials/add_path.py @@ -0,0 +1,10 @@ +################################### +# Adds a path to the main folder. +# TODO: improve this +################################### + + +import sys + +# sys.path.insert(0, "path/to/iotools/optimization_tools") +sys.path.insert(0, "/home/dante/Documents/iotools/optimization_tools") diff --git a/optimization_tools/tutorials/evaluate_parameters.py b/optimization_tools/tutorials/evaluate_parameters.py new file mode 100644 index 0000000..426f4e6 --- /dev/null +++ b/optimization_tools/tutorials/evaluate_parameters.py @@ -0,0 +1,80 @@ +import add_path + +import numpy as np +import argparse + +from Benchmarking.benchmark_utils import evaluate_default_parameters, evaluate_parameters, \ + get_size_decrease, get_throughput_increase, get_memory_usage_decrease, get_performance +from Benchmarking.variables import benchmark_datafile_dict, compression_types, default_variable_values + + +# Add terminal arguments +parser = argparse.ArgumentParser() + +parser.add_argument('-benchmark', choices=benchmark_datafile_dict, type=str, + default="lhcb") +parser.add_argument('-compression_type', choices=compression_types, type=str, + default=default_variable_values["compression_type"]) +parser.add_argument('-cluster_size', type=int, + default=default_variable_values["cluster_size"]) +parser.add_argument('-page_size', type=int, + default=default_variable_values["page_size"]) +parser.add_argument('-cluster_bunch', type=int, + default=default_variable_values["cluster_bunch"]) +parser.add_argument('-evaluations', type=int, default=10) + +args = parser.parse_args() + +benchmark = args.benchmark +compression_type = args.compression_type +cluster_size = args.cluster_size +page_size = args.page_size +cluster_bunch = args.cluster_bunch +evaluations = args.evaluations + +# Get the default performance for the chosen benchmark +default_file_size, default_throughputs, default_memory_usages = evaluate_default_parameters( + benchmark, evaluations=evaluations) + +default_mean_throughput = np.mean(default_throughputs) +default_mean_memory_usage = np.mean(default_memory_usages) + +# Get performance of the given parameters on the chosen benchmark +generated_file_size, throughputs, memory_usages = evaluate_parameters(benchmark, compression_type, cluster_size, + page_size, cluster_bunch, evaluations=evaluations) + +mean_throughput = np.mean(throughputs) +mean_memory_usage = np.mean(memory_usages) + +# Get the relative performance of the parameters comapred to the default parameter set +size_decrease = get_size_decrease(generated_file_size, default_file_size) +throughput_increase = get_throughput_increase( + mean_throughput, default_mean_throughput) +memory_usage_decrease = get_memory_usage_decrease( + mean_memory_usage, default_mean_memory_usage) +performance = get_performance( + [size_decrease, throughput_increase, memory_usage_decrease]) + +# Print performance metrics of the default parameter set +print(f"#" * 100) +print(f"The size of the default file is {default_file_size}") +print( + f"The default parameters reached an average throughput of {default_mean_throughput:.2f}") +print( + f"The default parameters had an average maximum memory usage of {default_mean_memory_usage:.2f}") + +# Print performance metrics of the given parameter set +print(f"#" * 100) +print(f"The size of the generated file is {generated_file_size}") +print( + f"The new parameters reached an average throughput of {mean_throughput:.2f}") +print( + f"The new parameters had an average maximum memory usage of {mean_memory_usage:.2f}") + +# Print the relative performance of the given parameters +print(f"#" * 100) +print(f"The filesize decreased by {size_decrease:.2f}%") +print(f"The throughput increasd by {throughput_increase:.2f}%") +print(f"The memory usage decreased by {memory_usage_decrease:.2f}%") + +print(f"\nOverall performance is {performance:.2f}") diff --git a/optimization_tools/tutorials/run_annealer.py b/optimization_tools/tutorials/run_annealer.py new file mode 100644 index 0000000..c0bc1f7 --- /dev/null +++ b/optimization_tools/tutorials/run_annealer.py @@ -0,0 +1,35 @@ +import add_path + +from Benchmarking.Algorithms import Annealer +from Benchmarking.DataStructures.Configuration import getConfiguration + +# Example of how to run the simmulated annealer on all four benchmarks +# For all benchmarks, lz4 and zstd are optimizated seperately. + +evolution_steps = 200 + + +def run_annealer(benchmark: str, compression_type: str, evolution_steps: int, multi_change: bool = False): + # Create a configuration where the compression type is set to the given compression_type, and can never be mutated. + conf = getConfiguration(compression_type=compression_type, + compression_types=[compression_type]) + a = Annealer(configuration=conf, benchmark=benchmark, + multi_change=multi_change) + a.evolve(steps=evolution_steps) + + +# LHCb +run_annealer("lhcb", "lz4", evolution_steps, multi_change=False) +run_annealer("lhcb", "zstd", evolution_steps, multi_change=False) + +# Atlas +run_annealer("atlas", "lz4", evolution_steps, multi_change=False) +run_annealer("atlas", "zstd", evolution_steps, multi_change=False) + +# CMS +run_annealer("cms", "lz4", evolution_steps, multi_change=False) +run_annealer("cms", "zstd", evolution_steps, multi_change=False) + +# H1 +run_annealer("h1", "lz4", evolution_steps, multi_change=False) +run_annealer("h1", "zstd", evolution_steps, multi_change=False)