diff --git a/.gitignore b/.gitignore index a597f10..a49dfe4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,9 @@ __pycache__/ gurobi.log ks_engine/__pycache__/ instances/* +sphinx_doc/build/ *.csv *.dat *.sol -*.sh \ No newline at end of file +*.sh + diff --git a/ks.py b/ks.py index 075a760..502b877 100644 --- a/ks.py +++ b/ks.py @@ -39,7 +39,7 @@ def main(): else: if sol_file := conf["SOLUTION_FILE"]: sol.save_as_sol_file(sol_file) - + print("Solution:", sol.value) sol.debug.export_csv(conf["DEBUG"], False) diff --git a/ks_engine/__init__.py b/ks_engine/__init__.py index 34ac802..3ed0c31 100644 --- a/ks_engine/__init__.py +++ b/ks_engine/__init__.py @@ -2,35 +2,11 @@ # Copyright (c) 2019 Filippo Ranza -""" -ks_engine -========= - -An implementation of the Kernel Search Heuristic method - -Available functions -------------------- -kernel_search - run the Kernel Search Heuristic - -config_loader - load ks_engine (and eventually client code) configuration - from given YAML file - - - -Available subpackages ---------------------- -kernel_utils - contains some basic initial kernel generators - and a generator 'factory' - -bucket_utils - contains some basic bucket generators - and a generator 'factory' - -""" - from .kernel_search import kernel_search, KernelMethods from .config_loader import load_config -from .kernel_algorithms import * +from .kernel_algorithms import * # noqa + + +__all__ = ['kernel_search', + 'KernelMethods', + 'load_config'] diff --git a/ks_engine/config_loader.py b/ks_engine/config_loader.py index 619c371..3012f70 100644 --- a/ks_engine/config_loader.py +++ b/ks_engine/config_loader.py @@ -25,7 +25,14 @@ } -def check_config(conf): +def check_config(conf: dict): + """ + Check the provided configuration file. + + :param conf: A dictionary representing the configuration file. + :type conf: dict + :raises ValueError: If the type of every key in the dictionary is wrong. + """ for k, v in DEFAULT_CONF.items(): c = conf[k] if not (type(c) is type(v)): @@ -34,28 +41,19 @@ def check_config(conf): ) -def load_config(file_name): +def load_config(file_name: str): """ Load the configuration from the given file. - Parameters - ---------- - file_name : str - path to the YAML configuration file - - Raises - ------ - ValueError - if some variable in the given configuration - does not have the same type of the variables - in the default configuration. Other variables + :param file_name: path to the YAML configuration file. + :type file_name: str + :raises ValueError: if some variable in the given configuration + does not have the same type of the variables + in the default configuration. Other variables are not checked. - - Returns - ------- - config: dict - map configuration variable name into + :return: map configuration variable name into their values + :rtype: dict """ if not file_name: return DEFAULT_CONF diff --git a/ks_engine/feature_kernel.py b/ks_engine/feature_kernel.py index 33583f3..c9f4083 100644 --- a/ks_engine/feature_kernel.py +++ b/ks_engine/feature_kernel.py @@ -22,7 +22,21 @@ def init_feature_kernel(model, config): - + """ + Initialize a kernel set from the model using a particular configuration. + + A feature kernel is a particular way to build the kernel that uses some + particular machine learning technique to try to guess most important + feature. + + :param model: the model to solve. + :param config: a configuration to use in the resolution. + :type config: dict + :raises ValueError: if no solution is found in the subproblems. + :return: None, the initial kernel set and a + :class:`~ks_engine.solution.Solution`. + :rtype: tuple + """ preload_model = Model(model, config, False) model_count = config["FEATURE_KERNEL"]["COUNT"] abs_size = int(preload_model.model_size() * DEF_REL_SIZE) @@ -68,6 +82,24 @@ def init_feature_kernel(model, config): def build_kernel_and_values(solution_set, model_size, var_name_table: dict, policy): + """ + Build the kernel set. + + This building is made taking into account the solution_set that are used to + compute the importance of the features using a machine learning algorithm. + + :param solution_set: a set of different solution + from different subproblems. + :type solution_set: dict + :param model_size: the size of the model. + :type model_size: int + :param var_name_table: name-bool pair containing names of vars. + :type var_name_table: dict + :param policy: the policy to use to choose the size of the kernel. + :type policy: str + :return: a pair of variables in the kernel and a + :class:`~ks_engine.solution.Solution` with all variables. + """ features = compute_feature_importance(solution_set, model_size) var_couple = list(zip(var_name_table.keys(), features)) @@ -83,6 +115,16 @@ def build_kernel_and_values(solution_set, model_size, var_name_table: dict, poli def compute_feature_importance(solution_set, model_size): + """ + Compute the feature importance, using a random forest. + + :param solution_set: a set of different solution + from different subproblems. + :type solution_set: dict + :param model_size: the size of the model. + :type model_size: int + :return: the feature importence for the RF classifier. + """ instances, classes = build_sklean_instance(solution_set, model_size) classifier = RandomForestClassifier() classifier.fit(instances, classes) @@ -90,6 +132,16 @@ def compute_feature_importance(solution_set, model_size): def build_initial_kernel(var_name_table, kernel_var_names): + """ + Build the initial kernel set. + + :param var_name_table: name-bool pair containing names of vars. + :type var_name_table: dict + :param kernel_var_names: names of variable to set in the kernel + :type kernel_var_names: list + :return: var_name_table with value True for vars in the kernel. + :rtype: dict + """ for var in kernel_var_names: var_name_table[var] = True @@ -99,7 +151,23 @@ def build_initial_kernel(var_name_table, kernel_var_names): def generate_model_solutions( model, config, var_names, count, size, min_time, max_time ): - + """ + Generate a solution from a model, with respect to other params. + + :param model: the model to solve. + :param config: a configuration to use in the resolution. + :type config: dict + :param var_names: a name-bool pair of all variables. + :type var_names: dict + :param count: the number of iteration to execute on the model. + :type count: int + :param size: the size of the problem to solve. + :type size: int + :param min_time: minimum time to spent in execution. + :param max_time: maximum time to spent in execution. + :return: a iter-tuple iteration description pair. + :rtype: dict + """ time_limit = min_time logger = feature_logger_factory(config["FEATURE_KERNEL"].get("LOG_FILE")) solution_set = {} @@ -107,7 +175,7 @@ def generate_model_solutions( print("iter", k) if time_limit: config["TIME_LIMIT"] = time_limit - + selected = generate_random_sub_model(var_names, size) result = solve_sub_model(model, config, selected) logger.log_data(k, size, result) @@ -130,15 +198,24 @@ def generate_model_solutions( if time_limit < max_time: time_limit += 1 size = size_grow_function(size, len(var_names)) - + if size > len(var_names): - size = len(var_names) + size = len(var_names) logger.save() return solution_set def split_kernel_vars(var_couple, count): + """ + Split into kernel variables and not. + + :param var_couple: var-value pair. + :param count: number of variable to put into the kernel. + :type count: int + :return: all the variable in the kernel (i.e. count vars). + :rtype: list + """ var_couple.sort(key=lambda x: -x[1]) head = var_couple[:count] kernel = [n for n, i in head] @@ -146,6 +223,9 @@ def split_kernel_vars(var_couple, count): def build_sklean_instance(solutions, var_count): + """ + .. warning:: typo in the name. So, not documented now. + """ sol_count = len(solutions.values()) instances = np.ndarray((sol_count, var_count)) classes = np.ndarray(sol_count) @@ -157,6 +237,18 @@ def build_sklean_instance(solutions, var_count): def solve_sub_model(model, config, selected_vars): + """ + Solve a subproblem with respect to the model + without the selected vars and the configuration. + + :param model: the supermodel of the model to solve. + :param config: the configuration for the model. + :type config: dict + :param selected_vars: vars to disable. + :type selected_vars: dict + :return: if the subproblem has a relaxation a + tuple with its solution and the status, else None. + """ lin_model = Model(model, config, True) lin_model.disable_variables(selected_vars) stat = lin_model.run() @@ -179,7 +271,19 @@ def solve_sub_model(model, config, selected_vars): return None -def load_model(model, config, relax): + +def load_model(model, config, relax): + """ + Create the :class:`~ks_engine.model.Model` according to the config. + + :param model: the model to build as object. + :param config: the configuration for the model. + :type config: dict + :param relax: True if is a relaxation, else False. + :type relax: bool + :return: the :class:`~ks_engine.model.Model` object. + :rtype: :class:`~ks_engine.model.Model` + """ if relax: output = Model(model, config, True, False) else: @@ -190,7 +294,21 @@ def load_model(model, config, relax): return output + def generate_random_sub_model(var_names, count): + """ + Generate randomly a subproblem. + + :param var_names: a name-bool pair. + :type var_names: dict + :param count: the number of variables to exclude from the list of vars. + :type count: int + :return: a name-None pair representing variables to disable + :rtype: dict + + .. warning:: The returned variable must be EXCLUDED from the model, + NOT chosen. + """ if count == len(var_names): for k in var_names.keys(): var_names[k] = True @@ -200,23 +318,50 @@ def generate_random_sub_model(var_names, count): output = {k: None for k, v in var_names.items() if not v} return output + def random_select(var_names, count): + """ + Sample count variable from var_names + + :param var_names: a name-bool pair from which sample. + :type var_names: dict + :param count: the number of variables to sample. + :type count: int + :return: the variables pair with True if selected, else False. + :rtype: dict + """ for k in var_names.keys(): var_names[k] = False rng = secrets.SystemRandom() - + selected = rng.sample(var_names.keys(), count) - + for sel in selected: var_names[sel] = True return var_names + def get_variable_name_table(model): + """ + Compute the list of all names of the variable in the model. + + :param model: the model of which get all the variables names. + :return: a name-False pair. + :rtype: dict + """ return {var.varName: False for var in model.model.getVars()} def cache_solution(curr_sol, cache_file): + """ + Store the current solution to a cache file. + + :param curr_sol: the current solution to store. + :param cache_file: the cache file name. + :type cache_file: str + :return: the current solution with the previous solution values added. + """ if os.path.exists(cache_file): with open(cache_file, "rb") as file: prev_solution = pickle.load(file) @@ -234,6 +379,15 @@ def cache_solution(curr_sol, cache_file): def get_kernel_size(solution, policy): + """ + [Compute the size that the kernel must have with respect to a policy. + + :param solution: a solution from which get the kernel. + :param policy: the policy to use to choose the size. + :type policy: str + :return: the size that the kernel must have. + :rtype: int + """ vals = solution.values() infeasible = (v.model_size for v in vals if v.status == INFEASIBLE) feasible = (v.model_size for v in vals if v.status == FEASIBLE) @@ -251,9 +405,19 @@ def get_kernel_size(solution, policy): return output + def size_grow_function(curr_size, model_size): + """ + Compute the new size of the subproblem from the previous. + + :param curr_size: the current size of the model. + :type curr_size: int + :param model_size: the model size, that is the upper bound of curr_size. + :type model_size: int + :return: the new current size + :rtype: int + """ ratio = curr_size / model_size ratio = ratio ** (4 / 5) curr_size = int(model_size * ratio) return curr_size - diff --git a/ks_engine/kernel_algorithms/__init__.py b/ks_engine/kernel_algorithms/__init__.py index 4f7fb2a..d9f5172 100644 --- a/ks_engine/kernel_algorithms/__init__.py +++ b/ks_engine/kernel_algorithms/__init__.py @@ -8,3 +8,8 @@ kernel_sorters, bucket_sorters, ) + +__all__ = ['bucket_builders', + 'kernel_builders', + 'kernel_sorters', + 'bucket_sorters'] diff --git a/ks_engine/kernel_algorithms/algorithm_selection.py b/ks_engine/kernel_algorithms/algorithm_selection.py index 5cc956e..b2c9759 100644 --- a/ks_engine/kernel_algorithms/algorithm_selection.py +++ b/ks_engine/kernel_algorithms/algorithm_selection.py @@ -3,48 +3,36 @@ # Copyright (c) 2019 Filippo Ranza from .base_bucket import BUCKET_BUILDERS, fixed_size_bucket +from typing import Callable from .base_kernel import KERNEL_BUILDERS, base_kernel_builder -from .base_sort import * +from .base_sort import kernel_sort, bucket_sort, BUCKET_SORT, KERNEL_SORTERS class Selector: - """ Selector allows client code to safely choose between available - algorithms. - """ - - def __init__(self, base_store, default): - """ - Parameters - ---------- - base_store: dict - a dictionary mapping string into functions - - default: function - the default function between the available ones + algorithms. - """ + :param base_store: a dictionary mapping string into functions + :type base_store: dict + :param default: the default function between the available ones + :type default: Callable + """ + def __init__(self, base_store: dict, default: Callable): self.store = base_store self.default = default - def add_algorithm(self, name, function): + def add_algorithm(self, name: str, function: Callable): """ Insert a new algorithm in the store, if not present - - Parameters - ---------- - name: str - new algorithm name, must be different from + + :param name: new algorithm name, must be different from the corrently available - function: callable - the algorithm implementation, although not required it + :type name: str + :param function: the algorithm implementation, although not required it must have the same signature of the order functions - - Raises - ------ - ValueError if name is already pointing to a function - + :type function: Callable + :raises ValueError: if name is already pointing to a function """ try: self.store[name] @@ -53,19 +41,15 @@ def add_algorithm(self, name, function): else: raise ValueError(f"algorithm {name} is already installed") - def get_algorithm(self, name): + def get_algorithm(self, name: str): """ Return the algorithm pointed by given name - Parameters - ---------- - name: str - desidered method name - - Returns - ------- - the function associated to the given name if - it is available, None otherwise. + :param name: desidered method name + :type name: str + :return: the function associated to the given name if + it is available, None otherwise. + :rtype: Callable """ return self.store.get(name) diff --git a/ks_engine/kernel_algorithms/base_bucket.py b/ks_engine/kernel_algorithms/base_bucket.py index 4576d2a..14b68ff 100644 --- a/ks_engine/kernel_algorithms/base_bucket.py +++ b/ks_engine/kernel_algorithms/base_bucket.py @@ -6,6 +6,22 @@ def fixed_size_bucket(base, values, sorter, sorter_conf, size=1, count=0): + """ + Build a Generator of bucket of the same dimension. + + :param base: the variables to be inserted in the buckets + :type base: dict + :param values: the values to be inserted in the buckets + :param sorter: a sorting function + :param sorter_conf: a configuration for the sorter + :param size: the size of the buckets, defaults to 1 + :type size: int, optional + :param count: the number of buckets, defaults to 0 + :type count: int, optional + :raises ValueError: if the number of variables cannot fill the buckets + :yield: a bucket + :rtype: list + """ variables = sorter(base, values, **sorter_conf) length = len(variables) if count: @@ -21,6 +37,20 @@ def fixed_size_bucket(base, values, sorter, sorter_conf, size=1, count=0): def decresing_size_bucket(base, values, sorter, sorter_conf, count): + """ + Build a Generator of bucket of decreasing dimension. + + :param base: the variables to be inserted in the buckets + :type base: dict + :param values: the values to be inserted in the buckets + :param sorter: a sorting function + :param sorter_conf: a configuration for the sorter + :param count: the number of buckets, defaults to 0 + :type count: int, optional + :raises ValueError: if the number of variables cannot fill the buckets + :yield: a bucket + :rtype: list + """ variables = sorter(base, values, **sorter_conf) blocks = sum(1 << i for i in range(count)) length = len(variables) diff --git a/ks_engine/kernel_algorithms/base_kernel.py b/ks_engine/kernel_algorithms/base_kernel.py index 6c2af73..6862170 100644 --- a/ks_engine/kernel_algorithms/base_kernel.py +++ b/ks_engine/kernel_algorithms/base_kernel.py @@ -7,10 +7,17 @@ def base_kernel_builder(base, values, sorter, sorter_conf): + """ + .. todo:: Cannot understand the parameters + """ return base -def percentage_better_kernel_builder(base, value, sorter, sorter_conf, percentage): +def percentage_better_kernel_builder( + base, value, sorter, sorter_conf, percentage): + """ + .. todo:: Cannot understand the parameters + """ kernel_vars = sorter(base, value, **sorter_conf) last_taken = math.floor(len(kernel_vars) * percentage) for name in kernel_vars[last_taken:]: diff --git a/ks_engine/kernel_algorithms/base_sort.py b/ks_engine/kernel_algorithms/base_sort.py index 4f270ab..d083112 100644 --- a/ks_engine/kernel_algorithms/base_sort.py +++ b/ks_engine/kernel_algorithms/base_sort.py @@ -10,6 +10,14 @@ def cheb_nodes(count: int): + """ + Create a list of Chebyshev nodes + + :param count: the number of nodes + :type count: int + :return: the list of Chebyshev nodes + :rtype: numpy.ndarray + """ output = [0] * count for i, n in enumerate(range(1, count + 1)): tmp = (((2 * n) - 1) / (2 * count)) * np.pi @@ -21,48 +29,74 @@ def cheb_nodes(count: int): def kernel_sort(kernel: dict, values): + """ + Sort the elements of the kernel passed in in ascending order. + + :param kernel: the kernel of the problem + :type kernel: dict + :param values: the values of variable in the kernel + :return: The variables sorted occording to their values + :rtype: list + """ tmp = [k for k, v in kernel.items() if v] tmp.sort(key=lambda x: values.get_value(x)) return tmp def bucket_sort(kernel: dict, values): + """ + Sort the elements of the kernel passed in descending order. + + :param kernel: the kernel of the problem + :type kernel: dict + :param values: the values of variable in the kernel + :return: The variables sorted occording to their values + :rtype: list + """ tmp = [k for k, v in kernel.items() if not v] tmp.sort(key=lambda x: -values.get_value(x)) return tmp def cheb_sort(kernel: dict, values): + """ + Sort the elements of the kernel passed using Chebyshev nodes for ordering. + + :param kernel: the kernel of the problem + :type kernel: dict + :param values: the values of variable in the kernel + :return: The variables sorted occording to their values + :rtype: list + """ tmp = [k for k, v in kernel.items() if not v] tmp.sort(key=lambda x: -values.get_value(x)) tmp = np.array(tmp) - head, tail = np.array_split(tmp, 2) - nodes = cheb_nodes(len(tmp)) + head, tail = np.array_split(tmp, 2) + nodes = cheb_nodes(len(tmp)) head_nodes, tail_nodes = np.array_split(nodes, 2) output = [] j, k = 0, 0 - + for _ in range(len(tmp)): done_tail = k >= len(tail_nodes) done_head = j >= len(head_nodes) if done_tail: - flag = True + flag = True elif done_head: - flag = False + flag = False else: if head_nodes[j] < tail_nodes[k]: flag = False else: flag = True - + if flag: output.append(head[j]) j += 1 else: output.append(tail[k]) k += 1 - - + return output diff --git a/ks_engine/kernel_search.py b/ks_engine/kernel_search.py index 0947646..49a735f 100644 --- a/ks_engine/kernel_search.py +++ b/ks_engine/kernel_search.py @@ -13,7 +13,19 @@ ) -def run_solution(model, config): +def run_solution(model: Model, config): + """ + Solve the passed model respecting the time limit declared + in the configuration. + + :param model: the model to solve. + :type model: Model + :param config: a configuration to use in the resolution. + :type config: dict + :raises RuntimeError: if the solution have timed out. + :return: True if the solution is optimal, else False. + :rtype: bool + """ begin = time.time_ns() stat = model.run() end = time.time_ns() @@ -31,6 +43,23 @@ def run_solution(model, config): def init_kernel(model, config, kernel_builder, kernel_sort, mps_file): + """ + Initialize a kernel set from the model using a particular configuration. + + :param model: the model to solve. + :param config: a configuration to use in the resolution. + :type config: dict + :param kernel_builder: the kernel builder method to use. + :type kernel_builder: Callable + :param kernel_sort: the kernel sorter method to use. + :type kernel_sort: Callable + :param mps_file: the name of the file containing the problem. + :type mps_file: str + :raises ValueError: if the relaxed problem has no solution. + :return: the solution value, the initial kernel set and a + :class:`~ks_engine.solution.Solution`. + :rtype: tuple + """ lp_model = Model(model, config, True) stat = run_solution(lp_model, config) @@ -48,7 +77,7 @@ def init_kernel(model, config, kernel_builder, kernel_sort, mps_file): int_model = Model(model, config, False) if config.get("PRELOAD_FILE"): int_model.preload_from_file() - + int_model.preload_solution(tmp_sol) int_model.disable_variables(kernel) stat = run_solution(int_model, config) @@ -61,11 +90,30 @@ def init_kernel(model, config, kernel_builder, kernel_sort, mps_file): def select_vars(base_kernel, bucket): + """ + Activate all variables inside the bucket, + so that will be used in the resolution. + + :param base_kernel: a var-isPresentInKernel pair representing + the kernel set. + :type base_kernel: dict + :param bucket: a list of variable, i.e. a bucket. + """ for var in bucket: base_kernel[var] = True def update_kernel(base_kernel, bucket, solution, null): + """ + Remove all null variables inside the bucket from the kernel set. + + :param base_kernel: a var-isPresentInKernel pair representing + the kernel set. + :type base_kernel: dict + :param bucket: a list of variable, i.e. a bucket. + :param solution: the current solution. + :param null: a value to consider as the null value, i.e. not assigned. + """ for var in bucket: if solution.get_value(var) == null: base_kernel[var] = False @@ -74,6 +122,24 @@ def update_kernel(base_kernel, bucket, solution, null): def run_extension( model, config, kernel, bucket, solution, bucket_index, iteration_index ): + """ + Insert new variables into the kernel set and compute a new solution + from this one. + + :param model: the model to solve. + :param config: a configuration to use in the resolution. + :type config: dict + :param kernel: a var-isPresentInKernel pair representing + the kernel set. + :type kernel: dict + :param bucket: a list of variable, i.e. a bucket. + :param solution: the current solution. + :param bucket_index: the index of the bucket in the bucket list. + :type bucket_index: int + :param iteration_index: the cardinal number of the iteration. + :type iteration_index: int + :return: a new solution computed using the new kernel set. + """ model = Model(model, config) model.disable_variables(kernel) model.add_bucket_contraints(solution, bucket) @@ -93,6 +159,25 @@ def run_extension( def initialize(model, conf, methods, mps_file): + """ + Compute an initial kernel set with its attached bucket. + + :param model: the model to solve. + :param conf: a configuration to use in the resolution. + :type conf: dict + :param methods: The collection of four methods: + + * Kernel Builder. + * Kernel Sorter. + * Bucket Builder. + * Bucket Sorter. + :type methods: namedtuple + :param mps_file: the name of the file containing the problem. + :type mps_file: str + :raises ValueError: if the kernel size raise to the model size. + :return: the current solution, the kernel set, the list of buckets. + :rtype: tuple + """ if conf.get("FEATURE_KERNEL"): curr_sol, base_kernel, values = init_feature_kernel(model, conf) else: @@ -103,7 +188,6 @@ def initialize(model, conf, methods, mps_file): if ill_kernel(base_kernel): raise ValueError("Kernel is large as the whole model") - buckets = methods.bucket_builder( base_kernel, values, @@ -113,14 +197,40 @@ def initialize(model, conf, methods, mps_file): ) return curr_sol, base_kernel, buckets + def ill_kernel(base_kernel): + """ + Detect if the kernel is ill or not. + + An ill kernel is a kernel that is as large as the model, i.e. is the model. + + :param base_kernel: a var-isPresentInKernel pair representing + the kernel set. + :type base_kernel: dict + :return: True if the kernel reached the length of the model, else False. + :rtype: bool + """ kernel_size = sum(1 for v in base_kernel.values() if v) model_size = len(base_kernel) return kernel_size == model_size def solve_buckets(model, config, curr_sol, base_kernel, buckets, iteration): - + """ + Pass and solve over all buckets updating the kernel. + + :param model: the model to solve. + :param config: a configuration to use in the resolution. + :type config: dict + :param curr_sol: the current solution that is present before the iteration. + :param base_kernel: a var-isPresentInKernel pair representing + the kernel set. + :type base_kernel: dict + :param buckets: the list of bucket over which iteration happen. + :param iteration: the number of the iteration. + :type iteration: int + :return: the solution found after iterate over all buckets. + """ for index, buck in enumerate(buckets): print(index) select_vars(base_kernel, buck) @@ -133,43 +243,27 @@ def solve_buckets(model, config, curr_sol, base_kernel, buckets, iteration): return curr_sol -def kernel_search(mps_file, config, kernel_methods): +def kernel_search(mps_file: str, config, kernel_methods): """ - Run Kernel Search Heuristic - - Parameters - ---------- - mps_file : str - The MIP problem instance file. - - config : dict - Kernel Search configuration - - kernel_methods: KernelMethods - The collection of four methods: - - Kernel Builder - - Kernel Sorter - - Bucket Builder - - Bucket Sorter - - Raises - ------ - ValueError - When the LP relaxation is unsolvable. + Run Kernel Search Heuristic. + + :param mps_file: The MIP problem instance file. + :type mps_file: str + :param config: Kernel Search configuration + :type config: dict + :param kernel_methods: The collection of four methods: + + * Kernel Builder. + * Kernel Sorter. + * Bucket Builder. + * Bucket Sorter. + :type kernel_methods: namedtuple + :raises ValueError: When the LP relaxation is unsolvable. In this case no feasible solution are available - - Returns - ------- - Value : float - Objective function value - - Variables: dict - Map variable name into its value - in the solution - + :return: Objective function value + :rtype: float """ - # init_feature_kernel(mps_file, config, None, None) # exit() diff --git a/ks_engine/logger.py b/ks_engine/logger.py index e522198..8d48cc4 100644 --- a/ks_engine/logger.py +++ b/ks_engine/logger.py @@ -11,9 +11,11 @@ def log_data(self, *data): def save(self): pass + class MockLogger(Logger): pass + class CSVFeatureLogger(Logger): def __init__(self, file_name): @@ -22,6 +24,13 @@ def __init__(self, file_name): self.file_name = file_name def log_data(self, iter_count, var_count, result): + """ + Insert into the log list a new record. + + :param iter_count: the number of the iteration. + :param var_count: the number of variable. + :param result: the result. + """ if result: _, result = result data = (iter_count, var_count, result) @@ -32,16 +41,20 @@ def save(self): csv_writer = csv.writer(file) for entry in self.log: csv_writer.writerow(entry) - -def feature_logger_factory(file_name): + +def feature_logger_factory(file_name=None): + """ + Construct a logger. + + If a file_name is provided than will be able to save in a CSV file. + + :param file_name: the name of the CSV file to use for saving, defaults to None + :type file_name: str, optional + :return: a logger object + :rtype: :class:`~Logger` + """ if file_name: return CSVFeatureLogger(file_name) else: return MockLogger() - - - - - - diff --git a/ks_engine/model.py b/ks_engine/model.py index 473837d..bfbc1ba 100644 --- a/ks_engine/model.py +++ b/ks_engine/model.py @@ -20,16 +20,31 @@ "MIP_GAP": "MIPGap", } + def reset_time_limit(config): + """ + Reset the time limit to the default time limit. + + :param config: the actual configuration. + :type config: dict + :return: the new value if changed, else None. + """ if config["TIME_LIMIT"] != DEFAULT_CONF["TIME_LIMIT"]: - output = config["TIME_LIMIT"] - config["TIME_LIMIT"] = DEFAULT_CONF["TIME_LIMIT"] + output = config["TIME_LIMIT"] + config["TIME_LIMIT"] = DEFAULT_CONF["TIME_LIMIT"] else: output = None return output def create_env(config): + """ + Create a gurobi Env and set this with config options. + + :param config: a configuration to set the gurobi Env. + :type config: dict + :return: the gurobi env + """ env = gurobipy.Env() if not config["LOG"]: env.setParam("OutputFlag", 0) @@ -37,16 +52,24 @@ def create_env(config): for k, v in GUROBI_PARAMS.items(): def_val = DEFAULT_CONF[k] conf = config[k] - if conf != def_val: + if conf != def_val: env.setParam(v, conf) return env def model_loarder(mps_file, config): + """ + Load a mps file, and presolve if the config file suggest it. + + :param mps_file: a mps file + :param config: the configuration for the run. + :type config: dict + :return: a model instance. + """ presolve = config["PRESOLVE"] if presolve: - tl = reset_time_limit(config) + tl = reset_time_limit(config) model = gurobipy.read(mps_file, env=create_env(config)) model.setParam("Presolve", 2) model.update() @@ -76,11 +99,19 @@ def __init__(self, model, config, linear_relax=False, one_solution=False): self.model = self.model.relax() def preload_from_file(self): + """ + Preload the solution file from the model if exist. + """ if self.sol_file and os.path.isfile(self.sol_file): self.model.read(self.sol_file) - def preload_solution(self, sol=None): + """ + Preload a solution from local internal variables, if exists. + + :param sol: the current internal solution, defaults to None + :type sol: [type], optional + """ if not self.preload or sol is None: return @@ -88,17 +119,36 @@ def preload_solution(self, sol=None): self.model.getVarByName(name).start = value def run(self): + """ + Optimize the current model. + + :return: True if it is optimal, else False. + :rtype: bool + """ self.model.optimize() stat = self.model.status self.stat = stat return stat == gurobipy.GRB.status.OPTIMAL def disable_variables(self, base_kernel, value=0): + """ + Disable search over variable that the base_kernel indicate as not in the search space. + + :param base_kernel: a kernel with variables. + :param value: a value to which set variables to disable, defaults to 0 + :type value: int, optional + """ for name, _ in filter(lambda x: not x[1], base_kernel.items()): var = self.model.getVarByName(name) self.model.addConstr(var == value) def add_bucket_contraints(self, solution, bucket): + """ + Add to a bucket the constraint that the sum of all variables inside the bucket must be >= 1. + + :param solution: a solution for the model. + :param bucket: the bucket where the constraint must be added. + """ self.model.addConstr( gurobipy.quicksum(self.model.getVarByName(var) for var in bucket) >= 1 ) @@ -106,6 +156,12 @@ def add_bucket_contraints(self, solution, bucket): self.model.setParam("Cutoff", solution.value) def build_solution(self, prev_sol=None): + """ + Build a new solution. + + :param prev_sol: the previous solution, defaults to None + :return: the best solution between the current and the previous. + """ gen = ((var.varName, var.x) for var in self.model.getVars()) if prev_sol: prev_sol.update(self.model.objVal, gen) @@ -115,10 +171,29 @@ def build_solution(self, prev_sol=None): return prev_sol def get_base_variables(self, null_value=0.0): + """ + Returns all variables of the model. + + :param null_value: a value to be considered as the null one, defaults to 0.0 + :type null_value: float, optional + :return: a dict with all variables name as keys and as value if they are different from the null_value + :rtype: dict + """ gen = ((var.varName, var.x != null_value) for var in self.model.getVars()) return dict(gen) def build_lp_solution(self, null_value=0.0): + """ + Build the linear programming model. + + Use the current value of the variables if those are not set at the + null_value, else use the reduced cost. + + :param null_value: a value to be considered as the null one, defaults to 0.0 + :type null_value: float, optional + :return: a Solution object. + :rtype: :class:`~ks_engine.solution.Solution` + """ gen = self._lp_sol_generator(null_value) return Solution(self.model.objVal, gen) @@ -130,6 +205,16 @@ def _lp_sol_generator(self, null_value): yield var.varName, var.x def build_debug(self, kernel_size, bucket_size): + """ + Build a :class:`~ks_engine.solution.DebugData` object. + + :param kernel_size: the size of the kernel. + :type kernel_size: int + :param bucket_size: the size of the bucket. + :type bucket_size: int + :return: a new instance for the debug. + :rtype: :class:`~ks_engine.solution.DebugData` + """ return DebugData( value=self.model.objVal, time=self.model.getAttr("Runtime"), @@ -139,14 +224,32 @@ def build_debug(self, kernel_size, bucket_size): ) def model_size(self): + """ + Returns the size of the model. + + :return: the size of the, that is the number of variables. + :rtype: int + """ tmp = self.model.getVars() output = len(tmp) return output def reach_solution_limit(self): + """ + Check if the solution is optimal or reached the setted limit. + + :return: True if one of the two situation is reached, else False. + :rtype: bool + """ time_limit = self.stat == gurobipy.GRB.status.SOLUTION_LIMIT optimal = self.stat == gurobipy.GRB.status.OPTIMAL return time_limit or optimal def reach_time_limit(self): + """ + Check if the maximum time is elapsed. + + :return: True if the maximum time is elapsed, else False. + :rtype: bool + """ return self.stat == gurobipy.GRB.status.TIME_LIMIT diff --git a/ks_engine/solution.py b/ks_engine/solution.py index 0395932..f41506c 100644 --- a/ks_engine/solution.py +++ b/ks_engine/solution.py @@ -20,6 +20,12 @@ def __init__(self): self.max_iter = 0 def add_data(self, data, index): + """ + Extends the debug informations. + + :param data: the new data to be added. + :param index: the index of the new data to be added. + """ self.store[index] = data if self.max_bucket < index.bucket: @@ -28,7 +34,15 @@ def add_data(self, data, index): if self.max_iter < index.iteration: self.max_iter = index.iteration - def export_csv(self, file_name, compress): + def export_csv(self, file_name: str, compress): + """ + Save as a csv file the current debug information in csv format. + + :param file_name: the name of the file whera save data. + :type file_name: str + :param compress: determine if the csv must be compressed in gz. + :type compress: bool + """ csv = self.get_csv() if compress: csv = bytes(csv, "UTF-8") @@ -39,6 +53,12 @@ def export_csv(self, file_name, compress): file.write(csv) def get_csv(self): + """ + Returns the current debug information in csv format. + + :return: a string representing the csv file. + :rtype: str + """ out = "bucket,iteration,value,time,nodes,kernel_size,bucket_size" for k, v in self.store.items(): tmp = f"{k.bucket},{k.iteration},{v.value},{v.time},{v.nodes},{v.kernel_size},{v.bucket_size}" @@ -46,16 +66,33 @@ def get_csv(self): return out def bucket_iter(self, iteration): + """ + Yields information relative to a particular iteration. + + :param iteration: the iteration whose information are of interest. + :yield: a bucket-value pair from all the stored ones. + """ for k, v in self.store.items(): if k.iteration == iteration: yield k.bucket, v def iteration_iter(self, bucket): + """ + Yields information relative to a particular bucket. + + :param bucket: the bucket whose information are of interest. + :yield: a iteration-value pair from all the stored ones. + """ for k, v in self.store.items(): if k.bucket == bucket: yield k.iteration, v def full_iter(self): + """ + Yields name-value pair of debug informations. + + :yield: a name-value pair from all the stored ones. + """ for k, v in self.store.items(): yield k, v @@ -67,22 +104,51 @@ def __init__(self, value, var_iter): self.debug = DebugInfo() def get_value(self, name): + """ + Returns the value associated with a var name. + + :param name: the name of the variable whose value is searched. + :return: the value of the variable. + """ return self.vars[name] def update(self, value, var_iter): + """ + Update the value and variable values of the current solution. + + :param value: value of the objective function. + :param var_iter: list of var-value pair. + """ self.value = value for k, v in var_iter: self.vars[k] = v def update_debug_info(self, index, debug_info): + """ + Extends data to the debug of the run. + + :param index: the index in a dict where the new info will be saved. + :param debug_info: the information to store. + """ self.debug.add_data(debug_info, index) def variables(self): + """ + Returns the variables values. + + :return: A numpy array representing the values + :rtype: numpy.array + """ vals = self.vars.values() list_vals = list(vals) return np.array(list_vals) - def save_as_sol_file(self, file_name): + def save_as_sol_file(self, file_name: str): + """ + Save persinstently the solution. + + :param file_name: the name of the file. + """ file_name = get_solution_file_name(file_name) with open(file_name, "w") as file: @@ -91,11 +157,17 @@ def save_as_sol_file(self, file_name): def get_solution_file_name(file_name): - if file_name is None: + """ + Return the name of the file in which save the solution + + :param file_name: the name of the file. + :return: A .sol filename + :rtype: str + """ + if file_name is None: return None - + if file_name.endswith('.sol'): return file_name else: return f"{file_name}.sol" - diff --git a/sphinx_doc/Makefile b/sphinx_doc/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/sphinx_doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/sphinx_doc/README.md b/sphinx_doc/README.md new file mode 100644 index 0000000..1c756b5 --- /dev/null +++ b/sphinx_doc/README.md @@ -0,0 +1,21 @@ +## Doc building + +To build the doc are necessary some additional packages. These can be installed e.g. with pip: + +``` +pip install sphinx +pip install sphinx_rtd_theme +``` + +A good practice is to install these package locally (with --user) or in a virtualenv (venv). Anyway, it's not mandatory. + +If the build of the doc is make locally it's strongly recommended to create a separete folder inside +the sphinx_doc folder, let's call this new folder build. + +To build the doc as a html "website" from the sphinx_doc folder: + +``` +sphinx-build source build +``` + +In this way, the resulting "website" will be in the build folder. Be free to change the destination folder name. diff --git a/sphinx_doc/make.bat b/sphinx_doc/make.bat new file mode 100644 index 0000000..6247f7e --- /dev/null +++ b/sphinx_doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/sphinx_doc/source/apidoc/config_loader.rst b/sphinx_doc/source/apidoc/config_loader.rst new file mode 100644 index 0000000..8db12c0 --- /dev/null +++ b/sphinx_doc/source/apidoc/config_loader.rst @@ -0,0 +1,8 @@ +config\_loader module +-------------------------------- + +.. automodule:: ks_engine.config_loader + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/feature_kernel.rst b/sphinx_doc/source/apidoc/feature_kernel.rst new file mode 100644 index 0000000..0c012b4 --- /dev/null +++ b/sphinx_doc/source/apidoc/feature_kernel.rst @@ -0,0 +1,8 @@ +feature\_kernel module +--------------------------------- + +.. automodule:: ks_engine.feature_kernel + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/kernel_algorithms/algorithm_selection.rst b/sphinx_doc/source/apidoc/kernel_algorithms/algorithm_selection.rst new file mode 100644 index 0000000..26dd775 --- /dev/null +++ b/sphinx_doc/source/apidoc/kernel_algorithms/algorithm_selection.rst @@ -0,0 +1,8 @@ +algorithm\_selection module +--------------------------------------------------------- + +.. automodule:: ks_engine.kernel_algorithms.algorithm_selection + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/kernel_algorithms/base_bucket.rst b/sphinx_doc/source/apidoc/kernel_algorithms/base_bucket.rst new file mode 100644 index 0000000..7133347 --- /dev/null +++ b/sphinx_doc/source/apidoc/kernel_algorithms/base_bucket.rst @@ -0,0 +1,8 @@ +base\_bucket module +------------------------------------------------- + +.. automodule:: ks_engine.kernel_algorithms.base_bucket + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/kernel_algorithms/base_kernel.rst b/sphinx_doc/source/apidoc/kernel_algorithms/base_kernel.rst new file mode 100644 index 0000000..ea8ac96 --- /dev/null +++ b/sphinx_doc/source/apidoc/kernel_algorithms/base_kernel.rst @@ -0,0 +1,8 @@ +base\_kernel module +------------------------------------------------- + +.. automodule:: ks_engine.kernel_algorithms.base_kernel + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/kernel_algorithms/base_sort.rst b/sphinx_doc/source/apidoc/kernel_algorithms/base_sort.rst new file mode 100644 index 0000000..699349e --- /dev/null +++ b/sphinx_doc/source/apidoc/kernel_algorithms/base_sort.rst @@ -0,0 +1,8 @@ +base\_sort module +----------------------------------------------- + +.. automodule:: ks_engine.kernel_algorithms.base_sort + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/kernel_algorithms/kernel_algorithms.rst b/sphinx_doc/source/apidoc/kernel_algorithms/kernel_algorithms.rst new file mode 100644 index 0000000..3f80ae2 --- /dev/null +++ b/sphinx_doc/source/apidoc/kernel_algorithms/kernel_algorithms.rst @@ -0,0 +1,16 @@ +kernel\_algorithms package +===================================== + +.. automodule:: ks_engine.kernel_algorithms + :members: + :undoc-members: + :show-inheritance: + + +.. toctree:: + :maxdepth: 4 + + algorithm_selection + base_bucket + base_kernel + base_sort diff --git a/sphinx_doc/source/apidoc/kernel_search.rst b/sphinx_doc/source/apidoc/kernel_search.rst new file mode 100644 index 0000000..92f0296 --- /dev/null +++ b/sphinx_doc/source/apidoc/kernel_search.rst @@ -0,0 +1,8 @@ +kernel\_search module +-------------------------------- + +.. automodule:: ks_engine.kernel_search + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/ks_engine.rst b/sphinx_doc/source/apidoc/ks_engine.rst new file mode 100644 index 0000000..2791c46 --- /dev/null +++ b/sphinx_doc/source/apidoc/ks_engine.rst @@ -0,0 +1,21 @@ +ks\_engine package +================== + +.. automodule:: ks_engine + :members: + :undoc-members: + :show-inheritance: + :exclude-members: KernelMethods, kernel_search, load_config + +.. toctree:: + :maxdepth: 4 + + config_loader + feature_kernel + kernel_search + logger + model + solution + + kernel_algorithms/kernel_algorithms + diff --git a/sphinx_doc/source/apidoc/logger.rst b/sphinx_doc/source/apidoc/logger.rst new file mode 100644 index 0000000..0b48145 --- /dev/null +++ b/sphinx_doc/source/apidoc/logger.rst @@ -0,0 +1,8 @@ +logger module +------------------------ + +.. automodule:: ks_engine.logger + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/model.rst b/sphinx_doc/source/apidoc/model.rst new file mode 100644 index 0000000..b955933 --- /dev/null +++ b/sphinx_doc/source/apidoc/model.rst @@ -0,0 +1,8 @@ +model module +----------------------- + +.. automodule:: ks_engine.model + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/apidoc/solution.rst b/sphinx_doc/source/apidoc/solution.rst new file mode 100644 index 0000000..6336345 --- /dev/null +++ b/sphinx_doc/source/apidoc/solution.rst @@ -0,0 +1,8 @@ +solution module +-------------------------- + +.. automodule:: ks_engine.solution + :members: + :undoc-members: + :show-inheritance: + diff --git a/sphinx_doc/source/conf.py b/sphinx_doc/source/conf.py new file mode 100644 index 0000000..30feace --- /dev/null +++ b/sphinx_doc/source/conf.py @@ -0,0 +1,59 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../../')) + + +# -- Project information ----------------------------------------------------- + +project = 'ks.py' +copyright = '2021, FilippoRanza' +author = 'FilippoRanza' + +# The full version, including alpha/beta/rc tags +release = '0.2.1' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.todo', + 'sphinx.ext.autodoc', + 'sphinx_rtd_theme'] + +todo_include_todos = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +html_theme_options = { 'collapse_navigation': False } + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] diff --git a/sphinx_doc/source/index.rst b/sphinx_doc/source/index.rst new file mode 100644 index 0000000..abdbbf2 --- /dev/null +++ b/sphinx_doc/source/index.rst @@ -0,0 +1,20 @@ +Welcome to ks.py's documentation! +================================= + +About +====== +This is an independent python implementation of the kernel search framework. + + +Alphabetical index +=================== + +* :ref:`genindex` + + +.. toctree:: + :maxdepth: 4 + :caption: API documentation + + apidoc/ks_engine +