Merge pull request #27 from chcwww/hyperparameter

Add Hyperparameter Search Support for Tree-based Models

Author: chcwww

docs/api/linear.rst

@@ -34,12 +34,17 @@ The simplest usage is::
 
 .. autofunction:: get_positive_labels
 
+.. autofunction:: linear_test
+
 .. autoclass:: FlatModel
    :members:
 
 .. autoclass:: TreeModel
    :members:
 
+.. autoclass:: EnsembleTreeModel
+   :members:
+
 Load Dataset
 ^^^^^^^^^^^^
 
@@ -101,3 +106,18 @@ Grid Search with Sklearn Estimators
    :members:
 
    .. automethod:: __init__
+
+Grid Search for Tree-Based Linear Method
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autoclass:: TreeGridParameter
+   :members:
+
+   .. automethod:: __init__
+
+.. autoclass:: TreeGridSearch
+   :members:
+
+   .. automethod:: __init__
+
+   .. automethod:: __call__

docs/examples/plot_linear_gridsearch_tutorial.py

@@ -1,7 +1,14 @@
 """
-Hyperparameter Search for Linear Methods
+Hyperparameter Search for One-vs-rest Linear Methods
 =============================================================
+.. warning::
+
+    If you are using the tree-based linear method,
+    please check `Hyperparameter Search for Tree-Based Linear Method  <../auto_examples/plot_tree_gridsearch_tutorial.html>`_.
+
 This guide helps users to tune the hyperparameters of the feature generation step and the linear model.
+In this guide, the following methods are available:
+``1vsrest``, ``thresholding``, ``cost_sensitive``, ``cost_sensitive_micro``, and ``binary_and_multiclass``.
 
 Here we show an example of tuning a linear text classifier with the `rcv1 dataset <https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html#rcv1v2%20(topics;%20full%20sets)>`_.
 Starting with loading and preprocessing of the data without using ``Preprocessor``:

docs/examples/plot_tree_gridsearch_tutorial.py

@@ -0,0 +1,141 @@
+"""
+Hyperparameter Search for Tree-Based Linear Method
+=============================================================
+.. warning::
+
+    If you are using the one-vs-rest linear methods,
+    please check `Hyperparameter Search for One-vs-rest Linear Methods  <../auto_examples/plot_linear_gridsearch_tutorial.html>`_.
+
+To apply tree-based linear methods,
+we first convert raw text into numerical TF-IDF features.
+During training, the method builds a label tree and trains linear classifiers.
+At inference, the model traverses the tree and selects
+only a few candidate labels at each level to speed up prediction.
+
+To improve model performance, we need to search the hyperparameter space.
+Therefore, in this guide, we help users tune the hyperparameters of the tree-based linear method.
+
+.. seealso::
+
+    `Implementation Document <https://www.csie.ntu.edu.tw/~cjlin/papers/libmultilabel/libmultilabel_implementation.pdf>`_:
+        For more details about the implementation of tree-based linear methods and hyperparameter search.
+
+Here we show an example of tuning a tree-based linear text classifier with the `rcv1 dataset <https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html#rcv1v2%20(topics;%20full%20sets)>`_.
+Starting with loading the data:
+"""
+
+import logging
+
+from libmultilabel import linear
+
+logging.basicConfig(level=logging.INFO)
+
+datasets = linear.load_dataset("txt", "data/rcv1/train.txt", "data/rcv1/test.txt")
+L = len(datasets["train"]["y"])
+
+######################################################################
+# Next, we set up the search space.
+
+import numpy as np
+
+dmax = 10
+K_factors = [-2, 5]
+search_space_dict = {
+    "ngram_range": [(1, 1), (1, 2), (1, 3)],
+    "stop_words": ["english"],
+    "dmax": [dmax],
+    "K": [max(2, int(np.round(np.power(L, 1 / dmax) * np.power(2.0, alpha) + 0.5))) for alpha in K_factors],
+    "s": [1],
+    "c": [0.5, 1, 2],
+    "B": [1],
+    "beam_width": [10],
+    "prob_A": [3]
+}
+
+######################################################################
+# Following the suggestions in the `implementation document <https://www.csie.ntu.edu.tw/~cjlin/papers/libmultilabel/libmultilabel_implementation.pdf>`_,
+# we define 18 configurations to build a simple yet strong baseline.
+#
+# The search space covers several key parts of the search process:
+#
+# - Text feature extraction: (``ngram_range``, ``stop_words``)
+#
+#       - We use the vectorizer ``TfidfVectorizer`` from ``sklearn`` to generate features from raw text.
+#
+# - Label tree structure: (``dmax``, ``K``)
+#
+#      - The depth and node degree of the label tree. Note that ``K`` is the number of clusters and is calculated using the formula from the `implementation document <https://www.csie.ntu.edu.tw/~cjlin/papers/libmultilabel/libmultilabel_implementation.pdf>`_.
+#
+# - Linear classifier: (``s``, ``c``, ``B``)
+#
+#       - We combined them into a LIBLINEAR option string for training linear classifiers. (see *train Usage* in `liblinear <https://github.com/cjlin1/liblinear>`__ README)
+#
+# - Prediction: (``beam_width``, ``prob_A``)
+#
+#       - The number of candidates considered and the parameter for the probability estimation function at each level during prediction.
+#
+# .. tip::
+#
+#     Available hyperparameters (and their defaults) are defined in the class variables of :py:class:`~libmultilabel.linear.TreeGridParameter`.
+#
+# In :py:class:`~libmultilabel.linear.TreeGridSearch`, we perform cross-validation for evaluation.
+# Specifically, we split the training data into ``n_folds``,
+# sequentially using each fold as the validation set while training on the remaining folds.
+# Finally, we aggregate the validation outputs from each fold and compute the ``monitor_metrics``.
+# Initialization requires the dataset, the number of cross-validation folds, and the evaluation metrics.
+
+n_folds = 3
+monitor_metrics = ["P@1", "P@3", "P@5"]
+search = linear.TreeGridSearch(datasets, n_folds, monitor_metrics)
+cv_scores = search(search_space_dict)
+
+######################################################################
+# ``cv_scores`` is a dictionary where keys are :py:class:`~libmultilabel.linear.TreeGridParameter` instances and values are the ``monitor_metrics`` results.
+#
+# Here we sort the results in descending order by the first metric in ``monitor_metrics``.
+# You can retrieve the best parameters after the grid search with the following code:
+
+sorted_cv_scores = sorted(cv_scores.items(), key=lambda x: x[1][monitor_metrics[0]], reverse=True)
+print(sorted_cv_scores)
+
+best_params, best_cv_scores = list(sorted_cv_scores)[0]
+print(best_params, best_cv_scores)
+
+######################################################################
+# The best parameters are::
+#
+#   {'ngram_range': (1, 3), 'stop_words': 'english', 'dmax': 10, 'K': 88, 's': 1, 'c': 1, 'B': 1, 'beam_width': 10, 'prob_A': 3}
+#
+# with best cross-validation scores::
+#
+#   {'P@1': 0.9669, 'P@3': 0.8137, 'P@5': 0.5640}
+#
+# We can then retrain using the best parameters,
+# and use :py:meth:`~libmultilabel.linear.linear_test` and :py:meth:`~libmultilabel.linear.get_metrics` to compute test performance.
+
+from dataclasses import asdict
+
+preprocessor = linear.Preprocessor(tfidf_params=asdict(best_params.tfidf))
+transformed_dataset = preprocessor.fit_transform(datasets)
+
+model = linear.train_tree(
+    transformed_dataset["train"]["y"],
+    transformed_dataset["train"]["x"],
+    best_params.linear_options,
+    **asdict(best_params.tree),
+)
+
+metrics, _, _, _ = linear.linear_test(
+    y = transformed_dataset["test"]["y"],
+    x = transformed_dataset["test"]["x"],
+    model = model,
+    metrics = linear.get_metrics(monitor_metrics, num_classes=-1),
+    predict_kwargs = asdict(best_params.predict),
+)
+
+print(metrics.compute())
+
+######################################################################
+# The result of the best parameters will look similar to::
+#
+#   {'P@1': 0.9554, 'P@3': 0.7968, 'P@5': 0.5576}

docs/search_retrain.rst

@@ -7,4 +7,5 @@ Hyperparameter Search
 
 
     ../auto_examples/plot_linear_gridsearch_tutorial
+    ../auto_examples/plot_tree_gridsearch_tutorial
     tutorials/Parameter_Selection_for_Neural_Networks

libmultilabel/linear/tree.py

@@ -4,6 +4,7 @@
 
 import numpy as np
 import scipy.sparse as sparse
+from scipy.special import log_expit
 from sparsekmeans import LloydKmeans, ElkanKmeans
 import sklearn.preprocessing
 from tqdm import tqdm
@@ -58,16 +59,35 @@ def __init__(
         self.multiclass = False
         self._model_separated = False # Indicates whether the model has been separated for pruning tree.
 
+    def sigmoid_A(self, x: np.ndarray, prob_A: int) -> np.ndarray:
+        """
+        Calculate log(sigmoid(prob_A * x)), which represents the probability of the positive class in binary classification.
+
+        Args:
+            x (np.ndarray): The decision value matrix with dimension number of instances * number of classes.
+            prob_A (int):
+                The hyperparameter used in the probability estimation function for
+                binary classification: sigmoid(prob_A * x).
+
+        Returns:
+            np.ndarray: A matrix with dimension number of instances * number of classes.
+        """
+        return log_expit(prob_A * x)
+
     def predict_values(
         self,
         x: sparse.csr_matrix,
         beam_width: int = 10,
+        prob_A: int = 3,
     ) -> np.ndarray:
         """Calculate the probability estimates associated with x.
 
         Args:
             x (sparse.csr_matrix): A matrix with dimension number of instances * number of features.
-            beam_width (int, optional): Number of candidates considered during beam search. Defaults to 10.
+            beam_width (int, optional): Number of candidates considered during beam search.
+            prob_A (int, optional):
+                The hyperparameter used in the probability estimation function for
+                binary classification: sigmoid(prob_A * decision_value_matrix).
 
         Returns:
             np.ndarray: A matrix with dimension number of instances * number of classes.
@@ -81,8 +101,8 @@ def predict_values(
             if not self._model_separated:
                 self._separate_model_for_pruning_tree()
                 self._model_separated = True
-            all_preds = self._prune_tree_and_predict_values(x, beam_width) # number of instances * (number of labels + total number of metalabels)
-        return np.vstack([self._beam_search(all_preds[i], beam_width) for i in range(all_preds.shape[0])])
+            all_preds = self._prune_tree_and_predict_values(x, beam_width, prob_A) # number of instances * (number of labels + total number of metalabels)
+        return np.vstack([self._beam_search(all_preds[i], beam_width, prob_A) for i in range(all_preds.shape[0])])
 
     def _separate_model_for_pruning_tree(self):
         """
@@ -113,7 +133,7 @@ def _separate_model_for_pruning_tree(self):
             )
             self.subtree_models.append(subtree_flatmodel)
 
-    def _prune_tree_and_predict_values(self, x: sparse.csr_matrix, beam_width: int) -> np.ndarray:
+    def _prune_tree_and_predict_values(self, x: sparse.csr_matrix, beam_width: int, prob_A: int) -> np.ndarray:
         """Calculates the selective decision values associated with instances x by evaluating only the most relevant subtrees.
 
         Only subtrees corresponding to the top beam_width candidates from the root are evaluated,
@@ -122,6 +142,9 @@ def _prune_tree_and_predict_values(self, x: sparse.csr_matrix, beam_width: int)
         Args:
             x (sparse.csr_matrix): A matrix with dimension number of instances * number of features.
             beam_width (int): Number of top candidate branches considered for prediction.
+            prob_A (int):
+                The hyperparameter used in the probability estimation function for
+                binary classification: sigmoid(prob_A * decision_value_matrix).
 
         Returns:
             np.ndarray: A matrix with dimension number of instances * (number of labels + total number of metalabels).
@@ -132,7 +155,7 @@ def _prune_tree_and_predict_values(self, x: sparse.csr_matrix, beam_width: int)
 
         # Calculate root decision values and scores
         root_preds = linear.predict_values(self.root_model, x)
-        children_scores = 0.0 - np.square(np.maximum(0, 1 - root_preds))
+        children_scores = 0.0 + self.sigmoid_A(root_preds, prob_A)
 
         slice = np.s_[:, self.node_ptr[self.root.index] : self.node_ptr[self.root.index + 1]]
         all_preds[slice] = root_preds
@@ -159,12 +182,15 @@ def _prune_tree_and_predict_values(self, x: sparse.csr_matrix, beam_width: int)
 
         return all_preds
 
-    def _beam_search(self, instance_preds: np.ndarray, beam_width: int) -> np.ndarray:
+    def _beam_search(self, instance_preds: np.ndarray, beam_width: int, prob_A: int) -> np.ndarray:
         """Predict with beam search using cached probability estimates for a single instance.
 
         Args:
             instance_preds (np.ndarray): A vector of cached probability estimates of each node, has dimension number of labels + total number of metalabels.
             beam_width (int): Number of candidates considered.
+            prob_A (int, optional):
+                The hyperparameter used in the probability estimation function for
+                binary classification: sigmoid(prob_A * decision_value_matrix).
 
         Returns:
             np.ndarray: A vector with dimension number of classes.
@@ -182,7 +208,7 @@ def _beam_search(self, instance_preds: np.ndarray, beam_width: int) -> np.ndarra
                     continue
                 slice = np.s_[self.node_ptr[node.index] : self.node_ptr[node.index + 1]]
                 pred = instance_preds[slice]
-                children_score = score - np.square(np.maximum(0, 1 - pred))
+                children_score = score + self.sigmoid_A(pred, prob_A)
                 next_level.extend(zip(node.children, children_score.tolist()))
 
             cur_level = sorted(next_level, key=lambda pair: -pair[1])[:beam_width]
@@ -193,7 +219,7 @@ def _beam_search(self, instance_preds: np.ndarray, beam_width: int) -> np.ndarra
         for node, score in cur_level:
             slice = np.s_[self.node_ptr[node.index] : self.node_ptr[node.index + 1]]
             pred = instance_preds[slice]
-            scores[node.label_map] = np.exp(score - np.square(np.maximum(0, 1 - pred)))
+            scores[node.label_map] = np.exp(score + self.sigmoid_A(pred, prob_A))
         return scores
 
 
@@ -204,6 +230,7 @@ def train_tree(
     K=DEFAULT_K,
     dmax=DEFAULT_DMAX,
     verbose: bool = True,
+    root: Node = None,
 ) -> TreeModel:
     """Train a linear model for multi-label data using a divide-and-conquer strategy.
     The algorithm used is based on https://github.com/xmc-aalto/bonsai.
@@ -215,14 +242,16 @@ def train_tree(
         K (int, optional): Maximum degree of nodes in the tree. Defaults to 100.
         dmax (int, optional): Maximum depth of the tree. Defaults to 10.
         verbose (bool, optional): Output extra progress information. Defaults to True.
+        root (Node, optional): Pre-built tree root. Defaults to None.
 
     Returns:
         TreeModel: A model which can be used in predict_values.
     """
-    label_representation = (y.T * x).tocsr()
-    label_representation = sklearn.preprocessing.normalize(label_representation, norm="l2", axis=1)
-    root = _build_tree(label_representation, np.arange(y.shape[1]), 0, K, dmax)
-    root.is_root = True
+    if root is None:
+        label_representation = (y.T * x).tocsr()
+        label_representation = sklearn.preprocessing.normalize(label_representation, norm="l2", axis=1)
+        root = _build_tree(label_representation, np.arange(y.shape[1]), 0, K, dmax)
+        root.is_root = True
 
     num_nodes = 0
     # Both type(x) and type(y) are sparse.csr_matrix