added optimization and predictive unc to KNN

skopt k selection ; variance across predictions

Author: Richard Michael

__init__.py

@@ -17,7 +17,7 @@ def UncertainRFFactory(representation, alphabet):
     return UncertainRandomForest()
 
 def KNNFactory(representation, alphabet):
-    return KNN()
+    return KNN(optimize=True)
 
 
 optimize = True

algorithm_factories.py

@@ -1,26 +1,59 @@
 import numpy as np
+from typing import Tuple
 from sklearn.neighbors import KNeighborsRegressor
+from sklearn.model_selection import cross_val_score
 from algorithms.abstract_algorithm import AbstractAlgorithm
+from skopt.space import Integer
+from skopt.utils import use_named_args
+from skopt import gp_minimize
 
 
 class KNN(AbstractAlgorithm):
-    def __init__(self):
+    def __init__(self, optimize: bool=False, k_max: int=100, opt_budget: int=100, seed=42) -> None:
         self.model = None
-        self.optimize = False
+        self.optimize = optimize
+        self.seed = seed
+        if self.optimize:
+            self.k_max = k_max
+            self.opt_budget = opt_budget
+            self.opt_space = [
+            Integer(1, self.k_max, name="n_neighbors"),
+        ]
 
-    def get_name(self):
+    def get_name(self) -> str:
         return "KNN"
 
-    def train(self, X, Y):
+    def train(self, X: np.ndarray, Y: np.ndarray) -> None:
         assert(Y.shape[1] == 1)
         self.model = KNeighborsRegressor(n_neighbors=int(np.ceil(0.3*len(X))), n_jobs=-1)  # use all processors
         Y = Y.squeeze() if Y.shape[0] > 1 else Y
+        if self.optimize:
+            self.k_max = int(len(X)) # all data is maximal possible 
+            @use_named_args(self.opt_space)
+            def _opt_objective(**params):
+                self.model.set_params(**params)
+                return -np.mean(cross_val_score(self.model, X, Y, cv=5, n_jobs=-1, scoring="neg_mean_absolute_error"))
+            res_gp = gp_minimize(_opt_objective, self.opt_space, n_calls=self.opt_budget, random_state=self.seed)
+            print(f"Score: {res_gp.fun}")
+            print(f"Parameters: k={res_gp.x[0]}")
+            self.model = KNeighborsRegressor(n_neighbors=res_gp.x[0], n_jobs=-1) 
         self.model.fit(X, Y)
 
-    def predict(self, X):
+    def predict(self, X) -> Tuple[np.array, np.array]:
+        """
+        Returns:
+            pred - model predictions
+            unc - model variance as E[(f(x) - E[f(x)])**2]
+        """
         pred = self.model.predict(X).reshape(-1, 1)
-        unc = np.zeros(pred.shape)
+        unc = np.mean(np.square(pred-np.mean(pred)), axis=1).reshape(-1, 1)
+        assert pred.shape == unc.shape
         return pred, unc
 
     def predict_f(self, X: np.ndarray):
         return self.predict(X)
+
+
+
+
+

algorithms/KNN.py

@@ -10,20 +10,20 @@
 
 
 datasets = ["MTH3", "TIMB", "CALM", "1FQG", "UBQT", "BRCA", "TOXI"] # "MTH3", "TIMB", "CALM", "1FQG", "UBQT", "BRCA", "TOXI"
-representations = [EVE_DENSITY, EVE, TRANSFORMER, ONE_HOT, ESM] # VAE_AUX, VAE_RAND, TRANSFORMER, VAE, ONE_HOT, ESM, EVE, VAE_AUX EXTRA 1D rep: VAE_DENSITY
+# datasets = ["TOXI"] # "MTH3", "TIMB", "CALM", "1FQG", "UBQT", "BRCA", "TOXI"
+representations = [TRANSFORMER, ONE_HOT, ESM, EVE, EVE_DENSITY] # VAE_AUX, VAE_RAND, TRANSFORMER, VAE, ONE_HOT, ESM, EVE, VAE_AUX EXTRA 1D rep: VAE_DENSITY
 MOCK = False
 # Protocols: RandomSplitterFactory, BlockSplitterFactory, PositionalSplitterFactory, BioSplitterFactory, FractionalSplitterFactory
 protocol_factories = [RandomSplitterFactory, PositionalSplitterFactory]
+# protocol_factories = [PositionalSplitterFactory]
 # protocol_factories = [FractionalSplitterFactory]
-# protocol_factories = [WeightedTaskSplitterFactory]
 # protocol_factories = [BioSplitterFactory("TOXI", 1, 2), BioSplitterFactory("TOXI", 2, 2), BioSplitterFactory("TOXI", 2, 3), BioSplitterFactory("TOXI", 3, 3), BioSplitterFactory("TOXI", 3, 4)]
 # [BioSplitterFactory("TOXI", 1, 2), BioSplitterFactory("TOXI", 2, 2), BioSplitterFactory("TOXI", 2, 3), BioSplitterFactory("TOXI", 3, 3), BioSplitterFactory("TOXI", 3, 4)]:
 
 # Methods: # KNNFactory, RandomForestFactory, UncertainRFFactory, GPSEFactory, GPLinearFactory, GPMaternFactory
 # method_factories = [get_key_for_factory(f) for f in [KNNFactory, RandomForestFactory]]
-method_factories = [get_key_for_factory(f) for f in [KNNFactory, RandomForestFactory, UncertainRFFactory, GPSEFactory, GPLinearFactory, GPMaternFactory]]
+method_factories = [get_key_for_factory(f) for f in [KNNFactory]]
 
-# TODO: rerun with KNN and RF for sanity check after data-load refactor:
 experiment_iterator = product(datasets, representations, protocol_factories, method_factories)
 def run_experiments():
     for dataset, representation, protocol_factory, factory_key in experiment_iterator:
@@ -76,9 +76,9 @@ def run_augmentation_experiments():
 
 
 if __name__ == "__main__":
-    # run_experiments()
+    run_experiments() # TODO: toxi all
     # ABLATION STUDY: (dim-reduction, augmentation, threshold):
     # run_dim_reduction_experiments()
-    run_augmentation_experiments()
+    # run_augmentation_experiments()
     #run_threshold_experiments()