Try completing test with Keras

Also adds a classification test probably this will be overkill for GH actions
but try it.

Author: pobonomo

tests/test_keras/keras_cases.py

@@ -4,9 +4,9 @@
 
 """
 
-import os
 from abc import ABC, abstractmethod
 
+import tensorflow as tf
 import keras
 
 
@@ -32,25 +32,11 @@ def __init__(
         self,
         dataset,
     ):
-        self.basedir = os.path.join(os.path.dirname(__file__), "..", "predictors")
         self.dataset = dataset
 
         # Filled with get data if needed
         self._data = None
 
-        keras_version_file = f"{dataset}_keras_version"
-
-        try:
-            with open(os.path.join(self.basedir, keras_version_file)) as file_in:
-                version = file_in.read().strip()
-        except FileNotFoundError:
-            version = None
-        if version != keras.__version__:
-            print(f"Keras version changed. Regenerate predictors for {dataset}")
-            self.build_all_predictors()
-            with open(os.path.join(self.basedir, keras_version_file), "w") as file_out:
-                print(keras.__version__, file=file_out)
-
     def __iter__(self):
         return self.all_tested_layers.__iter__()
 
@@ -69,39 +55,25 @@ def data(self):
             self.load_data()
         return self._data
 
-    def predictor_file(self, predictor):
-        return f"{self.dataset}_{layers_as_string(predictor)}.keras"
-
-    def build_predictor(self, layers):
+    def get_case(self, layers):
         """Build model for one predictor"""
         X, y = self.data
         predictor = self.compile(layers)
         predictor.fit(X, y)
 
-        predictor.save(self.predictor_file(layers))
         return predictor
 
-    def build_all_predictors(self):
-        """Build all the predictor for this case.
-        (Done when we have a new sklearn version)"""
-        for predictor in self:
-            self.build_predictor(predictor)
-
-    def get_case(self, predictor):
-        filename = self.predictor_file(predictor)
-        try:
-            return keras.saving.load_model(os.path.join(self.basedir, filename))
-        except ValueError:
-            return self.build_predictor(predictor)
-
 
 class HousingCases(Cases):
     """Base class to have cases for testing regression models on diabetes set
 
     This is appropriate for testing a regression with a single output."""
 
     def __init__(self):
-        self.all_tested_layers = [[keras.layers.Dense(16, activation="relu")]]
+        self.all_tested_layers = [
+            [keras.layers.Dense(16, activation="relu")],
+            [keras.layers.Dense(16, activation="sigmoid")],
+        ]
         super().__init__("housing")
         self.load_data()
 
@@ -117,3 +89,33 @@ def compile(self, layers):
         )
         nn.compile(loss="mean_squared_error", optimizer="adam")
         return nn
+
+
+class MNISTCases(Cases):
+    """Base class to have cases for testing regression models on diabetes set
+
+    This is appropriate for testing a regression with a single output."""
+
+    def __init__(self):
+        self.all_tested_layers = [
+            [keras.layers.Dense(20, activation="relu")],
+            [keras.layers.Dense(20, activation="sigmoid")],
+        ]
+        super().__init__("housing")
+        self.load_data()
+
+    def load_data(self):
+        (X_train, y_train), (_, _) = keras.datasets.fashion_mnist.load_data()
+        X_train = tf.reshape(tf.cast(X_train, tf.float32) / 255.0, [-1, 28 * 28])
+        self._data = (X_train, y_train)
+
+    def compile(self, layers):
+        nn = keras.models.Sequential(
+            [keras.layers.InputLayer((28 * 28,))] + layers + [keras.layers.Dense(10)]
+        )
+        nn.compile(
+            optimizer="adam",
+            loss=tf.keras.losses.SparseCategoricalCrossentropy(),
+            metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
+        )
+        return nn

tests/test_keras/test_keras_formulations.py

@@ -4,7 +4,7 @@
 from joblib import load
 
 from ..fixed_formulation import FixedRegressionModel
-from .keras_cases import HousingCases
+from .keras_cases import HousingCases, MNISTCases
 
 VERBOSE = False
 
@@ -45,3 +45,13 @@ def test_housing_keras(self):
             onecase = {"predictor": regressor, "nonconvex": 0}
             self.do_one_case(onecase, X, 5, "all")
             self.do_one_case(onecase, X, 6, "pairs")
+
+    def test_mnist_keras(self):
+        cases = MNISTCases()
+
+        X = cases._data[0].numpy()
+        for case in cases:
+            regressor = cases.get_case(case)
+            onecase = {"predictor": regressor, "nonconvex": 0}
+            self.do_one_case(onecase, X, 5, "all")
+            self.do_one_case(onecase, X, 6, "pairs")