fix mfom tests: simulate multi-label dataset

Author: Vanova

experiments/runner.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # OGA-TS model runner
 
-export PYTHONPATH="/home/vano/wrkdir/projects/Python/attribute_detection/:$PYTHONPATH"
+export PYTHONPATH="`pwd`/:$PYTHONPATH"
 source activate ai # activate conda environment
 
 model=sed_ogits

src/model/objectives.py

@@ -33,12 +33,12 @@ def mfom_eer_normalized(y_true, y_pred):
     # smooth false negative and false positive rates
     fnr = K.log(K.sum(fn, axis=0) + 1.) - K.log(P + 1.)
     fpr = K.log(K.sum(fp, axis=0) + 1.) - K.log(N + 1.)
-    fnr = K.exp(fnr)
-    fpr = K.exp(fpr)
-    smooth_eer = fpr + .5 * K.abs(fnr - fpr)
     # debug output
     # fnr = K.print_tensor(K.exp(fnr), message="FNR is: ")
     # fpr = K.print_tensor(K.exp(fpr), message="FPR is: ")
+    fnr = K.exp(fnr)
+    fpr = K.exp(fpr)
+    smooth_eer = fpr + .5 * K.abs(fnr - fpr)
     return K.mean(smooth_eer)
 
 

tests/model/test_mfom_2d.py

@@ -1,83 +1,100 @@
+"""
+    Simulate multi-label classification.
+"""
 import numpy as np
 import keras.backend as K
 from keras.models import Model
 from keras.layers import Dense, Activation, Input
+import keras.regularizers as regs
+import keras.constraints as constraints
 import matplotlib.pyplot as plt
+from sklearn.datasets import make_multilabel_classification
+from sklearn.model_selection import train_test_split
+from sklearn import preprocessing
 import src.model.mfom as mfom
 import src.utils.metrics as MT
 import src.model.objectives as obj
 
-np.random.seed(777)
+RANDOM_SEED = 777
+np.random.seed(RANDOM_SEED)
 
 
-def generate_dataset(output_dim=14, num_examples=10000):
-    """
-    Summation of two binary numbers.
-    Input is two binary numbers, stacked in one vector.
-    Output is an integer number.
-    """
+def generate_dataset(n_smp=300, ratio=0.3, n_feat=2, n_cls=2):
+    x, y = make_multilabel_classification(n_samples=n_smp, n_features=n_feat,
+                                          n_classes=n_cls, n_labels=1,
+                                          allow_unlabeled=False,
+                                          random_state=RANDOM_SEED)
+    scaler = preprocessing.StandardScaler()
+    x = scaler.fit_transform(x)
+    x_tr, x_tst, y_tr, y_tst = train_test_split(x, y, test_size=ratio, random_state=RANDOM_SEED)
+    return x_tr, x_tst, y_tr, y_tst
 
-    def int2vec(x, dim=output_dim):
-        out = np.zeros(dim)
-        binrep = np.array(list(np.binary_repr(x))).astype('int')
-        out[-len(binrep):] = binrep
-        return out
 
-    x_left_int = (np.random.rand(num_examples) * 2 ** (output_dim - 1)).astype('int')
-    x_right_int = (np.random.rand(num_examples) * 2 ** (output_dim - 1)).astype('int')
-    y_int = x_left_int + x_right_int
-
-    x = list()
-    for i in range(len(x_left_int)):
-        x.append(np.concatenate((int2vec(x_left_int[i]), int2vec(x_right_int[i]))))
-
-    y = list()
-    for i in range(len(y_int)):
-        y.append(int2vec(y_int[i]))
-
-    x = np.array(x)
-    y = np.array(y)
-    return x, y
-
-
-if __name__ == '__main__':
-    dim = 14
-    nclass = 7
-
-    # Input block
-    feat_input = Input(shape=(dim,), name='main_input')
+def mfom_model(in_dim, nclass):
+    # input block
+    feat_input = Input(shape=(in_dim,), name='main_input')
     # layer 1
-    x = Dense(30, name='dense1')(feat_input)
+    x = Dense(10, name='dense1')(feat_input)
     x = Activation(activation='sigmoid', name='act1')(x)
+    # layer 2
+    x = Dense(10, name='dense2')(x)
+    x = Activation(activation='sigmoid', name='act2')(x)
     # output layer
     x = Dense(nclass, name='pre_activation')(x)
     y_pred = Activation(activation='sigmoid', name='output')(x)
 
+    # === MFoM head ===
     # misclassification layer, feed Y
     y_true = Input(shape=(nclass,), name='y_true')
     psi = mfom.UvZMisclassification(name='uvz_misclass')([y_true, y_pred])
 
     # class Loss function layer
-    out = mfom.SmoothErrorCounter(name='smooth_error_counter')(psi)
+    # NOTE: you may want to add regularization or constraints
+    out = mfom.SmoothErrorCounter(name='smooth_error_counter',
+                                  # alpha_constraint=constraints.min_max_norm(min_value=-4., max_value=4.),
+                                  # alpha_regularizer=regs.l1(0.001),
+                                  # beta_constraint=constraints.min_max_norm(min_value=-4., max_value=4.),
+                                  # beta_regularizer=regs.l1(0.001)
+                                  )(psi)
 
     # compile model
     model = Model(input=[y_true, feat_input], output=out)
-    model.compile(loss=obj.mfom_eer_normalized, optimizer='Adam')
-    model.summary()
+    return model
 
-    # train
-    X, Y = generate_dataset(output_dim=nclass)
-    hist = model.fit([Y, X], Y, nb_epoch=100, batch_size=16)
 
-    # calc accuracy: we cut MFoM head, up to sigmoid output
+def cut_mfom(model):
+    # calc accuracy: cut MFoM head, up to sigmoid output
     input = model.get_layer(name='main_input').output
     out = model.get_layer(name='output').output
-    cut_model = Model(input=input, output=out)
-    y_pred = cut_model.predict(X)
-    eer_val = MT.eer(y_true=Y.flatten(), y_pred=y_pred.flatten())
+    cut_net = Model(input=input, output=out)
+    return cut_net
+
+
+if __name__ == '__main__':
+    dim = 20
+    nclass = 10
+
+    # mfom model
+    model = mfom_model(dim, nclass)
+    model.compile(loss=obj.mfom_eer_normalized, optimizer='Adam')
+    model.summary()
+
+    # training on multi-label dataset
+    x_train, x_test, y_train, y_test = generate_dataset(n_smp=10000, n_feat=dim, n_cls=nclass)
+    mask = y_train.sum(axis=-1) != nclass
+    y_train = y_train[mask]
+    x_train = x_train[mask]
+    hist = model.fit([y_train, x_train], y_train, nb_epoch=10, batch_size=16)
+
+    # cut MFoM head
+    cut_model = cut_mfom(model)
+    y_pred = cut_model.predict(x_test)
+
+    # evaluate
+    eer_val = MT.eer(y_true=y_test.flatten(), y_pred=y_pred.flatten())
     print('EER: %.4f' % eer_val)
 
-    # history plot, alpha and beta params
+    # history plot, alpha and beta params of MFoM
     m = model.get_layer('smooth_error_counter')
     print('alpha: ', K.get_value(m.alpha))
     print('beta: ', K.get_value(m.beta))

tests/model/test_mfom_3d.py

@@ -47,7 +47,7 @@ def int2vec(x, dim=output_dim):
     x = Permute((2, 1))(feat_input)
     for _f in [256, 64]:
         x = TimeDistributed(Dense(_f))(x)
-        # x = Activation(activation='elu')(x)
+        x = Activation(activation='elu')(x)
         x = Dropout(0.5)(x)
     x = TimeDistributed(Dense(nclass))(x)
     y_pred = Activation(activation='tanh', name='output')(x)
@@ -62,7 +62,7 @@ def int2vec(x, dim=output_dim):
 
     # compile model
     model = Model(input=[y_true, feat_input], output=out)
-    model.compile(loss=obj.mfom_eer_normalized, optimizer='Adadelta') # Adam, Adadelta
+    model.compile(loss=obj.mfom_eer_normalized, optimizer='Adadelta')
     model.summary()
 
     # train
@@ -87,22 +87,11 @@ def int2vec(x, dim=output_dim):
     print('l_EER: %.4f' % eer_val)
     print(model.evaluate([all_Y, all_X], all_Y))
 
-    # TODO notice from the experiments:
-    # when we minimize obj.mfom_microf1 with psi = y_pred or psi = -y_pred + 0.5 in
-    # UvZMisclassification() layer, the smoothF1 is minimized !!! but EER is not at all.
-    # When we minimize obj.mfom_microf1 with psi = -y_pred + y_neg * unit_avg + y_true * zeros_avg,
-    # then both smoothF1 and EER are minimized :)
-
     # history plot, alpha and beta params
     m = model.get_layer('smooth_error_counter')
     print('alpha: ', K.get_value(m.alpha))
     print('beta: ', K.get_value(m.beta))
 
     # print stats of psi misclassification measure
-    # m = model.get_layer('uvz_misclass')
-    # print('stats_d: ', K.get_value(m.stats_psi))
     plt.plot(hist.history['loss'])
-    plt.show()
-
-    # TODO test MFoM with honest uvz-misclassification,
-    # TODO test for 2 classes and check Triplet loss symptoms
+    plt.show()