Added Inverse Square Root Linear Unit (ISRLU) activation layer

CODEOWNERS

@@ -26,6 +26,7 @@ keras_contrib/callbacks/snapshot.py @titu1994
 
 
 # layers
+keras_contrib/layers/advanced_activations/isrlu.py @SriRangaTarun
 keras_contrib/layers/advanced_activations/sinerelu.py @wilderrodrigues
 keras_contrib/layers/advanced_activations/swish.py @gabrieldemarmiesse
 keras_contrib/layers/convolutional/subpixelupscaling.py @titu1994

keras_contrib/layers/__init__.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+from .advanced_activations.isrlu import ISRLU
 from .advanced_activations.pelu import PELU
 from .advanced_activations.srelu import SReLU
 from .advanced_activations.swish import Swish

keras_contrib/layers/advanced_activations/isrlu.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+from keras import backend as K
+from keras.layers import Layer
+from keras_contrib.utils.test_utils import to_tuple
+from keras_contrib.utils.test_utils import is_tf_keras
+
+
+class ISRLU(Layer):
+    """Inverse Square Root Linear Unit
+    See: https://arxiv.org/pdf/1710.09967.pdf by AI Perf
+    Reference: https://en.wikipedia.org/wiki/Activation_function
+    Inverse Square Root Linear activation f(α, x):
+        x >= 0: x
+        x < 0:  x / sqrt(1 + α * x^2)
+    # Input shape
+        Arbitrary. Use the keyword argument `input_shape`
+        (tuple of integers, does not include the samples axis)
+        when using this layer as the first layer in a model.
+    # Output shape
+        Same shape as the input.
+    # Arguments
+        alpha: Value of the alpha weights (float)
+         NOTE : This function can become unstable for
+                negative values of α (it may return
+                NaNs). In particular, this happens when
+                α < 0 and x < -1/sqrt(α).
+                If this happens, try limiting the magnitude
+                of α below a certain threshold, such that
+                1 + α * x^2 is always positive.
+                Alternatively, you can normalize the inputs
+                into fixed ranges before passing them to ISRLU.
+                Adjust the value of α based on your specific
+                dataset and use-case.
+    # Example
+        model = Sequential()
+        model.add(Dense(5, input_shape=(15,))
+        model.add(ISRLU(alpha=-0.3))
+    """
+    def __init__(self,
+                 alpha=0.1,
+                 **kwargs):
+
+        super(ISRLU, self).__init__(**kwargs)
+        self.alpha = alpha
+        self.trainable = False
+
+    def alpha_initializer(self, input_shape, dtype='float32', **kwargs):
+        return self.alpha * K.ones(input_shape,
+                                   dtype=dtype)
+
+    def build(self, input_shape):
+        input_shape = to_tuple(input_shape)
+        new_input_shape = input_shape[1:]
+        self.alphas = self.add_weight(shape=new_input_shape,
+                                      name='{}_alphas'.format(self.name),
+                                      initializer=self.alpha_initializer,
+                                      trainable=self.trainable)
+        self.build = True
+
+    def call(self, x):
+        def inverse_quadratic_square_root(x):
+            return x / K.sqrt(1 + self.alphas * K.square(x))
+
+        return K.switch(K.less(x, K.zeros_like(x)), inverse_quadratic_square_root(x), x)
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def get_config(self):
+        config = {'alpha': self.alpha,
+                  'trainable': self.trainable}
+        base_config = super(ISRLU, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

tests/keras_contrib/layers/advanced_activations/test_isrlu.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+import pytest
+from keras_contrib.utils.test_utils import layer_test
+from keras_contrib.layers import ISRLU
+
+
+@pytest.mark.parametrize('alpha', [0.2, 0.3, -0.01])
+def test_isrlu(alpha):
+    layer_test(ISRLU,
+               kwargs={'alpha': alpha},
+               input_shape=(2, 3, 4))
+
+
+if __name__ == '__main__':
+    pytest.main([__file__])