added 12 with slides

Author: patrickloeber

11_softmax_and_crossentropy.py

@@ -11,7 +11,7 @@
 #                           sum = 1.0
 #
 
-# Softmax applies the exponential function to each element, and notmalizes
+# Softmax applies the exponential function to each element, and normalizes
 # by dividing by the sum of all these exponentials
 # -> squashes the output to be between 0 and 1 = probability
 # sum of all probabilities is 1
@@ -54,16 +54,15 @@ def cross_entropy(actual, predicted):
 loss = nn.CrossEntropyLoss()
 # loss(input, target)
 
-# target is of size nBatch = 1
+# target is of size nSamples = 1
 # each element has class label: 0, 1, or 2
 # Y (=target) contains class labels, not one-hot
 Y = torch.tensor([0])
 
-# input is of size nBatch x nClasses = 1 x 3
+# input is of size nSamples x nClasses = 1 x 3
 # y_pred (=input) must be raw, unnormalizes scores (logits) for each class, not softmax
 Y_pred_good = torch.tensor([[2.0, 1.0, 0.1]])
 Y_pred_bad = torch.tensor([[0.5, 2.0, 0.3]])
-
 l1 = loss(Y_pred_good, Y)
 l2 = loss(Y_pred_bad, Y)
 
@@ -98,6 +97,11 @@ def cross_entropy(actual, predicted):
 print(f'Batch Loss1:  {l1.item():.4f}')
 print(f'Batch Loss2: {l2.item():.4f}')
 
+# get predictions
+_, predictions1 = torch.max(Y_pred_good, 1)
+_, predictions2 = torch.max(Y_pred_bad, 1)
+print(f'Actual class: {Y}, Y_pred1: {predictions1}, Y_pred2: {predictions2}')
+
 # Binary classification
 class NeuralNet1(nn.Module):
     def __init__(self, input_size, hidden_size):

12_activation_functions.py

@@ -0,0 +1,74 @@
+# output = w*x + b
+# output = activation_function(output)
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+x = torch.tensor([-1.0, 1.0, 2.0, 3.0])
+
+# sofmax
+output = torch.softmax(x, dim=0)
+print(output)
+sm = nn.Softmax(dim=0)
+output = sm(x)
+print(output)
+
+# sigmoid 
+output = torch.sigmoid(x)
+print(output)
+s = nn.Sigmoid()
+output = s(x)
+print(output)
+
+#tanh
+output = torch.tanh(x)
+print(output)
+t = nn.Tanh()
+output = t(x)
+print(output)
+
+# relu
+output = torch.relu(x)
+print(output)
+relu = nn.ReLU()
+output = relu(x)
+print(output)
+
+# leaky relu
+output = F.leaky_relu(x)
+print(output)
+lrelu = nn.LeakyReLU()
+output = lrelu(x)
+print(output)
+
+#nn.ReLU() creates an nn.Module which you can add e.g. to an nn.Sequential model.
+#torch.relu on the other side is just the functional API call to the relu function,
+#so that you can add it e.g. in your forward method yourself.
+
+# option 1 (create nn modules)
+class NeuralNet(nn.Module):
+    def __init__(self, input_size, hidden_size):
+        super(NeuralNet, self).__init__()
+        self.linear1 = nn.Linear(input_size, hidden_size)
+        self.relu = nn.ReLU()
+        self.linear2 = nn.Linear(hidden_size, 1)
+        self.sigmoid = nn.Sigmoid()
+    
+    def forward(self, x):
+        out = self.linear1(x)
+        out = self.relu(out)
+        out = self.linear2(out)
+        out = self.sigmoid(out)
+        return out
+
+# option 2 (use activation functions directly in forward pass)
+class NeuralNet(nn.Module):
+    def __init__(self, input_size, hidden_size):
+        super(NeuralNet, self).__init__()
+        self.linear1 = nn.Linear(input_size, hidden_size)
+        self.linear2 = nn.Linear(hidden_size, 1)
+    
+    def forward(self, x):
+        out = torch.relu(self.linear1(x))
+        out = torch.sigmoid(self.linear2(out))
+        return out

12_plot_activations.py

@@ -0,0 +1,144 @@
+import numpy as np 
+import matplotlib.pyplot as plt
+
+##### Sigmoid
+sigmoid = lambda x: 1 / (1 + np.exp(-x))
+
+x=np.linspace(-10,10,10)
+
+y=np.linspace(-10,10,100)
+
+fig = plt.figure()
+plt.plot(y,sigmoid(y),'b', label='linspace(-10,10,100)')
+
+plt.grid(linestyle='--')
+
+plt.xlabel('X Axis')
+
+plt.ylabel('Y Axis')
+
+plt.title('Sigmoid Function')
+
+plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
+plt.yticks([-2, -1, 0, 1, 2])
+
+plt.ylim(-2, 2)
+plt.xlim(-4, 4)
+
+plt.show()
+#plt.savefig('sigmoid.png')
+
+fig = plt.figure()
+
+##### TanH
+tanh = lambda x: 2*sigmoid(2*x)-1
+
+x=np.linspace(-10,10,10)
+
+y=np.linspace(-10,10,100)
+
+plt.plot(y,tanh(y),'b', label='linspace(-10,10,100)')
+
+plt.grid(linestyle='--')
+
+plt.xlabel('X Axis')
+
+plt.ylabel('Y Axis')
+
+plt.title('TanH Function')
+
+plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
+plt.yticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
+
+plt.ylim(-4, 4)
+plt.xlim(-4, 4)
+
+plt.show()
+#plt.savefig('tanh.png')
+
+fig = plt.figure()
+
+##### ReLU
+relu = lambda x: np.where(x>=0, x, 0)
+
+x=np.linspace(-10,10,10)
+
+y=np.linspace(-10,10,1000)
+
+plt.plot(y,relu(y),'b', label='linspace(-10,10,100)')
+
+plt.grid(linestyle='--')
+
+plt.xlabel('X Axis')
+
+plt.ylabel('Y Axis')
+
+plt.title('ReLU')
+
+plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
+plt.yticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
+
+plt.ylim(-4, 4)
+plt.xlim(-4, 4)
+
+plt.show()
+#plt.savefig('relu.png')
+
+fig = plt.figure()
+
+##### Leaky ReLU
+leakyrelu = lambda x: np.where(x>=0, x, 0.1*x)
+
+x=np.linspace(-10,10,10)
+
+y=np.linspace(-10,10,1000)
+
+plt.plot(y,leakyrelu(y),'b', label='linspace(-10,10,100)')
+
+plt.grid(linestyle='--')
+
+plt.xlabel('X Axis')
+
+plt.ylabel('Y Axis')
+
+plt.title('Leaky ReLU')
+
+plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
+plt.yticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
+
+plt.ylim(-4, 4)
+plt.xlim(-4, 4)
+
+plt.show()
+#plt.savefig('lrelu.png')
+
+fig = plt.figure()
+
+
+##### Binary Step
+bstep = lambda x: np.where(x>=0, 1, 0)
+
+x=np.linspace(-10,10,10)
+
+y=np.linspace(-10,10,1000)
+
+plt.plot(y,bstep(y),'b', label='linspace(-10,10,100)')
+
+plt.grid(linestyle='--')
+
+plt.xlabel('X Axis')
+
+plt.ylabel('Y Axis')
+
+plt.title('Step Function')
+
+plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
+plt.yticks([-2, -1, 0, 1, 2])
+
+plt.ylim(-2, 2)
+plt.xlim(-4, 4)
+
+plt.show()
+#plt.savefig('step.png')
+
+print('done')