chapter02_mathematical-building-blocks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n",
    "\n",
    "**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n",
    "\n",
    "This notebook was generated for TensorFlow 2.6."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "# The mathematical building blocks of neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## A first look at a neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Loading the MNIST dataset in Keras**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                            \n",
      "                            \n",
      "                            \n",
      "                            \n",
      "                            \n",
      "                ==* +@%=    \n",
      "        ..-+*%%%%%#*%%*:    \n",
      "       .%%%%%%%%%%-::..     \n",
      "        #%%%%%**%%          \n",
      "        :+-%%#  .+          \n",
      "           +%-              \n",
      "           =%*              \n",
      "            *%:             \n",
      "            .%#+-           \n",
      "             :%%%=          \n",
      "              .*%%+         \n",
      "                -%%*        \n",
      "                 %%%:       \n",
      "              .=*%%#        \n",
      "            .+%%%%%*        \n",
      "           =#%%%%#:         \n",
      "         :#%%%%*:           \n",
      "       *#%%%%*:             \n",
      "    .*#%%%%%=               \n",
      "    =%%%#==                 \n",
      "                            \n",
      "                            \n",
      "                            \n"
     ]
    }
   ],
   "source": [
    "train_images.shape\n",
    "\n",
    "train_images[0]\n",
    "\n",
    "def array_to_ascii_art(array, chars=' .:-=+*#%@'):\n",
    "    # Get the minimum and maximum values of the array\n",
    "    min_value = min([min(row) for row in array])\n",
    "    max_value = max([max(row) for row in array])\n",
    "    # Get the number of characters to represent the values\n",
    "    num_chars = len(chars)\n",
    "    # Loop over the elements of the array and generate the ASCII art\n",
    "    art = []\n",
    "    for row in array:\n",
    "        art.append(''.join([chars[int((num_chars - 1) * (value - min_value) / (max_value - min_value))] for value in row]))\n",
    "    return '\\n'.join(art)\n",
    "\n",
    "\n",
    "print(array_to_ascii_art(train_images[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels\n",
    "\n",
    "train_labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([7, 2, 1, ..., 4, 5, 6], dtype=uint8)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**The network architecture**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "model = keras.Sequential([\n",
    "    layers.Dense(512, activation=\"relu\"),\n",
    "    layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**The compilation step**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"sparse_categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Preparing the image data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**\"Fitting\" the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.2639 - accuracy: 0.9237\n",
      "Epoch 2/5\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.1057 - accuracy: 0.9693\n",
      "Epoch 3/5\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0705 - accuracy: 0.9789\n",
      "Epoch 4/5\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0507 - accuracy: 0.9851\n",
      "Epoch 5/5\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0380 - accuracy: 0.9888\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x7f95e454b2d0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_images, train_labels, epochs=5, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Using the model to make predictions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 17ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([1.7831281e-08, 4.5046519e-10, 3.4956847e-07, 8.3210207e-06,\n",
       "       6.5046725e-11, 2.4568442e-07, 2.9225101e-12, 9.9998879e-01,\n",
       "       1.3051647e-07, 2.0811699e-06], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digits = test_images[0:10]\n",
    "predictions = model.predict(test_digits)\n",
    "predictions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0].argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9999888"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0][7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Evaluating the model on new data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 1s 4ms/step - loss: 0.0610 - accuracy: 0.9796\n",
      "test_acc: 0.9796000123023987\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = model.evaluate(test_images, test_labels)\n",
    "print(f\"test_acc: {test_acc}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Data representations for neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Scalars (rank-0 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(12)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "x = np.array(12)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Vectors (rank-1 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([12,  3,  6, 14,  7])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([12, 3, 6, 14, 7])\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Matrices (rank-2 tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[5, 78, 2, 34, 0],\n",
    "              [6, 79, 3, 35, 1],\n",
    "              [7, 80, 4, 36, 2]])\n",
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Rank-3 and higher-rank tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]],\n",
    "              [[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]],\n",
    "              [[5, 78, 2, 34, 0],\n",
    "               [6, 79, 3, 35, 1],\n",
    "               [7, 80, 4, 36, 2]]])\n",
    "x.ndim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Key attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.ndim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('uint8')"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Displaying the fourth digit**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAbYklEQVR4nO3df2zU9R3H8deB9ERsryulvZ4ULKigAl2G0jUq4mgoXUZAyCbqFjAEIitG7JymTkSdWSdmzOgq/rPB3ESYiUD0DxxW286tsIESxn50tOkEAi1I0l4pUhj97I+G2w6K8D3u+u4dz0fyTejd99N78/XSp1/67bc+55wTAAD9bJD1AACAKxMBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJq6yHuBcPT09OnTokNLT0+Xz+azHAQB45JxTZ2enQqGQBg268HnOgAvQoUOHlJ+fbz0GAOAyHThwQCNHjrzg8wMuQOnp6ZJ6B8/IyDCeBgDgVTgcVn5+fuTr+YUkLEDV1dV66aWX1NraqsLCQr366quaMmXKRded/We3jIwMAgQASexi30ZJyEUIGzduVEVFhVauXKlPPvlEhYWFKi0t1ZEjRxLxcgCAJJSQAK1evVqLFy/WQw89pFtuuUWvv/66rrnmGv3qV79KxMsBAJJQ3AN06tQp7dq1SyUlJf97kUGDVFJSooaGhvP27+7uVjgcjtoAAKkv7gH6/PPPdebMGeXm5kY9npubq9bW1vP2r6qqUiAQiGxcAQcAVwbzH0StrKxUR0dHZDtw4ID1SACAfhD3q+Cys7M1ePBgtbW1RT3e1tamYDB43v5+v19+vz/eYwAABri4nwGlpaVp8uTJqqmpiTzW09OjmpoaFRcXx/vlAABJKiE/B1RRUaEFCxbotttu05QpU/Tyyy+rq6tLDz30UCJeDgCQhBISoPvuu09Hjx7VM888o9bWVn31q1/V1q1bz7swAQBw5fI555z1EP8vHA4rEAioo6ODOyEAQBK61K/j5lfBAQCuTAQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwETcA/Tss8/K5/NFbePHj4/3ywAAktxVifikt956qz744IP/vchVCXkZAEASS0gZrrrqKgWDwUR8agBAikjI94D27dunUCikMWPG6MEHH9T+/fsvuG93d7fC4XDUBgBIfXEPUFFRkdatW6etW7dqzZo1amlp0V133aXOzs4+96+qqlIgEIhs+fn58R4JADAA+ZxzLpEv0N7ertGjR2v16tVatGjRec93d3eru7s78nE4HFZ+fr46OjqUkZGRyNEAAAkQDocVCAQu+nU84VcHZGZm6qabblJTU1Ofz/v9fvn9/kSPAQAYYBL+c0DHjx9Xc3Oz8vLyEv1SAIAkEvcAPf7446qrq9O///1v/elPf9K9996rwYMH6/7774/3SwEAkljc/wnu4MGDuv/++3Xs2DGNGDFCd955p7Zv364RI0bE+6UAAEks7gHasGFDvD8lACAFcS84AIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMBEwn8hHZBMduzY4XnNb37zG89r6uvrPa/Zu3ev5zWx+tnPfuZ5TSgU8rzmD3/4g+c13/ve9zyvKSoq8rwGiccZEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAExwN2ykpI0bN8a07tFHH/W85ujRo57XOOc8r5k2bZrnNZ9//rnnNZL0+OOPx7TOq1iOQyx/pw0bNnheg8TjDAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMMHNSNGv/vOf/3he85e//MXzmsWLF3teI0ldXV2e19x9992e16xYscLzmjvvvNPzmu7ubs9rJOk73/mO5zXvv/9+TK/l1W233dYvr4PE4wwIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADDBzUjRr3772996XrNo0aIETNK3GTNmeF6zceNGz2syMjI8r4lFLLNJ/Xdj0fz8fM9rFixYkIBJYIEzIACACQIEADDhOUD19fWaNWuWQqGQfD6fNm/eHPW8c07PPPOM8vLyNHToUJWUlGjfvn3xmhcAkCI8B6irq0uFhYWqrq7u8/lVq1bplVde0euvv64dO3Zo2LBhKi0t1cmTJy97WABA6vB8EUJZWZnKysr6fM45p5dffllPP/20Zs+eLUl64403lJubq82bN2v+/PmXNy0AIGXE9XtALS0tam1tVUlJSeSxQCCgoqIiNTQ09Lmmu7tb4XA4agMApL64Bqi1tVWSlJubG/V4bm5u5LlzVVVVKRAIRLZYLssEACQf86vgKisr1dHREdkOHDhgPRIAoB/ENUDBYFCS1NbWFvV4W1tb5Llz+f1+ZWRkRG0AgNQX1wAVFBQoGAyqpqYm8lg4HNaOHTtUXFwcz5cCACQ5z1fBHT9+XE1NTZGPW1patHv3bmVlZWnUqFFavny5XnjhBd14440qKCjQihUrFAqFNGfOnHjODQBIcp4DtHPnTt1zzz2RjysqKiT13p9p3bp1euKJJ9TV1aUlS5aovb1dd955p7Zu3aqrr746flMDAJKezznnrIf4f+FwWIFAQB0dHXw/aIB7+umnPa/5yU9+4nmNz+fzvKa8vNzzGkl64YUXPK8ZyO/Tm2++OaZ1//rXv+I8Sd/eeecdz2vO/owhBq5L/TpufhUcAODKRIAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABOefx0DUs/zzz8f07pY7mzt9/s9ryktLfW85sUXX/S8RpKGDh0a0zqvTp486XnN73//e89rPvvsM89rJCmWm+SvWLHC8xrubH1l4wwIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADDBzUhTTHt7u+c1r732Wkyv5fP5PK+J5caimzdv9rymPzU1NXle8+CDD3pes3PnTs9rYvXtb3/b85onnngiAZMglXEGBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCY4GakKebUqVOe1xw9ejQBk/TtlVde8bzmyJEjntesXbvW8xpJ2rJli+c1f/vb3zyv6ezs9Lwmlpu/DhoU2/9jfve73/W8ZtiwYTG9Fq5cnAEBAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACa4GWmKSUtL87wmJycnpteK5Sah119/vec1sdyEsz9dd911ntdkZGR4XnPo0CHPa7Kzsz2vkaRZs2bFtA7wgjMgAIAJAgQAMOE5QPX19Zo1a5ZCoZB8Pp82b94c9fzChQvl8/mitpkzZ8ZrXgBAivAcoK6uLhUWFqq6uvqC+8ycOVOHDx+ObG+99dZlDQkASD2eL0IoKytTWVnZl+7j9/sVDAZjHgoAkPoS8j2g2tpa5eTkaNy4cVq6dKmOHTt2wX27u7sVDoejNgBA6ot7gGbOnKk33nhDNTU1evHFF1VXV6eysjKdOXOmz/2rqqoUCAQiW35+frxHAgAMQHH/OaD58+dH/jxx4kRNmjRJY8eOVW1traZPn37e/pWVlaqoqIh8HA6HiRAAXAESfhn2mDFjlJ2draampj6f9/v9ysjIiNoAAKkv4QE6ePCgjh07pry8vES/FAAgiXj+J7jjx49Hnc20tLRo9+7dysrKUlZWlp577jnNmzdPwWBQzc3NeuKJJ3TDDTeotLQ0roMDAJKb5wDt3LlT99xzT+Tjs9+/WbBggdasWaM9e/bo17/+tdrb2xUKhTRjxgz9+Mc/lt/vj9/UAICk5zlA06ZNk3Pugs+///77lzUQLk9mZqbnNefezeJSfetb3/K85ssuyb+QG264wfOa2bNne14j9d7Jw6usrCzPa/7/Yp1LFcvNSGN5HaC/cC84AIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmIj7r+RG8ikqKopp3dGjR+M8SXKqr6/3vKaurs7zGp/P53nNmDFjPK8B+gtnQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACW5GClymL774wvOaWG4sGsua+fPne14D9BfOgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAE9yMFLhMpaWl1iMASYkzIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABDcjBS7T+++/bz0CkJQ4AwIAmCBAAAATngJUVVWl22+/Xenp6crJydGcOXPU2NgYtc/JkydVXl6u4cOH69prr9W8efPU1tYW16EBAMnPU4Dq6upUXl6u7du3a9u2bTp9+rRmzJihrq6uyD6PPfaY3n33Xb399tuqq6vToUOHNHfu3LgPDgBIbp4uQti6dWvUx+vWrVNOTo527dqlqVOnqqOjQ7/85S+1fv16feMb35AkrV27VjfffLO2b9+ur3/96/GbHACQ1C7re0AdHR2SpKysLEnSrl27dPr0aZWUlET2GT9+vEaNGqWGhoY+P0d3d7fC4XDUBgBIfTEHqKenR8uXL9cdd9yhCRMmSJJaW1uVlpamzMzMqH1zc3PV2tra5+epqqpSIBCIbPn5+bGOBABIIjEHqLy8XHv37tWGDRsua4DKykp1dHREtgMHDlzW5wMAJIeYfhB12bJleu+991RfX6+RI0dGHg8Ggzp16pTa29ujzoLa2toUDAb7/Fx+v19+vz+WMQAASczTGZBzTsuWLdOmTZv04YcfqqCgIOr5yZMna8iQIaqpqYk81tjYqP3796u4uDg+EwMAUoKnM6Dy8nKtX79eW7ZsUXp6euT7OoFAQEOHDlUgENCiRYtUUVGhrKwsZWRk6JFHHlFxcTFXwAEAongK0Jo1ayRJ06ZNi3p87dq1WrhwoSTp5z//uQYNGqR58+apu7tbpaWleu211+IyLAAgdXgKkHPuovtcffXVqq6uVnV1dcxDAcmkubnZegQgKXEvOACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJiI6TeiAvifu+66y/OaS7mzPJDqOAMCAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAExwM1LgMk2cONHzmhtvvNHzmubm5n5ZI0kjRoyIaR3gBWdAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJbkYKGHjqqac8r1m0aFG/vI4k/eIXv/C85pZbbonptXDl4gwIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADDBzUgBA3PnzvW8ZsOGDZ7XbNu2zfMaSXr22Wc9r1m7dq3nNcOGDfO8BqmDMyAAgAkCBAAw4SlAVVVVuv3225Wenq6cnBzNmTNHjY2NUftMmzZNPp8vanv44YfjOjQAIPl5ClBdXZ3Ky8u1fft2bdu2TadPn9aMGTPU1dUVtd/ixYt1+PDhyLZq1aq4Dg0ASH6eLkLYunVr1Mfr1q1TTk6Odu3apalTp0Yev+aaaxQMBuMzIQAgJV3W94A6OjokSVlZWVGPv/nmm8rOztaECRNUWVmpEydOXPBzdHd3KxwOR20AgNQX82XYPT09Wr58ue644w5NmDAh8vgDDzyg0aNHKxQKac+ePXryySfV2Niod955p8/PU1VVpeeeey7WMQAASSrmAJWXl2vv3r36+OOPox5fsmRJ5M8TJ05UXl6epk+frubmZo0dO/a8z1NZWamKiorIx+FwWPn5+bGOBQBIEjEFaNmyZXrvvfdUX1+vkSNHfum+RUVFkqSmpqY+A+T3++X3+2MZAwCQxDwFyDmnRx55RJs2bVJtba0KCgouumb37t2SpLy8vJgGBACkJk8BKi8v1/r167Vlyxalp6ertbVVkhQIBDR06FA1Nzdr/fr1+uY3v6nhw4drz549euyxxzR16lRNmjQpIX8BAEBy8hSgNWvWSOr9YdP/t3btWi1cuFBpaWn64IMP9PLLL6urq0v5+fmaN2+enn766bgNDABIDZ7/Ce7L5Ofnq66u7rIGAgBcGXzuYlXpZ+FwWIFAQB0dHcrIyLAeBxgwYvkZuR/96EcxvdZrr73mec1f//pXz2tuueUWz2sw8F3q13FuRgoAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmOBmpACAuOJmpACAAY0AAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAICJq6wHONfZW9OFw2HjSQAAsTj79ftitxodcAHq7OyUJOXn5xtPAgC4HJ2dnQoEAhd8fsDdDbunp0eHDh1Senq6fD5f1HPhcFj5+fk6cODAFX2nbI5DL45DL45DL45Dr4FwHJxz6uzsVCgU0qBBF/5Oz4A7Axo0aJBGjhz5pftkZGRc0W+wszgOvTgOvTgOvTgOvayPw5ed+ZzFRQgAABMECABgIqkC5Pf7tXLlSvn9futRTHEcenEcenEcenEceiXTcRhwFyEAAK4MSXUGBABIHQQIAGCCAAEATBAgAICJpAlQdXW1rr/+el199dUqKirSn//8Z+uR+t2zzz4rn88XtY0fP956rISrr6/XrFmzFAqF5PP5tHnz5qjnnXN65plnlJeXp6FDh6qkpET79u2zGTaBLnYcFi5ceN77Y+bMmTbDJkhVVZVuv/12paenKycnR3PmzFFjY2PUPidPnlR5ebmGDx+ua6+9VvPmzVNbW5vRxIlxKcdh2rRp570fHn74YaOJ+5YUAdq4caMqKiq0cuVKffLJJyosLFRpaamOHDliPVq/u/XWW3X48OHI9vHHH1uPlHBdXV0qLCxUdXV1n8+vWrVKr7zyil5//XXt2LFDw4YNU2lpqU6ePNnPkybWxY6DJM2cOTPq/fHWW2/144SJV1dXp/Lycm3fvl3btm3T6dOnNWPGDHV1dUX2eeyxx/Tuu+/q7bffVl1dnQ4dOqS5c+caTh1/l3IcJGnx4sVR74dVq1YZTXwBLglMmTLFlZeXRz4+c+aMC4VCrqqqynCq/rdy5UpXWFhoPYYpSW7Tpk2Rj3t6elwwGHQvvfRS5LH29nbn9/vdW2+9ZTBh/zj3ODjn3IIFC9zs2bNN5rFy5MgRJ8nV1dU553r/2w8ZMsS9/fbbkX3+8Y9/OEmuoaHBasyEO/c4OOfc3Xff7R599FG7oS7BgD8DOnXqlHbt2qWSkpLIY4MGDVJJSYkaGhoMJ7Oxb98+hUIhjRkzRg8++KD2799vPZKplpYWtba2Rr0/AoGAioqKrsj3R21trXJycjRu3DgtXbpUx44dsx4poTo6OiRJWVlZkqRdu3bp9OnTUe+H8ePHa9SoUSn9fjj3OJz15ptvKjs7WxMmTFBlZaVOnDhhMd4FDbibkZ7r888/15kzZ5Sbmxv1eG5urv75z38aTWWjqKhI69at07hx43T48GE999xzuuuuu7R3716lp6dbj2eitbVVkvp8f5x97koxc+ZMzZ07VwUFBWpubtZTTz2lsrIyNTQ0aPDgwdbjxV1PT4+WL1+uO+64QxMmTJDU+35IS0tTZmZm1L6p/H7o6zhI0gMPPKDRo0crFAppz549evLJJ9XY2Kh33nnHcNpoAz5A+J+ysrLInydNmqSioiKNHj1av/vd77Ro0SLDyTAQzJ8/P/LniRMnatKkSRo7dqxqa2s1ffp0w8kSo7y8XHv37r0ivg/6ZS50HJYsWRL588SJE5WXl6fp06erublZY8eO7e8x+zTg/wkuOztbgwcPPu8qlra2NgWDQaOpBobMzEzddNNNampqsh7FzNn3AO+P840ZM0bZ2dkp+f5YtmyZ3nvvPX300UdRv74lGAzq1KlTam9vj9o/Vd8PFzoOfSkqKpKkAfV+GPABSktL0+TJk1VTUxN5rKenRzU1NSouLjaczN7x48fV3NysvLw861HMFBQUKBgMRr0/wuGwduzYccW/Pw4ePKhjx46l1PvDOadly5Zp06ZN+vDDD1VQUBD1/OTJkzVkyJCo90NjY6P279+fUu+Hix2HvuzevVuSBtb7wfoqiEuxYcMG5/f73bp169zf//53t2TJEpeZmelaW1utR+tXP/jBD1xtba1raWlxf/zjH11JSYnLzs52R44csR4toTo7O92nn37qPv30UyfJrV692n366afus88+c84599Of/tRlZma6LVu2uD179rjZs2e7goIC98UXXxhPHl9fdhw6Ozvd448/7hoaGlxLS4v74IMP3Ne+9jV34403upMnT1qPHjdLly51gUDA1dbWusOHD0e2EydORPZ5+OGH3ahRo9yHH37odu7c6YqLi11xcbHh1PF3sePQ1NTknn/+ebdz507X0tLitmzZ4saMGeOmTp1qPHm0pAiQc869+uqrbtSoUS4tLc1NmTLFbd++3Xqkfnffffe5vLw8l5aW5q677jp33333uaamJuuxEu6jjz5yks7bFixY4JzrvRR7xYoVLjc31/n9fjd9+nTX2NhoO3QCfNlxOHHihJsxY4YbMWKEGzJkiBs9erRbvHhxyv1PWl9/f0lu7dq1kX2++OIL9/3vf9995Stfcddcc42799573eHDh+2GToCLHYf9+/e7qVOnuqysLOf3+90NN9zgfvjDH7qOjg7bwc/Br2MAAJgY8N8DAgCkJgIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADAxH8BB0q1GdOY6GMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "digit = train_images[4]\n",
    "plt.imshow(digit, cmap=plt.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels[4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Manipulating tensors in NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100, :, :]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90, 28, 28)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_slice = train_images[10:100, 0:28, 0:28]\n",
    "my_slice.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_slice = train_images[:, 14:, 14:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "my_slice = train_images[:, 7:-7, 7:-7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### The notion of data batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "batch = train_images[:128]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "batch = train_images[128:256]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "n = 3\n",
    "batch = train_images[128 * n:128 * (n + 1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Real-world examples of data tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Vector data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Timeseries data or sequence data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Image data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Video data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## The gears of neural networks: tensor operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Element-wise operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_relu(x):\n",
    "    assert len(x.shape) == 2\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] = max(x[i, j], 0)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_add(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert x.shape == y.shape\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] += y[i, j]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took: 0.01 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "x = np.random.random((20, 100))\n",
    "y = np.random.random((20, 100))\n",
    "\n",
    "t0 = time.time()\n",
    "for _ in range(1000):\n",
    "    z = x + y\n",
    "    z = np.maximum(z, 0.)\n",
    "print(\"Took: {0:.2f} s\".format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took: 1.62 s\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "for _ in range(1000):\n",
    "    z = naive_add(x, y)\n",
    "    z = naive_relu(z)\n",
    "print(\"Took: {0:.2f} s\".format(time.time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Broadcasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "X = np.random.random((32, 10))\n",
    "y = np.random.random((10,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "y = np.expand_dims(y, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "Y = np.concatenate([y] * 32, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_add_matrix_and_vector(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    x = x.copy()\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            x[i, j] += y[j]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "x = np.random.random((64, 3, 32, 10))\n",
    "y = np.random.random((32, 10))\n",
    "z = np.maximum(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Tensor product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = np.random.random((32,))\n",
    "y = np.random.random((32,))\n",
    "z = np.dot(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_vector_dot(x, y):\n",
    "    assert len(x.shape) == 1\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[0] == y.shape[0]\n",
    "    z = 0.\n",
    "    for i in range(x.shape[0]):\n",
    "        z += x[i] * y[i]\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_vector_dot(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 1\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    z = np.zeros(x.shape[0])\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(x.shape[1]):\n",
    "            z[i] += x[i, j] * y[j]\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_vector_dot(x, y):\n",
    "    z = np.zeros(x.shape[0])\n",
    "    for i in range(x.shape[0]):\n",
    "        z[i] = naive_vector_dot(x[i, :], y)\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def naive_matrix_dot(x, y):\n",
    "    assert len(x.shape) == 2\n",
    "    assert len(y.shape) == 2\n",
    "    assert x.shape[1] == y.shape[0]\n",
    "    z = np.zeros((x.shape[0], y.shape[1]))\n",
    "    for i in range(x.shape[0]):\n",
    "        for j in range(y.shape[1]):\n",
    "            row_x = x[i, :]\n",
    "            column_y = y[:, j]\n",
    "            z[i, j] = naive_vector_dot(row_x, column_y)\n",
    "    return z"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Tensor reshaping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "train_images = train_images.reshape((60000, 28 * 28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 2)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[0., 1.],\n",
    "             [2., 3.],\n",
    "             [4., 5.]])\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.],\n",
       "       [1.],\n",
       "       [2.],\n",
       "       [3.],\n",
       "       [4.],\n",
       "       [5.]])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = x.reshape((6, 1))\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 300)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.zeros((300, 20))\n",
    "x = np.transpose(x)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Geometric interpretation of tensor operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### A geometric interpretation of deep learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## The engine of neural networks: gradient-based optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### What's a derivative?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Derivative of a tensor operation: the gradient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Stochastic gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Chaining derivatives: The Backpropagation algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The chain rule"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### Automatic differentiation with computation graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### The gradient tape in TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "x = tf.Variable(0.)\n",
    "with tf.GradientTape() as tape:\n",
    "    y = 2 * x + 3\n",
    "grad_of_y_wrt_x = tape.gradient(y, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "x = tf.Variable(tf.random.uniform((2, 2)))\n",
    "with tf.GradientTape() as tape:\n",
    "    y = 2 * x + 3\n",
    "grad_of_y_wrt_x = tape.gradient(y, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "W = tf.Variable(tf.random.uniform((2, 2)))\n",
    "b = tf.Variable(tf.zeros((2,)))\n",
    "x = tf.random.uniform((2, 2))\n",
    "with tf.GradientTape() as tape:\n",
    "    y = tf.matmul(x, W) + b\n",
    "grad_of_y_wrt_W_and_b = tape.gradient(y, [W, b])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Looking back at our first example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = keras.Sequential([\n",
    "    layers.Dense(512, activation=\"relu\"),\n",
    "    layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"rmsprop\",\n",
    "              loss=\"sparse_categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.2563 - accuracy: 0.9258\n",
      "Epoch 2/5\n",
      "469/469 [==============================] - 2s 5ms/step - loss: 0.1040 - accuracy: 0.9695\n",
      "Epoch 3/5\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0690 - accuracy: 0.9794\n",
      "Epoch 4/5\n",
      "469/469 [==============================] - 2s 4ms/step - loss: 0.0505 - accuracy: 0.9845\n",
      "Epoch 5/5\n",
      "469/469 [==============================] - 2s 5ms/step - loss: 0.0377 - accuracy: 0.9884\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x23d512ab208>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_images, train_labels, epochs=5, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Reimplementing our first example from scratch in TensorFlow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A simple Dense class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "class NaiveDense:\n",
    "    def __init__(self, input_size, output_size, activation):\n",
    "        self.activation = activation\n",
    "\n",
    "        w_shape = (input_size, output_size)\n",
    "        w_initial_value = tf.random.uniform(w_shape, minval=0, maxval=1e-1)\n",
    "        self.W = tf.Variable(w_initial_value)\n",
    "\n",
    "        b_shape = (output_size,)\n",
    "        b_initial_value = tf.zeros(b_shape)\n",
    "        self.b = tf.Variable(b_initial_value)\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        return self.activation(tf.matmul(inputs, self.W) + self.b)\n",
    "\n",
    "    @property\n",
    "    def weights(self):\n",
    "        return [self.W, self.b]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A simple Sequential class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "class NaiveSequential:\n",
    "    def __init__(self, layers):\n",
    "        self.layers = layers\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        x = inputs\n",
    "        for layer in self.layers:\n",
    "           x = layer(x)\n",
    "        return x\n",
    "\n",
    "    @property\n",
    "    def weights(self):\n",
    "       weights = []\n",
    "       for layer in self.layers:\n",
    "           weights += layer.weights\n",
    "       return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model = NaiveSequential([\n",
    "    NaiveDense(input_size=28 * 28, output_size=512, activation=tf.nn.relu),\n",
    "    NaiveDense(input_size=512, output_size=10, activation=tf.nn.softmax)\n",
    "])\n",
    "assert len(model.weights) == 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "#### A batch generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class BatchGenerator:\n",
    "    def __init__(self, images, labels, batch_size=128):\n",
    "        assert len(images) == len(labels)\n",
    "        self.index = 0\n",
    "        self.images = images\n",
    "        self.labels = labels\n",
    "        self.batch_size = batch_size\n",
    "        self.num_batches = math.ceil(len(images) / batch_size)\n",
    "\n",
    "    def next(self):\n",
    "        images = self.images[self.index : self.index + self.batch_size]\n",
    "        labels = self.labels[self.index : self.index + self.batch_size]\n",
    "        self.index += self.batch_size\n",
    "        return images, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Running one training step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def one_training_step(model, images_batch, labels_batch):\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions = model(images_batch)\n",
    "        per_sample_losses = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "            labels_batch, predictions)\n",
    "        average_loss = tf.reduce_mean(per_sample_losses)\n",
    "    gradients = tape.gradient(average_loss, model.weights)\n",
    "    update_weights(gradients, model.weights)\n",
    "    return average_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "learning_rate = 1e-3\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    for g, w in zip(gradients, weights):\n",
    "        w.assign_sub(g * learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import optimizers\n",
    "\n",
    "optimizer = optimizers.SGD(learning_rate=1e-3)\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    optimizer.apply_gradients(zip(gradients, weights))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### The full training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def fit(model, images, labels, epochs, batch_size=128):\n",
    "    for epoch_counter in range(epochs):\n",
    "        print(f\"Epoch {epoch_counter}\")\n",
    "        batch_generator = BatchGenerator(images, labels)\n",
    "        for batch_counter in range(batch_generator.num_batches):\n",
    "            images_batch, labels_batch = batch_generator.next()\n",
    "            loss = one_training_step(model, images_batch, labels_batch)\n",
    "            if batch_counter % 100 == 0:\n",
    "                print(f\"loss at batch {batch_counter}: {loss:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n",
      "loss at batch 0: 4.78\n",
      "loss at batch 100: 2.24\n",
      "loss at batch 200: 2.20\n",
      "loss at batch 300: 2.07\n",
      "loss at batch 400: 2.20\n",
      "Epoch 1\n",
      "loss at batch 0: 1.94\n",
      "loss at batch 100: 1.89\n",
      "loss at batch 200: 1.83\n",
      "loss at batch 300: 1.70\n",
      "loss at batch 400: 1.82\n",
      "Epoch 2\n",
      "loss at batch 0: 1.62\n",
      "loss at batch 100: 1.60\n",
      "loss at batch 200: 1.52\n",
      "loss at batch 300: 1.43\n",
      "loss at batch 400: 1.50\n",
      "Epoch 3\n",
      "loss at batch 0: 1.37\n",
      "loss at batch 100: 1.36\n",
      "loss at batch 200: 1.26\n",
      "loss at batch 300: 1.22\n",
      "loss at batch 400: 1.27\n",
      "Epoch 4\n",
      "loss at batch 0: 1.17\n",
      "loss at batch 100: 1.18\n",
      "loss at batch 200: 1.06\n",
      "loss at batch 300: 1.06\n",
      "loss at batch 400: 1.10\n",
      "Epoch 5\n",
      "loss at batch 0: 1.02\n",
      "loss at batch 100: 1.04\n",
      "loss at batch 200: 0.92\n",
      "loss at batch 300: 0.94\n",
      "loss at batch 400: 0.98\n",
      "Epoch 6\n",
      "loss at batch 0: 0.91\n",
      "loss at batch 100: 0.93\n",
      "loss at batch 200: 0.81\n",
      "loss at batch 300: 0.85\n",
      "loss at batch 400: 0.90\n",
      "Epoch 7\n",
      "loss at batch 0: 0.82\n",
      "loss at batch 100: 0.84\n",
      "loss at batch 200: 0.73\n",
      "loss at batch 300: 0.78\n",
      "loss at batch 400: 0.83\n",
      "Epoch 8\n",
      "loss at batch 0: 0.76\n",
      "loss at batch 100: 0.78\n",
      "loss at batch 200: 0.67\n",
      "loss at batch 300: 0.73\n",
      "loss at batch 400: 0.77\n",
      "Epoch 9\n",
      "loss at batch 0: 0.70\n",
      "loss at batch 100: 0.72\n",
      "loss at batch 200: 0.62\n",
      "loss at batch 300: 0.68\n",
      "loss at batch 400: 0.73\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255\n",
    "\n",
    "fit(model, train_images, train_labels, epochs=10, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Evaluating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.82\n"
     ]
    }
   ],
   "source": [
    "predictions = model(test_images)\n",
    "predictions = predictions.numpy()\n",
    "predicted_labels = np.argmax(predictions, axis=1)\n",
    "matches = predicted_labels == test_labels\n",
    "print(f\"accuracy: {matches.mean():.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Summary"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "chapter02_mathematical-building-blocks.i",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "vscode": {
   "interpreter": {
    "hash": "b09ec625f77bf4fd762565a912b97636504ad6ec901eb2d0f4cf5a7de23e1ee5"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}