008 - Epochs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./datasets/ud730/mnist/train-images-idx3-ubyte.gz\n",
      "Extracting ./datasets/ud730/mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting ./datasets/ud730/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./datasets/ud730/mnist/t10k-labels-idx1-ubyte.gz\n",
      "Epoch: 0    - Cost: 1.36     Valid Accuracy: 0.816\n",
      "Epoch: 1    - Cost: 1.06     Valid Accuracy: 0.856\n",
      "Epoch: 2    - Cost: 0.92     Valid Accuracy: 0.871\n",
      "Epoch: 3    - Cost: 0.831    Valid Accuracy: 0.881\n",
      "Epoch: 4    - Cost: 0.767    Valid Accuracy: 0.885\n",
      "Epoch: 5    - Cost: 0.716    Valid Accuracy: 0.889\n",
      "Epoch: 6    - Cost: 0.674    Valid Accuracy: 0.892\n",
      "Epoch: 7    - Cost: 0.637    Valid Accuracy: 0.894\n",
      "Epoch: 8    - Cost: 0.606    Valid Accuracy: 0.898\n",
      "Epoch: 9    - Cost: 0.578    Valid Accuracy: 0.899\n",
      "Epoch: 10   - Cost: 0.554    Valid Accuracy: 0.9  \n",
      "Epoch: 11   - Cost: 0.532    Valid Accuracy: 0.901\n",
      "Epoch: 12   - Cost: 0.513    Valid Accuracy: 0.901\n",
      "Epoch: 13   - Cost: 0.495    Valid Accuracy: 0.902\n",
      "Epoch: 14   - Cost: 0.48     Valid Accuracy: 0.904\n",
      "Epoch: 15   - Cost: 0.466    Valid Accuracy: 0.903\n",
      "Epoch: 16   - Cost: 0.453    Valid Accuracy: 0.905\n",
      "Epoch: 17   - Cost: 0.441    Valid Accuracy: 0.905\n",
      "Epoch: 18   - Cost: 0.43     Valid Accuracy: 0.905\n",
      "Epoch: 19   - Cost: 0.421    Valid Accuracy: 0.906\n",
      "Epoch: 20   - Cost: 0.412    Valid Accuracy: 0.906\n",
      "Epoch: 21   - Cost: 0.403    Valid Accuracy: 0.907\n",
      "Epoch: 22   - Cost: 0.396    Valid Accuracy: 0.908\n",
      "Epoch: 23   - Cost: 0.389    Valid Accuracy: 0.908\n",
      "Epoch: 24   - Cost: 0.382    Valid Accuracy: 0.908\n",
      "Epoch: 25   - Cost: 0.376    Valid Accuracy: 0.909\n",
      "Epoch: 26   - Cost: 0.37     Valid Accuracy: 0.909\n",
      "Epoch: 27   - Cost: 0.365    Valid Accuracy: 0.91 \n",
      "Epoch: 28   - Cost: 0.359    Valid Accuracy: 0.91 \n",
      "Epoch: 29   - Cost: 0.355    Valid Accuracy: 0.91 \n",
      "Epoch: 30   - Cost: 0.35     Valid Accuracy: 0.91 \n",
      "Epoch: 31   - Cost: 0.346    Valid Accuracy: 0.911\n",
      "Epoch: 32   - Cost: 0.342    Valid Accuracy: 0.911\n",
      "Epoch: 33   - Cost: 0.338    Valid Accuracy: 0.912\n",
      "Epoch: 34   - Cost: 0.335    Valid Accuracy: 0.912\n",
      "Epoch: 35   - Cost: 0.331    Valid Accuracy: 0.912\n",
      "Epoch: 36   - Cost: 0.328    Valid Accuracy: 0.913\n",
      "Epoch: 37   - Cost: 0.325    Valid Accuracy: 0.913\n",
      "Epoch: 38   - Cost: 0.322    Valid Accuracy: 0.913\n",
      "Epoch: 39   - Cost: 0.319    Valid Accuracy: 0.913\n",
      "Epoch: 40   - Cost: 0.316    Valid Accuracy: 0.913\n",
      "Epoch: 41   - Cost: 0.314    Valid Accuracy: 0.913\n",
      "Epoch: 42   - Cost: 0.311    Valid Accuracy: 0.914\n",
      "Epoch: 43   - Cost: 0.309    Valid Accuracy: 0.914\n",
      "Epoch: 44   - Cost: 0.307    Valid Accuracy: 0.914\n",
      "Epoch: 45   - Cost: 0.305    Valid Accuracy: 0.914\n",
      "Epoch: 46   - Cost: 0.302    Valid Accuracy: 0.914\n",
      "Epoch: 47   - Cost: 0.3      Valid Accuracy: 0.915\n",
      "Epoch: 48   - Cost: 0.298    Valid Accuracy: 0.916\n",
      "Epoch: 49   - Cost: 0.297    Valid Accuracy: 0.916\n",
      "Epoch: 50   - Cost: 0.295    Valid Accuracy: 0.917\n",
      "Epoch: 51   - Cost: 0.293    Valid Accuracy: 0.917\n",
      "Epoch: 52   - Cost: 0.291    Valid Accuracy: 0.917\n",
      "Epoch: 53   - Cost: 0.29     Valid Accuracy: 0.917\n",
      "Epoch: 54   - Cost: 0.288    Valid Accuracy: 0.917\n",
      "Epoch: 55   - Cost: 0.286    Valid Accuracy: 0.917\n",
      "Epoch: 56   - Cost: 0.285    Valid Accuracy: 0.917\n",
      "Epoch: 57   - Cost: 0.283    Valid Accuracy: 0.917\n",
      "Epoch: 58   - Cost: 0.282    Valid Accuracy: 0.917\n",
      "Epoch: 59   - Cost: 0.281    Valid Accuracy: 0.918\n",
      "Epoch: 60   - Cost: 0.279    Valid Accuracy: 0.918\n",
      "Epoch: 61   - Cost: 0.278    Valid Accuracy: 0.918\n",
      "Epoch: 62   - Cost: 0.277    Valid Accuracy: 0.918\n",
      "Epoch: 63   - Cost: 0.275    Valid Accuracy: 0.919\n",
      "Epoch: 64   - Cost: 0.274    Valid Accuracy: 0.919\n",
      "Epoch: 65   - Cost: 0.273    Valid Accuracy: 0.919\n",
      "Epoch: 66   - Cost: 0.272    Valid Accuracy: 0.919\n",
      "Epoch: 67   - Cost: 0.271    Valid Accuracy: 0.919\n",
      "Epoch: 68   - Cost: 0.269    Valid Accuracy: 0.919\n",
      "Epoch: 69   - Cost: 0.268    Valid Accuracy: 0.92 \n",
      "Epoch: 70   - Cost: 0.267    Valid Accuracy: 0.92 \n",
      "Epoch: 71   - Cost: 0.266    Valid Accuracy: 0.921\n",
      "Epoch: 72   - Cost: 0.265    Valid Accuracy: 0.921\n",
      "Epoch: 73   - Cost: 0.264    Valid Accuracy: 0.921\n",
      "Epoch: 74   - Cost: 0.263    Valid Accuracy: 0.921\n",
      "Epoch: 75   - Cost: 0.262    Valid Accuracy: 0.921\n",
      "Epoch: 76   - Cost: 0.261    Valid Accuracy: 0.921\n",
      "Epoch: 77   - Cost: 0.26     Valid Accuracy: 0.921\n",
      "Epoch: 78   - Cost: 0.259    Valid Accuracy: 0.921\n",
      "Epoch: 79   - Cost: 0.258    Valid Accuracy: 0.921\n",
      "Epoch: 80   - Cost: 0.257    Valid Accuracy: 0.921\n",
      "Epoch: 81   - Cost: 0.256    Valid Accuracy: 0.921\n",
      "Epoch: 82   - Cost: 0.256    Valid Accuracy: 0.921\n",
      "Epoch: 83   - Cost: 0.255    Valid Accuracy: 0.921\n",
      "Epoch: 84   - Cost: 0.254    Valid Accuracy: 0.921\n",
      "Epoch: 85   - Cost: 0.253    Valid Accuracy: 0.921\n",
      "Epoch: 86   - Cost: 0.252    Valid Accuracy: 0.921\n",
      "Epoch: 87   - Cost: 0.251    Valid Accuracy: 0.921\n",
      "Epoch: 88   - Cost: 0.251    Valid Accuracy: 0.921\n",
      "Epoch: 89   - Cost: 0.25     Valid Accuracy: 0.921\n",
      "Epoch: 90   - Cost: 0.249    Valid Accuracy: 0.921\n",
      "Epoch: 91   - Cost: 0.248    Valid Accuracy: 0.921\n",
      "Epoch: 92   - Cost: 0.248    Valid Accuracy: 0.922\n",
      "Epoch: 93   - Cost: 0.247    Valid Accuracy: 0.921\n",
      "Epoch: 94   - Cost: 0.246    Valid Accuracy: 0.922\n",
      "Epoch: 95   - Cost: 0.245    Valid Accuracy: 0.922\n",
      "Epoch: 96   - Cost: 0.245    Valid Accuracy: 0.922\n",
      "Epoch: 97   - Cost: 0.244    Valid Accuracy: 0.922\n",
      "Epoch: 98   - Cost: 0.243    Valid Accuracy: 0.922\n",
      "Epoch: 99   - Cost: 0.243    Valid Accuracy: 0.922\n",
      "Test Accuracy: 0.9171000123023987\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "def batches(batch_size, features, labels):\n",
    "    \"\"\"\n",
    "    Create batches of features and labels\n",
    "    :param batch_size: The batch size\n",
    "    :param features: List of features\n",
    "    :param labels: List of labels\n",
    "    :return: Batches of (Features, Labels)\n",
    "    \"\"\"\n",
    "    assert len(features) == len(labels)\n",
    "    output_batches = []\n",
    "    \n",
    "    sample_size = len(features)\n",
    "    for start_i in range(0, sample_size, batch_size):\n",
    "        end_i = start_i + batch_size\n",
    "        batch = [features[start_i:end_i], labels[start_i:end_i]]\n",
    "        output_batches.append(batch)\n",
    "        \n",
    "    return output_batches\n",
    "\n",
    "\n",
    "def print_epoch_stats(epoch_i, sess, last_features, last_labels):\n",
    "    \"\"\"\n",
    "    Print cost and validation accuracy of an epoch\n",
    "    \"\"\"\n",
    "    current_cost = sess.run(\n",
    "        cost,\n",
    "        feed_dict={features: last_features, labels: last_labels})\n",
    "    valid_accuracy = sess.run(\n",
    "        accuracy,\n",
    "        feed_dict={features: valid_features, labels: valid_labels})\n",
    "    print('Epoch: {:<4} - Cost: {:<8.3} Valid Accuracy: {:<5.3}'.format(\n",
    "        epoch_i,\n",
    "        current_cost,\n",
    "        valid_accuracy))\n",
    "\n",
    "n_input = 784  # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10  # MNIST total classes (0-9 digits)\n",
    "\n",
    "# Import MNIST data\n",
    "mnist = input_data.read_data_sets('./datasets/ud730/mnist', one_hot=True)\n",
    "\n",
    "# The features are already scaled and the data is shuffled\n",
    "train_features = mnist.train.images\n",
    "valid_features = mnist.validation.images\n",
    "test_features = mnist.test.images\n",
    "\n",
    "train_labels = mnist.train.labels.astype(np.float32)\n",
    "valid_labels = mnist.validation.labels.astype(np.float32)\n",
    "test_labels = mnist.test.labels.astype(np.float32)\n",
    "\n",
    "# Features and Labels\n",
    "features = tf.placeholder(tf.float32, [None, n_input])\n",
    "labels = tf.placeholder(tf.float32, [None, n_classes])\n",
    "\n",
    "# Weights & bias\n",
    "weights = tf.Variable(tf.random_normal([n_input, n_classes]))\n",
    "bias = tf.Variable(tf.random_normal([n_classes]))\n",
    "\n",
    "# Logits - xW + b\n",
    "logits = tf.add(tf.matmul(features, weights), bias)\n",
    "\n",
    "# Define loss and optimizer\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "# Calculate accuracy\n",
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "batch_size = 128\n",
    "epochs = 100\n",
    "learn_rate = 0.25\n",
    "\n",
    "train_batches = batches(batch_size, train_features, train_labels)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    # Training cycle\n",
    "    for epoch_i in range(epochs):\n",
    "\n",
    "        # Loop over all batches\n",
    "        for batch_features, batch_labels in train_batches:\n",
    "            train_feed_dict = {\n",
    "                features: batch_features,\n",
    "                labels: batch_labels,\n",
    "                learning_rate: learn_rate}\n",
    "            sess.run(optimizer, feed_dict=train_feed_dict)\n",
    "\n",
    "        # Print cost and validation accuracy of an epoch\n",
    "        print_epoch_stats(epoch_i, sess, batch_features, batch_labels)\n",
    "\n",
    "    # Calculate accuracy for test dataset\n",
    "    test_accuracy = sess.run(\n",
    "        accuracy,\n",
    "        feed_dict={features: test_features, labels: test_labels})\n",
    "\n",
    "print('Test Accuracy: {}'.format(test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}