3a_MLP_MNIST_160516.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\* *[Notice] I wrote thie code while following the examples in [Choi's Tesorflow-101 tutorial](https://github.com/sjchoi86/Tensorflow-101). And,  as I know, most of Choi's examples originally come from [Aymeric Damien's](https://github.com/aymericdamien/TensorFlow-Examples/) and  [Nathan Lintz's ](https://github.com/nlintz/TensorFlow-Tutorials) tutorials.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Multilayer Perceptron with MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "#%matplotlib inline  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist      = input_data.read_data_sets('data', one_hot=True)\n",
    "X_train   = mnist.train.images\n",
    "Y_train = mnist.train.labels\n",
    "X_test    = mnist.test.images\n",
    "Y_test  = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of (X_train, X_test, Y_train, Y_test)\n",
      "((55000, 784), (10000, 784), (55000, 10), (10000, 10))\n"
     ]
    }
   ],
   "source": [
    "dimX = X_train.shape[1]\n",
    "dimY = Y_train.shape[1]\n",
    "nTrain = X_train.shape[0]\n",
    "nTest = X_test.shape[0]\n",
    "print (\"Shape of (X_train, X_test, Y_train, Y_test)\")\n",
    "print (X_train.shape, X_test.shape, Y_train.shape, Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define my neural network structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nLayer0 = dimX\n",
    "nLayer1 = 256\n",
    "nLayer2 = 256\n",
    "nLayer3 =  dimY\n",
    "sigma_init = 0.1   # For randomized initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W = {\n",
    "    'W1': tf.Variable(tf.random_normal([nLayer0, nLayer1], stddev = sigma_init)),\n",
    "    'W2': tf.Variable(tf.random_normal([nLayer1, nLayer2], stddev = sigma_init)),\n",
    "    'W3': tf.Variable(tf.random_normal([nLayer2, nLayer3], stddev = sigma_init))\n",
    "}\n",
    "b = {\n",
    "    'b1': tf.Variable(tf.random_normal([nLayer1])),\n",
    "    'b2': tf.Variable(tf.random_normal([nLayer2])),\n",
    "    'b3': tf.Variable(tf.random_normal([nLayer3]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_myNN(_X, _W, _b):\n",
    "    Layer1 = tf.nn.sigmoid(tf.add(tf.matmul(_X,_W['W1']), _b['b1']))\n",
    "    Layer2 = tf.nn.sigmoid(tf.add(tf.matmul(Layer1,_W['W2']), _b['b2']))\n",
    "    Layer3 = tf.add(tf.matmul(Layer2,_W['W3']), _b['b3'])    \n",
    "    #Layer3 = tf.nn.sigmoid(tf.add(tf.matmul(Layer2,_W['W3']), _b['b3']))\n",
    "    return Layer3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, [None, dimX], name=\"input\")\n",
    "Y= tf.placeholder(tf.float32, [None, dimY], name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Y_pred = model_myNN(X, W, b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Define loss function, optimizer, measurer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use *softmax_cross_entropy()* and *AdamOptimizer()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(Y_pred, Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "#optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "training_epochs = 30\n",
    "display_epoch = 5\n",
    "batch_size = 100   # For each time, we will use 100 samples to update parameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(Y_pred, 1), tf.argmax(Y, 1))    \n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "We use *with* for load a TF session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(epoch 5)\n",
      "[Loss / Tranining Accuracy] 0.0764 / 0.9778\n",
      " \n",
      "(epoch 10)\n",
      "[Loss / Tranining Accuracy] 0.0241 / 0.9935\n",
      " \n",
      "(epoch 15)\n",
      "[Loss / Tranining Accuracy] 0.0087 / 0.9980\n",
      " \n",
      "(epoch 20)\n",
      "[Loss / Tranining Accuracy] 0.0030 / 0.9996\n",
      " \n",
      "(epoch 25)\n",
      "[Loss / Tranining Accuracy] 0.0011 / 0.9998\n",
      " \n",
      "(epoch 30)\n",
      "[Loss / Tranining Accuracy] 0.0002 / 1.0000\n",
      " \n",
      "(epoch 35)\n",
      "[Loss / Tranining Accuracy] 0.0001 / 1.0000\n",
      " \n",
      "(epoch 40)\n",
      "[Loss / Tranining Accuracy] 0.0001 / 1.0000\n",
      " \n",
      "(epoch 45)\n",
      "[Loss / Tranining Accuracy] 0.0000 / 1.0000\n",
      " \n",
      "(epoch 50)\n",
      "[Loss / Tranining Accuracy] 0.0002 / 1.0000\n",
      " \n",
      "[Test Accuracy]  0.9799\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.initialize_all_variables())\n",
    "\n",
    "    for epoch in range(training_epochs):\n",
    "        nBatch  = int(nTrain/batch_size)\n",
    "        myIdx =  np.random.permutation(nTrain)\n",
    "        for ii in range(nBatch):\n",
    "            X_batch = X_train[myIdx[ii*batch_size:(ii+1)*batch_size],:]\n",
    "            Y_batch = Y_train[myIdx[ii*batch_size:(ii+1)*batch_size],:]\n",
    "            #print X_batch.shape, Y_batch.shape\n",
    "            sess.run(optimizer, feed_dict={X:X_batch, Y:Y_batch})\n",
    "          \n",
    "        if (epoch+1) % display_epoch == 0:\n",
    "            loss_temp = sess.run(loss, feed_dict={X: X_train, Y:Y_train}) \n",
    "            accuracy_temp = accuracy.eval({X: X_train, Y:Y_train})\n",
    "            print \"(epoch {})\".format(epoch+1) \n",
    "            print \"[Loss / Tranining Accuracy] {:05.4f} / {:05.4f}\".format(loss_temp, accuracy_temp)\n",
    "            print \" \"\n",
    "            \n",
    "    print \"[Test Accuracy] \",  accuracy.eval({X: X_test, Y: Y_test})   "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}