1_LinearRegression_160516.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\* *[Notice] I wrote thie code while following the examples in [Choi's Tesorflow-101 tutorial](https://github.com/sjchoi86/Tensorflow-101). And,  as I know, most of Choi's examples originally come from [Aymeric Damien's](https://github.com/aymericdamien/TensorFlow-Examples/) and  [Nathan Lintz's ](https://github.com/nlintz/TensorFlow-Tutorials) tutorials.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "#%matplotlib inline  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set initial data\n",
    "\n",
    "My training data :  $y = 0.5x + 0.1 + \\sigma(0,0.1)$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W_ref  = 0.5\n",
    "b_ref = 0.1\n",
    "nData = 51\n",
    "noise_mu = 0\n",
    "noise_std = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = np.linspace(-2,2,nData)\n",
    "Y_test = W_ref * X_train + b_ref\n",
    "Y_train = Y_test + np.random.normal(noise_mu, noise_std, nData)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot the data using *matplotlib*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(1)\n",
    "plt.plot(X_train, Y_test, 'ro', label='True data')\n",
    "plt.plot(X_train, Y_train, 'bo', label='Training data')\n",
    "plt.axis('equal')\n",
    "plt.legend(loc='lower right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Write a TF graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, name=\"input\")\n",
    "Y= tf.placeholder(tf.float32, name=\"output\")\n",
    "W = tf.Variable(np.random.randn(), name=\"weight\")\n",
    "b = tf.Variable(np.random.randn(), name=\"bias\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_pred = tf.add(tf.mul(X, W), b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use a L2 loss function,  $loss = -\\Sigma (y'-y)^2$\n",
    "\n",
    "*reduce_mean(X)* returns the mean value for all elements of the tensor *X*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.square(Y-Y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate  = 0.005\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "training_epochs = 50  # We will repeat the learning process 2000 times\n",
    "display_epoch    = 5  # We will print the error at every 200 epochs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(epoch 5)\n",
      "[W, b / loss] 0.4960, 0.0644 / 0.0142\n",
      " \n",
      "(epoch 10)\n",
      "[W, b / loss] 0.5017, 0.1158 / 0.0113\n",
      " \n",
      "(epoch 15)\n",
      "[W, b / loss] 0.5021, 0.1195 / 0.0113\n",
      " \n",
      "(epoch 20)\n",
      "[W, b / loss] 0.5021, 0.1198 / 0.0113\n",
      " \n",
      "(epoch 25)\n",
      "[W, b / loss] 0.5021, 0.1198 / 0.0113\n",
      " \n",
      "(epoch 30)\n",
      "[W, b / loss] 0.5021, 0.1198 / 0.0113\n",
      " \n",
      "(epoch 35)\n",
      "[W, b / loss] 0.5021, 0.1198 / 0.0113\n",
      " \n",
      "(epoch 40)\n",
      "[W, b / loss] 0.5021, 0.1198 / 0.0113\n",
      " \n",
      "(epoch 45)\n",
      "[W, b / loss] 0.5021, 0.1198 / 0.0113\n",
      " \n",
      "(epoch 50)\n",
      "[W, b / loss] 0.5021, 0.1198 / 0.0113\n",
      " \n",
      "[Final: W, b] 0.5021, 0.1198\n",
      "[Final: W, b] 0.5000, 0.1000\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(training_epochs):\n",
    "    for (x,y) in zip(X_train, Y_train):\n",
    "        sess.run(optimizer, feed_dict={X:x, Y:y})\n",
    "        \n",
    "    # Print the result\n",
    "    if (epoch+1) % display_epoch == 0:\n",
    "        W_temp = sess.run(W)\n",
    "        b_temp = sess.run(b)\n",
    "        loss_temp = sess.run(loss, feed_dict={X: X_train, Y:Y_train})  \n",
    "        print \"(epoch {})\".format(epoch+1) \n",
    "        print \"[W, b / loss] {:05.4f}, {:05.4f} / {:05.4f}\".format(W_temp, b_temp, loss_temp) \n",
    "        print \" \"\n",
    "\n",
    "# Final results        \n",
    "W_result = sess.run(W)\n",
    "b_result = sess.run(b)            \n",
    "print \"[Final: W, b] {:05.4f}, {:05.4f}\".format(W_result, b_result)\n",
    "print \"[Final: W, b] {:05.4f}, {:05.4f}\".format(W_ref, b_ref)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.figure(2)\n",
    "plt.plot(X_train, Y_test, 'ro', label='True data')\n",
    "plt.plot(X_train, Y_train, 'bo', label='Training data')\n",
    "plt.plot(X_train, W_result*X_train+b_result, 'g-', linewidth=3, label='Regression result')\n",
    "plt.axis('equal')\n",
    "plt.legend(loc='lower right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}