Lesson 11 - Convolutional Neural Nets - Solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Nets\n",
    "\n",
    "This type of neural nets are predominantly (and heavily) used in image processing.\n",
    "https://www.youtube.com/watch?v=BcEapJEKz3M\n",
    "\n",
    "## Useful terms:\n",
    "\n",
    "1. Convolution\n",
    "2. Max pooling\n",
    "2. Softmax\n",
    "3. Cross Entropy\n",
    "\n",
    "## Further Readings:\n",
    "https://ujjwalkarn.me/2016/08/11/intuitive-explanation-convnets/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Conv2D, MaxPool2D, Flatten\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "mnist = input_data.read_data_sets('./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x, y = mnist.train.next_batch(20)\n",
    "num_pixels = x.shape[1]\n",
    "width = 28\n",
    "height = 28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 784)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "784"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "28*28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8, 8, 8, 0, 7, 1, 4, 8, 7, 8, 4, 9, 1, 0, 5, 0, 5, 7, 7, 4], dtype=uint8)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADwFJREFUeJzt3X+MFHWax/HPI4pR3IDIMCKLN2swZ4jJYdJBE8i5/tjV\nNSJuomb9g3AGmU1EcxqjAhc9/MMETllFvayZPRE06KJZCfxhzgA5oiTnhoag/PA80IxBAjLGjbhB\n5ZDn/phiM+j0t4bu6q6G5/1KJt1dT9XUYzsfqru/XfU1dxeAeM4ouwEA5SD8QFCEHwiK8ANBEX4g\nKMIPBEX4gaAIPxAU4QeCOrOVOxszZox3dXW1cpdAKL29vfriiy9sKOs2FH4zu1HSUknDJP2Huy9K\nrd/V1aVqtdrILgEkVCqVIa9b98t+Mxsm6d8l/UrSJEl3mtmken8fgNZq5D3/FEl73P0Tdz8i6Y+S\nZhTTFoBmayT84yXtHfD4s2zZCcys28yqZlbt6+trYHcAitT0T/vdvcfdK+5e6ejoaPbuAAxRI+Hf\nJ2nCgMc/zZYBOAU0Ev7Nki41s5+Z2XBJv5G0tpi2ADRb3UN97n7UzO6V9Lb6h/qWufvOwjoD0FQN\njfO7+1uS3iqoFwAtxNd7gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQ\nhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxA\nUIQfCKqhWXrNrFfS15K+l3TU3StFNAWg+RoKf+Yad/+igN8DoIV42Q8E1Wj4XdJ6M9tiZt1FNASg\nNRp92T/N3feZ2VhJ68zsf9z9nYErZP8odEvSxRdf3ODuABSloSO/u+/Lbg9KWi1pyiDr9Lh7xd0r\nHR0djewOQIHqDr+ZjTCznxy/L+mXknYU1RiA5mrkZX+npNVmdvz3vOru/1lIVwCaru7wu/snkv6h\nwF5wGtq7d2/N2pEjR1rYyYneeOONZH3kyJHJ+s0335ysT5gw4aR7ajWG+oCgCD8QFOEHgiL8QFCE\nHwiK8ANBFXFWH9rY0aNHk/WdO3cm6++//36y/vTTTyfrH330Uc3ad999l9x22LBhyfqYMWOSdXev\nWcu+n1K3Rx99NFmfPn16sv7SSy81tP8icOQHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAY528DX331\nVbK+e/fuZP3VV1+tWdu1a1dy23Xr1iXreaZNm5asP/nkkzVrV111VXLbESNGJOuXXXZZst5MS5Ys\nSdYffvjhZJ1xfgClIfxAUIQfCIrwA0ERfiAowg8ERfiBoBjnH6JvvvmmZu3tt99Obrty5cpkff36\n9cl63vcAOjs7a9auvfba5LbLly9P1m+44YZkffTo0cn6mWeemn9ihw4dStY3bdqUrD/22GNFttMU\nHPmBoAg/EBThB4Ii/EBQhB8IivADQRF+IKjcQVgzWybpZkkH3f3ybNloSaskdUnqlXSHu/+leW02\n30MPPZSsr127tmZtz549RbdzgosuuihZf/zxx2vWZs+eXXQ7p4W8+Qzyrruf9/9k3rx5J91Tqw3l\nyL9c0o0/WDZP0gZ3v1TShuwxgFNIbvjd/R1JX/5g8QxJK7L7KyTdWnBfAJqs3vf8ne6+P7t/QFLt\n75cCaEsNf+Dn/ROi1ZwUzcy6zaxqZtW+vr5GdwegIPWG/3MzGydJ2e3BWiu6e4+7V9y90tHRUefu\nABSt3vCvlTQruz9L0ppi2gHQKrnhN7PXJP23pL83s8/MbLakRZJ+YWa7JV2fPQZwCrHUHOZFq1Qq\nXq1WW7a/k7Fs2bJk/dxzz61Z27t3b3LbhQsXJuupawVI6Xnmpcbmml+1alWyfssttyTrZ599dt37\nbraPP/64Zm3KlCnJbW+88Yej2ydasWJFsl7WdQwqlYqq1eqQ/iD4hh8QFOEHgiL8QFCEHwiK8ANB\nEX4gKIb6CvDAAw8k62PHjk3Wb7vttmR91KhRyXpqSGvBggXJbd99991kPa+3V155JVlvZMjr2LFj\nyfrGjRuT9dtvv71mLW8oL28K7eHDhyfrZWGoD0Auwg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+4Hp6\nepL1vKmm586dm6w/8sgjNWt5Y+VPPfVU3b9bkubPn1+zlvff1a7j+HkY5weQi/ADQRF+ICjCDwRF\n+IGgCD8QFOEHgirn+sJoG93d3cl63lTW9957b7J++PDhmrX33nsvue3mzZuT9UWL0tNF5E27Hh1H\nfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IKvd8fjNbJulmSQfd/fJs2UJJcyT1ZastcPe38nbG+fyn\nnzPOSB8/UtOHT5w4MbntCy+8kKxfc801yXpERZ/Pv1zSYDMcPO3uk7Of3OADaC+54Xf3dyR92YJe\nALRQI+/57zOzD8xsmZmdX1hHAFqi3vD/XtIlkiZL2i9pSa0VzazbzKpmVu3r66u1GoAWqyv87v65\nu3/v7sck/UHSlMS6Pe5ecfdKR0dHvX0CKFhd4TezcQMe/lrSjmLaAdAquaf0mtlrkn4uaYyZfSbp\nXyX93MwmS3JJvZJ+28QeATRBbvjd/c5BFr/YhF5Qgh070i/a8uaxb2Teh2nTpiXrjOM3F9/wA4Ii\n/EBQhB8IivADQRF+ICjCDwTFpbtPA99++23N2uLFi5PbPvHEE8n6yJEjk/WtW7cm688//3zN2ptv\nvpncdunSpcn6eeedl6wjjSM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwTFOP8pYPv27cn63XffXbOW\nd6n05557Llm/6667kvVzzjknWb/66qtr1pYvX57cdvXq1cn6zJkzk3WkceQHgiL8QFCEHwiK8ANB\nEX4gKMIPBEX4gaAY52+Bo0ePJuuvv/56sj5nzpxkfdKkSTVrn376aXLb8ePHJ+upKbab7YILLiht\n3xFw5AeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoHLH+c1sgqSXJXVKckk97r7UzEZLWiWpS1KvpDvc\n/S/Na/XUlTeOn3deeup8fSl9bfyzzjoruW2zpc7JnzhxYnLb66+/vuh2MMBQjvxHJT3o7pMkXSVp\nrplNkjRP0gZ3v1TShuwxgFNEbvjdfb+7b83ufy3pQ0njJc2QtCJbbYWkW5vVJIDindR7fjPrknSF\npD9L6nT3/VnpgPrfFgA4RQw5/GZ2nqQ/Sbrf3Q8NrLm7q//zgMG26zazqplV+/r6GmoWQHGGFH4z\nO0v9wV/p7sdnV/zczMZl9XGSDg62rbv3uHvF3SsdHR1F9AygALnht/7Tul6U9KG7/25Aaa2kWdn9\nWZLWFN8egGYZyim9UyXNlLTdzLZlyxZIWiTpdTObLelTSXc0p8X2l3dp7bxTcpcsWZKs33///Sfd\nU6vkXX57zZrax4QFCxYktx0+fHg9LWGIcsPv7psk1Tqp+7pi2wHQKnzDDwiK8ANBEX4gKMIPBEX4\ngaAIPxAUl+4uwDPPPJOspy6tLUlz584tsp2TcuzYsWR948aNyfqDDz6YrKe+1Xnfffclt0VzceQH\ngiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAY52+B6dOnJ+uNXl47NQX4tm3batYk6dlnn03WV65cmazP\nnz8/WU+N5Xd2ctnHMnHkB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgrH+mrdaoVCperVZbtr9WOXDg\nQLI+derUZP3w4cPJ+hVXXJGsb926tWbtyJEjyW0XL16crM+YMSNZHzt2bLKO1qpUKqpWq7UutX8C\njvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EFTu+fxmNkHSy5I6JbmkHndfamYLJc2R1JetusDd32pW\no+3swgsvTNa3bNnSUH3dunXJ+pVXXlmzds899yS3TV1XH6e3oVzM46ikB919q5n9RNIWMzv+1/i0\nuz/VvPYANEtu+N19v6T92f2vzexDSeOb3RiA5jqp9/xm1iXpCkl/zhbdZ2YfmNkyMzu/xjbdZlY1\ns2pfX99gqwAowZDDb2bnSfqTpPvd/ZCk30u6RNJk9b8yWDLYdu7e4+4Vd6/w/hJoH0MKv5mdpf7g\nr3T3NyXJ3T939+/d/ZikP0ia0rw2ARQtN/xmZpJelPShu/9uwPJxA1b7taQdxbcHoFmG8mn/VEkz\nJW03s+PXgV4g6U4zm6z+4b9eSb9tSoengVGjRiXr1113XUN1oB5D+bR/k6TBzg8OOaYPnC74hh8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiColk7RbWZ9kj4d\nsGiMpC9a1sDJadfe2rUvid7qVWRvf+fuQ7peXkvD/6Odm1XdvVJaAwnt2lu79iXRW73K6o2X/UBQ\nhB8Iquzw95S8/5R27a1d+5LorV6l9Fbqe34A5Sn7yA+gJKWE38xuNLOPzGyPmc0ro4dazKzXzLab\n2TYzq5bcyzIzO2hmOwYsG21m68xsd3Y76DRpJfW20Mz2Zc/dNjO7qaTeJpjZf5nZLjPbaWb/nC0v\n9blL9FXK89byl/1mNkzS/0r6haTPJG2WdKe772ppIzWYWa+kiruXPiZsZv8o6a+SXnb3y7Nl/ybp\nS3dflP3Deb67P9ImvS2U9NeyZ27OJpQZN3BmaUm3SvonlfjcJfq6QyU8b2Uc+adI2uPun7j7EUl/\nlDSjhD7anru/I+nLHyyeIWlFdn+F+v94Wq5Gb23B3fe7+9bs/teSjs8sXepzl+irFGWEf7ykvQMe\nf6b2mvLbJa03sy1m1l12M4PozKZNl6QDkjrLbGYQuTM3t9IPZpZum+eunhmvi8YHfj82zd0nS/qV\npLnZy9u25P3v2dppuGZIMze3yiAzS/9Nmc9dvTNeF62M8O+TNGHA459my9qCu+/Lbg9KWq32m334\n8+OTpGa3B0vu52/aaebmwWaWVhs8d+0043UZ4d8s6VIz+5mZDZf0G0lrS+jjR8xsRPZBjMxshKRf\nqv1mH14raVZ2f5akNSX2coJ2mbm51szSKvm5a7sZr9295T+SblL/J/4fS/qXMnqo0dclkt7PfnaW\n3Zuk19T/MvD/1P/ZyGxJF0jaIGm3pPWSRrdRb69I2i7pA/UHbVxJvU1T/0v6DyRty35uKvu5S/RV\nyvPGN/yAoPjADwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUP8PxWG1i9BwSPUAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88b24b45c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(x[1].reshape((28,28)),cmap='Greys')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic logistic multiclass classification:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEmhJREFUeJzt3X+s3fV93/Hna3ZDSTMSCHcWu6azt7rtDGrU4FGvraqs\nroSTVDWTUuS0DV7GQBW0y6ZJLXTS+GOyRLRpbdEGkxUyTBfFtWg2vLV0RU6zbOoMuzRpjKGUuxCC\nXYNvSRamVKUzee+P86E73M819/qc43t87edDOjqf8/5+P9/v58O17ut+f5wvqSokSRr2l6Y9AEnS\n+cdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUmf9tAcwqiuvvLI2bdo07WFI0pry\n5JNP/klVzSy33poNh02bNjE3NzftYUjSmpLkhZWs52klSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAk\ndQwHSVLHcJAkdQwHSVJnzX5Dehyb7vzNv2h/5Z4PTnEkknR+8shBktQxHCRJHcNBktQxHCRJHcNB\nktRZNhySfDLJqSRPDdX+RZI/TPKlJP8hybuGlt2VZD7Js0luGKpfl+RoW3ZvkrT6JUl+vdUfT7Jp\nslOUJJ2tlRw5PAjsXFR7DLi2qr4P+CPgLoAkW4HdwDWtz31J1rU+9wO3Alva641t3gJ8vaq+C/hl\n4OOjTkaSNBnLhkNVfR742qLa71TV6fbxCLCxtXcBB6rqtap6HpgHrk9yFXBZVR2pqgIeAm4c6rO/\ntR8GdrxxVCFJmo5JXHP4+8CjrT0LvDi07Hirzbb24vqb+rTA+Qbw7qV2lOS2JHNJ5hYWFiYwdEnS\nUsYKhyT/FDgNfGoyw3lrVbWvqrZV1baZmWX//9iSpBGNHA5J/h7w48BPt1NFACeAq4dW29hqJ/j/\np56G62/qk2Q98E7glVHHJUka30jhkGQn8AvAT1TVnw4tOgTsbncgbWZw4fmJqjoJvJpke7uecDPw\nyFCfPa39IeCzQ2EjSZqCZR+8l+TTwPuAK5McB+5mcHfSJcBj7drxkar62ao6luQg8DSD0013VNXr\nbVO3M7jz6VIG1yjeuE7xAPBrSeYZXPjePZmpSZJGtWw4VNWHlyg/8Bbr7wX2LlGfA65dov5nwE8u\nNw5J0urxG9KSpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7h\nIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnq\nLBsOST6Z5FSSp4ZqVyR5LMlz7f3yoWV3JZlP8mySG4bq1yU52pbdmyStfkmSX2/1x5NsmuwUJUln\nayVHDg8COxfV7gQOV9UW4HD7TJKtwG7gmtbnviTrWp/7gVuBLe31xjZvAb5eVd8F/DLw8VEnI0ma\njGXDoao+D3xtUXkXsL+19wM3DtUPVNVrVfU8MA9cn+Qq4LKqOlJVBTy0qM8b23oY2PHGUYUkaTpG\nveawoapOtvZLwIbWngVeHFrveKvNtvbi+pv6VNVp4BvAu5faaZLbkswlmVtYWBhx6JKk5Yx9Qbod\nCdQExrKSfe2rqm1VtW1mZmY1dilJF6VRw+HldqqI9n6q1U8AVw+tt7HVTrT24vqb+iRZD7wTeGXE\ncUmSJmDUcDgE7GntPcAjQ/Xd7Q6kzQwuPD/RTkG9mmR7u55w86I+b2zrQ8Bn29GIJGlK1i+3QpJP\nA+8DrkxyHLgbuAc4mOQW4AXgJoCqOpbkIPA0cBq4o6peb5u6ncGdT5cCj7YXwAPAryWZZ3Dhe/dE\nZiZJGtmy4VBVHz7Doh1nWH8vsHeJ+hxw7RL1PwN+crlxSJJWj9+QliR1DAdJUsdwkCR1DAdJUsdw\nkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1\nDAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1xgqHJP84ybEkTyX5dJJvT3JFkseSPNfeLx9a/64k\n80meTXLDUP26JEfbsnuTZJxxSZLGM3I4JJkF/iGwraquBdYBu4E7gcNVtQU43D6TZGtbfg2wE7gv\nybq2ufuBW4Et7bVz1HFJksY37mml9cClSdYDbwf+GNgF7G/L9wM3tvYu4EBVvVZVzwPzwPVJrgIu\nq6ojVVXAQ0N9JElTMHI4VNUJ4F8CXwVOAt+oqt8BNlTVybbaS8CG1p4FXhzaxPFWm23txXVJ0pSM\nc1rpcgZHA5uBvwp8R5KfGV6nHQnUWCN88z5vSzKXZG5hYWFSm5UkLTLOaaUfA56vqoWq+r/AZ4Af\nBF5up4po76fa+ieAq4f6b2y1E629uN6pqn1Vta2qts3MzIwxdEnSWxknHL4KbE/y9nZ30Q7gGeAQ\nsKetswd4pLUPAbuTXJJkM4MLz0+0U1CvJtnetnPzUB9J0hSsH7VjVT2e5GHg94HTwBeAfcA7gINJ\nbgFeAG5q6x9LchB4uq1/R1W93jZ3O/AgcCnwaHtJkqZk5HAAqKq7gbsXlV9jcBSx1Pp7gb1L1OeA\na8cZiyRpcvyGtCSpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySp\nYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhI\nkjpjhUOSdyV5OMkfJnkmyd9OckWSx5I8194vH1r/riTzSZ5NcsNQ/bokR9uye5NknHFJksYz7pHD\nrwK/XVXfC7wHeAa4EzhcVVuAw+0zSbYCu4FrgJ3AfUnWte3cD9wKbGmvnWOOS5I0hpHDIck7gR8B\nHgCoqj+vqv8N7AL2t9X2Aze29i7gQFW9VlXPA/PA9UmuAi6rqiNVVcBDQ30kSVMwzpHDZmAB+HdJ\nvpDkE0m+A9hQVSfbOi8BG1p7FnhxqP/xVptt7cV1SdKUjBMO64H3AvdX1fcD36SdQnpDOxKoMfbx\nJkluSzKXZG5hYWFSm5UkLTJOOBwHjlfV4+3zwwzC4uV2qoj2fqotPwFcPdR/Y6udaO3F9U5V7auq\nbVW1bWZmZoyhS5LeysjhUFUvAS8m+Z5W2gE8DRwC9rTaHuCR1j4E7E5ySZLNDC48P9FOQb2aZHu7\nS+nmoT6SpClYP2b/nwc+leRtwJeBjzIInINJbgFeAG4CqKpjSQ4yCJDTwB1V9Xrbzu3Ag8ClwKPt\nJUmakrHCoaq+CGxbYtGOM6y/F9i7RH0OuHacsUiSJsdvSEuSOoaDJKljOEiSOoaDJKljOEiSOoaD\nJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKlj\nOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKkzdjgkWZfkC0n+c/t8RZLHkjzX3i8fWveuJPNJnk1y\nw1D9uiRH27J7k2TccUmSRjeJI4ePAc8Mfb4TOFxVW4DD7TNJtgK7gWuAncB9Sda1PvcDtwJb2mvn\nBMYlSRrRWOGQZCPwQeATQ+VdwP7W3g/cOFQ/UFWvVdXzwDxwfZKrgMuq6khVFfDQUB9J0hSMe+Tw\nK8AvAN8aqm2oqpOt/RKwobVngReH1jvearOtvbguSZqSkcMhyY8Dp6rqyTOt044EatR9LLHP25LM\nJZlbWFiY1GYlSYuMc+TwQ8BPJPkKcAD40ST/Hni5nSqivZ9q658Arh7qv7HVTrT24nqnqvZV1baq\n2jYzMzPG0CVJb2XkcKiqu6pqY1VtYnCh+bNV9TPAIWBPW20P8EhrHwJ2J7kkyWYGF56faKegXk2y\nvd2ldPNQH0nSFKw/B9u8BziY5BbgBeAmgKo6luQg8DRwGrijql5vfW4HHgQuBR5tL0nSlEwkHKrq\nc8DnWvsVYMcZ1tsL7F2iPgdcO4mxSJLG5zekJUkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkd\nw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS\n1DEcJEkdw0GS1DEcJEkdw0GS1Bk5HJJcneR3kzyd5FiSj7X6FUkeS/Jce798qM9dSeaTPJvkhqH6\ndUmOtmX3Jsl405IkjWOcI4fTwD+pqq3AduCOJFuBO4HDVbUFONw+05btBq4BdgL3JVnXtnU/cCuw\npb12jjEuSdKYRg6HqjpZVb/f2v8HeAaYBXYB+9tq+4EbW3sXcKCqXquq54F54PokVwGXVdWRqirg\noaE+kqQpmMg1hySbgO8HHgc2VNXJtuglYENrzwIvDnU73mqzrb24LkmakrHDIck7gN8A/lFVvTq8\nrB0J1Lj7GNrXbUnmkswtLCxMarOSpEXGCock38YgGD5VVZ9p5ZfbqSLa+6lWPwFcPdR9Y6udaO3F\n9U5V7auqbVW1bWZmZpyhS5Lewjh3KwV4AHimqv7V0KJDwJ7W3gM8MlTfneSSJJsZXHh+op2CejXJ\n9rbNm4f6SJKmYP0YfX8I+AhwNMkXW+2XgHuAg0luAV4AbgKoqmNJDgJPM7jT6Y6qer31ux14ELgU\neLS9JElTMnI4VNV/B870fYQdZ+izF9i7RH0OuHbUsUiSJstvSEuSOoaDJKljOEiSOoaDJKljOEiS\nOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOuM8sluStEo23fmbf9H+yj0fPOf7\n88hBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJnfPm8RlJdgK/CqwD\nPlFV90x5SJI0VcOPzFht58WRQ5J1wL8B3g9sBT6cZOt0RyVJF6/zIhyA64H5qvpyVf05cADYNeUx\nSdJF63w5rTQLvDj0+TjwA6ux4zM96XClh3Nn2+dMT1McZxxn+4TGlexrJdtcPLaVjPts1znT/lYy\n7pXMZ1LbfKuf09n+t5zU3M52X2fb90zGfWLo2f7cVrLvsx33uL8XLgSpqmmPgSQfAnZW1T9onz8C\n/EBV/dyi9W4Dbmsfvwd4dsRdXgn8yYh91yrnfHFwzheHceb816pqZrmVzpcjhxPA1UOfN7bam1TV\nPmDfuDtLMldV28bdzlrinC8OzvnisBpzPl+uOfxPYEuSzUneBuwGDk15TJJ00Tovjhyq6nSSnwP+\nC4NbWT9ZVcemPCxJumidF+EAUFW/BfzWKu1u7FNTa5Bzvjg454vDOZ/zeXFBWpJ0fjlfrjlIks4j\nF3Q4JNmZ5Nkk80nuXGJ5ktzbln8pyXunMc5JWsGcf7rN9WiS30vynmmMc5KWm/PQen8ryel26/Sa\ntpI5J3lfki8mOZbkv672GCdpBf+u35nkPyX5gzbfj05jnJOU5JNJTiV56gzLz+3vr6q6IF8MLmz/\nL+CvA28D/gDYumidDwCPAgG2A49Pe9yrMOcfBC5v7fdfDHMeWu+zDK5rfWja416Fn/O7gKeB72yf\n/8q0x32O5/tLwMdbewb4GvC2aY99zHn/CPBe4KkzLD+nv78u5COHlTySYxfwUA0cAd6V5KrVHugE\nLTvnqvq9qvp6+3iEwXdK1rKVPnrl54HfAE6t5uDOkZXM+aeAz1TVVwGqai3PeyXzLeAvJwnwDgbh\ncHp1hzlZVfV5BvM4k3P6++tCDoelHskxO8I6a8nZzucWBn95rGXLzjnJLPB3gftXcVzn0kp+zt8N\nXJ7kc0meTHLzqo1u8lYy338N/E3gj4GjwMeq6lurM7ypOae/v86bW1m1upL8HQbh8MPTHssq+BXg\nF6vqW4M/LC8K64HrgB3ApcD/SHKkqv5ousM6Z24Avgj8KPA3gMeS/LeqenW6w1q7LuRwWMkjOVb0\n2I41ZEXzSfJ9wCeA91fVK6s0tnNlJXPeBhxowXAl8IEkp6vqP67OECduJXM+DrxSVd8Evpnk88B7\ngLUYDiuZ70eBe2pwMn4+yfPA9wJPrM4Qp+Kc/v66kE8rreSRHIeAm9tV/+3AN6rq5GoPdIKWnXOS\n7wQ+A3zkAvkrctk5V9XmqtpUVZuAh4Hb13AwwMr+bT8C/HCS9UnezuApx8+s8jgnZSXz/SqDoySS\nbGDwYM4vr+ooV985/f11wR451BkeyZHkZ9vyf8vgzpUPAPPAnzL462PNWuGc/xnwbuC+9pf06VrD\nDy1b4ZwvKCuZc1U9k+S3gS8B32Lwf1dc8pbI890Kf8b/HHgwyVEGd+/8YlWt6Se1Jvk08D7gyiTH\ngbuBb4PV+f3lN6QlSZ0L+bSSJGlEhoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqfP/AEtN\nrMQsVxsaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88f00f2940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(x.ravel(),100)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train, y_train = mnist.train.next_batch(1000)\n",
    "x_test, y_test = mnist.test.next_batch(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic = LogisticRegression()\n",
    "logistic.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 1, 0, 8, 2, 6, 1, 9, 1, 0], dtype=uint8)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = logistic.predict(x_test)\n",
    "y_pred[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predicting the probabilities for the first 3 images:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  1.00464678e-05,   7.34788286e-05,   5.55601362e-01,\n",
       "          3.76713648e-05,   7.78980604e-07,   3.08947507e-04,\n",
       "          4.43483003e-01,   5.25947388e-10,   4.75286972e-04,\n",
       "          9.42502338e-06],\n",
       "       [  6.84467773e-04,   9.13792521e-01,   1.68375885e-03,\n",
       "          7.05362594e-03,   8.33105878e-05,   6.42039023e-04,\n",
       "          2.64083947e-04,   2.07489885e-04,   7.53772835e-02,\n",
       "          2.11419043e-04],\n",
       "       [  5.42632046e-01,   5.23614019e-05,   2.52195485e-06,\n",
       "          7.87723555e-06,   7.00215717e-03,   3.52044691e-01,\n",
       "          8.40985294e-02,   1.41449207e-02,   5.24138014e-07,\n",
       "          1.43702060e-05]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic.predict_proba(x_test[:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Accuracy of the predictions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.88"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.count_nonzero(y_pred == y_test)/len(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Keras Multilayered Perceptron (Neural Net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_data_generator(batch_size):\n",
    "    while(1):\n",
    "        x, y = mnist.train.next_batch(batch_size)\n",
    "        yield x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Zero hidden layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is important to note that when we do classification problems we use the **Categorical Crossentropy Loss**. When its only two classes we can use Logistic Loss (Binary Crossentropy Loss). Finally for regression problems we use **Mean Squared Error**.\n",
    "\n",
    "The Cross Entropy loss is defined as:\n",
    "$$\\mathcal{L} = -\\frac{1}{N}\\sum_i \\mathcal{I}(y_i=1)\\log(p_{i1})+\\mathcal{I}(y_i=2)\\log(1-p_{i2})+\\cdots++\\mathcal{I}(y_i=K)\\log(1-p_{iK})$$\n",
    "where $N$ is the number of training instances, $K$ is the number of classes and $p_{ik}$ is the probability that instance $i$ belongs to $k$.\n",
    "\n",
    "Softmax takes a $D$ dimensional vector and squeezes them through a function such that we have $D$ outputs whos values are positive and sums to one.\n",
    "$$\n",
    "\\text{softmax}(\\mathbf{y})_d = \\frac{\\exp(-y_d)}{\\exp(-y_1)+...+\\exp(-y_D)}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(units=10, input_dim=num_pixels, activation='softmax'))\n",
    "model.compile(optimizer='adadelta', loss='sparse_categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "214/214 [==============================] - 1s - loss: 1.3301     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f889a830128>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 256\n",
    "model.fit_generator(train_data_generator(batch_size=batch_size), mnist.train.num_examples//batch_size, epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  32/1000 [..............................] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([6, 1, 0, 7, 9, 6, 1, 9, 1, 0])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict_classes(x_test)\n",
    "y_pred[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.849"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.count_nonzero(y_pred == y_test)/len(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1 Hidden Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(units=50, input_dim=num_pixels, activation='relu'))\n",
    "model.add(Dense(units=10, activation='softmax'))\n",
    "model.compile(optimizer='adadelta', loss='sparse_categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "214/214 [==============================] - 1s - loss: 0.8011     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f888d79b828>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 256\n",
    "model.fit_generator(train_data_generator(batch_size=batch_size), mnist.train.num_examples//batch_size, epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  32/1000 [..............................] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.905"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict_classes(x_test)\n",
    "np.count_nonzero(y_pred == y_test)/len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_2 (Dense)              (None, 50)                39250     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                510       \n",
      "=================================================================\n",
      "Total params: 39,760.0\n",
      "Trainable params: 39,760\n",
      "Non-trainable params: 0.0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39250"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "784*50+50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolution Neural Networks (CNN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Points to note **\n",
    "1. One CNN, connected to **one** node above is simply a Dense layer with most weights set to zero.\n",
    "2. The same CNN, connected to multiple nodes is weight tying/ sharing.\n",
    "\n",
    "Consider the following convolution mask:\n",
    "<img src='https://ujwlkarn.files.wordpress.com/2016/07/screen-shot-2016-07-24-at-11-25-24-pm.png?w=74&h=64'>\n",
    "<img src='https://ujwlkarn.files.wordpress.com/2016/07/convolution_schematic.gif?w=536&h=392'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Need to reshape the x values to a 28x28 image\n",
    "def train_data_generator(batch_size):\n",
    "    while(1):\n",
    "        x, y = mnist.train.next_batch(batch_size)\n",
    "        yield x.reshape((-1,28,28,1)), y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3,3), padding='same', activation='relu', input_shape = (width,height,1)))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "model.compile(optimizer='adadelta', loss='sparse_categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 28, 28, 32)        320       \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                250890    \n",
      "=================================================================\n",
      "Total params: 251,210.0\n",
      "Trainable params: 251,210.0\n",
      "Non-trainable params: 0.0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25088"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "28*28*32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "214/214 [==============================] - 20s - loss: 0.4835    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8885850828>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 256\n",
    "model.fit_generator(train_data_generator(batch_size=batch_size), mnist.train.num_examples//batch_size, epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 992/1000 [============================>.] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.93"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict_classes(x_test.reshape((-1,28,28,1)))\n",
    "np.count_nonzero(y_pred == y_test)/len(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using the max pooling layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3,3), padding='same', activation='relu', input_shape = (width,height,1)))\n",
    "model.add(MaxPool2D(pool_size=(2, 2)))\n",
    "# model.add(Conv2D(32, kernel_size=(3,3), padding='same', activation='relu'))\n",
    "# model.add(MaxPool2D(pool_size=(2, 2)))\n",
    "# model.add(Conv2D(32, kernel_size=(3,3), padding='same', activation='relu'))\n",
    "# model.add(MaxPool2D(pool_size=(2, 2)))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "model.compile(optimizer='adadelta', loss='sparse_categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_2 (Conv2D)            (None, 28, 28, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 14, 14, 32)        0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 6272)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                62730     \n",
      "=================================================================\n",
      "Total params: 63,050.0\n",
      "Trainable params: 63,050.0\n",
      "Non-trainable params: 0.0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "214/214 [==============================] - 21s - loss: 0.6134    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8883e76b70>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 256\n",
    "model.fit_generator(train_data_generator(batch_size=batch_size), mnist.train.num_examples//batch_size, epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 832/1000 [=======================>......] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.911"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict_classes(x_test.reshape((-1,28,28,1)))\n",
    "np.count_nonzero(y_pred == y_test)/len(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi Layer Convolutional Network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3,3), padding='same', activation='relu', input_shape = (width,height,1)))\n",
    "model.add(MaxPool2D(pool_size=(2, 2)))\n",
    "model.add(Conv2D(32, kernel_size=(3,3), padding='same', activation='relu'))\n",
    "model.add(MaxPool2D(pool_size=(2, 2)))\n",
    "model.add(Conv2D(32, kernel_size=(3,3), padding='same', activation='relu'))\n",
    "model.add(MaxPool2D(pool_size=(2, 2)))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "model.compile(optimizer='adadelta', loss='sparse_categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_9 (Conv2D)            (None, 28, 28, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_8 (MaxPooling2 (None, 14, 14, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_10 (Conv2D)           (None, 14, 14, 32)        9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_9 (MaxPooling2 (None, 7, 7, 32)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_11 (Conv2D)           (None, 7, 7, 32)          9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_10 (MaxPooling (None, 3, 3, 32)          0         \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 288)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 10)                2890      \n",
      "=================================================================\n",
      "Total params: 21,706.0\n",
      "Trainable params: 21,706.0\n",
      "Non-trainable params: 0.0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "214/214 [==============================] - 53s - loss: 0.6373    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8877f9fe48>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 256\n",
    "model.fit_generator(train_data_generator(batch_size=batch_size), mnist.train.num_examples//batch_size, epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 0s     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.96"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict_classes(x_test.reshape((-1,28,28,1)))\n",
    "np.count_nonzero(y_pred == y_test)/len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "latex_envs": {
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 0
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}