Chapter 11.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Xavier and He Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.dense.Dense at 0x7f82161c6160>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation=\"relu\", kernel_initializer=\"he_normal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.dense.Dense at 0x7f8218056e20>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# He initialization with a uniform distribution\n",
    "\n",
    "he_avg_init = keras.initializers.VarianceScaling(scale = 2., mode = \"fan_avg\",\n",
    "                                                distribution = \"uniform\")\n",
    "keras.layers.Dense(10, activation = \"sigmoid\", kernel_initializer = he_avg_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
    "X_train_full = X_train_full / 255.0\n",
    "X_test = X_test / 255.0\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "# leaky ReLU\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.LeakyReLU(),\n",
    "    keras.layers.Dense(100, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.LeakyReLU(),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1719/1719 [==============================] - 4s 2ms/step - loss: 1.2819 - accuracy: 0.6229 - val_loss: 0.8886 - val_accuracy: 0.7160\n",
      "Epoch 2/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.7955 - accuracy: 0.7362 - val_loss: 0.7130 - val_accuracy: 0.7656\n",
      "Epoch 3/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.6816 - accuracy: 0.7721 - val_loss: 0.6427 - val_accuracy: 0.7902\n",
      "Epoch 4/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.6217 - accuracy: 0.7944 - val_loss: 0.5900 - val_accuracy: 0.8066\n",
      "Epoch 5/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.5832 - accuracy: 0.8075 - val_loss: 0.5582 - val_accuracy: 0.8200\n",
      "Epoch 6/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.5553 - accuracy: 0.8156 - val_loss: 0.5349 - val_accuracy: 0.8238\n",
      "Epoch 7/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.5338 - accuracy: 0.8224 - val_loss: 0.5156 - val_accuracy: 0.8306\n",
      "Epoch 8/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.5172 - accuracy: 0.8273 - val_loss: 0.5079 - val_accuracy: 0.8282\n",
      "Epoch 9/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.5040 - accuracy: 0.8290 - val_loss: 0.4895 - val_accuracy: 0.8386\n",
      "Epoch 10/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.4924 - accuracy: 0.8320 - val_loss: 0.4817 - val_accuracy: 0.8398\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train, epochs=10,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PReLU\n",
    "\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.PReLU(),\n",
    "    keras.layers.Dense(100, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.PReLU(),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1719/1719 [==============================] - 4s 2ms/step - loss: 1.3461 - accuracy: 0.6209 - val_loss: 0.9255 - val_accuracy: 0.7186\n",
      "Epoch 2/10\n",
      "1719/1719 [==============================] - 4s 2ms/step - loss: 0.8197 - accuracy: 0.7355 - val_loss: 0.7305 - val_accuracy: 0.7628\n",
      "Epoch 3/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.6965 - accuracy: 0.7693 - val_loss: 0.6565 - val_accuracy: 0.7880\n",
      "Epoch 4/10\n",
      "1719/1719 [==============================] - 4s 2ms/step - loss: 0.6331 - accuracy: 0.7909 - val_loss: 0.6003 - val_accuracy: 0.8048\n",
      "Epoch 5/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.5917 - accuracy: 0.8057 - val_loss: 0.5656 - val_accuracy: 0.8180\n",
      "Epoch 6/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.5618 - accuracy: 0.8136 - val_loss: 0.5406 - val_accuracy: 0.8238\n",
      "Epoch 7/10\n",
      "1719/1719 [==============================] - 4s 2ms/step - loss: 0.5390 - accuracy: 0.8205 - val_loss: 0.5196 - val_accuracy: 0.8312\n",
      "Epoch 8/10\n",
      "1719/1719 [==============================] - 4s 2ms/step - loss: 0.5213 - accuracy: 0.8258 - val_loss: 0.5114 - val_accuracy: 0.8316\n",
      "Epoch 9/10\n",
      "1719/1719 [==============================] - 4s 2ms/step - loss: 0.5070 - accuracy: 0.8288 - val_loss: 0.4916 - val_accuracy: 0.8376\n",
      "Epoch 10/10\n",
      "1719/1719 [==============================] - 3s 2ms/step - loss: 0.4945 - accuracy: 0.8315 - val_loss: 0.4826 - val_accuracy: 0.8396\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train, epochs=10,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.special import erfc\n",
    "\n",
    "# alpha and scale to self normalize with mean 0 and standard deviation 1\n",
    "# (see equation 14 in the paper):\n",
    "alpha_0_1 = -np.sqrt(2 / np.pi) / (erfc(1/np.sqrt(2)) * np.exp(1/2) - 1)\n",
    "scale_0_1 = (1 - erfc(1 / np.sqrt(2)) * np.sqrt(np.e)) * np.sqrt(2 * np.pi) * (2 * erfc(np.sqrt(2))*np.e**2 + np.pi*erfc(1/np.sqrt(2))**2*np.e - 2*(2+np.pi)*erfc(1/np.sqrt(2))*np.sqrt(np.e)+np.pi+2)**(-1/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selu(z, scale=scale_0_1, alpha=alpha_0_1):\n",
    "    return scale * elu(z, alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.dense.Dense at 0x7f81f7c4edf0>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation=\"selu\",\n",
    "                   kernel_initializer=\"lecun_normal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential()\n",
    "model.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model.add(keras.layers.Dense(300, activation=\"selu\",\n",
    "                             kernel_initializer=\"lecun_normal\"))\n",
    "for layer in range(99):\n",
    "    model.add(keras.layers.Dense(100, activation=\"selu\",\n",
    "                                 kernel_initializer=\"lecun_normal\"))\n",
    "model.add(keras.layers.Dense(10, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pixel_means = X_train.mean(axis=0, keepdims=True)\n",
    "pixel_stds = X_train.std(axis=0, keepdims=True)\n",
    "X_train_scaled = (X_train - pixel_means) / pixel_stds\n",
    "X_valid_scaled = (X_valid - pixel_means) / pixel_stds\n",
    "X_test_scaled = (X_test - pixel_means) / pixel_stds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1719/1719 [==============================] - 18s 9ms/step - loss: 1.1344 - accuracy: 0.5573 - val_loss: 1.1278 - val_accuracy: 0.4936\n",
      "Epoch 2/5\n",
      "1719/1719 [==============================] - 16s 9ms/step - loss: 0.8060 - accuracy: 0.6918 - val_loss: 0.7098 - val_accuracy: 0.7254\n",
      "Epoch 3/5\n",
      "1719/1719 [==============================] - 16s 9ms/step - loss: 0.6977 - accuracy: 0.7395 - val_loss: 0.6509 - val_accuracy: 0.7710\n",
      "Epoch 4/5\n",
      "1719/1719 [==============================] - 16s 9ms/step - loss: 0.6434 - accuracy: 0.7605 - val_loss: 0.5453 - val_accuracy: 0.7982\n",
      "Epoch 5/5\n",
      "1719/1719 [==============================] - 16s 9ms/step - loss: 0.5673 - accuracy: 0.7885 - val_loss: 0.5220 - val_accuracy: 0.8074\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train, epochs=5,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batch Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape = [28, 28]),\n",
    "    keras.layers.BatchNormalization(), # batch normalization before each hidden layer's activation function\n",
    "    keras.layers.Dense(300, activation = \"elu\", kernel_initializer = \"he_normal\"),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Dense(100, activation = \"elu\", kernel_initializer = \"he_normal\"),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Dense(10, activation = \"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " flatten_3 (Flatten)         (None, 784)               0         \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 784)              3136      \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " dense_110 (Dense)           (None, 300)               235500    \n",
      "                                                                 \n",
      " batch_normalization_1 (Batc  (None, 300)              1200      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dense_111 (Dense)           (None, 100)               30100     \n",
      "                                                                 \n",
      " batch_normalization_2 (Batc  (None, 100)              400       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " dense_112 (Dense)           (None, 10)                1010      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 271,346\n",
      "Trainable params: 268,978\n",
      "Non-trainable params: 2,368\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('batch_normalization/gamma:0', True),\n",
       " ('batch_normalization/beta:0', True),\n",
       " ('batch_normalization/moving_mean:0', False),\n",
       " ('batch_normalization/moving_variance:0', False)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(var.name, var.trainable) for var in model.layers[1].variables]\n",
    "# first BN layer\n",
    "# 2 are trainable while 2 are not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Dense(300, use_bias=False), # remove bias term from previous layer\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(\"relu\"),\n",
    "    keras.layers.Dense(100, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(\"relu\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(clipvalue = 1.0)\n",
    "model.compile(loss = \"mse\", optimizer = optimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transfering Learning with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_dataset(X, y):\n",
    "    y_5_or_6 = (y == 5) | (y == 6) # sandals or shirts\n",
    "    y_A = y[~y_5_or_6]\n",
    "    y_A[y_A > 6] -= 2 # class indices 7, 8, 9 should be moved to 5, 6, 7\n",
    "    y_B = (y[y_5_or_6] == 6).astype(np.float32) # binary classification task: is it a shirt (class 6)?\n",
    "    return ((X[~y_5_or_6], y_A),\n",
    "            (X[y_5_or_6], y_B))\n",
    "\n",
    "(X_train_A, y_train_A), (X_train_B, y_train_B) = split_dataset(X_train, y_train)\n",
    "(X_valid_A, y_valid_A), (X_valid_B, y_valid_B) = split_dataset(X_valid, y_valid)\n",
    "(X_test_A, y_test_A), (X_test_B, y_test_B) = split_dataset(X_test, y_test)\n",
    "X_train_B = X_train_B[:200]\n",
    "y_train_B = y_train_B[:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 0, 5, 7, 7, 7, 4, 4, 3, 4, 0, 1, 6, 3, 4, 3, 2, 6, 5, 3, 4, 5,\n",
       "       1, 3, 4, 2, 0, 6, 7, 1], dtype=uint8)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_A[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 0., 0., 0., 0.,\n",
       "       0., 0., 1., 1., 0., 0., 1., 1., 0., 1., 1., 1., 1.], dtype=float32)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_B[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_A = keras.models.Sequential()\n",
    "model_A.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "for n_hidden in (300, 100, 50, 50, 50):\n",
    "    model_A.add(keras.layers.Dense(n_hidden, activation=\"selu\"))\n",
    "model_A.add(keras.layers.Dense(8, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_A.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "                optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "1375/1375 [==============================] - 4s 2ms/step - loss: 0.5926 - accuracy: 0.8103 - val_loss: 0.3892 - val_accuracy: 0.8675\n",
      "Epoch 2/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.3523 - accuracy: 0.8788 - val_loss: 0.3290 - val_accuracy: 0.8819\n",
      "Epoch 3/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.3170 - accuracy: 0.8896 - val_loss: 0.3012 - val_accuracy: 0.8989\n",
      "Epoch 4/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2973 - accuracy: 0.8973 - val_loss: 0.2895 - val_accuracy: 0.9023\n",
      "Epoch 5/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2835 - accuracy: 0.9020 - val_loss: 0.2774 - val_accuracy: 0.9066\n",
      "Epoch 6/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2729 - accuracy: 0.9060 - val_loss: 0.2732 - val_accuracy: 0.9066\n",
      "Epoch 7/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2641 - accuracy: 0.9090 - val_loss: 0.2721 - val_accuracy: 0.9088\n",
      "Epoch 8/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2572 - accuracy: 0.9126 - val_loss: 0.2588 - val_accuracy: 0.9143\n",
      "Epoch 9/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2518 - accuracy: 0.9133 - val_loss: 0.2562 - val_accuracy: 0.9141\n",
      "Epoch 10/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2469 - accuracy: 0.9154 - val_loss: 0.2540 - val_accuracy: 0.9155\n",
      "Epoch 11/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2422 - accuracy: 0.9175 - val_loss: 0.2497 - val_accuracy: 0.9148\n",
      "Epoch 12/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2382 - accuracy: 0.9190 - val_loss: 0.2514 - val_accuracy: 0.9128\n",
      "Epoch 13/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2350 - accuracy: 0.9198 - val_loss: 0.2444 - val_accuracy: 0.9165\n",
      "Epoch 14/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2315 - accuracy: 0.9214 - val_loss: 0.2414 - val_accuracy: 0.9173\n",
      "Epoch 15/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2287 - accuracy: 0.9215 - val_loss: 0.2447 - val_accuracy: 0.9188\n",
      "Epoch 16/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2254 - accuracy: 0.9222 - val_loss: 0.2382 - val_accuracy: 0.9195\n",
      "Epoch 17/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2231 - accuracy: 0.9232 - val_loss: 0.2407 - val_accuracy: 0.9183\n",
      "Epoch 18/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2201 - accuracy: 0.9243 - val_loss: 0.2427 - val_accuracy: 0.9153\n",
      "Epoch 19/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2178 - accuracy: 0.9255 - val_loss: 0.2329 - val_accuracy: 0.9203\n",
      "Epoch 20/20\n",
      "1375/1375 [==============================] - 3s 2ms/step - loss: 0.2156 - accuracy: 0.9262 - val_loss: 0.2332 - val_accuracy: 0.9205\n"
     ]
    }
   ],
   "source": [
    "history = model_A.fit(X_train_A, y_train_A, epochs=20,\n",
    "                    validation_data=(X_valid_A, y_valid_A))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_A.save(\"my_model_A.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_B = keras.models.Sequential()\n",
    "model_B.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "for n_hidden in (300, 100, 50, 50, 50):\n",
    "    model_B.add(keras.layers.Dense(n_hidden, activation=\"selu\"))\n",
    "model_B.add(keras.layers.Dense(1, activation=\"sigmoid\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_B.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=keras.optimizers.SGD(learning_rate=1e-3),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "7/7 [==============================] - 1s 38ms/step - loss: 0.9573 - accuracy: 0.4650 - val_loss: 0.6314 - val_accuracy: 0.6004\n",
      "Epoch 2/20\n",
      "7/7 [==============================] - 0s 15ms/step - loss: 0.5692 - accuracy: 0.7450 - val_loss: 0.4784 - val_accuracy: 0.8529\n",
      "Epoch 3/20\n",
      "7/7 [==============================] - 0s 16ms/step - loss: 0.4503 - accuracy: 0.8650 - val_loss: 0.4102 - val_accuracy: 0.8945\n",
      "Epoch 4/20\n",
      "7/7 [==============================] - 0s 16ms/step - loss: 0.3879 - accuracy: 0.8950 - val_loss: 0.3647 - val_accuracy: 0.9178\n",
      "Epoch 5/20\n",
      "7/7 [==============================] - 0s 16ms/step - loss: 0.3435 - accuracy: 0.9250 - val_loss: 0.3300 - val_accuracy: 0.9320\n",
      "Epoch 6/20\n",
      "7/7 [==============================] - 0s 16ms/step - loss: 0.3081 - accuracy: 0.9300 - val_loss: 0.3019 - val_accuracy: 0.9402\n",
      "Epoch 7/20\n",
      "7/7 [==============================] - 0s 19ms/step - loss: 0.2800 - accuracy: 0.9350 - val_loss: 0.2804 - val_accuracy: 0.9422\n",
      "Epoch 8/20\n",
      "7/7 [==============================] - 0s 17ms/step - loss: 0.2564 - accuracy: 0.9450 - val_loss: 0.2606 - val_accuracy: 0.9473\n",
      "Epoch 9/20\n",
      "7/7 [==============================] - 0s 11ms/step - loss: 0.2362 - accuracy: 0.9550 - val_loss: 0.2428 - val_accuracy: 0.9523\n",
      "Epoch 10/20\n",
      "7/7 [==============================] - 0s 11ms/step - loss: 0.2188 - accuracy: 0.9600 - val_loss: 0.2281 - val_accuracy: 0.9544\n",
      "Epoch 11/20\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.2036 - accuracy: 0.9700 - val_loss: 0.2150 - val_accuracy: 0.9584\n",
      "Epoch 12/20\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1898 - accuracy: 0.9700 - val_loss: 0.2036 - val_accuracy: 0.9584\n",
      "Epoch 13/20\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.1773 - accuracy: 0.9750 - val_loss: 0.1931 - val_accuracy: 0.9615\n",
      "Epoch 14/20\n",
      "7/7 [==============================] - 0s 11ms/step - loss: 0.1668 - accuracy: 0.9800 - val_loss: 0.1838 - val_accuracy: 0.9635\n",
      "Epoch 15/20\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1570 - accuracy: 0.9900 - val_loss: 0.1746 - val_accuracy: 0.9686\n",
      "Epoch 16/20\n",
      "7/7 [==============================] - 0s 17ms/step - loss: 0.1481 - accuracy: 0.9900 - val_loss: 0.1674 - val_accuracy: 0.9686\n",
      "Epoch 17/20\n",
      "7/7 [==============================] - 0s 17ms/step - loss: 0.1406 - accuracy: 0.9900 - val_loss: 0.1604 - val_accuracy: 0.9706\n",
      "Epoch 18/20\n",
      "7/7 [==============================] - 0s 11ms/step - loss: 0.1334 - accuracy: 0.9900 - val_loss: 0.1539 - val_accuracy: 0.9706\n",
      "Epoch 19/20\n",
      "7/7 [==============================] - 0s 11ms/step - loss: 0.1268 - accuracy: 0.9900 - val_loss: 0.1482 - val_accuracy: 0.9716\n",
      "Epoch 20/20\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.1208 - accuracy: 0.9900 - val_loss: 0.1431 - val_accuracy: 0.9716\n"
     ]
    }
   ],
   "source": [
    "history = model_B.fit(X_train_B, y_train_B, epochs=20,\n",
    "                      validation_data=(X_valid_B, y_valid_B))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_A = keras.models.load_model(\"my_model_A.h5\")\n",
    "model_B_on_A = keras.models.Sequential(model_A.layers[:-1]) # reuse all layers except for the output layer\n",
    "model_B_on_A.add(keras.layers.Dense(1, activation = \"sigmoid\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# when training model_b_on_A, it affects model_A\n",
    "# to avoid, use clone model_A before reusing its layers\n",
    "\n",
    "model_A_clone = keras.models.clone_model(model_A)\n",
    "model_A_clone.set_weights(model_A.get_weights())\n",
    "model_B_on_A = keras.models.Sequential(model_A.layers[:-1]) # reuse all layers except for the output layer\n",
    "model_B_on_A.add(keras.layers.Dense(1, activation = \"sigmoid\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n",
      "7/7 [==============================] - 1s 33ms/step - loss: 0.0810 - accuracy: 0.9900 - val_loss: 0.0969 - val_accuracy: 0.9878\n",
      "Epoch 2/4\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0724 - accuracy: 0.9950 - val_loss: 0.0894 - val_accuracy: 0.9868\n",
      "Epoch 3/4\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0655 - accuracy: 0.9950 - val_loss: 0.0832 - val_accuracy: 0.9868\n",
      "Epoch 4/4\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0598 - accuracy: 0.9950 - val_loss: 0.0782 - val_accuracy: 0.9878\n",
      "Epoch 1/16\n",
      "7/7 [==============================] - 1s 36ms/step - loss: 0.0566 - accuracy: 0.9950 - val_loss: 0.0778 - val_accuracy: 0.9878\n",
      "Epoch 2/16\n",
      "7/7 [==============================] - 0s 16ms/step - loss: 0.0562 - accuracy: 0.9950 - val_loss: 0.0774 - val_accuracy: 0.9878\n",
      "Epoch 3/16\n",
      "7/7 [==============================] - 0s 17ms/step - loss: 0.0557 - accuracy: 0.9950 - val_loss: 0.0770 - val_accuracy: 0.9878\n",
      "Epoch 4/16\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.0553 - accuracy: 0.9950 - val_loss: 0.0766 - val_accuracy: 0.9878\n",
      "Epoch 5/16\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.0549 - accuracy: 0.9950 - val_loss: 0.0762 - val_accuracy: 0.9878\n",
      "Epoch 6/16\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.0545 - accuracy: 0.9950 - val_loss: 0.0759 - val_accuracy: 0.9878\n",
      "Epoch 7/16\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.0541 - accuracy: 0.9950 - val_loss: 0.0755 - val_accuracy: 0.9878\n",
      "Epoch 8/16\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.0538 - accuracy: 0.9950 - val_loss: 0.0751 - val_accuracy: 0.9878\n",
      "Epoch 9/16\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0534 - accuracy: 0.9950 - val_loss: 0.0748 - val_accuracy: 0.9878\n",
      "Epoch 10/16\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0530 - accuracy: 0.9950 - val_loss: 0.0744 - val_accuracy: 0.9878\n",
      "Epoch 11/16\n",
      "7/7 [==============================] - 0s 12ms/step - loss: 0.0526 - accuracy: 0.9950 - val_loss: 0.0741 - val_accuracy: 0.9878\n",
      "Epoch 12/16\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0523 - accuracy: 0.9950 - val_loss: 0.0737 - val_accuracy: 0.9878\n",
      "Epoch 13/16\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0519 - accuracy: 0.9950 - val_loss: 0.0734 - val_accuracy: 0.9878\n",
      "Epoch 14/16\n",
      "7/7 [==============================] - 0s 15ms/step - loss: 0.0515 - accuracy: 0.9950 - val_loss: 0.0730 - val_accuracy: 0.9878\n",
      "Epoch 15/16\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0511 - accuracy: 0.9950 - val_loss: 0.0727 - val_accuracy: 0.9878\n",
      "Epoch 16/16\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0508 - accuracy: 0.9950 - val_loss: 0.0724 - val_accuracy: 0.9878\n"
     ]
    }
   ],
   "source": [
    "# freese the reused layers during the first few epochs, giving the new layer some time to learn reasonable weights\n",
    "\n",
    "for layer in model_B_on_A.layers[:-1]:\n",
    "    layer.trainable = False\n",
    "\n",
    "model_B_on_A.compile(loss = \"binary_crossentropy\", optimizer = \"sgd\",\n",
    "                    metrics = [\"accuracy\"])\n",
    "\n",
    "# train the model for few epochs, then unfreeze the reused layers\n",
    "\n",
    "history = model_B_on_A.fit(X_train_B, y_train_B, epochs = 4,\n",
    "                          validation_data = (X_valid_B, y_valid_B))\n",
    "\n",
    "for layer in model_B_on_A.layers[:-1]:\n",
    "    layer.trainable = True\n",
    "    \n",
    "optimizer = keras.optimizers.SGD(learning_rate = 1e-4)\n",
    "model_B_on_A.compile(loss = \"binary_crossentropy\", optimizer = optimizer,\n",
    "                    metrics = [\"accuracy\"])\n",
    "history = model_B_on_A.fit(X_train_B, y_train_B, epochs = 16,\n",
    "                          validation_data = (X_valid_B, y_valid_B))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63/63 [==============================] - 0s 2ms/step - loss: 0.0627 - accuracy: 0.9935\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.06274233013391495, 0.9934999942779541]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_B_on_A.evaluate(X_test_B, y_test_B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Momentum Optimization\n",
    "\n",
    "optimizer = keras.optimizers.SGD(learning_rate=0.001, momentum = 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nesterov Accelerated Gradient\n",
    "\n",
    "optimizer = keras.optimizers.SGD(learning_rate = 0.001, momentum = 0.9, nesterov = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AdaGrad\n",
    "\n",
    "optimizer = keras.optimizers.Adagrad(learning_rate=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RMSProp\n",
    "\n",
    "optimizer = keras.optimizers.RMSprop(learning_rate=0.001, rho=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adam Optimization\n",
    "\n",
    "optimizer = keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adamax Optimization\n",
    "\n",
    "optimizer = keras.optimizers.Adamax(learning_rate=0.001, beta_1=0.9, beta_2=0.999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nadam Optimization\n",
    "\n",
    "optimizer = keras.optimizers.Nadam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Avoiding Overfitting Through Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l1 and l2 Regularization\n",
    "\n",
    "# l2\n",
    "layer = keras.layers.Dense(100, activation = \"elu\",\n",
    "                          kernel_initializer = \"he_normal\",\n",
    "                          kernel_regularizer = keras.regularizers.l2(0.01))\n",
    "\n",
    "# keras.regularizers.l1() for l1 regularization\n",
    "# keras.regularizers.l1_l2() for both"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply same regularizer to all layers in network\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "RegularizedDense = partial(keras.layers.Dense,\n",
    "                           activation=\"elu\",\n",
    "                           kernel_initializer=\"he_normal\",\n",
    "                           kernel_regularizer=keras.regularizers.l2(0.01))\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    RegularizedDense(300),\n",
    "    RegularizedDense(100),\n",
    "    RegularizedDense(10, activation=\"softmax\")\n",
    "])\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "n_epochs = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dropout rate\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape = [28, 28]),\n",
    "    keras.layers.Dropout(rate = 0.2),\n",
    "    keras.layers.Dense(300, activation = \"elu\", kernel_initializer = \"he_normal\"),\n",
    "    keras.layers.Dropout(rate = 0.2),\n",
    "    keras.layers.Dense(100, activation = \"elu\", kernel_initializer = \"he_normal\"),\n",
    "    keras.layers.Dropout(rate = 0.2),\n",
    "    keras.layers.Dense(10, activation = \"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Monte Carlo (MC) Dropout = averaging over multiple prediction with dropout\n",
    "\n",
    "import numpy as np\n",
    "y_probas = np.stack([model(X_test_scaled, training = True)\n",
    "                    for sample in range(100)])\n",
    "y_proba = y_probas.mean(axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.13, 0.08, 0.05, 0.1 , 0.12, 0.07, 0.21, 0.03, 0.11, 0.1 ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(model.predict(X_test_scaled[:1]), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0.04, 0.1 , 0.09, 0.01, 0.06, 0.01, 0.49, 0.  , 0.18, 0.01]],\n",
       "\n",
       "       [[0.06, 0.05, 0.06, 0.01, 0.02, 0.04, 0.1 , 0.47, 0.16, 0.02]],\n",
       "\n",
       "       [[0.03, 0.09, 0.05, 0.31, 0.06, 0.02, 0.17, 0.02, 0.22, 0.04]],\n",
       "\n",
       "       [[0.04, 0.35, 0.13, 0.17, 0.02, 0.02, 0.06, 0.  , 0.19, 0.02]],\n",
       "\n",
       "       [[0.14, 0.06, 0.02, 0.06, 0.05, 0.03, 0.19, 0.01, 0.36, 0.07]],\n",
       "\n",
       "       [[0.11, 0.01, 0.06, 0.02, 0.03, 0.08, 0.38, 0.  , 0.16, 0.15]],\n",
       "\n",
       "       [[0.05, 0.02, 0.05, 0.02, 0.32, 0.04, 0.32, 0.  , 0.02, 0.17]],\n",
       "\n",
       "       [[0.16, 0.21, 0.02, 0.07, 0.03, 0.03, 0.2 , 0.06, 0.09, 0.12]],\n",
       "\n",
       "       [[0.06, 0.13, 0.01, 0.04, 0.33, 0.08, 0.05, 0.14, 0.15, 0.01]],\n",
       "\n",
       "       [[0.  , 0.01, 0.  , 0.02, 0.02, 0.  , 0.14, 0.  , 0.08, 0.73]],\n",
       "\n",
       "       [[0.14, 0.06, 0.  , 0.03, 0.17, 0.18, 0.17, 0.02, 0.22, 0.01]],\n",
       "\n",
       "       [[0.07, 0.1 , 0.01, 0.1 , 0.01, 0.06, 0.32, 0.03, 0.3 , 0.01]],\n",
       "\n",
       "       [[0.09, 0.19, 0.06, 0.18, 0.03, 0.08, 0.01, 0.01, 0.19, 0.15]],\n",
       "\n",
       "       [[0.05, 0.  , 0.08, 0.05, 0.18, 0.25, 0.18, 0.01, 0.05, 0.15]],\n",
       "\n",
       "       [[0.04, 0.  , 0.01, 0.  , 0.04, 0.06, 0.55, 0.07, 0.05, 0.18]],\n",
       "\n",
       "       [[0.66, 0.01, 0.02, 0.01, 0.04, 0.05, 0.12, 0.02, 0.04, 0.04]],\n",
       "\n",
       "       [[0.37, 0.09, 0.02, 0.02, 0.01, 0.08, 0.29, 0.05, 0.04, 0.03]],\n",
       "\n",
       "       [[0.04, 0.25, 0.02, 0.22, 0.03, 0.01, 0.29, 0.  , 0.02, 0.12]],\n",
       "\n",
       "       [[0.01, 0.02, 0.03, 0.02, 0.03, 0.07, 0.18, 0.01, 0.12, 0.51]],\n",
       "\n",
       "       [[0.09, 0.14, 0.11, 0.02, 0.18, 0.03, 0.1 , 0.  , 0.04, 0.3 ]],\n",
       "\n",
       "       [[0.04, 0.06, 0.02, 0.06, 0.02, 0.01, 0.34, 0.02, 0.42, 0.01]],\n",
       "\n",
       "       [[0.03, 0.17, 0.03, 0.14, 0.2 , 0.19, 0.05, 0.02, 0.05, 0.13]],\n",
       "\n",
       "       [[0.13, 0.03, 0.  , 0.04, 0.15, 0.3 , 0.13, 0.08, 0.08, 0.05]],\n",
       "\n",
       "       [[0.02, 0.03, 0.05, 0.19, 0.04, 0.1 , 0.24, 0.  , 0.26, 0.06]],\n",
       "\n",
       "       [[0.03, 0.04, 0.22, 0.19, 0.08, 0.07, 0.05, 0.  , 0.07, 0.26]],\n",
       "\n",
       "       [[0.13, 0.01, 0.01, 0.03, 0.05, 0.06, 0.06, 0.51, 0.03, 0.13]],\n",
       "\n",
       "       [[0.  , 0.42, 0.02, 0.06, 0.05, 0.02, 0.35, 0.  , 0.05, 0.03]],\n",
       "\n",
       "       [[0.  , 0.  , 0.06, 0.02, 0.09, 0.02, 0.7 , 0.  , 0.07, 0.03]],\n",
       "\n",
       "       [[0.01, 0.82, 0.05, 0.04, 0.01, 0.04, 0.01, 0.  , 0.  , 0.03]],\n",
       "\n",
       "       [[0.01, 0.15, 0.09, 0.38, 0.05, 0.05, 0.11, 0.02, 0.05, 0.1 ]],\n",
       "\n",
       "       [[0.  , 0.01, 0.01, 0.03, 0.04, 0.01, 0.69, 0.03, 0.05, 0.13]],\n",
       "\n",
       "       [[0.04, 0.03, 0.  , 0.05, 0.27, 0.04, 0.34, 0.12, 0.01, 0.1 ]],\n",
       "\n",
       "       [[0.02, 0.  , 0.02, 0.58, 0.17, 0.03, 0.03, 0.01, 0.04, 0.09]],\n",
       "\n",
       "       [[0.02, 0.15, 0.04, 0.09, 0.28, 0.  , 0.06, 0.02, 0.21, 0.12]],\n",
       "\n",
       "       [[0.04, 0.18, 0.03, 0.11, 0.05, 0.08, 0.14, 0.04, 0.33, 0.  ]],\n",
       "\n",
       "       [[0.03, 0.02, 0.23, 0.02, 0.22, 0.1 , 0.06, 0.01, 0.03, 0.28]],\n",
       "\n",
       "       [[0.04, 0.2 , 0.2 , 0.07, 0.  , 0.22, 0.07, 0.  , 0.09, 0.1 ]],\n",
       "\n",
       "       [[0.01, 0.05, 0.02, 0.06, 0.01, 0.02, 0.13, 0.02, 0.62, 0.05]],\n",
       "\n",
       "       [[0.48, 0.02, 0.07, 0.03, 0.04, 0.1 , 0.11, 0.1 , 0.03, 0.01]],\n",
       "\n",
       "       [[0.01, 0.  , 0.  , 0.  , 0.05, 0.  , 0.75, 0.  , 0.16, 0.02]],\n",
       "\n",
       "       [[0.31, 0.  , 0.08, 0.07, 0.03, 0.18, 0.02, 0.  , 0.02, 0.28]],\n",
       "\n",
       "       [[0.02, 0.13, 0.03, 0.01, 0.04, 0.  , 0.54, 0.  , 0.11, 0.11]],\n",
       "\n",
       "       [[0.02, 0.53, 0.05, 0.13, 0.07, 0.02, 0.09, 0.01, 0.02, 0.05]],\n",
       "\n",
       "       [[0.02, 0.12, 0.01, 0.04, 0.05, 0.08, 0.31, 0.17, 0.17, 0.03]],\n",
       "\n",
       "       [[0.03, 0.01, 0.  , 0.08, 0.28, 0.05, 0.04, 0.05, 0.37, 0.08]],\n",
       "\n",
       "       [[0.03, 0.03, 0.  , 0.02, 0.09, 0.07, 0.08, 0.58, 0.05, 0.04]],\n",
       "\n",
       "       [[0.05, 0.22, 0.01, 0.03, 0.06, 0.01, 0.05, 0.01, 0.08, 0.48]],\n",
       "\n",
       "       [[0.45, 0.05, 0.1 , 0.03, 0.07, 0.12, 0.02, 0.06, 0.02, 0.09]],\n",
       "\n",
       "       [[0.32, 0.01, 0.  , 0.05, 0.11, 0.09, 0.24, 0.02, 0.15, 0.01]],\n",
       "\n",
       "       [[0.17, 0.04, 0.24, 0.03, 0.01, 0.14, 0.05, 0.02, 0.18, 0.12]],\n",
       "\n",
       "       [[0.04, 0.22, 0.2 , 0.07, 0.03, 0.01, 0.03, 0.  , 0.24, 0.16]],\n",
       "\n",
       "       [[0.03, 0.02, 0.03, 0.03, 0.57, 0.03, 0.05, 0.02, 0.14, 0.07]],\n",
       "\n",
       "       [[0.01, 0.01, 0.04, 0.2 , 0.05, 0.15, 0.37, 0.01, 0.12, 0.04]],\n",
       "\n",
       "       [[0.08, 0.57, 0.03, 0.01, 0.  , 0.02, 0.17, 0.03, 0.01, 0.08]],\n",
       "\n",
       "       [[0.01, 0.21, 0.06, 0.38, 0.03, 0.08, 0.03, 0.05, 0.12, 0.03]],\n",
       "\n",
       "       [[0.12, 0.  , 0.1 , 0.05, 0.03, 0.4 , 0.02, 0.03, 0.25, 0.  ]],\n",
       "\n",
       "       [[0.02, 0.13, 0.14, 0.05, 0.  , 0.02, 0.31, 0.02, 0.02, 0.28]],\n",
       "\n",
       "       [[0.14, 0.18, 0.01, 0.05, 0.04, 0.01, 0.39, 0.16, 0.01, 0.02]],\n",
       "\n",
       "       [[0.46, 0.04, 0.01, 0.03, 0.07, 0.01, 0.2 , 0.07, 0.05, 0.07]],\n",
       "\n",
       "       [[0.04, 0.06, 0.06, 0.41, 0.31, 0.03, 0.04, 0.  , 0.05, 0.01]],\n",
       "\n",
       "       [[0.06, 0.  , 0.03, 0.04, 0.04, 0.25, 0.08, 0.03, 0.14, 0.33]],\n",
       "\n",
       "       [[0.21, 0.  , 0.04, 0.  , 0.02, 0.09, 0.12, 0.46, 0.03, 0.02]],\n",
       "\n",
       "       [[0.01, 0.21, 0.01, 0.04, 0.02, 0.08, 0.15, 0.04, 0.42, 0.02]],\n",
       "\n",
       "       [[0.05, 0.02, 0.16, 0.03, 0.11, 0.01, 0.37, 0.01, 0.19, 0.04]],\n",
       "\n",
       "       [[0.02, 0.03, 0.06, 0.04, 0.52, 0.05, 0.02, 0.12, 0.08, 0.06]],\n",
       "\n",
       "       [[0.24, 0.07, 0.07, 0.14, 0.02, 0.09, 0.23, 0.01, 0.03, 0.1 ]],\n",
       "\n",
       "       [[0.  , 0.02, 0.01, 0.15, 0.51, 0.02, 0.12, 0.11, 0.06, 0.  ]],\n",
       "\n",
       "       [[0.02, 0.21, 0.12, 0.04, 0.05, 0.14, 0.23, 0.1 , 0.01, 0.08]],\n",
       "\n",
       "       [[0.01, 0.12, 0.02, 0.08, 0.55, 0.01, 0.09, 0.02, 0.07, 0.04]],\n",
       "\n",
       "       [[0.01, 0.11, 0.01, 0.21, 0.04, 0.37, 0.14, 0.01, 0.03, 0.06]],\n",
       "\n",
       "       [[0.01, 0.25, 0.32, 0.14, 0.03, 0.02, 0.21, 0.01, 0.01, 0.01]],\n",
       "\n",
       "       [[0.05, 0.18, 0.07, 0.16, 0.01, 0.07, 0.29, 0.1 , 0.07, 0.01]],\n",
       "\n",
       "       [[0.37, 0.02, 0.01, 0.01, 0.03, 0.27, 0.14, 0.06, 0.02, 0.07]],\n",
       "\n",
       "       [[0.22, 0.01, 0.01, 0.05, 0.1 , 0.04, 0.27, 0.02, 0.01, 0.27]],\n",
       "\n",
       "       [[0.08, 0.12, 0.06, 0.05, 0.1 , 0.09, 0.06, 0.01, 0.36, 0.06]],\n",
       "\n",
       "       [[0.02, 0.12, 0.  , 0.01, 0.24, 0.03, 0.22, 0.16, 0.02, 0.19]],\n",
       "\n",
       "       [[0.14, 0.03, 0.05, 0.12, 0.19, 0.08, 0.16, 0.01, 0.1 , 0.12]],\n",
       "\n",
       "       [[0.84, 0.03, 0.03, 0.02, 0.01, 0.01, 0.03, 0.03, 0.01, 0.  ]],\n",
       "\n",
       "       [[0.02, 0.05, 0.01, 0.02, 0.02, 0.03, 0.22, 0.  , 0.35, 0.27]],\n",
       "\n",
       "       [[0.54, 0.01, 0.03, 0.09, 0.03, 0.1 , 0.03, 0.01, 0.02, 0.13]],\n",
       "\n",
       "       [[0.21, 0.02, 0.34, 0.02, 0.07, 0.02, 0.1 , 0.01, 0.11, 0.08]],\n",
       "\n",
       "       [[0.06, 0.02, 0.04, 0.22, 0.3 , 0.04, 0.18, 0.01, 0.01, 0.12]],\n",
       "\n",
       "       [[0.06, 0.05, 0.17, 0.57, 0.02, 0.04, 0.06, 0.01, 0.02, 0.03]],\n",
       "\n",
       "       [[0.68, 0.02, 0.07, 0.01, 0.03, 0.01, 0.12, 0.01, 0.04, 0.01]],\n",
       "\n",
       "       [[0.3 , 0.02, 0.03, 0.01, 0.08, 0.33, 0.12, 0.02, 0.04, 0.04]],\n",
       "\n",
       "       [[0.22, 0.09, 0.01, 0.19, 0.13, 0.1 , 0.16, 0.04, 0.05, 0.02]],\n",
       "\n",
       "       [[0.01, 0.08, 0.01, 0.08, 0.02, 0.  , 0.78, 0.  , 0.01, 0.01]],\n",
       "\n",
       "       [[0.12, 0.03, 0.07, 0.05, 0.18, 0.04, 0.37, 0.05, 0.05, 0.04]],\n",
       "\n",
       "       [[0.01, 0.24, 0.09, 0.08, 0.18, 0.14, 0.05, 0.08, 0.1 , 0.03]],\n",
       "\n",
       "       [[0.1 , 0.35, 0.19, 0.03, 0.07, 0.02, 0.14, 0.  , 0.06, 0.06]],\n",
       "\n",
       "       [[0.09, 0.05, 0.04, 0.01, 0.06, 0.05, 0.36, 0.17, 0.16, 0.02]],\n",
       "\n",
       "       [[0.  , 0.06, 0.  , 0.42, 0.08, 0.04, 0.1 , 0.07, 0.01, 0.23]],\n",
       "\n",
       "       [[0.09, 0.17, 0.15, 0.03, 0.12, 0.01, 0.25, 0.05, 0.1 , 0.04]],\n",
       "\n",
       "       [[0.02, 0.09, 0.  , 0.03, 0.36, 0.21, 0.2 , 0.02, 0.03, 0.04]],\n",
       "\n",
       "       [[0.08, 0.01, 0.04, 0.03, 0.12, 0.03, 0.52, 0.02, 0.03, 0.13]],\n",
       "\n",
       "       [[0.09, 0.08, 0.07, 0.04, 0.08, 0.04, 0.35, 0.02, 0.03, 0.2 ]],\n",
       "\n",
       "       [[0.05, 0.  , 0.  , 0.03, 0.34, 0.  , 0.03, 0.01, 0.09, 0.45]],\n",
       "\n",
       "       [[0.04, 0.03, 0.07, 0.08, 0.28, 0.04, 0.11, 0.  , 0.02, 0.34]],\n",
       "\n",
       "       [[0.05, 0.  , 0.01, 0.03, 0.02, 0.02, 0.71, 0.06, 0.  , 0.09]],\n",
       "\n",
       "       [[0.07, 0.05, 0.03, 0.01, 0.28, 0.15, 0.23, 0.06, 0.06, 0.06]]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(y_probas[:, :1], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.11, 0.1 , 0.06, 0.09, 0.11, 0.07, 0.2 , 0.05, 0.11, 0.11]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(y_proba[:1], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.16, 0.13, 0.07, 0.11, 0.13, 0.08, 0.18, 0.1 , 0.11, 0.12]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# standard deviation of the probability estimates\n",
    "\n",
    "y_std = y_probas.std(axis = 0)\n",
    "np.round(y_std[:1], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1167"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = np.argmax(y_proba, axis=1)\n",
    "accuracy = np.sum(y_pred == y_test) / len(y_test)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MCDropout(keras.layers.Dropout):\n",
    "    def call(self, inputs):\n",
    "        return super().call(inputs, training=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MCAlphaDropout(keras.layers.AlphaDropout):\n",
    "    def call(self, inputs):\n",
    "        return super().call(inputs, training=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "mc_model = keras.models.Sequential([\n",
    "    MCAlphaDropout(layer.rate) if isinstance(layer, keras.layers.AlphaDropout) else layer\n",
    "    for layer in model.layers\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " flatten_3 (Flatten)         (None, 784)               0         \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 784)               0         \n",
      "                                                                 \n",
      " dense_8 (Dense)             (None, 300)               235500    \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 300)               0         \n",
      "                                                                 \n",
      " dense_9 (Dense)             (None, 100)               30100     \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 100)               0         \n",
      "                                                                 \n",
      " dense_10 (Dense)            (None, 10)                1010      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 266,610\n",
      "Trainable params: 266,610\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "mc_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True)\n",
    "mc_model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "mc_model.set_weights(model.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.13, 0.08, 0.05, 0.1 , 0.12, 0.07, 0.21, 0.03, 0.11, 0.1 ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(np.mean([mc_model.predict(X_test_scaled[:1]) for sample in range(100)], axis=0), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Max-Norm Regularization\n",
    "\n",
    "layer = keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                           kernel_constraint=keras.constraints.max_norm(1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "1719/1719 [==============================] - 6s 3ms/step - loss: 0.4738 - accuracy: 0.8335 - val_loss: 0.3767 - val_accuracy: 0.8622\n",
      "Epoch 2/2\n",
      "1719/1719 [==============================] - 6s 3ms/step - loss: 0.3538 - accuracy: 0.8718 - val_loss: 0.3763 - val_accuracy: 0.8694\n"
     ]
    }
   ],
   "source": [
    "MaxNormDense = partial(keras.layers.Dense,\n",
    "                       activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       kernel_constraint=keras.constraints.max_norm(1.))\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    MaxNormDense(300),\n",
    "    MaxNormDense(100),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "n_epochs = 2\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}