Lesson 18 - Fake News Classification - Solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recurrent Neural Nets - Fake News\n",
    "\n",
    "The RNN (LSTM) architechture that we are using is shown below, a many to one RNN.\n",
    "\n",
    "![RNNs](./images/manytoone.jpg)\n",
    "\n",
    "<img src='https://media.giphy.com/media/l0Iyau7QcKtKUYIda/giphy.gif'>\n",
    "\n",
    "We achieve 87% accuracy in a test set. However, the article in Second reference claims to have 93% accuracy. The main difference is that they seem to use a Bag of Words Model, which loses the order of words when sending into the ML algorithm. Also\n",
    "\n",
    "## References:\n",
    "1. Data: https://github.com/GeorgeMcIntire/fake_real_news_dataset\n",
    "2. Classification using Scikit Learn: https://blog.kjamistan.com/comparing-scikit-learn-text-classifiers-on-a-fake-news-dataset/\n",
    "3. Glove vectors: https://nlp.stanford.edu/projects/glove/\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tqdm in /root/miniconda3/lib/python3.6/site-packages\n",
      "Fetching package metadata ...........\n",
      "Solving package specifications: .\n",
      "\n",
      "# All requested packages already installed.\n",
      "# packages in environment at /root/miniconda3:\n",
      "#\n",
      "pillow                    4.2.1                    py36_0  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "!pip install tqdm\n",
    "!conda install -y Pillow\n",
    "\n",
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import re\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization, LSTM, Embedding, Reshape\n",
    "from keras.models import load_model, model_from_json\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import os\n",
    "import urllib\n",
    "\n",
    "from urllib.request import urlretrieve\n",
    "\n",
    "from os import mkdir, makedirs, remove, listdir\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "from utilties import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "folder_path = 'data'\n",
    "\n",
    "file = './data/fakenews.zip'\n",
    "url = 'https://github.com/GeorgeMcIntire/fake_real_news_dataset/raw/master/fake_or_real_news.csv.zip'\n",
    "downloadData(file, url)\n",
    "\n",
    "#################################    \n",
    "# Download GLOVE vector dataset\n",
    "#################################\n",
    "file = './data/glove.6B.zip'\n",
    "url = 'http://nlp.stanford.edu/data/glove.6B.zip'\n",
    "downloadData(file, url)\n",
    "\n",
    "with open('./data/glove.6B.50d.txt','rb') as f:\n",
    "    lines = f.readlines()\n",
    "    \n",
    "glove_weights = np.zeros((len(lines), 50))\n",
    "words = []\n",
    "for i, line in enumerate(lines):\n",
    "    word_weights = line.split()\n",
    "    words.append(word_weights[0])\n",
    "    weight = word_weights[1:]\n",
    "    glove_weights[i] = np.array([float(w) for w in weight])\n",
    "word_vocab = [w.decode(\"utf-8\") for w in words]\n",
    "\n",
    "word2glove = dict(zip(word_vocab, glove_weights))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocessing steps: lower case, remove urls, some punctuations etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.engine.topology import Layer\n",
    "import keras.backend as K\n",
    "from keras import initializers\n",
    "import numpy as np\n",
    "\n",
    "class Embedding2(Layer):\n",
    "\n",
    "    def __init__(self, input_dim, output_dim, fixed_weights, embeddings_initializer='uniform', \n",
    "                 input_length=None, **kwargs):\n",
    "        kwargs['dtype'] = 'int32'\n",
    "        if 'input_shape' not in kwargs:\n",
    "            if input_length:\n",
    "                kwargs['input_shape'] = (input_length,)\n",
    "            else:\n",
    "                kwargs['input_shape'] = (None,)\n",
    "        super(Embedding2, self).__init__(**kwargs)\n",
    "    \n",
    "        self.input_dim = input_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.embeddings_initializer = embeddings_initializer\n",
    "        self.fixed_weights = fixed_weights\n",
    "        self.num_trainable = input_dim - len(fixed_weights)\n",
    "        self.input_length = input_length\n",
    "        \n",
    "        w_mean = fixed_weights.mean(axis=0)\n",
    "        w_std = fixed_weights.std(axis=0)\n",
    "        self.variable_weights = w_mean + w_std*np.random.randn(self.num_trainable, output_dim)\n",
    "\n",
    "    def build(self, input_shape, name='embeddings'):        \n",
    "        fixed_weight = K.variable(self.fixed_weights, name=name+'_fixed')\n",
    "        variable_weight = K.variable(self.variable_weights, name=name+'_var')\n",
    "        \n",
    "        self._trainable_weights.append(variable_weight)\n",
    "        self._non_trainable_weights.append(fixed_weight)\n",
    "        \n",
    "        self.embeddings = K.concatenate([fixed_weight, variable_weight], axis=0)\n",
    "        \n",
    "        self.built = True\n",
    "\n",
    "    def call(self, inputs):\n",
    "        if K.dtype(inputs) != 'int32':\n",
    "            inputs = K.cast(inputs, 'int32')\n",
    "        out = K.gather(self.embeddings, inputs)\n",
    "        return out\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        if not self.input_length:\n",
    "            input_length = input_shape[1]\n",
    "        else:\n",
    "            input_length = self.input_length\n",
    "        return (input_shape[0], input_length, self.output_dim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6335, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>you can smell hillarys fear</td>\n",
       "      <td>daniel greenfield a shillman journalism fellow...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>watch the exact moment paul ryan committed pol...</td>\n",
       "      <td>google pinterest digg linkedin reddit stumbleu...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kerry to go to paris in gesture of sympathy</td>\n",
       "      <td>u . s . secretary of state john f . kerry said...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bernie supporters on twitter erupt in anger ag...</td>\n",
       "      <td>kaydee king kaydeeking november 9 2016 the les...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the battle of new york why this primary matters</td>\n",
       "      <td>its primary day in new york and frontrunners h...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                        you can smell hillarys fear   \n",
       "1  watch the exact moment paul ryan committed pol...   \n",
       "2        kerry to go to paris in gesture of sympathy   \n",
       "3  bernie supporters on twitter erupt in anger ag...   \n",
       "4    the battle of new york why this primary matters   \n",
       "\n",
       "                                                text label  \n",
       "0  daniel greenfield a shillman journalism fellow...  FAKE  \n",
       "1  google pinterest digg linkedin reddit stumbleu...  FAKE  \n",
       "2  u . s . secretary of state john f . kerry said...  REAL  \n",
       "3  kaydee king kaydeeking november 9 2016 the les...  FAKE  \n",
       "4  its primary day in new york and frontrunners h...  REAL  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/fake_or_real_news.csv')\n",
    "df.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "df.title = df.title.str.lower()\n",
    "df.text = df.text.str.lower()\n",
    "\n",
    "df.title = df.title.str.replace(r'http[\\w:/\\.]+','<URL>') # remove urls\n",
    "df.text = df.text.str.replace(r'http[\\w:/\\.]+','<URL>') # remove urls\n",
    "df.title = df.title.str.replace(r'[^\\.\\w\\s]','') #remove everything but characters and punctuation\n",
    "df.text = df.text.str.replace(r'[^\\.\\w\\s]','') #remove everything but characters and punctuation\n",
    "df.title = df.title.str.replace(r'\\.\\.+','.') #replace multple periods with a single one\n",
    "df.text = df.text.str.replace(r'\\.\\.+','.') #replace multple periods with a single one\n",
    "df.title = df.title.str.replace(r'\\.',' . ') #replace periods with a single one\n",
    "df.text = df.text.str.replace(r'\\.',' . ') #replace multple periods with a single one\n",
    "df.title = df.title.str.replace(r'\\s\\s+',' ') #replace multple white space with a single one\n",
    "df.text = df.text.str.replace(r'\\s\\s+',' ') #replace multple white space with a single one\n",
    "df.title = df.title.str.strip() \n",
    "df.text = df.text.str.strip() \n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get all the unique words. We will only consider words that have been used more than 5 times. Finally from this we create a dictionary mapping words to integers.\n",
    "\n",
    "Once this is done we will create a list of reviews where the words are converted to ints."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fraction of unique words in glove vectors:  0.668536737092\n",
      "The number of unique words are:  86806\n",
      "The first review looks like this: \n",
      "[4447, 12884, 5, 59768, 2855, 1229, 24, 0, 573, 483, 8, 5, 56, 243, 2518, 2856, 10, 1455, 1642, 1]\n",
      "And once this is converted back to words, it looks like: \n",
      "daniel greenfield a <Other> journalism fellow at the freedom center is a new york writer focusing on radical islam .\n"
     ]
    }
   ],
   "source": [
    "all_text = ' '.join(df.text.values)\n",
    "words = all_text.split()\n",
    "u_words = Counter(words).most_common()\n",
    "u_words_counter = u_words\n",
    "u_words_frequent = [word[0] for word in u_words if word[1]>5] # we will only consider words that have been used more than 5 times\n",
    "\n",
    "u_words_total = [k for k,v in u_words_counter]\n",
    "word_vocab = dict(zip(word_vocab, range(len(word_vocab))))\n",
    "word_in_glove = np.array([w in word_vocab for w in u_words_total])\n",
    "\n",
    "words_in_glove = [w for w,is_true in zip(u_words_total,word_in_glove) if is_true]\n",
    "words_not_in_glove = [w for w,is_true in zip(u_words_total,word_in_glove) if not is_true]\n",
    "\n",
    "print('Fraction of unique words in glove vectors: ', sum(word_in_glove)/len(word_in_glove))\n",
    "\n",
    "# # create the dictionary\n",
    "word2num = dict(zip(words_in_glove,range(len(words_in_glove))))\n",
    "len_glove_words = len(word2num)\n",
    "freq_words_not_glove = [w for w in words_not_in_glove if w in u_words_frequent]\n",
    "b = dict(zip(freq_words_not_glove,range(len(word2num), len(word2num)+len(freq_words_not_glove))))\n",
    "word2num = dict(**word2num, **b)\n",
    "word2num['<Other>'] = len(word2num)\n",
    "num2word = dict(zip(word2num.values(), word2num.keys()))\n",
    "\n",
    "int_text = [[word2num[word] if word in word2num else word2num['<Other>'] \n",
    "             for word in content.split()] for content in df.text.values]\n",
    "\n",
    "print('The number of unique words are: ', len(u_words))\n",
    "print('The first review looks like this: ')\n",
    "print(int_text[0][:20])\n",
    "print('And once this is converted back to words, it looks like: ')\n",
    "print(' '.join([num2word[i] for i in int_text[0][:20]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADm9JREFUeJzt3F/I3uV9x/H3Z0nrpK1DlyyEGPZEyEkUZmvIhJXSIatW\nD2JhlPRg5kDMQFda2A7ielBPAumgHcimI0Uxjq42W1sMWDdsVpAdaHws1pi4zFgj5iGatMLsTty0\n3x3cV+zdJ0/6/L/v5LneL/hxX/f39+e+fhc/ns/z+3PfqSokSX36rXF3QJI0PoaAJHXMEJCkjhkC\nktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOrx92B2axZs6YmJibG3Q1JuqQ8//zzP6uqtbMtd9GH\nwMTEBJOTk+PuhiRdUpK8PpflvBwkSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pgh\nIEkdu+i/MbwYE7ufmLF+cu9tI+6JJF2cPBOQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKlj\nhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYI\nSFLHDAFJ6pghIEkdMwQkqWOGgCR1bNYQSLIxyY+SHEtyNMmXWv2qJE8leaW9Xjm0zr1JTiQ5nuTm\nofoNSY60efcnyfLsliRpLuZyJvAe8JdVtQW4EbgnyRZgN3CoqjYDh9p72rwdwLXALcADSVa1bT0I\n3AVsbtMtS7gvkqR5mjUEqup0Vf24tX8BvAxsALYD+9ti+4HbW3s78FhVvVtVrwEngG1J1gNXVNUz\nVVXAo0PrSJLGYF73BJJMAB8HngXWVdXpNutNYF1rbwDeGFrtVKttaO3pdUnSmMw5BJJ8FPgu8OWq\nemd4XvvPvpaqU0l2JZlMMnn27Nml2qwkaZo5hUCSDzEIgG9V1fda+a12iYf2eqbVp4CNQ6tf3WpT\nrT29fp6q2ldVW6tq69q1a+e6L5KkeZrL00EBHgJerqpvDM06COxs7Z3A40P1HUkuS7KJwQ3gw+3S\n0TtJbmzbvGNoHUnSGKyewzJ/BPwZcCTJC63218Be4ECSO4HXgc8DVNXRJAeAYwyeLLqnqt5v690N\nPAJcDjzZJknSmMwaAlX1H8CFnue/6QLr7AH2zFCfBK6bTwclScvHbwxLUscMAUnqmCEgSR0zBCSp\nY4aAJHVsLo+IrjgTu5+YsX5y720j7okkjZdnApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAk\ndcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLH\nDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljs4ZAkoeT\nnEny0lDtviRTSV5o061D8+5NciLJ8SQ3D9VvSHKkzbs/SZZ+dyRJ8zGXM4FHgFtmqP9tVV3fph8A\nJNkC7ACubes8kGRVW/5B4C5gc5tm2qYkaYRmDYGqehp4e47b2w48VlXvVtVrwAlgW5L1wBVV9UxV\nFfAocPtCOy1JWhqLuSfwxSQvtstFV7baBuCNoWVOtdqG1p5en1GSXUkmk0yePXt2EV2UJP0mCw2B\nB4FrgOuB08DXl6xHQFXtq6qtVbV17dq1S7lpSdKQBYVAVb1VVe9X1S+BbwLb2qwpYOPQole32lRr\nT69LksZoQSHQrvGf8zng3JNDB4EdSS5LsonBDeDDVXUaeCfJje2poDuAxxfRb0nSElg92wJJvg18\nGliT5BTwVeDTSa4HCjgJ/DlAVR1NcgA4BrwH3FNV77dN3c3gSaPLgSfbJEkao1lDoKq+MEP5od+w\n/B5gzwz1SeC6efVOkrSs/MawJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1\nzBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscM\nAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHVo+7AxeTid1PzFg/ufe2EfdEkkbD\nMwFJ6pghIEkdMwQkqWOGgCR1zBCQpI7NGgJJHk5yJslLQ7WrkjyV5JX2euXQvHuTnEhyPMnNQ/Ub\nkhxp8+5PkqXfHUnSfMzlTOAR4JZptd3AoaraDBxq70myBdgBXNvWeSDJqrbOg8BdwOY2Td+mJGnE\nZg2BqnoaeHtaeTuwv7X3A7cP1R+rqner6jXgBLAtyXrgiqp6pqoKeHRoHUnSmCz0nsC6qjrd2m8C\n61p7A/DG0HKnWm1Da0+vS5LGaNE3htt/9rUEfflAkl1JJpNMnj17dik3LUkastAQeKtd4qG9nmn1\nKWDj0HJXt9pUa0+vz6iq9lXV1qraunbt2gV2UZI0m4WGwEFgZ2vvBB4fqu9IclmSTQxuAB9ul47e\nSXJjeyrojqF1JEljMusPyCX5NvBpYE2SU8BXgb3AgSR3Aq8DnweoqqNJDgDHgPeAe6rq/bapuxk8\naXQ58GSbJEljNGsIVNUXLjDrpgssvwfYM0N9ErhuXr2TJC0rvzEsSR0zBCSpY4aAJHXMEJCkjhkC\nktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJ\nHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQx\nQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1bPW4O3ApmNj9xIz1k3tvG3FPJGlpLepMIMnJJEeS\nvJBkstWuSvJUklfa65VDy9+b5ESS40luXmznJUmLsxSXg/64qq6vqq3t/W7gUFVtBg619yTZAuwA\nrgVuAR5IsmoJPl+StEDLcU9gO7C/tfcDtw/VH6uqd6vqNeAEsG0ZPl+SNEeLDYECfpjk+SS7Wm1d\nVZ1u7TeBda29AXhjaN1TrXaeJLuSTCaZPHv27CK7KEm6kMXeGP5kVU0l+T3gqST/OTyzqipJzXej\nVbUP2AewdevWea8vSZqbRZ0JVNVUez0DfJ/B5Z23kqwHaK9n2uJTwMah1a9uNUnSmCw4BJJ8JMnH\nzrWBzwAvAQeBnW2xncDjrX0Q2JHksiSbgM3A4YV+viRp8RZzOWgd8P0k57bzT1X1r0meAw4kuRN4\nHfg8QFUdTXIAOAa8B9xTVe8vqveSpEVZcAhU1U+BP5ih/nPgpgusswfYs9DPlCQtLX82QpI6ZghI\nUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1\nzBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOrR53By5lE7ufmLF+cu9t\nI+6JJC2MZwKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpYz4iugx8dFTSpcIzAUnqmCEgSR0zBCSp\nY94TGKEL3SsA7xdIGo+RnwkkuSXJ8SQnkuwe9edLkn5lpGcCSVYBfw/8CXAKeC7Jwao6Nsp+XIx+\n01nCfHhGIWk+Rn0msA04UVU/rar/BR4Dto+4D5KkZtT3BDYAbwy9PwX84Yj7sKLN9zsKS/WdBr8b\nIV2aLsobw0l2Abva2/9JcnyBm1oD/GxpenVpy9c+aM5pTIaWX6rPvVh5jJzPMTnfpTgmvz+XhUYd\nAlPAxqH3V7far6mqfcC+xX5Yksmq2rrY7awkjsmvczzO55icbyWPyajvCTwHbE6yKcmHgR3AwRH3\nQZLUjPRMoKreS/IXwL8Bq4CHq+roKPsgSfqVkd8TqKofAD8Y0cct+pLSCuSY/DrH43yOyflW7Jik\nqsbdB0nSmPjbQZLUsRUZAr39NEWSk0mOJHkhyWSrXZXkqSSvtNcrh5a/t43N8SQ3D9VvaNs5keT+\nJBnH/sxXkoeTnEny0lBtyfY/yWVJvtPqzyaZGOX+LcQFxuS+JFPtOHkhya1D81b0mCTZmORHSY4l\nOZrkS63e9XECQFWtqInBDedXgWuADwM/AbaMu1/LvM8ngTXTan8D7G7t3cDXWntLG5PLgE1trFa1\neYeBG4EATwKfHfe+zXH/PwV8AnhpOfYfuBv4h9beAXxn3Pu8wDG5D/irGZZd8WMCrAc+0dofA/6r\n7XfXx0lVrcgzAX+aYmA7sL+19wO3D9Ufq6p3q+o14ASwLcl64IqqeqYGR/GjQ+tc1KrqaeDtaeWl\n3P/hbf0LcNPFfpZ0gTG5kBU/JlV1uqp+3Nq/AF5m8AsGXR8nsDIvB8300xQbxtSXUSngh0meb9+2\nBlhXVadb+01gXWtfaHw2tPb0+qVqKff/g3Wq6j3gv4HfXZ5uL7svJnmxXS46d+mjqzFpl2k+DjyL\nx8mKDIEefbKqrgc+C9yT5FPDM9t/LN0+Btb7/g95kMFl0uuB08DXx9ud0UvyUeC7wJer6p3heb0e\nJysxBOb00xQrSVVNtdczwPcZXBJ7q5260l7PtMUvND5TrT29fqlayv3/YJ0kq4HfAX6+bD1fJlX1\nVlW9X1W/BL7J4DiBTsYkyYcYBMC3qup7rdz9cbISQ6Crn6ZI8pEkHzvXBj4DvMRgn3e2xXYCj7f2\nQWBHe5JhE7AZONxOid9JcmO7jnnH0DqXoqXc/+Ft/Snw7+2/xkvKuT92zecYHCfQwZi0/j8EvFxV\n3xia5XEy7jvTyzEBtzK4+/8q8JVx92eZ9/UaBk8x/AQ4em5/GVyLPAS8AvwQuGpona+0sTnO0BNA\nwFYGfxheBf6O9mXCi30Cvs3g8sb/MbhGe+dS7j/w28A/M7g5eBi4Ztz7vMAx+UfgCPAigz9Y63sZ\nE+CTDC71vAi80KZbez9OqspvDEtSz1bi5SBJ0hwZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKlj\nhoAkdez/AZWXL1UekkIfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7feb47834748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist([len(t) for t in int_text],50)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of articles greater than 500 in length is:  3714\n",
      "The number of articles less than 50 in length is:  440\n"
     ]
    }
   ],
   "source": [
    "print('The number of articles greater than 500 in length is: ', np.sum(np.array([len(t)>500 for t in int_text])))\n",
    "print('The number of articles less than 50 in length is: ', np.sum(np.array([len(t)<50 for t in int_text])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You cannot pass differing lengths of sentences to the algorithm. Hence we shall prepad the sentence with `<PAD>`. Sequences less than 500 in length will be prepadded and sequences that are longer than 500 will be truncated. It is assumed that the sentiment of the review can be asserted from the first 500 words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num2word[len(word2num)] = '<PAD>'\n",
    "word2num['<PAD>'] = len(word2num)\n",
    "\n",
    "for i, t in enumerate(int_text):\n",
    "    if len(t)<500:\n",
    "        int_text[i] = [word2num['<PAD>']]*(500-len(t)) + t\n",
    "    elif len(t)>500:\n",
    "        int_text[i] = t[:500]\n",
    "    else:\n",
    "        continue\n",
    "\n",
    "x = np.array(int_text)\n",
    "y = (df.label.values=='REAL').astype('int')\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A real news article:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'u . s . secretary of state john f . kerry said monday that he will stop in paris later this week amid criticism that no top american officials attended sundays unity march against terrorism . kerry said he expects to arrive in paris thursday evening as he heads home after a week abroad . he said he will fly to france at the conclusion of a series of meetings scheduled for thursday in sofia bulgaria . he plans to meet the next day with foreign minister laurent fabius and president francois hollande then return to washington . the visit by kerry who has family and childhood ties to the country and speaks fluent french could address some of the criticism that the united states snubbed france in its darkest hour in many years . the french press on monday was filled with questions about why neither president obama nor kerry attended sundays march as about 40 leaders of other nations did . obama was said to have stayed away because his own security needs can be taxing on a country and kerry had prior commitments . among roughly 40 leaders who did attend was israeli prime minister benjamin netanyahu no stranger to intense security who marched beside hollande through the city streets . the highest ranking u . s . officials attending the march were jane hartley the ambassador to france and victoria nuland the assistant secretary of state for european affairs . attorney general eric h . holder jr . was in paris for meetings with law enforcement officials but did not participate in the march . kerry spent sunday at a business summit hosted by indias prime minister narendra modi . the united states is eager for india to relax stringent laws that function as barriers to foreign investment and hopes modis government will act to open the huge indian market for more american businesses . in a news conference kerry brushed aside criticism that the united states had not sent a more senior official to paris as quibbling a little bit . he noted that many staffers of the american embassy in paris attended the march including the ambassador . he said he had wanted to be present at the march himself but could not because of his prior commitments in india . but that is why i am going there on the way home to make it crystal clear how passionately we feel about the events that have taken place there he said . and i dont think the people of france have any doubts about americas understanding of what happened of our personal sense of loss and our deep commitment to the people of france in this moment of trauma .'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.label=='REAL'].text.values[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A fake news article:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'daniel greenfield a shillman journalism fellow at the freedom center is a new york writer focusing on radical islam . in the final stretch of the election hillary rodham clinton has gone to war with the fbi . the word unprecedented has been thrown around so often this election that it ought to be retired . but its still unprecedented for the nominee of a major political party to go war with the fbi . but thats exactly what hillary and her people have done . coma patients just waking up now and watching an hour of cnn from their hospital beds would assume that fbi director james comey is hillarys opponent in this election . the fbi is under attack by everyone from obama to cnn . hillarys people have circulated a letter attacking comey . there are currently more media hit pieces lambasting him than targeting trump . it wouldnt be too surprising if the clintons or their allies were to start running attack ads against the fbi . the fbis leadership is being warned that the entire leftwing establishment will form a lynch mob if they continue going after hillary . and the fbis credibility is being attacked by the media and the democrats to preemptively head off the results of the investigation of the clinton foundation and hillary clinton . the covert struggle between fbi agents and obamas doj people has gone explosively public . the new york times has compared comey to j . edgar hoover . its bizarre headline james comey role recalls hoovers fbi fairly or not practically admits up front that its spouting nonsense . the boston globe has published a column calling for comeys resignation . not to be outdone time has an editorial claiming that the scandal is really an attack on all women . james carville appeared on msnbc to remind everyone that he was still alive and insane . he accused comey of coordinating with house republicans and the kgb . and you thought the vast right wing conspiracy was a stretch . countless media stories charge comey with violating procedure . do you know whats a procedural violation emailing classified information stored on your bathroom server . senator harry reid has sent comey a letter accusing him of violating the hatch act . the hatch act is a nice idea that has as much relevance in the age of obama as the tenth amendment . but the cable news spectrum quickly filled with media hacks glancing at the wikipedia article on the hatch act under the table while accusing the fbi director of one of the most awkward conspiracies against hillary ever . if james comey is really out to hurt hillary he picked one hell of a strange way to do it . not too long ago democrats were breathing a sigh of relief when he gave hillary clinton a pass in a prominent public statement . if he really were out to elect trump by keeping the email scandal going why did he trash the investigation was he on the payroll of house republicans and the kgb back then and playing it coy or was it a sudden development where vladimir putin and paul ryan talked him into taking a look at anthony weiners computer either comey is the most cunning fbi director that ever lived or hes just awkwardly trying to navigate a political mess that has trapped him between a doj leadership whose political futures are tied to hillarys victory and his own bureau whose apolitical agents just want to be allowed to do their jobs . the only truly mysterious thing is why hillary and her associates decided to go to war with a respected federal agency . most americans like the fbi while hillary clinton enjoys a 60 unfavorable rating . and its an interesting question . hillarys old strategy was to lie and deny that the fbi even had a criminal investigation underway . instead her associates insisted that it was a security review . the fbi corrected her and she shrugged it off . but the old breezy denial approach has given way to a savage assault on the fbi . pretending that nothing was wrong was a bad strategy but it was a better one that picking a fight with the fbi while lunatic clinton associates try to claim that the fbi is really the kgb . there are two possible explanations . hillary clinton might be arrogant enough to lash out at the fbi now that she believes that victory is near . the same kind of hubris that led her to plan her victory fireworks display could lead her to declare a war on the fbi for irritating her during the final miles of her campaign . but the other explanation is that her people panicked . going to war with the fbi is not the behavior of a smart and focused presidential campaign . its an act of desperation . when a presidential candidate decides that her only option is to try and destroy the credibility of the fbi thats not hubris its fear of what the fbi might be about to reveal about her . during the original fbi investigation hillary clinton was confident that she could ride it out . and she had good reason for believing that . but that hillary clinton is gone . in her place is a paranoid wreck . within a short space of time the positive clinton campaign promising to unite the country has been replaced by a desperate and flailing operation that has focused all its energy on fighting the fbi . theres only one reason for such bizarre behavior . the clinton campaign has decided that an fbi investigation of the latest batch of emails poses a threat to its survival . and so its gone all in on fighting the fbi . its an unprecedented step born of fear . its hard to know whether that fear is justified . but the existence of that fear already tells us a whole lot . clinton loyalists rigged the old investigation . they knew the outcome ahead of time as well as they knew the debate questions . now suddenly they are no longer in control . and they are afraid . you can smell the fear . the fbi has wiretaps from the investigation of the clinton foundation . its finding new emails all the time . and clintonworld panicked . the spinmeisters of clintonworld have claimed that the email scandal is just so much smoke without fire . all thats here is the appearance of impropriety without any of the substance . but this isnt how you react to smoke . its how you respond to a fire . the misguided assault on the fbi tells us that hillary clinton and her allies are afraid of a revelation bigger than the fundamental illegality of her email setup . the email setup was a preemptive cover up . the clinton campaign has panicked badly out of the belief right or wrong that whatever crime the illegal setup was meant to cover up is at risk of being exposed . the clintons have weathered countless scandals over the years . whatever they are protecting this time around is bigger than the usual corruption bribery sexual assaults and abuses of power that have followed them around throughout the years . this is bigger and more damaging than any of the allegations that have already come out . and they dont want fbi investigators anywhere near it . the campaign against comey is pure intimidation . its also a warning . any senior fbi people who value their careers are being warned to stay away . the democrats are closing ranks around their nominee against the fbi . its an ugly and unprecedented scene . it may also be their last stand . hillary clinton has awkwardly wound her way through numerous scandals in just this election cycle . but shes never shown fear or desperation before . now that has changed . whatever she is afraid of it lies buried in her emails with huma abedin . and it can bring her down like nothing else has .'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.label=='FAKE'].text.values[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Many to One LSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic Method:\n",
    "This method is no different to the method utilised in the sentiment analysis lesson."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, None, 50)          2988500   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 64)                29440     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 3,018,005.0\n",
      "Trainable params: 3,018,005\n",
      "Non-trainable params: 0.0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(len(word2num), 50)) # , batch_size=batch_size\n",
    "model.add(LSTM(64))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  1 / 5\n",
      "Train on 5701 samples, validate on 634 samples\n",
      "Epoch 1/1\n",
      "5701/5701 [==============================] - 43s - loss: 0.6816 - acc: 0.6764 - val_loss: 0.4216 - val_acc: 0.8312\n",
      "Epoch  2 / 5\n",
      "Train on 5701 samples, validate on 634 samples\n",
      "Epoch 1/1\n",
      "5701/5701 [==============================] - 41s - loss: 0.3978 - acc: 0.8467 - val_loss: 0.3006 - val_acc: 0.9054\n",
      "Epoch  3 / 5\n",
      "Train on 5701 samples, validate on 634 samples\n",
      "Epoch 1/1\n",
      "5701/5701 [==============================] - 40s - loss: 0.3299 - acc: 0.8834 - val_loss: 0.2884 - val_acc: 0.8959\n",
      "Epoch  4 / 5\n",
      "Train on 5701 samples, validate on 634 samples\n",
      "Epoch 1/1\n",
      "5701/5701 [==============================] - 39s - loss: 0.2465 - acc: 0.9216 - val_loss: 0.2858 - val_acc: 0.9101\n",
      "Epoch  5 / 5\n",
      "Train on 5701 samples, validate on 634 samples\n",
      "Epoch 1/1\n",
      "5701/5701 [==============================] - 39s - loss: 0.1458 - acc: 0.9519 - val_loss: 0.3718 - val_acc: 0.8785\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "epochs = 5\n",
    "model.fit(X_train, y_train, batch_size=batch_size, epochs=1, validation_data=(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Method 2: Fixed Embeddings\n",
    "This is where we use the `Embedding2` class to which we give a set of weights which remain the same through training. Note especially the number of trainable parameters in the summary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding2_2 (Embedding2)    (None, None, 50)          2988500   \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 64)                29440     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 3,018,005.0\n",
      "Trainable params: 116,355\n",
      "Non-trainable params: 2,901,650.0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding2(len(word2num), 50,\n",
    "                    fixed_weights=np.array([word2glove[w] for w in words_in_glove]))) # , batch_size=batch_size\n",
    "model.add(LSTM(64))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "# rmsprop = keras.optimizers.RMSprop(lr=1e-4)\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I may heave cheated and run the following block 3 times. Good thing about Keras is that it remembers the last learning rate and goes from there."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5701 samples, validate on 634 samples\n",
      "Epoch 1/15\n",
      "5701/5701 [==============================] - 35s - loss: 0.4407 - acc: 0.8020 - val_loss: 0.4284 - val_acc: 0.8186\n",
      "Epoch 2/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.4446 - acc: 0.7809 - val_loss: 0.4648 - val_acc: 0.7871\n",
      "Epoch 3/15\n",
      "5701/5701 [==============================] - 38s - loss: 0.4454 - acc: 0.7960 - val_loss: 0.4176 - val_acc: 0.8139\n",
      "Epoch 4/15\n",
      "5701/5701 [==============================] - 38s - loss: 0.4182 - acc: 0.8065 - val_loss: 0.4217 - val_acc: 0.8312\n",
      "Epoch 5/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.4272 - acc: 0.8155 - val_loss: 0.3892 - val_acc: 0.8344\n",
      "Epoch 6/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.4431 - acc: 0.8009 - val_loss: 0.4041 - val_acc: 0.8076\n",
      "Epoch 7/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.4293 - acc: 0.8035 - val_loss: 0.4148 - val_acc: 0.8391\n",
      "Epoch 8/15\n",
      "5701/5701 [==============================] - 38s - loss: 0.4366 - acc: 0.8081 - val_loss: 0.5276 - val_acc: 0.7461\n",
      "Epoch 9/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.4180 - acc: 0.8148 - val_loss: 0.3716 - val_acc: 0.8454\n",
      "Epoch 10/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.4151 - acc: 0.8181 - val_loss: 0.4656 - val_acc: 0.7886\n",
      "Epoch 11/15\n",
      "5701/5701 [==============================] - 38s - loss: 0.3882 - acc: 0.8332 - val_loss: 0.4031 - val_acc: 0.8470\n",
      "Epoch 12/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.3763 - acc: 0.8460 - val_loss: 0.3771 - val_acc: 0.8454\n",
      "Epoch 13/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.3826 - acc: 0.8302 - val_loss: 0.3658 - val_acc: 0.8549\n",
      "Epoch 14/15\n",
      "5701/5701 [==============================] - 37s - loss: 0.3496 - acc: 0.8521 - val_loss: 0.3223 - val_acc: 0.8722\n",
      "Epoch 15/15\n",
      "5701/5701 [==============================] - 36s - loss: 0.3767 - acc: 0.8455 - val_loss: 0.3267 - val_acc: 0.8738\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7feb018766a0>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 128\n",
    "model.fit(X_train, y_train, batch_size=batch_size, epochs=15, validation_data=(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.83911932]], dtype=float32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = \"North korea is testing out missiles on americans living overseas .\".lower()\n",
    "sentence_num = [word2num[w] if w in word2num else word2num['<Other>'] for w in sentence.split()]\n",
    "sentence_num = [word2num['<PAD>']]*(500-len(sentence_num)) + sentence_num\n",
    "sentence_num = np.array(sentence_num)\n",
    "model.predict(sentence_num[None,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'president trump is the greatest president of all time period .'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join([num2word[w] for w in sentence_num])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.84094501]], dtype=float32)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = \"The chemicals in the water is turning the freaking frogs gay says cnn . \".lower()\n",
    "sentence_num = [word2num[w] if w in word2num else word2num['<Other>'] for w in sentence.split()]\n",
    "sentence_num = [word2num['<PAD>']]*(500-len(sentence_num)) + sentence_num\n",
    "sentence_num = np.array(sentence_num)\n",
    "model.predict(sentence_num[None,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.74489152]], dtype=float32)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = \"President Trump is the greatest president of all time period .\".lower()\n",
    "sentence_num = [word2num[w] if w in word2num else word2num['<Other>'] for w in sentence.split()]\n",
    "sentence_num = [word2num['<PAD>']]*(0) + sentence_num\n",
    "sentence_num = np.array(sentence_num)\n",
    "model.predict(sentence_num[None,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "634/634 [==============================] - 1s     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.32666532286334116, 0.87381703394820642]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}