eng_data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from IPython.display import display, HTML\n",
    "import os\n",
    "import sys\n",
    "import csv\n",
    "import time\n",
    "import statistics\n",
    "import numpy as np\n",
    "import string\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows: 3570\n"
     ]
    }
   ],
   "source": [
    "#import and parse data, match with labels\n",
    "rawEngTwitData = open('data/tweetdata.txt', 'r')\n",
    "twitDataLabels = pd.read_excel('data/tweetlabels.xlsx', sheetname='sentiment2')\n",
    "twitDataListDict = []\n",
    "i = 0\n",
    "for line in rawEngTwitData:\n",
    "    try:\n",
    "        twitObj = json.loads(line)\n",
    "    except ValueError, e:\n",
    "        continue\n",
    "    #Empty object\n",
    "    if('created_at'not in twitObj):\n",
    "        continue\n",
    "    else:\n",
    "        twitDataRowDict = {}\n",
    "        #Get data fields\n",
    "        i += 1\n",
    "        twitID = twitObj['id']\n",
    "        twitIDString = twitObj['id_str']\n",
    "        twitText = twitObj['text']\n",
    "        #print('twitID: ' + str(twitID))\n",
    "        #Find corresponding label from label dataframe\n",
    "        labelRows = twitDataLabels[twitDataLabels.id == twitID]\n",
    "        if(len(labelRows.index) > 0):\n",
    "            label = labelRows['sentiment'].iloc[0]\n",
    "            if(label == 0):\n",
    "                label = -1\n",
    "            elif(label == 1):\n",
    "                label = -1\n",
    "            elif(label == -1):\n",
    "                label = 1\n",
    "            #Store extracted data\n",
    "            twitDataRowDict['ID'] = i\n",
    "            #twitDataRowDict['TwitIDString'] = twitIDString\n",
    "            twitDataRowDict['content'] = twitText\n",
    "            twitDataRowDict['depressed'] = label\n",
    "            twitDataListDict.append(twitDataRowDict)\n",
    "print(\"Number of rows: \" + str(len(twitDataListDict)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>content</th>\n",
       "      <th>depressed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Hey, look - I found my social anxiety again. W...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>RT @shannonpurser: Depression sucks. I'm in th...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>RT @HRoyalThighness: Lol my social anxiety goe...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>How to Deal with Stress, Anxiety and Bipolar D...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>RT @COCONUTOILBAE: when you want to be friends...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>@sabbunny I went there last Monday and I almos...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>RT @loopzoop: Hello? 2 hour depression nap? Do...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>Gossip Girl and Donald Trump give me so much a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>RT @lucasbavid: Me: I am gonna make art today,...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>I'm glad I understand what it is now tho. I us...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>RT @humansofny: “I don’t think I’m going to mi...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>RT @RTFFacts: According to studies, high-anxie...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>Anyone looking for organic products that can h...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>RT @oraltwjnk: Depression https://t.co/g4QIZjnRj1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>UR PRECIOUS AND ADORABLE HBAHDDHADDBAJ https:/...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>RT @pettyblackboy: Me and my homegirls taking ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>I am delighted to pledge support for people wi...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>It’s time for physicists to talk about mental ...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>RT @bpHopeMag: The Ultimate Guide To Social An...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>I like how we believe and know many depression...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>RT @_Liviiiiii: it's fucking me https://t.co/8...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>y'all i was having an anxiety attack yesterday...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23</td>\n",
       "      <td>When your anxiety makes you constantly feel na...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>RT @humansofny: “I don’t think I’m going to mi...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25</td>\n",
       "      <td>How food can improve your mental health - BBC ...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26</td>\n",
       "      <td>RT @pettyblackboy: Me and my homegirls taking ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>27</td>\n",
       "      <td>RT @roinnslainte: \"The NFMHS project has been ...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>28</td>\n",
       "      <td>i gotta beat this depression. beat it in the f...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>29</td>\n",
       "      <td>Depression cure? Taking THIS every day could h...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30</td>\n",
       "      <td>RT @pettyblackboy: Me and my homegirls taking ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3540</th>\n",
       "      <td>3545</td>\n",
       "      <td>RT @pettyblackboy: Me and my homegirls taking ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3541</th>\n",
       "      <td>3546</td>\n",
       "      <td>Max helps so much with my anxiety 🐶💕</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3542</th>\n",
       "      <td>3547</td>\n",
       "      <td>RT @GIRLposts: when you want to be friends w p...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3543</th>\n",
       "      <td>3548</td>\n",
       "      <td>my anxiety disorder convincing my subconscious...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3544</th>\n",
       "      <td>3549</td>\n",
       "      <td>RT @spicaterribIe: otabeks th scary lookin guy...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3545</th>\n",
       "      <td>3550</td>\n",
       "      <td>RT @lifeaseva: Can we all stop treating anxiet...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3546</th>\n",
       "      <td>3551</td>\n",
       "      <td>RT @COCONUTOILBAE: when you want to be friends...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3547</th>\n",
       "      <td>3552</td>\n",
       "      <td>my anxiety has ruined so many things for me in...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3548</th>\n",
       "      <td>3553</td>\n",
       "      <td>RT @COCONUTOILBAE: when you want to be friends...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3549</th>\n",
       "      <td>3554</td>\n",
       "      <td>RT @kashanacauley: Economic anxiety strikes ye...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3550</th>\n",
       "      <td>3555</td>\n",
       "      <td>ik its so weird how its diff but ik a few peop...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3551</th>\n",
       "      <td>3556</td>\n",
       "      <td>RT @ZulterithArt: Out of my home. In with the ...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3552</th>\n",
       "      <td>3557</td>\n",
       "      <td>RT @poIIIux: me: has seasonal depression every...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3553</th>\n",
       "      <td>3558</td>\n",
       "      <td>depression doesn't just come and go. it's debi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3554</th>\n",
       "      <td>3559</td>\n",
       "      <td>1. My dogs are the best medication for my anxi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3555</th>\n",
       "      <td>3560</td>\n",
       "      <td>RT @madiitiice: i love you phil i'm so proud o...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3556</th>\n",
       "      <td>3561</td>\n",
       "      <td>RT @ChloeHendricks: I tweet for the privileged...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3557</th>\n",
       "      <td>3562</td>\n",
       "      <td>RT @Gail_MacKean: Do you have lived experience...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3558</th>\n",
       "      <td>3563</td>\n",
       "      <td>RT @spicaterribIe: otabeks th scary lookin guy...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3559</th>\n",
       "      <td>3564</td>\n",
       "      <td>I have never experienced anxiety like this.</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3560</th>\n",
       "      <td>3565</td>\n",
       "      <td>Same ☹️ https://t.co/1cVMQdRfhU</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3561</th>\n",
       "      <td>3566</td>\n",
       "      <td>RT @humansofny: “I don’t think I’m going to mi...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3562</th>\n",
       "      <td>3567</td>\n",
       "      <td>RT @awktaku: I have terrible post-con depressi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3563</th>\n",
       "      <td>3568</td>\n",
       "      <td>RT @philuhmena: 7 months ago today I was raped...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3564</th>\n",
       "      <td>3569</td>\n",
       "      <td>RT @ChiIIVlbes: mental health is a serious issue.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3565</th>\n",
       "      <td>3570</td>\n",
       "      <td>I never get this much anxiety until I'm home :/</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3566</th>\n",
       "      <td>3571</td>\n",
       "      <td>Hospital staff warn of more 'adverse outcomes'...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3567</th>\n",
       "      <td>3572</td>\n",
       "      <td>RT @Jniewalker: มินย้อกถามซอฮยอนว่า depression...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3568</th>\n",
       "      <td>3573</td>\n",
       "      <td>RT @philuhmena: 7 months ago today I was raped...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3569</th>\n",
       "      <td>3574</td>\n",
       "      <td>my anxiety is thru the fucking roof.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3570 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ID                                            content  depressed\n",
       "0        1  Hey, look - I found my social anxiety again. W...          1\n",
       "1        2  RT @shannonpurser: Depression sucks. I'm in th...          1\n",
       "2        3  RT @HRoyalThighness: Lol my social anxiety goe...          1\n",
       "3        4  How to Deal with Stress, Anxiety and Bipolar D...          1\n",
       "4        5  RT @COCONUTOILBAE: when you want to be friends...         -1\n",
       "5        6  @sabbunny I went there last Monday and I almos...          1\n",
       "6        7  RT @loopzoop: Hello? 2 hour depression nap? Do...          1\n",
       "7        8  Gossip Girl and Donald Trump give me so much a...          1\n",
       "8        9  RT @lucasbavid: Me: I am gonna make art today,...          1\n",
       "9       10  I'm glad I understand what it is now tho. I us...          1\n",
       "10      11  RT @humansofny: “I don’t think I’m going to mi...         -1\n",
       "11      12  RT @RTFFacts: According to studies, high-anxie...          1\n",
       "12      13  Anyone looking for organic products that can h...          1\n",
       "13      14  RT @oraltwjnk: Depression https://t.co/g4QIZjnRj1          1\n",
       "14      15  UR PRECIOUS AND ADORABLE HBAHDDHADDBAJ https:/...         -1\n",
       "15      16  RT @pettyblackboy: Me and my homegirls taking ...          1\n",
       "16      17  I am delighted to pledge support for people wi...         -1\n",
       "17      18  It’s time for physicists to talk about mental ...         -1\n",
       "18      19  RT @bpHopeMag: The Ultimate Guide To Social An...          1\n",
       "19      20  I like how we believe and know many depression...         -1\n",
       "20      21  RT @_Liviiiiii: it's fucking me https://t.co/8...         -1\n",
       "21      22  y'all i was having an anxiety attack yesterday...          1\n",
       "22      23  When your anxiety makes you constantly feel na...          1\n",
       "23      24  RT @humansofny: “I don’t think I’m going to mi...         -1\n",
       "24      25  How food can improve your mental health - BBC ...         -1\n",
       "25      26  RT @pettyblackboy: Me and my homegirls taking ...          1\n",
       "26      27  RT @roinnslainte: \"The NFMHS project has been ...         -1\n",
       "27      28  i gotta beat this depression. beat it in the f...          1\n",
       "28      29  Depression cure? Taking THIS every day could h...         -1\n",
       "29      30  RT @pettyblackboy: Me and my homegirls taking ...          1\n",
       "...    ...                                                ...        ...\n",
       "3540  3545  RT @pettyblackboy: Me and my homegirls taking ...          1\n",
       "3541  3546             Max helps so much with my anxiety 🐶💕          1\n",
       "3542  3547  RT @GIRLposts: when you want to be friends w p...         -1\n",
       "3543  3548  my anxiety disorder convincing my subconscious...          1\n",
       "3544  3549  RT @spicaterribIe: otabeks th scary lookin guy...          1\n",
       "3545  3550  RT @lifeaseva: Can we all stop treating anxiet...         -1\n",
       "3546  3551  RT @COCONUTOILBAE: when you want to be friends...         -1\n",
       "3547  3552  my anxiety has ruined so many things for me in...         -1\n",
       "3548  3553  RT @COCONUTOILBAE: when you want to be friends...         -1\n",
       "3549  3554  RT @kashanacauley: Economic anxiety strikes ye...          1\n",
       "3550  3555  ik its so weird how its diff but ik a few peop...          1\n",
       "3551  3556  RT @ZulterithArt: Out of my home. In with the ...         -1\n",
       "3552  3557  RT @poIIIux: me: has seasonal depression every...          1\n",
       "3553  3558  depression doesn't just come and go. it's debi...          1\n",
       "3554  3559  1. My dogs are the best medication for my anxi...          1\n",
       "3555  3560  RT @madiitiice: i love you phil i'm so proud o...         -1\n",
       "3556  3561  RT @ChloeHendricks: I tweet for the privileged...          1\n",
       "3557  3562  RT @Gail_MacKean: Do you have lived experience...          1\n",
       "3558  3563  RT @spicaterribIe: otabeks th scary lookin guy...          1\n",
       "3559  3564        I have never experienced anxiety like this.         -1\n",
       "3560  3565                    Same ☹️ https://t.co/1cVMQdRfhU         -1\n",
       "3561  3566  RT @humansofny: “I don’t think I’m going to mi...         -1\n",
       "3562  3567  RT @awktaku: I have terrible post-con depressi...          1\n",
       "3563  3568  RT @philuhmena: 7 months ago today I was raped...         -1\n",
       "3564  3569  RT @ChiIIVlbes: mental health is a serious issue.          1\n",
       "3565  3570    I never get this much anxiety until I'm home :/          1\n",
       "3566  3571  Hospital staff warn of more 'adverse outcomes'...          1\n",
       "3567  3572  RT @Jniewalker: มินย้อกถามซอฮยอนว่า depression...          1\n",
       "3568  3573  RT @philuhmena: 7 months ago today I was raped...         -1\n",
       "3569  3574               my anxiety is thru the fucking roof.          1\n",
       "\n",
       "[3570 rows x 3 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Convert to dataframe and save as csv\n",
    "twitDataDf = pd.DataFrame(twitDataListDict)\n",
    "twitDataDf.to_csv('cleaned_data/eng_data_cleaned.csv', encoding='utf-8')\n",
    "display(twitDataDf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}