NLTK Stanford core nlp.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pos-tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.tag import StanfordPOSTagger\n",
    "path_to_jar = 'files/stanford-postagger.jar'\n",
    "english_model='files/english-left3words-distsim.tagger'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "java_path = \"C:/Program Files/Java/jdk1.8.0_111/bin/java.exe\"\n",
    "os.environ['JAVAHOME'] = java_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tagger = StanfordPOSTagger(english_model, path_to_jar, encoding='utf-8')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>Pos-Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>My</td>\n",
       "      <td>PRP$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>name</td>\n",
       "      <td>NN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>is</td>\n",
       "      <td>VBZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Emna</td>\n",
       "      <td>NNP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Amor</td>\n",
       "      <td>NNP</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  token Pos-Tag\n",
       "0    My    PRP$\n",
       "1  name      NN\n",
       "2    is     VBZ\n",
       "3  Emna     NNP\n",
       "4  Amor     NNP"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_sent = tagger.tag(\"My name is Emna Amor\".split())\n",
    "df=pd.DataFrame(tagged_sent,columns=['token','Pos-Tag'])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.parse import stanford\n",
    "os.environ['STANFORD_PARSER'] = 'files/stanford-parser.jar'\n",
    "os.environ['STANFORD_MODELS'] = 'files/stanford-parser-3.8.0-models.jar'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "english_model='files/englishFactored.ser.gz'\n",
    "parser=stanford.StanfordParser(model_path=english_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences=parser.raw_parse((\"Barack Obama watched a game in Germany, the 2nd of august 1992.\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Tree('S', [Tree('NP', [Tree('NNP', ['Barack']), Tree('NNP', ['Obama'])]), Tree('VP', [Tree('VBD', ['watched']), Tree('NP', [Tree('DT', ['a']), Tree('NN', ['game'])]), Tree('PP', [Tree('IN', ['in']), Tree('NP', [Tree('NP', [Tree('NNP', ['Germany'])]), Tree(',', [',']), Tree('NP', [Tree('NP', [Tree('DT', ['the']), Tree('NN', ['2nd'])]), Tree('PP', [Tree('IN', ['of']), Tree('NP', [Tree('NNP', ['august']), Tree('CD', ['1992'])])])])])])]), Tree('.', ['.'])])]\n"
     ]
    }
   ],
   "source": [
    "# you can draw the tree (sentence by sentence) or you can list te tree \n",
    "# here we have just one sentence\n",
    "for line in sentences:\n",
    "        #line.draw()\n",
    "        print(list(line))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.tag.stanford import StanfordNERTagger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ner_jar='files/stanford-ner.jar'\n",
    "classifier='files/english.muc.7class.distsim.crf.ser.gz'\n",
    "st = StanfordNERTagger(classifier,ner_jar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text=\"Harold Frederick Shipman is the most dangerous serial killer in 1970-2008 period.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "#clean_text=cleanupText(text)\n",
    "#textt=re.split('[- :]', text)\n",
    "classified_text = st.tag(text.split())\n",
    "#print(classified_text)\n",
    "df=pd.DataFrame(classified_text,columns=['token','entity'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Harold</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Frederick</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Shipman</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>is</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>most</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>dangerous</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>serial</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>killer</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>in</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1970-2008</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>period.</td>\n",
       "      <td>TIME</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        token  entity\n",
       "0      Harold  PERSON\n",
       "1   Frederick  PERSON\n",
       "2     Shipman  PERSON\n",
       "3          is       O\n",
       "4         the       O\n",
       "5        most       O\n",
       "6   dangerous       O\n",
       "7      serial       O\n",
       "8      killer       O\n",
       "9          in       O\n",
       "10  1970-2008    DATE\n",
       "11    period.    TIME"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Harold</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Frederick</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Shipman</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1970-2008</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>period.</td>\n",
       "      <td>TIME</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        token  entity\n",
       "0      Harold  PERSON\n",
       "1   Frederick  PERSON\n",
       "2     Shipman  PERSON\n",
       "10  1970-2008    DATE\n",
       "11    period.    TIME"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df['entity'] != 'O']\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## coref resolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# i will use core nlp server for coreference resolution \n",
    "# you can run the server using this command \n",
    "#java -mx4g -cp \"*\" edu.stanford.nlp.pipeline.StanfordCoreNLPServer\n",
    "\n",
    "# you can use the server for parsing , part of speech tagging or any other tool of stanford core nlp\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pycorenlp import StanfordCoreNLP\n",
    "nlp = StanfordCoreNLP('http://localhost:9000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parsed_text(text):\n",
    "    # The StanfordCoreNLP server is running on http://127.0.0.1:9000\n",
    "    nlp = StanfordCoreNLP('http://127.0.0.1:9000')\n",
    "    # Json response of all the annotations\n",
    "    output = nlp.annotate(text, properties={\n",
    "    'annotators': 'tokenize, ssplit, pos, depparse, parse,dcoref',\n",
    "    'outputFormat': 'json'\n",
    "    })\n",
    "    #json_string=output['sentences'][0]['parse']\n",
    "    #s =json_string.replace('\\n', ' ').replace('\\r', '')\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "s=parsed_text(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the most dangerous serial killer in 1970-2008 period refers to Harold Frederick Shipman\n"
     ]
    }
   ],
   "source": [
    "print (s['corefs']['1'][1]['text'],\"refers to \"+s['corefs']['1'][0]['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}