Step7_gpt2Rescore.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This notebook uses the publicly available GPT-2 language model from OpenAI (https://github.com/openai/gpt-2) to\n",
    "#rescore the candidate sentences generated by the kaldi bigram language model (Step 6). This improves performance because\n",
    "#GPT-2 is a much more powerful language model than the bigram model. \n",
    "\n",
    "#To run this you will need to download the GPT-2 model (1558M version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "#suppress all tensorflow warnings (largely related to compatability with v2)\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
    "\n",
    "#point this towards the top level dataset directory\n",
    "rootDir = os.path.expanduser('~') + '/handwritingBCIData/'\n",
    "\n",
    "#point this towards the code directory\n",
    "repoDir = os.getcwd() + '/'\n",
    "\n",
    "#defines which train/test partition to use\n",
    "cvPart = 'HeldOutTrials'\n",
    "\n",
    "#point this to wherever you downloaded the 1558M GPT-2 model\n",
    "gptModelDir = os.path.expanduser(os.path.expandvars('~/gpt-2/models'))\n",
    "\n",
    "#defines which datasets to process\n",
    "dataDirs = ['t5.2019.05.08','t5.2019.11.25','t5.2019.12.09','t5.2019.12.11','t5.2019.12.18',\n",
    "            't5.2019.12.20','t5.2020.01.06','t5.2020.01.08','t5.2020.01.13','t5.2020.01.15']\n",
    "\n",
    "#this prevents tensorflow from taking over more than one gpu on a multi-gpu machine\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]='0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#load GPT-2\n",
    "import json\n",
    "import gpt2.model, gpt2.encoder\n",
    "\n",
    "batch_size = 1\n",
    "model_name = '1558M'\n",
    "\n",
    "enc = gpt2.encoder.get_encoder(model_name, gptModelDir)\n",
    "hparams = gpt2.model.default_hparams()\n",
    "with open(os.path.join(gptModelDir, model_name, 'hparams.json')) as f:\n",
    "    hparams.override_from_dict(json.load(f))\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "X = tf.placeholder(tf.int32, [batch_size, None])\n",
    "logits = gpt2.model.model(hparams, X, past=None, scope='model', reuse=False)\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "ckpt = tf.train.latest_checkpoint(os.path.join(gptModelDir, model_name))\n",
    "saver.restore(sess, ckpt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " --t5.2019.05.08-- \n",
      "#2\n",
      "True:    you want me to sing?\n",
      "Decoded: you want me to sing?\n",
      "\n",
      "#5\n",
      "True:    have you ever seen a large cat fold itself into a tiny shoe box?\n",
      "Decoded: have you ever seen a large cat fold itself into a tiny shoe box?\n",
      "\n",
      "#21\n",
      "True:    the jeep was thirsty so i stopped for gas on the edge of town.\n",
      "Decoded: the jeep was thirsty so i stopped for gas on the edge of town.\n",
      "\n",
      "#33\n",
      "True:    because less than a quarter mile from where i'm standing right now is where alla's body was dumped.\n",
      "Decoded: because less than a quarter mile from where i'm standing right now is where alla's body was dumped.\n",
      "\n",
      "#34\n",
      "True:    there are moments when gentle background singing brings the song close to gospel.\n",
      "Decoded: there are moments when gentle background singing brings the song close to gospel.\n",
      "\n",
      "#39\n",
      "True:    so you could literally see here's what every single person in the company gets paid.\n",
      "Decoded: so you could literally see here's what every single person in the company gets paid.\n",
      "\n",
      "#54\n",
      "True:    i told you we were going to bring this sucker home, and we brought it home.\n",
      "Decoded: i told you we were going to bring this sucker home, and we brought it home.\n",
      "\n",
      "#90\n",
      "True:    what rent do you pay for your housing?\n",
      "Decoded: what rent do you pay for your housing?\n",
      "\n",
      "#91\n",
      "True:    plainly, there is a clear and present need to secure information in the public and private sectors.\n",
      "Decoded: plainly, there is a clear and present need to secure information in the public and private sectors.\n",
      "\n",
      "#98\n",
      "True:    i thought i could fix it but got trapped by my ego and refusal to fail.\n",
      "Decoded: i thought i could fix it but got trapped by my ego and refusal to fail.\n",
      "\n",
      " --t5.2019.11.25-- \n",
      "#5\n",
      "True:    then it wobbled under her weight and tilted forwards.\n",
      "Decoded: then it wobble'd under her weight and tilted forwards.\n",
      "\n",
      "#21\n",
      "True:    and my brother's in hong kong all that week.\n",
      "Decoded: and my brother's in hong kong all that week.\n",
      "\n",
      "#31\n",
      "True:    then we slipped into the kitchen , got up on the sink, and hauled ourselves into the little attic.\n",
      "Decoded: then we slipped into the kitchen , got up on the sink, and hauled ourselves into the little attic.\n",
      "\n",
      "#33\n",
      "True:    he didn't like being tied up to the women.\n",
      "Decoded: he didn't like being tied up to the women.\n",
      "\n",
      "#34\n",
      "True:    you can rent money.\n",
      "Decoded: you can rent money.\n",
      "\n",
      "#39\n",
      "True:    the only peace that you'll find here lies in a kind of pagan solitude.\n",
      "Decoded: the only peace that you'll find here lies in a kind of pagan solitude.\n",
      "\n",
      "#61\n",
      "True:    his pot was in his hand still, it preceded him like the torch of justice.\n",
      "Decoded: his pot was in his hand still, it preceded him like the torch of justice.\n",
      "\n",
      " --t5.2019.12.09-- \n",
      "#5\n",
      "True:    maybe get your horses to swim across.\n",
      "Decoded: may be get your horses to swim across.\n",
      "\n",
      "#21\n",
      "True:    i thought they were staying the night.\n",
      "Decoded: i thought they were staying the night.\n",
      "\n",
      "#31\n",
      "True:    it seemed a poor thing to have descended to.\n",
      "Decoded: it seemed a poor thing to have descended to.\n",
      "\n",
      "#33\n",
      "True:    his sudden talk of alibis made the whole thing sound serious.\n",
      "Decoded: this sudden talk of alibi's made the whole thing sound serious.\n",
      "\n",
      "#34\n",
      "True:    is the policy still to sit it out and wait rather than try and force your way into the prison?\n",
      "Decoded: is the policy still to sit it out and wait rather than try and force your way into the prison?\n",
      "\n",
      "#39\n",
      "True:    i accept that it precludes class a development.\n",
      "Decoded: i accept that it precludes class a development.\n",
      "\n",
      "#68\n",
      "True:    at that moment she heard rapid footsteps.\n",
      "Decoded: at that moment she heard rapid footsteps.\n",
      "\n",
      " --t5.2019.12.11-- \n",
      "#5\n",
      "True:    but i have to hear the whole tale.\n",
      "Decoded: but i have to hear the whole tale.\n",
      "\n",
      "#21\n",
      "True:    this fuzzy, probabilistic approach is a sensible one.\n",
      "Decoded: this fuzzy, probabilistic approach is a sensible one.\n",
      "\n",
      "#31\n",
      "True:    he needed to develop his mind.\n",
      "Decoded: he needed to develop his mind.\n",
      "\n",
      "#33\n",
      "True:    people here are also adopting a healthier diet.\n",
      "Decoded: people here are also adopting a healthier diet.\n",
      "\n",
      "#34\n",
      "True:    the royal navy had left its mark.\n",
      "Decoded: the royal navy had left its mark.\n",
      "\n",
      "#39\n",
      "True:    a long shower relaxed her still further.\n",
      "Decoded: a long shower relaxed her still further.\n",
      "\n",
      "#61\n",
      "True:    thrown in nuts and carrots and blend.\n",
      "Decoded: thrown in nuts and carrots and blend.\n",
      "\n",
      " --t5.2019.12.18-- \n",
      "#5\n",
      "True:    he lifted his hand out of the water to smooth his hair.\n",
      "Decoded: he lifted his hand out of the water to smooth his hair.\n",
      "\n",
      "#21\n",
      "True:    the servants would know whether it was possible to get back down the valley.\n",
      "Decoded: the servants would know whether it was possible to get back down the valley.\n",
      "\n",
      "#31\n",
      "True:    why are you prepared to play such a desperate game, to put your whole future in jeopardy like this?\n",
      "Decoded: why are you prepared to play such a desperate game, to put your whole future in jeopardy like this?\n",
      "\n",
      "#33\n",
      "True:    you had round red cheeks, just like a wooden doll.\n",
      "Decoded: you had round red cheeks, just like a wooden doll.\n",
      "\n",
      "#34\n",
      "True:    i have seven apples, seven oranges and seven bananas.\n",
      "Decoded: i have seven apples, seven oranges and seven bananas.\n",
      "\n",
      "#39\n",
      "True:    grumpy wizards make a toxic brew for the jovial queen.\n",
      "Decoded: grumpy wizards make a toxic brew for the jovial queen.\n",
      "\n",
      "#61\n",
      "True:    he reckons his success on the field depends on learning the lingo.\n",
      "Decoded: he reckons his success on the field depends on learning the lingo.\n",
      "\n",
      " --t5.2019.12.20-- \n",
      "#5\n",
      "True:    i'm not tough enough to toe the line.\n",
      "Decoded: i'm not tough enough to toe the line.\n",
      "\n",
      "#21\n",
      "True:    a lot of old folk are getting caught and we must stamp it out.\n",
      "Decoded: a lot of old folk are getting caught and we must stamp it out.\n",
      "\n",
      "#31\n",
      "True:    these agents are now being tested in clinical trials with people at risk.\n",
      "Decoded: these agents are now being tested in clinical trials with people at risk.\n",
      "\n",
      "#33\n",
      "True:    the way he said it made the name sound like orchids and honey.\n",
      "Decoded: the way he said it made the name sound like orchids and honey.\n",
      "\n",
      "#34\n",
      "True:    pack my box with five dozen liquor jugs.\n",
      "Decoded: pack my box with five dozen liquor jugs.\n",
      "\n",
      "#39\n",
      "True:    state officials are always under threat, always being kidnapped.\n",
      "Decoded: state officials are always under threat, always being kidnapped.\n",
      "\n",
      "#61\n",
      "True:    that's when i threw up on the carpet.\n",
      "Decoded: that's when i threw up on the carpet.\n",
      "\n",
      " --t5.2020.01.06-- \n",
      "#5\n",
      "True:    they breathe air and they cannot swim.\n",
      "Decoded: they breathe air and they can not swim.\n",
      "\n",
      "#21\n",
      "True:    from the terrace, a broad flight of central steps led down to lawns and formal flowerbeds.\n",
      "Decoded: from the terrace, a broad flight of central steps led down to lawns and formal flower beds.\n",
      "\n",
      "#31\n",
      "True:    the corpses lay uncovered in the open on plastic sheets.\n",
      "Decoded: the corpses lay uncovered in the open on plastic sheets.\n",
      "\n",
      "#39\n",
      "True:    shame on us for not caring.\n",
      "Decoded: shame on us for not caring\n",
      "\n",
      "#61\n",
      "True:    the other battalions were infected by the panic and also fled.\n",
      "Decoded: the other battalions were infected by the panic and also fled.\n",
      "\n",
      " --t5.2020.01.08-- \n",
      "#5\n",
      "True:    on the eve of the council the worst of horrors was revealed.\n",
      "Decoded: on the eve of the council the worst of horrors was revealed.\n",
      "\n",
      "#21\n",
      "True:    a small coal fire burned in the grate.\n",
      "Decoded: a small coal fire burned in the grate\n",
      "\n",
      "#31\n",
      "True:    the result is a lighter weight fabric, but one which still shows off the yarn.\n",
      "Decoded: the result is a lighter weight fabric, but one which still shows off the yarn.\n",
      "\n",
      "#33\n",
      "True:    perhaps they would accept his old currency.\n",
      "Decoded: perhaps they would accept his old currency.\n",
      "\n",
      "#34\n",
      "True:    already the grass was becoming crisp with frost.\n",
      "Decoded: already the grass was becoming crisp with frost.\n",
      "\n",
      "#39\n",
      "True:    it is the necessary foundation for growth of small businesses.\n",
      "Decoded: it is the necessary foundation for growth of small businesses.\n",
      "\n",
      "#61\n",
      "True:    celia constantly refers to her mother's drowning accident.\n",
      "Decoded: celia constantly referred to her mother's drowning accident.\n",
      "\n",
      " --t5.2020.01.13-- \n",
      "#22\n",
      "True:    crazy frederick bought many very exquisite opal jewels.\n",
      "Decoded: crazy frederick bought many very exquisite opal jewels\n",
      "\n",
      "#25\n",
      "True:    sphinx of black quartz, judge my vow.\n",
      "Decoded: sphinx of black quartz, judge my vow.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#121\n",
      "True:    residents were warned to prepare flood defences.\n",
      "Decoded: residents were warned to prepare flood defences.\n",
      "\n",
      " --t5.2020.01.15-- \n",
      "#22\n",
      "True:    imagine a star with a mass ten times that of the sun.\n",
      "Decoded: imagine a star with a mass ten times that of the sun.\n",
      "\n",
      "#25\n",
      "True:    the elder boys are not deterred, however.\n",
      "Decoded: the elder boys are not deterred, however.\n",
      "\n",
      "#121\n",
      "True:    you never used to swear, you know.\n",
      "Decoded: you never used to swear, you know.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Rescore the bigram language model's candidate sentences and choose the new best sentence.\n",
    "#This takes a little while, as we are processing ALL sentences from all datasets (both the train AND test sentences).\n",
    "#We also run each candidate sentence one at a time through GPT-2 \n",
    "#(but could probably speed this up by running multiple through at the same time).\n",
    "\n",
    "from kaldiReadWrite import readKaldiLatticeFile\n",
    "import numpy as np\n",
    "import scipy.io\n",
    "from rnnEval import wer\n",
    "import warnings\n",
    "\n",
    "#this stops scipy.io.savemat from throwing a warning about empty entries\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "for dataDir in dataDirs:\n",
    "    #process ALL sentences from this dataset (both train & test)\n",
    "    print(' --' + dataDir + '-- ')\n",
    "\n",
    "    sentenceDat = scipy.io.loadmat(rootDir+'Datasets/'+dataDir+'/sentences.mat')\n",
    "    cvPartFile = scipy.io.loadmat(rootDir+'RNNTrainingSteps/trainTestPartitions_'+cvPart+'.mat')\n",
    "    valIdx = cvPartFile[dataDir+'_test']\n",
    "    \n",
    "    kaldiDir = rootDir+'RNNTrainingSteps/Step6_ApplyBigramLM/'+cvPart+'/KaldiOutput/'+dataDir \n",
    "    nFiles = int(len(os.listdir(kaldiDir))/9)   \n",
    "    \n",
    "    decSentences = []\n",
    "    allErrCounts = []\n",
    "    \n",
    "    for fileIdx in range(nFiles):\n",
    "        #load kaldi bigram output\n",
    "        nbestFile = kaldiDir+'/'+str(fileIdx)+'_transcript.txt'\n",
    "        acFile = kaldiDir+'/'+str(fileIdx)+'_best_acscore.ark'\n",
    "        lmFile = kaldiDir+'/'+str(fileIdx)+'_best_lmscore.ark'\n",
    "\n",
    "        nums, content = readKaldiLatticeFile(nbestFile, 'string')\n",
    "        _, acScore = readKaldiLatticeFile(acFile, 'numeric')\n",
    "        _, lmScore = readKaldiLatticeFile(lmFile, 'numeric')\n",
    "        \n",
    "        #rescoring\n",
    "        lmRescore = np.zeros(lmScore.shape)\n",
    "        for y in range(len(content)):\n",
    "            if content[y]=='':\n",
    "                content[y] = ' '\n",
    "                \n",
    "            encText = enc.encode(content[y][0].upper() + content[y][1:])\n",
    "            encText.insert(0,50256) #special 'endoftext' symbol\n",
    "            encText.append(50256)\n",
    "\n",
    "            out = sess.run(logits, feed_dict={X: [encText]})\n",
    "\n",
    "            for x in range(out['logits'].shape[0]):\n",
    "                for t in range(out['logits'].shape[1]):\n",
    "                    out['logits'][x,t,:] = np.exp(out['logits'][x,t,:])/np.sum(np.exp(out['logits'][x,t,:]))\n",
    "            out['logits'] = np.log(out['logits'])\n",
    "\n",
    "            logSum = 0\n",
    "            for x in range(1,out['logits'].shape[1]):\n",
    "                logSum += out['logits'][0,x-1,encText[x]]\n",
    "\n",
    "            lmRescore[y] = -logSum\n",
    "\n",
    "        newBest = np.argmin(acScore + 2.0*lmRescore)\n",
    "        decSent = content[newBest]\n",
    "        decSentences.append(decSent)\n",
    "\n",
    "        #compute char/word error rates\n",
    "        trueText = sentenceDat['sentencePrompt'][fileIdx,0][0]\n",
    "        trueText = trueText.replace('>',' ')\n",
    "        trueText = trueText.replace('~','.')\n",
    "        trueText = trueText.replace('#','')\n",
    "        \n",
    "        charErrs = wer(trueText, decSent)\n",
    "        wordErrs = wer(trueText.split(), decSent.split())\n",
    "        allErrCounts.append(np.array([charErrs, len(trueText), wordErrs, len(trueText.split())]))\n",
    "\n",
    "        #print results from the held-out sentences\n",
    "        if fileIdx in valIdx:\n",
    "            print('#' + str(fileIdx))\n",
    "            print('True:    ' + trueText)\n",
    "            print('Decoded: ' + decSent)\n",
    "            print('')\n",
    "\n",
    "    #save error rates & decoded sentences\n",
    "    concatCounts = np.stack(allErrCounts, axis=0)\n",
    "    \n",
    "    saveDict = {}\n",
    "    saveDict['decSentences'] = decSentences\n",
    "    saveDict['trueSentences'] = sentenceDat['sentencePrompt']\n",
    "    saveDict['charCounts'] = concatCounts[:,1]\n",
    "    saveDict['charErrors'] = concatCounts[:,0]\n",
    "    saveDict['wordCounts'] = concatCounts[:,3]\n",
    "    saveDict['wordErrors'] = concatCounts[:,2]\n",
    "\n",
    "    scipy.io.savemat(rootDir + 'RNNTrainingSteps/Step7_GPT2Rescore/' + cvPart + '/' + dataDir + '_errCounts.mat', saveDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Character error rate: 0.34%\n",
      "Word error rate: 1.97%\n"
     ]
    }
   ],
   "source": [
    "#Summarize character/word error rate and word error rate across all sessions\n",
    "allErrCounts = []\n",
    "\n",
    "for dataDir in dataDirs:\n",
    "    dat = scipy.io.loadmat(rootDir + 'RNNTrainingSteps/Step7_GPT2Rescore/' + cvPart + '/' + dataDir + '_errCounts.mat')\n",
    "    cvPartFile = scipy.io.loadmat(rootDir+'RNNTrainingSteps/trainTestPartitions_'+cvPart+'.mat')\n",
    "    valIdx = cvPartFile[dataDir+'_test']\n",
    "    \n",
    "    if len(valIdx)==0:\n",
    "        continue\n",
    "        \n",
    "    valIdx = valIdx[0,:]\n",
    "    allErrCounts.append(np.stack([dat['charCounts'][0,valIdx],\n",
    "                         dat['charErrors'][0,valIdx],\n",
    "                         dat['wordCounts'][0,valIdx],\n",
    "                         dat['wordErrors'][0,valIdx]],axis=0).T)\n",
    "    \n",
    "concatErrCounts = np.squeeze(np.concatenate(allErrCounts, axis=0))\n",
    "cer = 100*(np.sum(concatErrCounts[:,1]) / np.sum(concatErrCounts[:,0]))\n",
    "wer = 100*(np.sum(concatErrCounts[:,3]) / np.sum(concatErrCounts[:,2]))\n",
    "\n",
    "print('Character error rate: %1.2f%%' % float(cer))\n",
    "print('Word error rate: %1.2f%%' % float(wer))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}