q.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gridworld import *\n",
    "from tqdm import trange\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "#Normal Q-learning\n",
    "\n",
    "gw = gridworld((5, 5), (0, 4), (4, 4), [(o, 4) for o in range(1, 4)], 0.9, 0, 1, 0)\n",
    "y = 0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Empirically estimate risk probability\n",
    "num_sims = 10000\n",
    "def estimate_rr(pi):\n",
    "    risky = 0\n",
    "    rewards = []\n",
    "    discounted = []\n",
    "    for i in trange(num_sims):\n",
    "        s = gw.reset()\n",
    "        reward = 0\n",
    "        d = False\n",
    "        j = 0\n",
    "        while(j < 1000):\n",
    "            a = pi[s[0],s[1]]\n",
    "            s1,r,d = gw.step(a)\n",
    "            reward += r\n",
    "            if d == True:\n",
    "                if s1 in gw.error_states:\n",
    "                    risky += 1\n",
    "                    discounted_reward = 0\n",
    "                else:\n",
    "                    discounted_reward = pow(y, j) * r\n",
    "                break\n",
    "            s = s1\n",
    "            j += 1\n",
    "        rewards.append(reward)\n",
    "        discounted.append(discounted_reward)\n",
    "    return sum(discounted)/float(num_sims), sum(rewards)/float(num_sims), float(risky)/num_sims    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100000/100000 [00:20<00:00, 4971.62it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3  4\n",
       "0  ➡  ➡  ⬇  ⬇  ⬅\n",
       "1  ⬇  ➡  ⬇  ⬇  E\n",
       "2  ➡  ➡  ➡  ⬇  E\n",
       "3  ⬆  ➡  ➡  ⬇  E\n",
       "4  ➡  ⬆  ➡  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100000/100000 [00:39<00:00, 2562.46it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬆</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>⬇</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3  4\n",
       "0  ➡  ⬇  ⬅  ⬅  ⬅\n",
       "1  ⬇  ⬇  ⬅  ⬅  E\n",
       "2  ➡  ⬇  ⬇  ⬆  E\n",
       "3  ⬇  ➡  ⬇  ⬅  E\n",
       "4  ➡  ⬆  ➡  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3  4\n",
       "0  -  -  -  -  S\n",
       "1  -  -  -  -  E\n",
       "2  -  -  -  -  E\n",
       "3  -  -  -  -  E\n",
       "4  -  -  -  -  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.06</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.18</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.14</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.18</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.15</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.19</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.09</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.25</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0     1     2     3      4\n",
       "0  0.06  0.13  0.16  0.14  0.093\n",
       "1  0.11  0.13  0.15  0.18      E\n",
       "2  0.14  0.11  0.21  0.18      E\n",
       "3  0.15  0.16  0.19  0.19      E\n",
       "4  0.09  0.17  0.18  0.25      G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>⬇</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3  4\n",
       "0  ➡  ⬇  ⬅  ⬇  ⬅\n",
       "1  ⬇  ⬇  ⬇  ⬇  E\n",
       "2  ➡  ⬇  ➡  ⬇  E\n",
       "3  ⬇  ➡  ⬇  ⬇  E\n",
       "4  ➡  ⬆  ➡  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:00<00:00, 15927.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean reward (empirical) 0.638443296904\n",
      "Successful Episodes  0.8662\n",
      "Risk probab  0.1338\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "trials = 10\n",
    "Q_list = []\n",
    "reward_trial_list = []\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "# Set learning parameters\n",
    "lr = 0.9\n",
    "num_episodes = 100000\n",
    "\n",
    "for k in range(trials):\n",
    "    #Initialize table with all zeros\n",
    "    Q = np.zeros((gw.M,gw.N,4))\n",
    "    jList = []\n",
    "    rList = []\n",
    "    for i in trange(num_episodes):\n",
    "        #print('Episode', i)\n",
    "        #Reset environment and get first new observation\n",
    "        s = gw.reset()\n",
    "        rAll = 0\n",
    "        d = False\n",
    "        j = 0\n",
    "        #The Q-Table learning algorithm\n",
    "        while j < 1000:\n",
    "            j+=1\n",
    "            #Choose an action by greedily (with noise) picking from Q table\n",
    "            a = np.argmax(Q[s[0],s[1],:] + np.random.randn(1,4)*(1./(i+1)))\n",
    "#             if np.random.rand() < epsilon:\n",
    "#                 # take random action\n",
    "#                 a = np.random.choice([0,1,2,3])\n",
    "#             else:\n",
    "#                 # take action according to the q function table\n",
    "#                 a = np.argmax(Q[s[0],s[1],:])\n",
    "            #Get new state and reward from environment\n",
    "            s1,r,d = gw.step(a)\n",
    "\n",
    "            #Update Q-Table with new knowledge\n",
    "            Q[s[0], s[1],a] = Q[s[0], s[1],a] + lr*(r + y*np.max(Q[s1[0], s1[1],:]) - Q[s[0], s[1],a])\n",
    "\n",
    "            rAll += r\n",
    "            s = s1\n",
    "            if d == True:\n",
    "                break\n",
    "        jList.append(j)    \n",
    "        rList.append(rAll)\n",
    "        \n",
    "    Q_list.append(Q)\n",
    "#     v = np.mean(Q, axis = 2)\n",
    "#     gw.print_v(v)\n",
    "\n",
    "    pi = np.argmax(Q, axis = 2)\n",
    "    gw.print_policy(pi)\n",
    "#     reward_trial_list.append(sum(rList)/float(num_episodes))\n",
    "#     print \"Score over time: \" +  str(sum(rList)/float(num_episodes))\n",
    "\n",
    "#     window = int(num_episodes/10)\n",
    "\n",
    "#     plt.figure(figsize=[9,16])\n",
    "#     plt.subplot(411)\n",
    "#     plt.plot(pd.Series(jList).rolling(window).mean())\n",
    "#     plt.title('Step Moving Average ({}-episode window)'.format(window))\n",
    "#     plt.ylabel('Moves')\n",
    "#     plt.xlabel('Episode')\n",
    "\n",
    "#     plt.subplot(412)\n",
    "#     plt.plot(pd.Series(rList).rolling(window).mean())\n",
    "#     plt.title('Reward Moving Average ({}-episode window)'.format(window))\n",
    "#     plt.ylabel('Reward')\n",
    "#     plt.xlabel('Episode')\n",
    "\n",
    "#     plt.tight_layout(pad=2)\n",
    "#     plt.show()\n",
    "\n",
    "gw.print_gw()\n",
    "meanQ = np.mean(np.array(Q_list), axis = 0)\n",
    "meanv = np.mean(meanQ, axis = 2)\n",
    "gw.print_v(meanv)\n",
    "meanpi = np.argmax(meanQ, axis = 2)\n",
    "gw.print_policy(meanpi)\n",
    "E_dis_reward, E_reward, E_risk = estimate_rr(meanpi)\n",
    "print \"Mean reward (empirical)\", E_dis_reward\n",
    "print \"Successful Episodes \", E_reward\n",
    "print \"Risk probab \", E_risk"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}