dist_q.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gridworld import *\n",
    "from tqdm import trange\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "#Normal Q-learning\n",
    "\n",
    "\n",
    "gw = gridworld((4, 4), (0, 3), (3, 3), [(o, 3) for o in range(1, 3)], 0.9, 0, 100, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.16227766, 2.23606798, 1.41421356, 1.        ],\n",
       "       [3.        , 2.        , 1.        , 0.        ],\n",
       "       [3.        , 2.        , 1.        , 0.        ],\n",
       "       [3.16227766, 2.23606798, 1.41421356, 1.        ]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "3.1622776601683795"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([[5.        , 3.53553391, 2.23606798, 1.58113883],\n",
       "       [4.74341649, 3.16227766, 1.58113883, 0.        ],\n",
       "       [4.74341649, 3.16227766, 1.58113883, 0.        ],\n",
       "       [5.        , 3.53553391, 2.23606798, 1.58113883]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def dist_error_state(state, error_state):\n",
    "    return ( np.sqrt( (state[0] - error_state[0])**2 + (state[1] - error_state[1])**2 ) )\n",
    "\n",
    "risk_states = np.zeros((gw.M, gw.N))\n",
    "for i in range(gw.M):\n",
    "    for j in range(gw.N):\n",
    "#         if (i, j) in gw.error_states:\n",
    "#             risk_states[i,j] == 0.\n",
    "#         if (i, j) == gw.goal:\n",
    "#             risk_states[i,j] == 0.\n",
    "#         else:\n",
    "        sm = 100.\n",
    "        for e in gw.error_states:\n",
    "            #sm += np.exp( - dist_error_state([i,j], e) )\n",
    "            sm = min(sm, dist_error_state([i,j], e))\n",
    "        risk_states[i,j] = sm\n",
    "risk_states\n",
    "np.random.seed(1)\n",
    "\n",
    "np.min(risk_states)\n",
    "np.max(risk_states)\n",
    "risk_states = (risk_states - np.min(risk_states))/(np.max(risk_states) - np.min(risk_states)) * 5\n",
    "risk_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Empirically estimate safety\n",
    "num_sims = 10000\n",
    "def estimate_rr(pi):\n",
    "    rewards = []\n",
    "    discounted = []\n",
    "    risks = []\n",
    "    discounted_reward = 0\n",
    "    for i in range(num_sims):\n",
    "        s = gw.reset()\n",
    "        reward = 0\n",
    "        d = False\n",
    "        j = 0\n",
    "        risk_dist = 0\n",
    "        while(j < 1000):\n",
    "            a = pi[s[0],s[1]]\n",
    "            s1,r,d = gw.step(a)\n",
    "            reward += r\n",
    "            risk_dist += risk_states[s[0], s[1]]                     \n",
    "            if d == True:\n",
    "                if s1 in gw.error_states:\n",
    "                    discounted_reward = 0\n",
    "                else:\n",
    "                    discounted_reward = pow(y, j) * r\n",
    "                break\n",
    "            s = s1\n",
    "            j += 1\n",
    "        rewards.append(reward)\n",
    "        discounted.append(discounted_reward)\n",
    "        risks.append(risk_dist)\n",
    "    return sum(discounted)/float(num_sims), sum(rewards)/float(num_sims), sum(risks)/float(num_sims)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:02<00:00, 4939.11it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ⬇  ⬇  ⬇  ⬅\n",
       "1  ⬇  ⬇  ⬅  E\n",
       "2  ➡  ⬇  ⬅  E\n",
       "3  ⬆  ➡  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:02<00:00, 3544.93it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬆</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>⬇</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ⬇  ⬅  ⬅  ⬅\n",
       "1  ⬇  ⬆  ⬇  E\n",
       "2  ⬇  ➡  ⬇  E\n",
       "3  ➡  ➡  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:01<00:00, 6747.76it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ➡  ➡  ⬇  ⬅\n",
       "1  ⬇  ➡  ⬇  E\n",
       "2  ➡  ➡  ⬇  E\n",
       "3  ➡  ⬆  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  -  -  -  S\n",
       "1  -  -  -  E\n",
       "2  -  -  -  E\n",
       "3  -  -  -  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14.54</td>\n",
       "      <td>15.38</td>\n",
       "      <td>19.41</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15.80</td>\n",
       "      <td>19.13</td>\n",
       "      <td>20.20</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.75</td>\n",
       "      <td>21.23</td>\n",
       "      <td>21.52</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19.21</td>\n",
       "      <td>22.07</td>\n",
       "      <td>25.00</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0      1      2   3\n",
       "0  14.54  15.38  19.41  18\n",
       "1  15.80  19.13  20.20   E\n",
       "2  19.75  21.23  21.52   E\n",
       "3  19.21  22.07  25.00   G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ⬇  ⬅  ⬇  ⬅\n",
       "1  ⬇  ⬇  ⬇  E\n",
       "2  ➡  ➡  ⬇  E\n",
       "3  ➡  ➡  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean reward (empirical) 69.9906138205\n",
      "Successful Episodes  89.41\n",
      "Risk probab  10.836533508745259\n"
     ]
    }
   ],
   "source": [
    "trials = 3\n",
    "Q_list = []\n",
    "reward_trial_list = []\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "# Set learning parameters\n",
    "lr = 0.9\n",
    "num_episodes = 10000\n",
    "\n",
    "for k in range(trials):\n",
    "    #Initialize table with all zeros\n",
    "    Q = np.zeros((gw.M,gw.N,4))\n",
    "    jList = []\n",
    "    rList = []\n",
    "    for i in trange(num_episodes):\n",
    "        #print('Episode', i)\n",
    "        #Reset environment and get first new observation\n",
    "        s = gw.reset()\n",
    "        rAll = 0\n",
    "        d = False\n",
    "        j = 0\n",
    "        #The Q-Table learning algorithm\n",
    "        while j < 1000:\n",
    "            j+=1\n",
    "            #Choose an action by greedily (with noise) picking from Q table\n",
    "            a = np.argmax(Q[s[0],s[1],:] + np.random.randn(1,4)*(1./(i+1)))\n",
    "#             if np.random.rand() < epsilon:\n",
    "#                 # take random action\n",
    "#                 a = np.random.choice([0,1,2,3])\n",
    "#             else:\n",
    "#                 # take action according to the q function table\n",
    "#                 a = np.argmax(Q[s[0],s[1],:])\n",
    "            #Get new state and reward from environment\n",
    "            s1,r,d = gw.step(a)\n",
    "\n",
    "            #Update Q-Table with new knowledge\n",
    "            Q[s[0], s[1],a] = Q[s[0], s[1],a] + lr*(r + y*np.max(Q[s1[0], s1[1],:]) - Q[s[0], s[1],a])\n",
    "\n",
    "            rAll += r\n",
    "            s = s1\n",
    "            if d == True:\n",
    "                break\n",
    "        jList.append(j)    \n",
    "        rList.append(rAll)\n",
    "        \n",
    "    Q_list.append(Q)\n",
    "#     v = np.mean(Q, axis = 2)\n",
    "#     gw.print_v(v)\n",
    "\n",
    "    pi = np.argmax(Q, axis = 2)\n",
    "    gw.print_policy(pi)\n",
    "#     reward_trial_list.append(sum(rList)/float(num_episodes))\n",
    "#     print \"Score over time: \" +  str(sum(rList)/float(num_episodes))\n",
    "\n",
    "#     window = int(num_episodes/10)\n",
    "\n",
    "#     plt.figure(figsize=[9,16])\n",
    "#     plt.subplot(411)\n",
    "#     plt.plot(pd.Series(jList).rolling(window).mean())\n",
    "#     plt.title('Step Moving Average ({}-episode window)'.format(window))\n",
    "#     plt.ylabel('Moves')\n",
    "#     plt.xlabel('Episode')\n",
    "\n",
    "#     plt.subplot(412)\n",
    "#     plt.plot(pd.Series(rList).rolling(window).mean())\n",
    "#     plt.title('Reward Moving Average ({}-episode window)'.format(window))\n",
    "#     plt.ylabel('Reward')\n",
    "#     plt.xlabel('Episode')\n",
    "\n",
    "#     plt.tight_layout(pad=2)\n",
    "#     plt.show()\n",
    "\n",
    "gw.print_gw()\n",
    "meanQ = np.mean(np.array(Q_list), axis = 0)\n",
    "meanv = np.mean(meanQ, axis = 2)\n",
    "gw.print_v(meanv)\n",
    "meanpi = np.argmax(meanQ, axis = 2)\n",
    "gw.print_policy(meanpi)\n",
    "E_dis_reward, E_reward, E_risk = estimate_rr(meanpi)\n",
    "print \"Mean reward (empirical)\", E_dis_reward\n",
    "print \"Successful Episodes \", E_reward\n",
    "print \"Safety \", E_risk"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}