dist_q2005.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gridworld import *\n",
    "from tqdm import trange\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "#S-Learning\n",
    "\n",
    "# gw = gridworld((5, 5), (0, 4), (4, 4), [(o, 4) for o in range(1, 4)], 0.9, 0, 100, 0)\n",
    "# y = .95\n",
    "gw = gridworld((4, 4), (0, 3), (3, 3), [(o, 3) for o in range(1, 3)], 0.9, 0, 100, 0)\n",
    "y = .95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set learning parameters\n",
    "lr = .7\n",
    "num_episodes = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dist_error_state(state, error_state):\n",
    "    return ( np.sqrt( (state[0] - error_state[0])**2 + (state[1] - error_state[1])**2 ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk_states = np.zeros((gw.M, gw.N))\n",
    "for i in range(gw.M):\n",
    "    for j in range(gw.N):\n",
    "#         if (i, j) in gw.error_states:\n",
    "#             risk_states[i,j] == 0.\n",
    "#         if (i, j) == gw.goal:\n",
    "#             risk_states[i,j] == 0.\n",
    "#         else:\n",
    "        sm = 100.\n",
    "        for e in gw.error_states:\n",
    "            #sm += np.exp( - dist_error_state([i,j], e) )\n",
    "            sm = min(sm, dist_error_state([i,j], e))\n",
    "        risk_states[i,j] = sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.16227766, 2.23606798, 1.41421356, 1.        ],\n",
       "       [3.        , 2.        , 1.        , 0.        ],\n",
       "       [3.        , 2.        , 1.        , 0.        ],\n",
       "       [3.16227766, 2.23606798, 1.41421356, 1.        ]])"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "3.1622776601683795"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([[5.        , 3.53553391, 2.23606798, 1.58113883],\n",
       "       [4.74341649, 3.16227766, 1.58113883, 0.        ],\n",
       "       [4.74341649, 3.16227766, 1.58113883, 0.        ],\n",
       "       [5.        , 3.53553391, 2.23606798, 1.58113883]])"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "risk_states\n",
    "np.random.seed(1)\n",
    "\n",
    "np.min(risk_states)\n",
    "np.max(risk_states)\n",
    "risk_states = (risk_states - np.min(risk_states))/(np.max(risk_states) - np.min(risk_states)) * 5\n",
    "risk_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Empirically estimate safety\n",
    "num_sims = 10000\n",
    "def estimate_rr(pi):\n",
    "    rewards = []\n",
    "    discounted = []\n",
    "    risks = []\n",
    "    discounted_reward = 0\n",
    "    for i in range(num_sims):\n",
    "        s = gw.reset()\n",
    "        reward = 0\n",
    "        d = False\n",
    "        j = 0\n",
    "        risk_dist = 0\n",
    "        while(j < 1000):\n",
    "            a = pi[s[0],s[1]]\n",
    "            s1,r,d = gw.step(a)\n",
    "            reward += r\n",
    "            risk_dist += risk_states[s[0], s[1]]                     \n",
    "            if d == True:\n",
    "                if s1 in gw.error_states:\n",
    "                    discounted_reward = 0\n",
    "                else:\n",
    "                    discounted_reward = pow(y, j) * r\n",
    "                break\n",
    "            s = s1\n",
    "            j += 1\n",
    "        rewards.append(reward)\n",
    "        discounted.append(discounted_reward)\n",
    "        risks.append(risk_dist)\n",
    "    return sum(discounted)/float(num_sims), sum(rewards)/float(num_sims), sum(risks)/float(num_sims)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n0.18, 0.2 - little risk\\n0.25 - trade-off\\n0.3, 0.35 - too risky\\n'"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Episode', 1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>⬆</td>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ⬇  ⬅  ⬅  ⬅\n",
       "1  ⬇  ⬅  ⬇  E\n",
       "2  ➡  ➡  ⬇  E\n",
       "3  ⬆  ⬆  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean reward (empirical) 57.2007726773\n",
      "Successful Episodes  92.08\n",
      "Safety  31.97490564602524\n",
      "Score over time: 91.57\n",
      "('Episode', 1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬅</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬆</td>\n",
       "      <td>⬇</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ⬆  ➡  ⬇  ⬅\n",
       "1  ➡  ➡  ⬅  E\n",
       "2  ➡  ➡  ⬇  E\n",
       "3  ➡  ⬆  ⬇  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean reward (empirical) 8.07166596181\n",
      "Successful Episodes  53.07\n",
      "Safety  102.72774198276305\n",
      "Score over time: 89.1\n",
      "('Episode', 1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>⬇</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬆</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ⬇  ➡  ⬇  ⬅\n",
       "1  ➡  ➡  ⬇  E\n",
       "2  ➡  ⬆  ⬇  E\n",
       "3  ➡  ➡  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean reward (empirical) 70.1355976095\n",
      "Successful Episodes  89.56\n",
      "Safety  10.497443249185972\n",
      "Score over time: 88.51\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  -  -  -  S\n",
       "1  -  -  -  E\n",
       "2  -  -  -  E\n",
       "3  -  -  -  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>42.07</td>\n",
       "      <td>39.52</td>\n",
       "      <td>38.49</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34.72</td>\n",
       "      <td>32.96</td>\n",
       "      <td>22.85</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33.77</td>\n",
       "      <td>27.48</td>\n",
       "      <td>17.00</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36.81</td>\n",
       "      <td>25.32</td>\n",
       "      <td>16.65</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0      1      2   3\n",
       "0  42.07  39.52  38.49  33\n",
       "1  34.72  32.96  22.85   E\n",
       "2  33.77  27.48  17.00   E\n",
       "3  36.81  25.32  16.65   G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>⬇</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬆</td>\n",
       "      <td>⬇</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬆</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ⬇  ➡  ⬅  ⬅\n",
       "1  ➡  ➡  ⬇  E\n",
       "2  ➡  ⬆  ⬇  E\n",
       "3  ➡  ⬆  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean reward (empirical) 31.2868399171\n",
      "Successful Episodes  88.0\n",
      "Safety  75.5380481845976\n"
     ]
    }
   ],
   "source": [
    "trials = 3\n",
    "Q_list = []\n",
    "reward_trial_list = []\n",
    "risk_probab_list = []\n",
    "trajecs = []\n",
    "factor = 0.16\n",
    "'''\n",
    "0.18, 0.2 - little risk\n",
    "0.25 - trade-off\n",
    "0.3, 0.35 - too risky\n",
    "'''\n",
    "for k in range(trials):\n",
    "    Q = np.zeros((gw.M, gw.N, 4))\n",
    "    Q_ = np.zeros((gw.M, gw.N, 4))\n",
    "    mainQ = np.zeros((gw.M, gw.N, 4))\n",
    "    jList = []\n",
    "    rList = []\n",
    "    risky = 0\n",
    "    for i in range(num_episodes):\n",
    "        if i%10000 == 1:\n",
    "            print('Episode', i)\n",
    "        #Reset environment and get first new observation\n",
    "        s = gw.reset()\n",
    "        rAll = 0\n",
    "        d = False\n",
    "        j = 0\n",
    "        temp_tr = []\n",
    "        #The Q-Table learning algorithm\n",
    "        while j < 100:\n",
    "            j+=1\n",
    "            #Choose an action by greedily (with noise) picking from Q table\n",
    "            a = np.argmax(Q[s[0],s[1],:] + np.random.randn(1,4)*(1./(i+1)))\n",
    "            #Get new state and reward from environment\n",
    "            s1,r,d = gw.step(a)\n",
    "            temp_tr.append(s1)\n",
    "\n",
    "#             r_ = 0\n",
    "#             if s1 in gw.error_states:\n",
    "#                 r_ = 1\n",
    "#                 risky += 1\n",
    "\n",
    "            #Update Q-Table with new knowledge\n",
    "            risky += risk_states[s[0], s[1]]\n",
    "            Q[s[0], s[1],a] = Q[s[0], s[1],a] + lr*(r + y*np.max(Q[s1[0], s1[1],:]) - Q[s[0], s[1],a])\n",
    "            Q_[s[0], s[1],a] = Q_[s[0], s[1],a] + lr*(risk_states[s[0], s[1]] + y*np.max(Q_[s1[0], s1[1],:]) - Q_[s[0], s[1],a])\n",
    "            mainQ = factor * Q + Q_\n",
    "            #print np.mean(mainQ, axis = 2)\n",
    "            #print()\n",
    "            rAll += r\n",
    "            s = s1\n",
    "            if d == True:\n",
    "                break\n",
    "        jList.append(j)\n",
    "        rList.append(rAll)\n",
    "        trajecs.append(temp_tr)\n",
    "    Q_list.append(mainQ)\n",
    "\n",
    "#     v = np.mean(mainQ, axis = 2)\n",
    "#     gw.print_v(v)\n",
    "\n",
    "    pi = np.argmax(mainQ, axis = 2)\n",
    "    gw.print_policy(pi)\n",
    "    E_dis_reward, E_reward, E_risk = estimate_rr(pi)\n",
    "    print \"Mean reward (empirical)\", E_dis_reward\n",
    "    print \"Successful Episodes \", E_reward\n",
    "    print \"Safety \", E_risk\n",
    "    reward_trial_list.append(sum(rList)/float(num_episodes))\n",
    "    print \"Score over time: \" +  str(sum(rList)/float(num_episodes))\n",
    "    risk_probab_list.append(risky/float(num_episodes))\n",
    "#     print \"Risk probab: \" +  str(risky/float(num_episodes))\n",
    "    \n",
    "#     window = int(num_episodes/10)\n",
    "\n",
    "#     plt.figure(figsize=[9,16])\n",
    "#     plt.subplot(411)\n",
    "#     plt.plot(pd.Series(jList).rolling(window).mean())\n",
    "#     plt.title('Step Moving Average ({}-episode window)'.format(window))\n",
    "#     plt.ylabel('Moves')\n",
    "#     plt.xlabel('Episode')\n",
    "\n",
    "#     plt.subplot(412)\n",
    "#     plt.plot(pd.Series(rList).rolling(window).mean())\n",
    "#     plt.title('Reward Moving Average ({}-episode window)'.format(window))\n",
    "#     plt.ylabel('Reward')\n",
    "#     plt.xlabel('Episode')\n",
    "\n",
    "#     plt.tight_layout(pad=2)\n",
    "#     plt.show()meanQ = np.mean(np.array(Q_list), axis = 0)\n",
    "\n",
    "\n",
    "gw.print_gw()\n",
    "meanQ = np.mean(np.array(Q_list), axis = 0)\n",
    "maxv = np.mean(meanQ, axis = 2)\n",
    "gw.print_v(maxv)\n",
    "meanpi = np.argmax(meanQ, axis = 2)\n",
    "gw.print_policy(meanpi)\n",
    "E_dis_reward, E_reward, E_risk = estimate_rr(meanpi)\n",
    "print \"Mean reward (empirical)\", E_dis_reward\n",
    "print \"Successful Episodes \", E_reward\n",
    "print \"Safety \", E_risk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>⬅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>⬇</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬅</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>➡</td>\n",
       "      <td>⬇</td>\n",
       "      <td>⬆</td>\n",
       "      <td>E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>➡</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3\n",
       "0  ⬇  ⬇  ⬅  ⬅\n",
       "1  ⬇  ⬇  ⬅  E\n",
       "2  ➡  ⬇  ⬆  E\n",
       "3  ➡  ➡  ➡  G"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEoZJREFUeJzt3X+MXeV95/H3Z+2SJulubIKFUtu79ipWKifabtgRcZVVFeEKTIhi/kizRN3iUu/6j6XbtNtVarp/WJsUCbRVaVAbVlbsxlQRBLnZxWpoWItQpSstDkOICOCwjPgR2wI8jQ3pFjWp0+/+cR/TGz8ztpk7nhnPvF/S6J7zPc+593l0rPnMec65x6kqJEka9o/muwOSpIXHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVJn+Xx3YKYuu+yyWrdu3Xx3Q5IuKo899thfVdWqc7W7aMNh3bp1jI+Pz3c3JOmikuTF82nntJIkqWM4SJI6hoMkqWM4SJI6hoMkqXPOcEiyN8nxJE8O1f5bku8keSLJ/0iyYmjbLUkmkjyT5Jqh+pZWm0iyc6i+PsmhVv9Skktmc4CSpDfvfM4cvgBsOaN2EHhfVf0L4P8CtwAk2QjcALy37fO5JMuSLAP+CLgW2Ah8orUFuB24o6reDZwEto80IknSyM4ZDlX1deDEGbX/VVWn2uojwJq2vBW4t6p+UFXPAxPAle1noqqeq6ofAvcCW5MEuArY3/bfB1w/4pgkSSOajWsOvwr8eVteDRwZ2na01aarvxN4dShoTtenlGRHkvEk45OTk7PQdUnSVEb6hnSS/wKcAr44O905u6raDewGGBsbq7n4TIB1O7/yxvILt103Vx8rSfNmxuGQ5FeAjwCbq+r0L+pjwNqhZmtajWnq3wNWJFnezh6G20uS5smMppWSbAE+BXy0ql4f2nQAuCHJW5KsBzYA3wAeBTa0O5MuYXDR+kALlYeBj7X9twH3z2wokqTZcj63st4D/B/gPUmOJtkO/CHwj4GDSb6V5L8DVNVTwH3A08BXgZur6kftrODXgAeBw8B9rS3AbwP/KckEg2sQe2Z1hJKkN+2c00pV9YkpytP+Aq+qW4Fbp6g/ADwwRf05BnczSZIWCL8hLUnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqnDMckuxNcjzJk0O1S5McTPJse13Z6klyZ5KJJE8kuWJon22t/bNJtg3V/1WSb7d97kyS2R6kJOnNOZ8zhy8AW86o7QQeqqoNwENtHeBaYEP72QHcBYMwAXYBHwCuBHadDpTW5t8P7XfmZ0mS5tg5w6Gqvg6cOKO8FdjXlvcB1w/V766BR4AVSd4FXAMcrKoTVXUSOAhsadv+SVU9UlUF3D30XpKkeTLTaw6XV9VLbfll4PK2vBo4MtTuaKudrX50irokaR6NfEG6/cVfs9CXc0qyI8l4kvHJycm5+EhJWpJmGg6vtCkh2uvxVj8GrB1qt6bVzlZfM0V9SlW1u6rGqmps1apVM+y6JOlcZhoOB4DTdxxtA+4fqt/Y7lraBLzWpp8eBK5OsrJdiL4aeLBt+36STe0upRuH3kuSNE+Wn6tBknuADwGXJTnK4K6j24D7kmwHXgQ+3po/AHwYmABeB24CqKoTST4DPNrafbqqTl/k/g8M7oh6K/Dn7UeSNI/OGQ5V9YlpNm2eom0BN0/zPnuBvVPUx4H3nasfkqS54zekJUkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1BkpHJL8ZpKnkjyZ5J4kP5lkfZJDSSaSfCnJJa3tW9r6RNu+buh9bmn1Z5JcM9qQJEmjmnE4JFkN/DowVlXvA5YBNwC3A3dU1buBk8D2tst24GSr39HakWRj2++9wBbgc0mWzbRfkqTRjTqttBx4a5LlwNuAl4CrgP1t+z7g+ra8ta3Ttm9Okla/t6p+UFXPAxPAlSP2S5I0ghmHQ1UdA34P+C6DUHgNeAx4tapOtWZHgdVteTVwpO17qrV/53B9in0kSfNglGmllQz+6l8P/DTwdgbTQhdMkh1JxpOMT05OXsiPkqQlbZRppV8Anq+qyar6O+DLwAeBFW2aCWANcKwtHwPWArTt7wC+N1yfYp8fU1W7q2qsqsZWrVo1QtclSWczSjh8F9iU5G3t2sFm4GngYeBjrc024P62fKCt07Z/raqq1W9odzOtBzYA3xihX5KkES0/d5OpVdWhJPuBbwKngMeB3cBXgHuT/G6r7Wm77AH+JMkEcILBHUpU1VNJ7mMQLKeAm6vqRzPtlyRpdDMOB4Cq2gXsOqP8HFPcbVRVfwv84jTvcytw6yh9kSTNHr8hLUnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqjBQOSVYk2Z/kO0kOJ/m5JJcmOZjk2fa6srVNkjuTTCR5IskVQ++zrbV/Nsm2UQclSRrNqGcOnwW+WlU/A/wscBjYCTxUVRuAh9o6wLXAhvazA7gLIMmlwC7gA8CVwK7TgSJJmh8zDock7wB+HtgDUFU/rKpXga3AvtZsH3B9W94K3F0DjwArkrwLuAY4WFUnquokcBDYMtN+SZJGN8qZw3pgEvjjJI8n+XyStwOXV9VLrc3LwOVteTVwZGj/o602XV2SNE9GCYflwBXAXVX1fuBv+IcpJACqqoAa4TN+TJIdScaTjE9OTs7W20qSzjBKOBwFjlbVoba+n0FYvNKmi2ivx9v2Y8Daof3XtNp09U5V7a6qsaoaW7Vq1QhdlySdzYzDoapeBo4keU8rbQaeBg4Ap+842gbc35YPADe2u5Y2Aa+16acHgauTrGwXoq9uNUnSPFk+4v7/EfhikkuA54CbGATOfUm2Ay8CH29tHwA+DEwAr7e2VNWJJJ8BHm3tPl1VJ0bslyRpBCOFQ1V9CxibYtPmKdoWcPM077MX2DtKXyRJs8dvSEuSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKkz6v8Epyms2/mVN5ZfuO26eeyJJM2MZw6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpM7I4ZBkWZLHk/xZW1+f5FCSiSRfSnJJq7+lrU+07euG3uOWVn8myTWj9kmSNJrZOHP4JHB4aP124I6qejdwEtje6tuBk61+R2tHko3ADcB7gS3A55Ism4V+SZJmaKRwSLIGuA74fFsPcBWwvzXZB1zflre2ddr2za39VuDeqvpBVT0PTABXjtIvSdJoRj1z+APgU8Dft/V3Aq9W1am2fhRY3ZZXA0cA2vbXWvs36lPsI0maBzMOhyQfAY5X1WOz2J9zfeaOJONJxicnJ+fqYyVpyRnlzOGDwEeTvADcy2A66bPAiiSnHwW+BjjWlo8BawHa9ncA3xuuT7HPj6mq3VU1VlVjq1atGqHrkqSzmXE4VNUtVbWmqtYxuKD8tar6JeBh4GOt2Tbg/rZ8oK3Ttn+tqqrVb2h3M60HNgDfmGm/JEmjuxD/2c9vA/cm+V3gcWBPq+8B/iTJBHCCQaBQVU8luQ94GjgF3FxVP7oA/ZIknadZCYeq+gvgL9ryc0xxt1FV/S3wi9Psfytw62z0RZI0Or8hLUnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqXIj/JnTJWLfzK28sv3DbdfPYE0maXZ45SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqTPjcEiyNsnDSZ5O8lSST7b6pUkOJnm2va5s9SS5M8lEkieSXDH0Xtta+2eTbBt9WJKkUYxy5nAK+K2q2ghsAm5OshHYCTxUVRuAh9o6wLXAhvazA7gLBmEC7AI+AFwJ7DodKJKk+THjcKiql6rqm235r4HDwGpgK7CvNdsHXN+WtwJ318AjwIok7wKuAQ5W1YmqOgkcBLbMtF+SpNHNyjWHJOuA9wOHgMur6qW26WXg8ra8GjgytNvRVpuuPtXn7EgynmR8cnJyNrouSZrCyA/eS/JTwJ8Cv1FV30/yxraqqiQ16mcMvd9uYDfA2NjYrL3vXPFBfZIuFiOdOST5CQbB8MWq+nIrv9Kmi2ivx1v9GLB2aPc1rTZdXZI0T0a5WynAHuBwVf3+0KYDwOk7jrYB9w/Vb2x3LW0CXmvTTw8CVydZ2S5EX91qkqR5Msq00geBXwa+neRbrfY7wG3AfUm2Ay8CH2/bHgA+DEwArwM3AVTViSSfAR5t7T5dVSdG6JckaUQzDoeq+t9Aptm8eYr2Bdw8zXvtBfbOtC9zafi6gSQtVn5DWpLUMRwkSR3DQZLUGfl7Djo7r1FIuhh55iBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6izJ7zn4/ypI0tl55iBJ6hgOkqSO4SBJ6hgOkqTOkrwgvdB4gVzSQmM4zJI3+/RVn9YqaSFzWkmS1PHMYYh/zUvSgGcOkqSO4SBJ6hgOkqSO4SBJ6nhBeoE520VxvwMhaa4smDOHJFuSPJNkIsnO+e6PJC1lCyIckiwD/gi4FtgIfCLJxvntlSQtXQtlWulKYKKqngNIci+wFXh6Xnu1wEw35eR0k6TZtlDCYTVwZGj9KPCBufjgxfDFt/MZgwEi6c1YKOFwXpLsAHa01f+X5Jk3sftlwF/Nfq8WtDfGnNvnuSdza0kf6yVmKY571DH/s/NptFDC4Riwdmh9Tav9mKraDeyeyQckGa+qsZl17+K0FMcMS3PcS3HMsDTHPVdjXhAXpIFHgQ1J1ie5BLgBODDPfZKkJWtBnDlU1akkvwY8CCwD9lbVU/PcLUlashZEOABU1QPAAxfwI2Y0HXWRW4pjhqU57qU4Zlia456TMaeq5uJzJEkXkYVyzUGStIAs+nBYKo/lSLI2ycNJnk7yVJJPtvqlSQ4meba9rpzvvs62JMuSPJ7kz9r6+iSH2jH/UrvJYVFJsiLJ/iTfSXI4yc8t9mOd5Dfbv+0nk9yT5CcX47FOsjfJ8SRPDtWmPLYZuLON/4kkV8xWPxZ1OCyxx3KcAn6rqjYCm4Cb21h3Ag9V1Qbgoba+2HwSODy0fjtwR1W9GzgJbJ+XXl1YnwW+WlU/A/wsg/Ev2mOdZDXw68BYVb2PwY0rN7A4j/UXgC1n1KY7ttcCG9rPDuCu2erEog4Hhh7LUVU/BE4/lmPRqaqXquqbbfmvGfyyWM1gvPtas33A9fPTwwsjyRrgOuDzbT3AVcD+1mQxjvkdwM8DewCq6odV9SqL/FgzuIHmrUmWA28DXmIRHuuq+jpw4ozydMd2K3B3DTwCrEjyrtnox2IPh6key7F6nvoyZ5KsA94PHAIur6qX2qaXgcvnqVsXyh8AnwL+vq2/E3i1qk619cV4zNcDk8Aft+m0zyd5O4v4WFfVMeD3gO8yCIXXgMdY/Mf6tOmO7QX7HbfYw2HJSfJTwJ8Cv1FV3x/eVoNb0xbN7WlJPgIcr6rH5rsvc2w5cAVwV1W9H/gbzphCWoTHeiWDv5LXAz8NvJ1+6mVJmKtju9jD4bwey7FYJPkJBsHwxar6ciu/cvo0s70en6/+XQAfBD6a5AUGU4ZXMZiLX9GmHmBxHvOjwNGqOtTW9zMIi8V8rH8BeL6qJqvq74AvMzj+i/1Ynzbdsb1gv+MWezgsmcdytLn2PcDhqvr9oU0HgG1teRtw/1z37UKpqluqak1VrWNwbL9WVb8EPAx8rDVbVGMGqKqXgSNJ3tNKmxk83n7RHmsG00mbkryt/Vs/PeZFfayHTHdsDwA3truWNgGvDU0/jWTRfwkuyYcZzEuffizHrfPcpQsiyb8G/hL4Nv8w//47DK473Af8U+BF4ONVdebFrotekg8B/7mqPpLknzM4k7gUeBz4t1X1g/ns32xL8i8ZXIS/BHgOuInBH3uL9lgn+a/Av2FwZ97jwL9jML++qI51knuADzF4+uorwC7gfzLFsW1B+YcMptheB26qqvFZ6cdiDwdJ0pu32KeVJEkzYDhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjr/HyUOYlXehNlYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = plt.hist([len(o) for o in trajecs], bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}