deep-reinforcement-learning-on-stock-data.ipynb

{"cells":[{"metadata":{"_cell_guid":"774f1f3d-5353-4a9f-858c-0cbd993577f8","_uuid":"0d30485734f2e1df31e62309ed20446b987dfea8","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"936eed8cc53b525db200513139019e94e0a89757"},"cell_type":"code","source":"import time\nimport copy\nimport numpy as np\nimport pandas as pd\nimport chainer\nimport chainer.functions as F\nimport chainer.links as L\nfrom plotly import tools\nfrom plotly.graph_objs import *\nfrom plotly.offline import init_notebook_mode, iplot, iplot_mpl\ninit_notebook_mode()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d5e73df952e3865515ba33fffabfea59b52c7e9"},"cell_type":"code","source":"data = pd.read_csv('../input/Data/Stocks/goog.us.txt')\ndata['Date'] = pd.to_datetime(data['Date'])\ndata = data.set_index('Date')\nprint(data.index.min(), data.index.max())\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10fddbff0b397113861a9a3c628c98773d3bbdd6"},"cell_type":"code","source":"date_split = '2016-01-01'\ntrain = data[:date_split]\ntest = data[date_split:]\nlen(train), len(test)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"e9d01773ac00cb8e280809042097ba1502d519e8"},"cell_type":"code","source":"def plot_train_test(train, test, date_split):\n    \n    data = [\n        Candlestick(x=train.index, open=train['Open'], high=train['High'], low=train['Low'], close=train['Close'], name='train'),\n        Candlestick(x=test.index, open=test['Open'], high=test['High'], low=test['Low'], close=test['Close'], name='test')\n    ]\n    layout = {\n         'shapes': [\n             {'x0': date_split, 'x1': date_split, 'y0': 0, 'y1': 1, 'xref': 'x', 'yref': 'paper', 'line': {'color': 'rgb(0,0,0)', 'width': 1}}\n         ],\n        'annotations': [\n            {'x': date_split, 'y': 1.0, 'xref': 'x', 'yref': 'paper', 'showarrow': False, 'xanchor': 'left', 'text': ' test data'},\n            {'x': date_split, 'y': 1.0, 'xref': 'x', 'yref': 'paper', 'showarrow': False, 'xanchor': 'right', 'text': 'train data '}\n        ]\n    }\n    figure = Figure(data=data, layout=layout)\n    iplot(figure)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"000f1c1c78598a1d39c56357fa04dbe94379bb35"},"cell_type":"code","source":"plot_train_test(train, test, date_split)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"1fc5d9a253b7cef891467c54bd61cbca3c623b80"},"cell_type":"code","source":"class Environment1:\n    \n    def __init__(self, data, history_t=90):\n        self.data = data\n        self.history_t = history_t\n        self.reset()\n        \n    def reset(self):\n        self.t = 0\n        self.done = False\n        self.profits = 0\n        self.positions = []\n        self.position_value = 0\n        self.history = [0 for _ in range(self.history_t)]\n        return [self.position_value] + self.history # obs\n    \n    def step(self, act):\n        reward = 0\n        \n        # act = 0: stay, 1: buy, 2: sell\n        if act == 1:\n            self.positions.append(self.data.iloc[self.t, :]['Close'])\n        elif act == 2: # sell\n            if len(self.positions) == 0:\n                reward = -1\n            else:\n                profits = 0\n                for p in self.positions:\n                    profits += (self.data.iloc[self.t, :]['Close'] - p)\n                reward += profits\n                self.profits += profits\n                self.positions = []\n        \n        # set next time\n        self.t += 1\n        self.position_value = 0\n        for p in self.positions:\n            self.position_value += (self.data.iloc[self.t, :]['Close'] - p)\n        self.history.pop(0)\n        self.history.append(self.data.iloc[self.t, :]['Close'] - self.data.iloc[(self.t-1), :]['Close'])\n        \n        # clipping reward\n        if reward > 0:\n            reward = 1\n        elif reward < 0:\n            reward = -1\n        \n        return [self.position_value] + self.history, reward, self.done # obs, reward, done","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fa94631ffc3408de7c1af0802283c32cda036a6"},"cell_type":"code","source":"env = Environment1(train)\nprint(env.reset())\nfor _ in range(3):\n    pact = np.random.randint(3)\n    print(env.step(pact))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"7e34a19d54e2888d5cb618e5b7e02b0a0fc633bd"},"cell_type":"code","source":"# DQN\n\ndef train_dqn(env):\n\n    class Q_Network(chainer.Chain):\n\n        def __init__(self, input_size, hidden_size, output_size):\n            super(Q_Network, self).__init__(\n                fc1 = L.Linear(input_size, hidden_size),\n                fc2 = L.Linear(hidden_size, hidden_size),\n                fc3 = L.Linear(hidden_size, output_size)\n            )\n\n        def __call__(self, x):\n            h = F.relu(self.fc1(x))\n            h = F.relu(self.fc2(h))\n            y = self.fc3(h)\n            return y\n\n        def reset(self):\n            self.zerograds()\n\n    Q = Q_Network(input_size=env.history_t+1, hidden_size=100, output_size=3)\n    Q_ast = copy.deepcopy(Q)\n    optimizer = chainer.optimizers.Adam()\n    optimizer.setup(Q)\n\n    epoch_num = 50\n    step_max = len(env.data)-1\n    memory_size = 200\n    batch_size = 20\n    epsilon = 1.0\n    epsilon_decrease = 1e-3\n    epsilon_min = 0.1\n    start_reduce_epsilon = 200\n    train_freq = 10\n    update_q_freq = 20\n    gamma = 0.97\n    show_log_freq = 5\n\n    memory = []\n    total_step = 0\n    total_rewards = []\n    total_losses = []\n\n    start = time.time()\n    for epoch in range(epoch_num):\n\n        pobs = env.reset()\n        step = 0\n        done = False\n        total_reward = 0\n        total_loss = 0\n\n        while not done and step < step_max:\n\n            # select act\n            pact = np.random.randint(3)\n            if np.random.rand() > epsilon:\n                pact = Q(np.array(pobs, dtype=np.float32).reshape(1, -1))\n                pact = np.argmax(pact.data)\n\n            # act\n            obs, reward, done = env.step(pact)\n\n            # add memory\n            memory.append((pobs, pact, reward, obs, done))\n            if len(memory) > memory_size:\n                memory.pop(0)\n\n            # train or update q\n            if len(memory) == memory_size:\n                if total_step % train_freq == 0:\n                    shuffled_memory = np.random.permutation(memory)\n                    memory_idx = range(len(shuffled_memory))\n                    for i in memory_idx[::batch_size]:\n                        batch = np.array(shuffled_memory[i:i+batch_size])\n                        b_pobs = np.array(batch[:, 0].tolist(), dtype=np.float32).reshape(batch_size, -1)\n                        b_pact = np.array(batch[:, 1].tolist(), dtype=np.int32)\n                        b_reward = np.array(batch[:, 2].tolist(), dtype=np.int32)\n                        b_obs = np.array(batch[:, 3].tolist(), dtype=np.float32).reshape(batch_size, -1)\n                        b_done = np.array(batch[:, 4].tolist(), dtype=np.bool)\n\n                        q = Q(b_pobs)\n                        maxq = np.max(Q_ast(b_obs).data, axis=1)\n                        target = copy.deepcopy(q.data)\n                        for j in range(batch_size):\n                            target[j, b_pact[j]] = b_reward[j]+gamma*maxq[j]*(not b_done[j])\n                        Q.reset()\n                        loss = F.mean_squared_error(q, target)\n                        total_loss += loss.data\n                        loss.backward()\n                        optimizer.update()\n\n                if total_step % update_q_freq == 0:\n                    Q_ast = copy.deepcopy(Q)\n\n            # epsilon\n            if epsilon > epsilon_min and total_step > start_reduce_epsilon:\n                epsilon -= epsilon_decrease\n\n            # next step\n            total_reward += reward\n            pobs = obs\n            step += 1\n            total_step += 1\n\n        total_rewards.append(total_reward)\n        total_losses.append(total_loss)\n\n        if (epoch+1) % show_log_freq == 0:\n            log_reward = sum(total_rewards[((epoch+1)-show_log_freq):])/show_log_freq\n            log_loss = sum(total_losses[((epoch+1)-show_log_freq):])/show_log_freq\n            elapsed_time = time.time()-start\n            print('\\t'.join(map(str, [epoch+1, epsilon, total_step, log_reward, log_loss, elapsed_time])))\n            start = time.time()\n            \n    return Q, total_losses, total_rewards\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09836d3d2f07a0edd74224a6bcd01192cf84caa6"},"cell_type":"code","source":"Q, total_losses, total_rewards = train_dqn(Environment1(train))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"73e0c0791daf22632f473ef371d2d62719331a2b"},"cell_type":"code","source":"def plot_loss_reward(total_losses, total_rewards):\n\n    figure = tools.make_subplots(rows=1, cols=2, subplot_titles=('loss', 'reward'), print_grid=False)\n    figure.append_trace(Scatter(y=total_losses, mode='lines', line=dict(color='skyblue')), 1, 1)\n    figure.append_trace(Scatter(y=total_rewards, mode='lines', line=dict(color='orange')), 1, 2)\n    figure['layout']['xaxis1'].update(title='epoch')\n    figure['layout']['xaxis2'].update(title='epoch')\n    figure['layout'].update(height=400, width=900, showlegend=False)\n    iplot(figure)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13cc8986e9d721d143c2a5a488865e2a27c882ec"},"cell_type":"code","source":"plot_loss_reward(total_losses, total_rewards)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"45856d723a14d02777475ac772593dc55319a9fe"},"cell_type":"code","source":"def plot_train_test_by_q(train_env, test_env, Q, algorithm_name):\n    \n    # train\n    pobs = train_env.reset()\n    train_acts = []\n    train_rewards = []\n\n    for _ in range(len(train_env.data)-1):\n        \n        pact = Q(np.array(pobs, dtype=np.float32).reshape(1, -1))\n        pact = np.argmax(pact.data)\n        train_acts.append(pact)\n            \n        obs, reward, done = train_env.step(pact)\n        train_rewards.append(reward)\n\n        pobs = obs\n        \n    train_profits = train_env.profits\n    \n    # test\n    pobs = test_env.reset()\n    test_acts = []\n    test_rewards = []\n\n    for _ in range(len(test_env.data)-1):\n    \n        pact = Q(np.array(pobs, dtype=np.float32).reshape(1, -1))\n        pact = np.argmax(pact.data)\n        test_acts.append(pact)\n            \n        obs, reward, done = test_env.step(pact)\n        test_rewards.append(reward)\n\n        pobs = obs\n        \n    test_profits = test_env.profits\n    \n    # plot\n    train_copy = train_env.data.copy()\n    test_copy = test_env.data.copy()\n    train_copy['act'] = train_acts + [np.nan]\n    train_copy['reward'] = train_rewards + [np.nan]\n    test_copy['act'] = test_acts + [np.nan]\n    test_copy['reward'] = test_rewards + [np.nan]\n    train0 = train_copy[train_copy['act'] == 0]\n    train1 = train_copy[train_copy['act'] == 1]\n    train2 = train_copy[train_copy['act'] == 2]\n    test0 = test_copy[test_copy['act'] == 0]\n    test1 = test_copy[test_copy['act'] == 1]\n    test2 = test_copy[test_copy['act'] == 2]\n    act_color0, act_color1, act_color2 = 'gray', 'cyan', 'magenta'\n\n    data = [\n        Candlestick(x=train0.index, open=train0['Open'], high=train0['High'], low=train0['Low'], close=train0['Close'], increasing=dict(line=dict(color=act_color0)), decreasing=dict(line=dict(color=act_color0))),\n        Candlestick(x=train1.index, open=train1['Open'], high=train1['High'], low=train1['Low'], close=train1['Close'], increasing=dict(line=dict(color=act_color1)), decreasing=dict(line=dict(color=act_color1))),\n        Candlestick(x=train2.index, open=train2['Open'], high=train2['High'], low=train2['Low'], close=train2['Close'], increasing=dict(line=dict(color=act_color2)), decreasing=dict(line=dict(color=act_color2))),\n        Candlestick(x=test0.index, open=test0['Open'], high=test0['High'], low=test0['Low'], close=test0['Close'], increasing=dict(line=dict(color=act_color0)), decreasing=dict(line=dict(color=act_color0))),\n        Candlestick(x=test1.index, open=test1['Open'], high=test1['High'], low=test1['Low'], close=test1['Close'], increasing=dict(line=dict(color=act_color1)), decreasing=dict(line=dict(color=act_color1))),\n        Candlestick(x=test2.index, open=test2['Open'], high=test2['High'], low=test2['Low'], close=test2['Close'], increasing=dict(line=dict(color=act_color2)), decreasing=dict(line=dict(color=act_color2)))\n    ]\n    title = '{}: train s-reward {}, profits {}, test s-reward {}, profits {}'.format(\n        algorithm_name,\n        int(sum(train_rewards)),\n        int(train_profits),\n        int(sum(test_rewards)),\n        int(test_profits)\n    )\n    layout = {\n        'title': title,\n        'showlegend': False,\n         'shapes': [\n             {'x0': date_split, 'x1': date_split, 'y0': 0, 'y1': 1, 'xref': 'x', 'yref': 'paper', 'line': {'color': 'rgb(0,0,0)', 'width': 1}}\n         ],\n        'annotations': [\n            {'x': date_split, 'y': 1.0, 'xref': 'x', 'yref': 'paper', 'showarrow': False, 'xanchor': 'left', 'text': ' test data'},\n            {'x': date_split, 'y': 1.0, 'xref': 'x', 'yref': 'paper', 'showarrow': False, 'xanchor': 'right', 'text': 'train data '}\n        ]\n    }\n    figure = Figure(data=data, layout=layout)\n    iplot(figure)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00243e4457ad8c355a466b1672fb5532d2ee3793"},"cell_type":"code","source":"plot_train_test_by_q(Environment1(train), Environment1(test), Q, 'DQN')","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"fb6210a03c8c90a3c314522fbbbf1a49711b7731"},"cell_type":"code","source":"# Double DQN\n\ndef train_ddqn(env):\n\n    class Q_Network(chainer.Chain):\n\n        def __init__(self, input_size, hidden_size, output_size):\n            super(Q_Network, self).__init__(\n                fc1 = L.Linear(input_size, hidden_size),\n                fc2 = L.Linear(hidden_size, hidden_size),\n                fc3 = L.Linear(hidden_size, output_size)\n            )\n\n        def __call__(self, x):\n            h = F.relu(self.fc1(x))\n            h = F.relu(self.fc2(h))\n            y = self.fc3(h)\n            return y\n\n        def reset(self):\n            self.zerograds()\n\n    Q = Q_Network(input_size=env.history_t+1, hidden_size=100, output_size=3)\n    Q_ast = copy.deepcopy(Q)\n    optimizer = chainer.optimizers.Adam()\n    optimizer.setup(Q)\n\n    epoch_num = 50\n    step_max = len(env.data)-1\n    memory_size = 200\n    batch_size = 50\n    epsilon = 1.0\n    epsilon_decrease = 1e-3\n    epsilon_min = 0.1\n    start_reduce_epsilon = 200\n    train_freq = 10\n    update_q_freq = 20\n    gamma = 0.97\n    show_log_freq = 5\n\n    memory = []\n    total_step = 0\n    total_rewards = []\n    total_losses = []\n\n    start = time.time()\n    for epoch in range(epoch_num):\n\n        pobs = env.reset()\n        step = 0\n        done = False\n        total_reward = 0\n        total_loss = 0\n\n        while not done and step < step_max:\n\n            # select act\n            pact = np.random.randint(3)\n            if np.random.rand() > epsilon:\n                pact = Q(np.array(pobs, dtype=np.float32).reshape(1, -1))\n                pact = np.argmax(pact.data)\n\n            # act\n            obs, reward, done = env.step(pact)\n\n            # add memory\n            memory.append((pobs, pact, reward, obs, done))\n            if len(memory) > memory_size:\n                memory.pop(0)\n\n            # train or update q\n            if len(memory) == memory_size:\n                if total_step % train_freq == 0:\n                    shuffled_memory = np.random.permutation(memory)\n                    memory_idx = range(len(shuffled_memory))\n                    for i in memory_idx[::batch_size]:\n                        batch = np.array(shuffled_memory[i:i+batch_size])\n                        b_pobs = np.array(batch[:, 0].tolist(), dtype=np.float32).reshape(batch_size, -1)\n                        b_pact = np.array(batch[:, 1].tolist(), dtype=np.int32)\n                        b_reward = np.array(batch[:, 2].tolist(), dtype=np.int32)\n                        b_obs = np.array(batch[:, 3].tolist(), dtype=np.float32).reshape(batch_size, -1)\n                        b_done = np.array(batch[:, 4].tolist(), dtype=np.bool)\n\n                        q = Q(b_pobs)\n                        \"\"\" <<< DQN -> Double DQN\n                        maxq = np.max(Q_ast(b_obs).data, axis=1)\n                        === \"\"\"\n                        indices = np.argmax(q.data, axis=1)\n                        maxqs = Q_ast(b_obs).data\n                        \"\"\" >>> \"\"\"\n                        target = copy.deepcopy(q.data)\n                        for j in range(batch_size):\n                            \"\"\" <<< DQN -> Double DQN\n                            target[j, b_pact[j]] = b_reward[j]+gamma*maxq[j]*(not b_done[j])\n                            === \"\"\"\n                            target[j, b_pact[j]] = b_reward[j]+gamma*maxqs[j, indices[j]]*(not b_done[j])\n                            \"\"\" >>> \"\"\"\n                        Q.reset()\n                        loss = F.mean_squared_error(q, target)\n                        total_loss += loss.data\n                        loss.backward()\n                        optimizer.update()\n\n                if total_step % update_q_freq == 0:\n                    Q_ast = copy.deepcopy(Q)\n\n            # epsilon\n            if epsilon > epsilon_min and total_step > start_reduce_epsilon:\n                epsilon -= epsilon_decrease\n\n            # next step\n            total_reward += reward\n            pobs = obs\n            step += 1\n            total_step += 1\n\n        total_rewards.append(total_reward)\n        total_losses.append(total_loss)\n\n        if (epoch+1) % show_log_freq == 0:\n            log_reward = sum(total_rewards[((epoch+1)-show_log_freq):])/show_log_freq\n            log_loss = sum(total_losses[((epoch+1)-show_log_freq):])/show_log_freq\n            elapsed_time = time.time()-start\n            print('\\t'.join(map(str, [epoch+1, epsilon, total_step, log_reward, log_loss, elapsed_time])))\n            start = time.time()\n            \n    return Q, total_losses, total_rewards","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a918dcbcc03decc0e58aae93a792b9b632ecdce3"},"cell_type":"code","source":"Q, total_losses, total_rewards = train_ddqn(Environment1(train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70144d9540d2b1a2af4370e3502a8c605ece2d5a"},"cell_type":"code","source":"plot_loss_reward(total_losses, total_rewards)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16710f3eb71db2588c6ad1343c54b00f0d31688f"},"cell_type":"code","source":"plot_train_test_by_q(Environment1(train), Environment1(test), Q, 'Double DQN')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"db727fd11a470328ee78f92f326bc6c4d34f1adf"},"cell_type":"code","source":"# Dueling Double DQN\n\ndef train_dddqn(env):\n\n    \"\"\" <<< Double DQN -> Dueling Double DQN\n    class Q_Network(chainer.Chain):\n\n        def __init__(self, input_size, hidden_size, output_size):\n            super(Q_Network, self).__init__(\n                fc1 = L.Linear(input_size, hidden_size),\n                fc2 = L.Linear(hidden_size, hidden_size),\n                fc3 = L.Linear(hidden_size, output_size)\n            )\n\n        def __call__(self, x):\n            h = F.relu(self.fc1(x))\n            h = F.relu(self.fc2(h))\n            y = self.fc3(h)\n            return y\n\n        def reset(self):\n            self.zerograds()\n    === \"\"\"\n    class Q_Network(chainer.Chain):\n\n        def __init__(self, input_size, hidden_size, output_size):\n            super(Q_Network, self).__init__(\n                fc1 = L.Linear(input_size, hidden_size),\n                fc2 = L.Linear(hidden_size, hidden_size),\n                fc3 = L.Linear(hidden_size, hidden_size//2),\n                fc4 = L.Linear(hidden_size, hidden_size//2),\n                state_value = L.Linear(hidden_size//2, 1),\n                advantage_value = L.Linear(hidden_size//2, output_size)\n            )\n            self.input_size = input_size\n            self.hidden_size = hidden_size\n            self.output_size = output_size\n\n        def __call__(self, x):\n            h = F.relu(self.fc1(x))\n            h = F.relu(self.fc2(h))\n            hs = F.relu(self.fc3(h))\n            ha = F.relu(self.fc4(h))\n            state_value = self.state_value(hs)\n            advantage_value = self.advantage_value(ha)\n            advantage_mean = (F.sum(advantage_value, axis=1)/float(self.output_size)).reshape(-1, 1)\n            q_value = F.concat([state_value for _ in range(self.output_size)], axis=1) + (advantage_value - F.concat([advantage_mean for _ in range(self.output_size)], axis=1))\n            return q_value\n\n        def reset(self):\n            self.zerograds()\n    \"\"\" >>> \"\"\"\n\n    Q = Q_Network(input_size=env.history_t+1, hidden_size=100, output_size=3)\n    Q_ast = copy.deepcopy(Q)\n    optimizer = chainer.optimizers.Adam()\n    optimizer.setup(Q)\n\n    epoch_num = 50\n    step_max = len(env.data)-1\n    memory_size = 200\n    batch_size = 50\n    epsilon = 1.0\n    epsilon_decrease = 1e-3\n    epsilon_min = 0.1\n    start_reduce_epsilon = 200\n    train_freq = 10\n    update_q_freq = 20\n    gamma = 0.97\n    show_log_freq = 5\n\n    memory = []\n    total_step = 0\n    total_rewards = []\n    total_losses = []\n\n    start = time.time()\n    for epoch in range(epoch_num):\n\n        pobs = env.reset()\n        step = 0\n        done = False\n        total_reward = 0\n        total_loss = 0\n\n        while not done and step < step_max:\n\n            # select act\n            pact = np.random.randint(3)\n            if np.random.rand() > epsilon:\n                pact = Q(np.array(pobs, dtype=np.float32).reshape(1, -1))\n                pact = np.argmax(pact.data)\n\n            # act\n            obs, reward, done = env.step(pact)\n\n            # add memory\n            memory.append((pobs, pact, reward, obs, done))\n            if len(memory) > memory_size:\n                memory.pop(0)\n\n            # train or update q\n            if len(memory) == memory_size:\n                if total_step % train_freq == 0:\n                    shuffled_memory = np.random.permutation(memory)\n                    memory_idx = range(len(shuffled_memory))\n                    for i in memory_idx[::batch_size]:\n                        batch = np.array(shuffled_memory[i:i+batch_size])\n                        b_pobs = np.array(batch[:, 0].tolist(), dtype=np.float32).reshape(batch_size, -1)\n                        b_pact = np.array(batch[:, 1].tolist(), dtype=np.int32)\n                        b_reward = np.array(batch[:, 2].tolist(), dtype=np.int32)\n                        b_obs = np.array(batch[:, 3].tolist(), dtype=np.float32).reshape(batch_size, -1)\n                        b_done = np.array(batch[:, 4].tolist(), dtype=np.bool)\n\n                        q = Q(b_pobs)\n                        \"\"\" <<< DQN -> Double DQN\n                        maxq = np.max(Q_ast(b_obs).data, axis=1)\n                        === \"\"\"\n                        indices = np.argmax(q.data, axis=1)\n                        maxqs = Q_ast(b_obs).data\n                        \"\"\" >>> \"\"\"\n                        target = copy.deepcopy(q.data)\n                        for j in range(batch_size):\n                            \"\"\" <<< DQN -> Double DQN\n                            target[j, b_pact[j]] = b_reward[j]+gamma*maxq[j]*(not b_done[j])\n                            === \"\"\"\n                            target[j, b_pact[j]] = b_reward[j]+gamma*maxqs[j, indices[j]]*(not b_done[j])\n                            \"\"\" >>> \"\"\"\n                        Q.reset()\n                        loss = F.mean_squared_error(q, target)\n                        total_loss += loss.data\n                        loss.backward()\n                        optimizer.update()\n\n                if total_step % update_q_freq == 0:\n                    Q_ast = copy.deepcopy(Q)\n\n            # epsilon\n            if epsilon > epsilon_min and total_step > start_reduce_epsilon:\n                epsilon -= epsilon_decrease\n\n            # next step\n            total_reward += reward\n            pobs = obs\n            step += 1\n            total_step += 1\n\n        total_rewards.append(total_reward)\n        total_losses.append(total_loss)\n\n        if (epoch+1) % show_log_freq == 0:\n            log_reward = sum(total_rewards[((epoch+1)-show_log_freq):])/show_log_freq\n            log_loss = sum(total_losses[((epoch+1)-show_log_freq):])/show_log_freq\n            elapsed_time = time.time()-start\n            print('\\t'.join(map(str, [epoch+1, epsilon, total_step, log_reward, log_loss, elapsed_time])))\n            start = time.time()\n            \n    return Q, total_losses, total_rewards","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6662bebe30b1ebf8fcb2aec06110ba9a7bcac47"},"cell_type":"code","source":"Q, total_losses, total_rewards = train_dddqn(Environment1(train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4228db250c11126e28335826986bba9fc9e3e082"},"cell_type":"code","source":"plot_loss_reward(total_losses, total_rewards)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"647731258374ac142e0c0c7b41e5f475cc1cd3d5"},"cell_type":"code","source":"plot_train_test_by_q(Environment1(train), Environment1(test), Q, 'Dueling Double DQN')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9ebb6e6d5060fe82f8ef100009ae1f521985d932"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}