jane-street-deep-reinforcement-learning-approach.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.019856,
     "end_time": "2020-12-05T09:50:48.936951",
     "exception": false,
     "start_time": "2020-12-05T09:50:48.917095",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Jane Street: Deep Reinforcement Learning Approach\n",
    "\n",
    "I try implementing a deep Q-Network to solve the prediction problem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_kg_hide-output": false,
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:48.987623Z",
     "iopub.status.busy": "2020-12-05T09:50:48.986654Z",
     "iopub.status.idle": "2020-12-05T09:50:51.622706Z",
     "shell.execute_reply": "2020-12-05T09:50:51.621932Z"
    },
    "papermill": {
     "duration": 2.668945,
     "end_time": "2020-12-05T09:50:51.622871",
     "exception": false,
     "start_time": "2020-12-05T09:50:48.953926",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import torch \n",
    "import torch.nn as nn \n",
    "import torch.nn.functional as F \n",
    "import torch.optim as optim \n",
    "from torch.distributions import Categorical\n",
    "\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "print(device)\n",
    "\n",
    "import os, gc, random\n",
    "if device == 'cuda':\n",
    "    import cudf\n",
    "    import cupy as cp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import janestreet\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import GroupKFold\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "from joblib import dump, load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:51.667976Z",
     "iopub.status.busy": "2020-12-05T09:50:51.667137Z",
     "iopub.status.idle": "2020-12-05T09:50:51.674813Z",
     "shell.execute_reply": "2020-12-05T09:50:51.674115Z"
    },
    "papermill": {
     "duration": 0.033925,
     "end_time": "2020-12-05T09:50:51.674952",
     "exception": false,
     "start_time": "2020-12-05T09:50:51.641027",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed_value):\n",
    "    random.seed(seed_value)\n",
    "    np.random.seed(seed_value)\n",
    "    torch.manual_seed(seed_value)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed_value)\n",
    "    \n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        \n",
    "seed_everything(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.01676,
     "end_time": "2020-12-05T09:50:51.709189",
     "exception": false,
     "start_time": "2020-12-05T09:50:51.692429",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_kg_hide-output": false,
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:51.752770Z",
     "iopub.status.busy": "2020-12-05T09:50:51.751897Z",
     "iopub.status.idle": "2020-12-05T09:50:51.783666Z",
     "shell.execute_reply": "2020-12-05T09:50:51.782802Z"
    },
    "papermill": {
     "duration": 0.057371,
     "end_time": "2020-12-05T09:50:51.783805",
     "exception": false,
     "start_time": "2020-12-05T09:50:51.726434",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# print('Loading...')\n",
    "# if device == 'cuda':\n",
    "#     train = cudf.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')\n",
    "# else:\n",
    "#     train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')\n",
    "train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv', nrows = 3)\n",
    "features = [c for c in train.columns if 'feature' in c]\n",
    "\n",
    "# print('Filling...')\n",
    "# f_mean = train[features[1:]].mean()\n",
    "# f_std = train[features[1:]].std()\n",
    "# train = train.query('weight > 0').reset_index(drop = True)\n",
    "# train[features[1:]] = train[features[1:]].fillna(f_mean)\n",
    "# train[features[1:]] = (train[features[1:]] - f_mean) / f_std\n",
    "# train['action'] = (train['resp'] > 0).astype('int')\n",
    "\n",
    "# print('Converting...')\n",
    "# if device == 'cuda':\n",
    "#     train = train.to_pandas()\n",
    "#     f_mean = f_mean.values.get()\n",
    "#     f_std = f_std.values.get()\n",
    "# else:\n",
    "#     f_mean = f_mean.values\n",
    "#     f_std = f_std.values\n",
    "# np.save('f_mean.npy', f_mean)\n",
    "# np.save('f_std.npy', f_std)\n",
    "\n",
    "# print('Finish.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:51.824777Z",
     "iopub.status.busy": "2020-12-05T09:50:51.823768Z",
     "iopub.status.idle": "2020-12-05T09:50:51.827496Z",
     "shell.execute_reply": "2020-12-05T09:50:51.826732Z"
    },
    "papermill": {
     "duration": 0.026445,
     "end_time": "2020-12-05T09:50:51.827628",
     "exception": false,
     "start_time": "2020-12-05T09:50:51.801183",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# def utility_score(date, weight, resp, action):\n",
    "#     count_i = len(np.unique(date))\n",
    "#     Pi = np.bincount(date, weight * resp * action)\n",
    "#     t = np.sum(Pi) / np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)\n",
    "#     u = np.clip(t, 0, 6) * np.sum(Pi)\n",
    "#     return u\n",
    "\n",
    "# def utility_score_pd(date, weight, resp, action):\n",
    "#     count_i = len(pd.unique(date))\n",
    "#     Pi = np.bincount(date, weight * resp * action)\n",
    "#     t = np.sum(Pi) / np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)\n",
    "#     u = np.clip(t, 0, 6) * np.sum(Pi)\n",
    "#     return u\n",
    "\n",
    "# def utility_score_max(date, weight, resp, action):\n",
    "#     count_i = date.max() + 1\n",
    "#     Pi = np.bincount(date, weight * resp * action)\n",
    "#     t = np.sum(Pi) / np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)\n",
    "#     u = np.clip(t, 0, 6) * np.sum(Pi)\n",
    "#     return u\n",
    "\n",
    "# def utility_score_last(date, weight, resp, action):\n",
    "#     count_i = date[-1] + 1\n",
    "#     Pi = np.bincount(date, weight * resp * action)\n",
    "#     t = np.sum(Pi) / np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)\n",
    "#     u = np.clip(t, 0, 6) * np.sum(Pi)\n",
    "#     return u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:51.868103Z",
     "iopub.status.busy": "2020-12-05T09:50:51.867337Z",
     "iopub.status.idle": "2020-12-05T09:50:51.870156Z",
     "shell.execute_reply": "2020-12-05T09:50:51.870744Z"
    },
    "papermill": {
     "duration": 0.025227,
     "end_time": "2020-12-05T09:50:51.870925",
     "exception": false,
     "start_time": "2020-12-05T09:50:51.845698",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# %timeit utility_score(train['date'].values, train['weight'].values, train['resp'].values, train['action'].values)\n",
    "# %timeit utility_score_pd(train['date'].values, train['weight'].values, train['resp'].values, train['action'].values)\n",
    "# %timeit utility_score_max(train['date'].values, train['weight'].values, train['resp'].values, train['action'].values)\n",
    "# %timeit utility_score_last(train['date'].values, train['weight'].values, train['resp'].values, train['action'].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.017149,
     "end_time": "2020-12-05T09:50:51.905732",
     "exception": false,
     "start_time": "2020-12-05T09:50:51.888583",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# DQN Model Functions\n",
    "\n",
    "modified from https://github.com/MoMe36/DuelingDDQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:51.956544Z",
     "iopub.status.busy": "2020-12-05T09:50:51.955685Z",
     "iopub.status.idle": "2020-12-05T09:50:51.959505Z",
     "shell.execute_reply": "2020-12-05T09:50:51.958610Z"
    },
    "papermill": {
     "duration": 0.036145,
     "end_time": "2020-12-05T09:50:51.959652",
     "exception": false,
     "start_time": "2020-12-05T09:50:51.923507",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class JSEnv:\n",
    "    \n",
    "    def __init__(self, df, feats):\n",
    "\n",
    "        self.n_samples = df.shape[0]\n",
    "        self.weight = torch.FloatTensor(df['weight'].values)\n",
    "        self.resp = torch.FloatTensor(df['resp'].values)\n",
    "        self.states = torch.FloatTensor(df[feats].values)\n",
    "        self.observation_space = df[feats].shape[1]\n",
    "        self.action_space = 2\n",
    "        self.idx = 0\n",
    "    \n",
    "    def reset(self):\n",
    "        self.idx = 0\n",
    "        return self.states[self.idx].view(1, -1)\n",
    "    \n",
    "    def step(self, action):\n",
    "        reward = self.weight[self.idx] * self.resp[self.idx] * action\n",
    "        self.idx += 1\n",
    "        if self.idx >= self.n_samples:\n",
    "            done = True\n",
    "            self.idx = 0\n",
    "        else:\n",
    "            done = False\n",
    "        info = 0\n",
    "        return self.states[self.idx].view(1, -1), reward, done, info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_kg_hide-input": false,
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:52.023929Z",
     "iopub.status.busy": "2020-12-05T09:50:52.022988Z",
     "iopub.status.idle": "2020-12-05T09:50:52.318804Z",
     "shell.execute_reply": "2020-12-05T09:50:52.317953Z"
    },
    "papermill": {
     "duration": 0.338982,
     "end_time": "2020-12-05T09:50:52.318943",
     "exception": false,
     "start_time": "2020-12-05T09:50:51.979961",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import gym \n",
    "import torch \n",
    "import random\n",
    "from argparse import ArgumentParser \n",
    "import os \n",
    "import pandas as pd \n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "from scipy.ndimage.filters import gaussian_filter1d\n",
    "from IPython.display import display, clear_output\n",
    "\n",
    "def save(agent, rewards): \n",
    "\n",
    "    torch.save(agent.q.state_dict(), 'Net.pt')\n",
    "\n",
    "#     plt.gca().cla()\n",
    "#     plt.plot(rewards, c = 'r', alpha = 0.3)\n",
    "#     plt.plot(gaussian_filter1d(rewards, sigma = 5), c = 'r', label = 'Rewards')\n",
    "#     plt.xlabel('Frames x 1000')\n",
    "#     plt.ylabel('Cumulative reward')\n",
    "#     plt.title('Dueling DDQN: JS')\n",
    "#     plt.legend()\n",
    "#     plt.savefig('reward.png')\n",
    "#     plt.show()\n",
    "    \n",
    "#     clear_output(wait = True)\n",
    "#     plt.pause(0.5)\n",
    "\n",
    "#     pd.DataFrame(rewards, columns = ['Reward']).to_csv('rewards.csv', index = False)\n",
    "\n",
    "class AgentConfig:\n",
    "\n",
    "    def __init__(self, \n",
    "                 epsilon_start = 1.,\n",
    "                 epsilon_final = 0.01,\n",
    "                 epsilon_decay = 8000,\n",
    "                 gamma = 0.99, \n",
    "                 lr = 1e-4, \n",
    "                 target_net_update_freq = 1000, \n",
    "                 memory_size = 100000, \n",
    "                 batch_size = 128, \n",
    "                 learning_starts = 5000,\n",
    "                 max_frames = 10000000): \n",
    "\n",
    "        self.epsilon_start = epsilon_start\n",
    "        self.epsilon_final = epsilon_final\n",
    "        self.epsilon_decay = epsilon_decay\n",
    "        self.epsilon_by_frame = lambda i: self.epsilon_final + (self.epsilon_start - self.epsilon_final) * np.exp(-1. * i / self.epsilon_decay)\n",
    "\n",
    "        self.gamma =gamma\n",
    "        self.lr =lr\n",
    "\n",
    "        self.target_net_update_freq =target_net_update_freq\n",
    "        self.memory_size =memory_size\n",
    "        self.batch_size =batch_size\n",
    "\n",
    "        self.learning_starts = learning_starts\n",
    "        self.max_frames = max_frames\n",
    "\n",
    "class ExperienceReplayMemory:\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.memory = []\n",
    "\n",
    "    def push(self, transition):\n",
    "        self.memory.append(transition)\n",
    "        if len(self.memory) > self.capacity:\n",
    "            del self.memory[0]\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        \n",
    "        batch = random.sample(self.memory, batch_size)\n",
    "        states = []\n",
    "        actions = []\n",
    "        rewards = []\n",
    "        next_states = [] \n",
    "        dones = []\n",
    "\n",
    "        for b in batch: \n",
    "            states.append(b[0])\n",
    "            actions.append(b[1])\n",
    "            rewards.append(b[2])\n",
    "            next_states.append(b[3])\n",
    "            dones.append(b[4])\n",
    "\n",
    "        return states, actions, rewards, next_states, dones\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_kg_hide-input": false,
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:52.370408Z",
     "iopub.status.busy": "2020-12-05T09:50:52.369536Z",
     "iopub.status.idle": "2020-12-05T09:50:52.373665Z",
     "shell.execute_reply": "2020-12-05T09:50:52.372860Z"
    },
    "papermill": {
     "duration": 0.036554,
     "end_time": "2020-12-05T09:50:52.373803",
     "exception": false,
     "start_time": "2020-12-05T09:50:52.337249",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn \n",
    "import torch.nn.functional as F \n",
    "import torch.optim as optim \n",
    "from torch.distributions import Categorical \n",
    "\n",
    "class DuelingNetwork(nn.Module): \n",
    "\n",
    "    def __init__(self, obs, ac): \n",
    "\n",
    "        super().__init__()\n",
    "\n",
    "        self.model = nn.Sequential(nn.utils.weight_norm(nn.Linear(obs, 512)),\n",
    "                                   nn.ReLU(), \n",
    "                                   nn.utils.weight_norm(nn.Linear(512, 256)),\n",
    "                                   nn.ReLU(),\n",
    "                                  )\n",
    "\n",
    "        self.value_head = nn.utils.weight_norm(nn.Linear(256, 1))\n",
    "        self.adv_head = nn.utils.weight_norm(nn.Linear(256, ac))\n",
    "\n",
    "    def forward(self, x): \n",
    "\n",
    "        out = self.model(x)\n",
    "\n",
    "        value = self.value_head(out)\n",
    "        adv = self.adv_head(out)\n",
    "\n",
    "        q_val = value + adv - adv.mean(1).reshape(-1,1)\n",
    "        return q_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_kg_hide-input": false,
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:52.437031Z",
     "iopub.status.busy": "2020-12-05T09:50:52.435871Z",
     "iopub.status.idle": "2020-12-05T09:50:52.440012Z",
     "shell.execute_reply": "2020-12-05T09:50:52.439330Z"
    },
    "papermill": {
     "duration": 0.047968,
     "end_time": "2020-12-05T09:50:52.440141",
     "exception": false,
     "start_time": "2020-12-05T09:50:52.392173",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "import torch \n",
    "import torch.nn as nn \n",
    "import torch.nn.functional as F \n",
    "import torch.optim as optim \n",
    "from torch.distributions import Categorical \n",
    "\n",
    "import numpy as np \n",
    "import gym \n",
    "import random \n",
    "\n",
    "class DuelingDDQN(nn.Module): \n",
    "\n",
    "    def __init__(self, obs, ac, config): \n",
    "\n",
    "        super().__init__()\n",
    "\n",
    "        self.q = DuelingNetwork(obs, ac).to(device)\n",
    "        self.target = DuelingNetwork(obs, ac).to(device)\n",
    "\n",
    "        self.target.load_state_dict(self.q.state_dict())\n",
    "\n",
    "        self.target_net_update_freq = config.target_net_update_freq\n",
    "        self.update_counter = 0\n",
    "\n",
    "    def get_action(self, x):\n",
    "        \n",
    "        x = torch.FloatTensor(x).to(device)\n",
    "        with torch.no_grad(): \n",
    "            a = self.q(x).max(1)[1]\n",
    "\n",
    "        return a.item()\n",
    "\n",
    "    def update_policy(self, adam, memory, params): \n",
    "\n",
    "        b_states, b_actions, b_rewards, b_next_states, b_masks = memory.sample(params.batch_size)\n",
    "\n",
    "        states = torch.FloatTensor(b_states).to(device)\n",
    "        actions = torch.LongTensor(b_actions).reshape(-1,1).to(device)\n",
    "        rewards = torch.FloatTensor(b_rewards).reshape(-1,1).to(device)\n",
    "        next_states = torch.FloatTensor(b_next_states).to(device)\n",
    "        masks = torch.FloatTensor(b_masks).reshape(-1,1).to(device)\n",
    "\n",
    "        current_q_values = self.q(states).gather(1, actions)\n",
    "\n",
    "        # print(current_q_values[:5])\n",
    "\n",
    "        with torch.no_grad():\n",
    "\n",
    "            max_next_q_vals = self.target(next_states).max(1)[0].reshape(-1,1)\n",
    "            # max_next_q_vals = self.\n",
    "        expected_q_vals = rewards + max_next_q_vals*0.99*masks\n",
    "        # print(expected_q_vals[:5])\n",
    "        loss = F.mse_loss(expected_q_vals, current_q_values)\n",
    "\n",
    "        # input(loss)\n",
    "\n",
    "        # print('\\n'*5)\n",
    "        \n",
    "        adam.zero_grad()\n",
    "        loss.backward()\n",
    "\n",
    "        for p in self.q.parameters(): \n",
    "            p.grad.data.clamp_(-1.,1.)\n",
    "        adam.step()\n",
    "\n",
    "        self.update_counter += 1\n",
    "        if self.update_counter % self.target_net_update_freq == 0: \n",
    "            self.update_counter = 0 \n",
    "            self.target.load_state_dict(self.q.state_dict())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.017897,
     "end_time": "2020-12-05T09:50:52.476448",
     "exception": false,
     "start_time": "2020-12-05T09:50:52.458551",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Train Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:52.520255Z",
     "iopub.status.busy": "2020-12-05T09:50:52.519427Z",
     "iopub.status.idle": "2020-12-05T09:50:52.523145Z",
     "shell.execute_reply": "2020-12-05T09:50:52.522368Z"
    },
    "papermill": {
     "duration": 0.028572,
     "end_time": "2020-12-05T09:50:52.523293",
     "exception": false,
     "start_time": "2020-12-05T09:50:52.494721",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# env = JSEnv(train, features)       \n",
    "# config = AgentConfig(epsilon_start = 1.,\n",
    "#                      epsilon_final = 0.01,\n",
    "#                      epsilon_decay = 8000,\n",
    "#                      gamma = 0.99, \n",
    "#                      lr = 1e-4, \n",
    "#                      target_net_update_freq = 1000, \n",
    "#                      memory_size = env.n_samples // 100, \n",
    "#                      batch_size = 128, \n",
    "#                      learning_starts = 5000,\n",
    "#                      max_frames = env.n_samples)\n",
    "# memory = ExperienceReplayMemory(config.memory_size)\n",
    "# agent = DuelingDDQN(env.observation_space, env.action_space, config)\n",
    "# adam = optim.Adam(agent.q.parameters(), lr = config.lr) \n",
    "\n",
    "# s = env.reset()\n",
    "# ep_reward = 0. \n",
    "# recap = []\n",
    "# cum_rewards = []\n",
    "\n",
    "# p_bar = tqdm(total = config.max_frames)\n",
    "# for frame in range(config.max_frames):\n",
    "\n",
    "#     epsilon = config.epsilon_by_frame(frame)\n",
    "\n",
    "#     if np.random.random() > epsilon: \n",
    "#         action = agent.get_action(s)\n",
    "#     else: \n",
    "#         action = np.random.randint(0, env.action_space)\n",
    "\n",
    "#     ns, r, done, infos = env.step(action)\n",
    "#     ep_reward += r \n",
    "#     if done:\n",
    "#         ns = env.reset()\n",
    "#         recap.append(ep_reward)\n",
    "#         p_bar.set_description('Rew: {:.3f}'.format(ep_reward))\n",
    "#         ep_reward = 0.\n",
    "\n",
    "#     memory.push((s.reshape(-1).numpy().tolist(), action, r, ns.reshape(-1).numpy().tolist(), 0. if done else 1.))\n",
    "#     s = ns  \n",
    "\n",
    "#     p_bar.update(1)\n",
    "\n",
    "#     if frame > config.learning_starts:\n",
    "#         agent.update_policy(adam, memory, config)\n",
    "\n",
    "#     if frame % 1000 == 0:\n",
    "#         print(f'{frame + 1}/{config.max_frames}:', ep_reward.item(), end = '\\r')\n",
    "# #         cum_rewards.append(ep_reward.item())\n",
    "#         save(agent, cum_rewards)\n",
    "\n",
    "# p_bar.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.018254,
     "end_time": "2020-12-05T09:50:52.560239",
     "exception": false,
     "start_time": "2020-12-05T09:50:52.541985",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:52.604578Z",
     "iopub.status.busy": "2020-12-05T09:50:52.603772Z",
     "iopub.status.idle": "2020-12-05T09:50:52.712384Z",
     "shell.execute_reply": "2020-12-05T09:50:52.711589Z"
    },
    "papermill": {
     "duration": 0.133572,
     "end_time": "2020-12-05T09:50:52.712517",
     "exception": false,
     "start_time": "2020-12-05T09:50:52.578945",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DuelingNetwork(\n",
       "  (model): Sequential(\n",
       "    (0): Linear(in_features=130, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=512, out_features=256, bias=True)\n",
       "    (3): ReLU()\n",
       "  )\n",
       "  (value_head): Linear(in_features=256, out_features=1, bias=True)\n",
       "  (adv_head): Linear(in_features=256, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checkpoint_path = './Net.pt'\n",
    "checkpoint_path = '../input/js-dqn/Net.pt'\n",
    "\n",
    "model = DuelingNetwork(len(features), 2).to(device)\n",
    "model.load_state_dict(torch.load(checkpoint_path, map_location = device))\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:52.757512Z",
     "iopub.status.busy": "2020-12-05T09:50:52.756579Z",
     "iopub.status.idle": "2020-12-05T09:50:52.777173Z",
     "shell.execute_reply": "2020-12-05T09:50:52.776343Z"
    },
    "papermill": {
     "duration": 0.045602,
     "end_time": "2020-12-05T09:50:52.777341",
     "exception": false,
     "start_time": "2020-12-05T09:50:52.731739",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# f_mean = np.load('./f_mean.npy')\n",
    "# f_std = np.load('./f_std.npy')\n",
    "\n",
    "f_mean = np.load('../input/js-dqn/f_mean.npy')\n",
    "f_std = np.load('../input/js-dqn/f_std.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:52.828044Z",
     "iopub.status.busy": "2020-12-05T09:50:52.827111Z",
     "iopub.status.idle": "2020-12-05T09:50:54.178436Z",
     "shell.execute_reply": "2020-12-05T09:50:54.177653Z"
    },
    "papermill": {
     "duration": 1.38168,
     "end_time": "2020-12-05T09:50:54.178572",
     "exception": false,
     "start_time": "2020-12-05T09:50:52.796892",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.70579064 -0.04924998]\n",
      " [ 0.70596105  0.48139542]\n",
      " [ 0.68286705  0.2586545 ]\n",
      " ...\n",
      " [ 0.7388593   0.5991102 ]\n",
      " [ 0.70749444  0.57120484]\n",
      " [ 0.55012345  0.16774449]]\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv('../input/jane-street-market-prediction/example_test.csv')\n",
    "test[features[1:]] = test[features[1:]].fillna(dict(zip(features[1:], f_mean)))\n",
    "test[features[1:]] = (test[features[1:]] - f_mean) / f_std\n",
    "pred = model(torch.FloatTensor(test[features].values).to(device)).detach().cpu().numpy()\n",
    "print(pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.01979,
     "end_time": "2020-12-05T09:50:54.220163",
     "exception": false,
     "start_time": "2020-12-05T09:50:54.200373",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Submitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:54.267575Z",
     "iopub.status.busy": "2020-12-05T09:50:54.266370Z",
     "iopub.status.idle": "2020-12-05T09:50:54.269899Z",
     "shell.execute_reply": "2020-12-05T09:50:54.269240Z"
    },
    "papermill": {
     "duration": 0.029363,
     "end_time": "2020-12-05T09:50:54.270073",
     "exception": false,
     "start_time": "2020-12-05T09:50:54.240710",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "env = janestreet.make_env()\n",
    "env_iter = env.iter_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_kg_hide-output": false,
    "execution": {
     "iopub.execute_input": "2020-12-05T09:50:54.323158Z",
     "iopub.status.busy": "2020-12-05T09:50:54.322364Z",
     "iopub.status.idle": "2020-12-05T09:54:58.156442Z",
     "shell.execute_reply": "2020-12-05T09:54:58.155378Z"
    },
    "papermill": {
     "duration": 243.865557,
     "end_time": "2020-12-05T09:54:58.156602",
     "exception": false,
     "start_time": "2020-12-05T09:50:54.291045",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4206b1fd05cb4e689cfb1b198ec23429",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for (test_df, pred_df) in tqdm(env_iter):\n",
    "    if test_df['weight'].item() > 0:\n",
    "        x_tt = test_df.loc[:, features].values\n",
    "        if np.isnan(x_tt[:, 1:].sum()):\n",
    "            x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:]) + np.isnan(x_tt[:, 1:]) * f_mean\n",
    "        x_tt[:, 1:] = (x_tt[:, 1:] - f_mean) / f_std\n",
    "        x_tt = torch.FloatTensor(x_tt).to(device)\n",
    "        pred_df.action = model(x_tt).max(1)[1].cpu().numpy().item()\n",
    "    else:\n",
    "        pred_df.action = 0\n",
    "    env.predict(pred_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 256.26247,
   "end_time": "2020-12-05T09:54:59.279525",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-12-05T09:50:43.017055",
   "version": "2.1.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "3928caaad7e549369cd0d319cc555c7a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3b0a594bf7164c5295d2a19296852fe2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4206b1fd05cb4e689cfb1b198ec23429": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_e0d9b16232084d038ba2ef09e3a949b6",
        "IPY_MODEL_e625311b700345c5a722b544c9b7e7dd"
       ],
       "layout": "IPY_MODEL_3928caaad7e549369cd0d319cc555c7a"
      }
     },
     "88469f45d21940539d9c1ce20745149e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "ab593116cd6345238a2398f81605749f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ccc7e203a63548529d20b593e9a2b789": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e0d9b16232084d038ba2ef09e3a949b6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ccc7e203a63548529d20b593e9a2b789",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_88469f45d21940539d9c1ce20745149e",
       "value": 1.0
      }
     },
     "e625311b700345c5a722b544c9b7e7dd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ab593116cd6345238a2398f81605749f",
       "placeholder": "​",
       "style": "IPY_MODEL_3b0a594bf7164c5295d2a19296852fe2",
       "value": " 15219/? [04:03&lt;00:00, 62.42it/s]"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}