stacking-logits.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:09:42.387789Z",
     "start_time": "2019-09-05T22:09:32.513804Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\"\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "import pickle as pk\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from fastai.vision import *\n",
    "from torch.nn import functional as F\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:09:42.398944Z",
     "start_time": "2019-09-05T22:09:42.392304Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0906_06-09-42\n"
     ]
    }
   ],
   "source": [
    "current_time = get_BJ_time()\n",
    "print(current_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:09:42.428697Z",
     "start_time": "2019-09-05T22:09:42.402517Z"
    }
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def seed_everything(seed):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "SEED = 2019\n",
    "seed_everything(SEED)\n",
    "\n",
    "deployment_dir = \"../output/inference\"\n",
    "\n",
    "def qk(y_pred, y):\n",
    "    k = torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0')\n",
    "    k[k != k] = 0\n",
    "    k[torch.isinf(k)] = 0\n",
    "    \n",
    "    return k\n",
    "\n",
    "df_2019_cv = pd.read_csv('../input/aptos-data-split/df_2019_cv.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:09:42.454359Z",
     "start_time": "2019-09-05T22:09:42.430634Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_code</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>path</th>\n",
       "      <th>is_valid1</th>\n",
       "      <th>is_valid2</th>\n",
       "      <th>is_valid3</th>\n",
       "      <th>is_valid4</th>\n",
       "      <th>is_valid5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000c1434d8d7</td>\n",
       "      <td>2</td>\n",
       "      <td>../input/aptos2019-blindness-detection/train_i...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>001639a390f0</td>\n",
       "      <td>4</td>\n",
       "      <td>../input/aptos2019-blindness-detection/train_i...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024cdab0c1e</td>\n",
       "      <td>1</td>\n",
       "      <td>../input/aptos2019-blindness-detection/train_i...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>002c21358ce6</td>\n",
       "      <td>0</td>\n",
       "      <td>../input/aptos2019-blindness-detection/train_i...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>005b95c28852</td>\n",
       "      <td>0</td>\n",
       "      <td>../input/aptos2019-blindness-detection/train_i...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id_code  diagnosis                                               path  \\\n",
       "0  000c1434d8d7          2  ../input/aptos2019-blindness-detection/train_i...   \n",
       "1  001639a390f0          4  ../input/aptos2019-blindness-detection/train_i...   \n",
       "2  0024cdab0c1e          1  ../input/aptos2019-blindness-detection/train_i...   \n",
       "3  002c21358ce6          0  ../input/aptos2019-blindness-detection/train_i...   \n",
       "4  005b95c28852          0  ../input/aptos2019-blindness-detection/train_i...   \n",
       "\n",
       "   is_valid1  is_valid2  is_valid3  is_valid4  is_valid5  \n",
       "0       True      False      False      False      False  \n",
       "1       True      False      False      False      False  \n",
       "2       True      False      False      False      False  \n",
       "3       True      False      False      False      False  \n",
       "4       True      False      False      False      False  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2019_cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:09:42.466194Z",
     "start_time": "2019-09-05T22:09:42.456684Z"
    }
   },
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('../input/aptos2019-blindness-detection/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-22T17:52:08.556127Z",
     "start_time": "2019-08-22T17:52:08.554197Z"
    }
   },
   "source": [
    "## Train logits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:09:54.641726Z",
     "start_time": "2019-09-05T22:09:54.638180Z"
    }
   },
   "outputs": [],
   "source": [
    "b3_models = [\"efficientnet-b3_0901_16-45-51_stage2_f1\", \"efficientnet-b3_0901_16-45-51_stage2_f2\",\n",
    "                  \"efficientnet-b3_0901_16-45-51_stage2_f3\", \"efficientnet-b3_0901_16-45-51_stage2_f4\",\n",
    "                  \"efficientnet-b3_0901_16-45-51_stage2_f5\"]\n",
    "\n",
    "b3_train_logits_list = []\n",
    "\n",
    "for i, m in enumerate(b3_models):\n",
    "    fold = i + 1\n",
    "    learn = load_learner(deployment_dir, \"{}.pkl\".format(m))\n",
    "    val_df = df_2019_cv[df_2019_cv[\"is_valid{}\".format(fold)]]\n",
    "    learn.data.add_test(ImageList.from_df(val_df,\n",
    "                                          '../input/aptos2019-blindness-detection',\n",
    "                                          cols=\"id_code\",\n",
    "                                          folder='train_images_ben_preprocessing_sigmaX10',\n",
    "                                          suffix='.png'))\n",
    "\n",
    "    logits,_ = learn.get_preds(DatasetType.Test)\n",
    "    logits = logits.numpy()\n",
    "    b3_train_logits_list.append(logits)\n",
    "    np.save(\"../output/stacking/{}_logits.npy\".format(m), logits)\n",
    "\n",
    "    print(logits.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:10:02.404786Z",
     "start_time": "2019-09-05T22:10:02.398354Z"
    }
   },
   "outputs": [],
   "source": [
    "b4_models = [\"efficientnet-b4_0820_01-09-57_stage2_f1\", \"efficientnet-b4_0820_01-09-57_stage2_f2\",\n",
    "                  \"efficientnet-b4_0820_01-09-57_stage2_f3\", \"efficientnet-b4_0820_01-09-57_stage2_f4\",\n",
    "                  \"efficientnet-b4_0821_00-02-25_stage2_f5\"]\n",
    "\n",
    "b4_train_logits_list = []\n",
    "for i, m in enumerate(b4_models):\n",
    "    fold = i + 1\n",
    "    learn = load_learner(deployment_dir, \"{}.pkl\".format(m))\n",
    "    val_df = df_2019_cv[df_2019_cv[\"is_valid{}\".format(fold)]]\n",
    "    learn.data.add_test(ImageList.from_df(val_df,\n",
    "                                          '../input/aptos2019-blindness-detection',\n",
    "                                          cols=\"id_code\",\n",
    "                                          folder='train_images_ben_preprocessing_sigmaX10',\n",
    "                                          suffix='.png'))\n",
    "\n",
    "    logits,_ = learn.get_preds(DatasetType.Test)\n",
    "    logits = logits.numpy()\n",
    "    b4_train_logits_list.append(logits)\n",
    "    np.save(\"../output/stacking/{}_logits.npy\".format(m), logits)\n",
    "\n",
    "    print(logits.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:10:08.018801Z",
     "start_time": "2019-09-05T22:10:08.015034Z"
    }
   },
   "outputs": [],
   "source": [
    "b5_models = [\"efficientnet-b5_0820_01-32-30_stage2_f1\", \"efficientnet-b5_0903_01-03-41_stage2_f2\",\n",
    "                  \"efficientnet-b5_0820_22-13-07_stage2_f3\", \"efficientnet-b5_0821_01-30-37_stage2_f4\",\n",
    "                  \"efficientnet-b5_0821_00-26-51_stage2_f5\"]\n",
    "\n",
    "b5_train_logits_list = []\n",
    "for i, m in enumerate(b5_models):\n",
    "    fold = i + 1\n",
    "    learn = load_learner(deployment_dir, \"{}.pkl\".format(m))\n",
    "    val_df = df_2019_cv[df_2019_cv[\"is_valid{}\".format(fold)]]\n",
    "    learn.data.add_test(ImageList.from_df(val_df,\n",
    "                                          '../input/aptos2019-blindness-detection',\n",
    "                                          cols=\"id_code\",\n",
    "                                          folder='train_images_ben_preprocessing_sigmaX10',\n",
    "                                          suffix='.png'))\n",
    "\n",
    "    logits,_ = learn.get_preds(DatasetType.Test)\n",
    "    logits = logits.numpy()\n",
    "    b5_train_logits_list.append(logits)\n",
    "    np.save(\"../output/stacking/{}_logits.npy\".format(m), logits)\n",
    "\n",
    "    print(logits.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Feature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-23T11:08:00.430735Z",
     "start_time": "2019-08-23T11:08:00.427585Z"
    }
   },
   "source": [
    "### Average"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-22T20:31:22.015853Z",
     "start_time": "2019-08-22T20:31:22.013172Z"
    }
   },
   "source": [
    "#### b3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-02T21:30:29.258704Z",
     "start_time": "2019-09-02T21:29:46.017018Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1928, 1)\n"
     ]
    }
   ],
   "source": [
    "b3_test_logits_list = []\n",
    "for m in b3_models:\n",
    "    learn = load_learner(deployment_dir, \"{}.pkl\".format(m))\n",
    "\n",
    "    learn.data.add_test(ImageList.from_df(test_df,\n",
    "                                      '../input/aptos2019-blindness-detection',\n",
    "                                      folder='test_images_ben_preprocessing_sigmaX10',\n",
    "                                      suffix='.png'))\n",
    "\n",
    "    logits,_ = learn.get_preds(DatasetType.Test)\n",
    "    logits = logits.numpy()\n",
    "    b3_test_logits_list.append(logits)\n",
    "    \n",
    "    np.save(\"../output/stacking/{}_logits_test.npy\".format(m), logits)\n",
    "    print(logits.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### b4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-02T21:31:01.865810Z",
     "start_time": "2019-09-02T21:30:29.262134Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1928, 1)\n"
     ]
    }
   ],
   "source": [
    "b4_test_logits_list = []\n",
    "for m in b4_models:\n",
    "    learn = load_learner(deployment_dir, \"{}.pkl\".format(m))\n",
    "\n",
    "    learn.data.add_test(ImageList.from_df(test_df,\n",
    "                                      '../input/aptos2019-blindness-detection',\n",
    "                                      folder='test_images_ben_preprocessing_sigmaX10',\n",
    "                                      suffix='.png'))\n",
    "\n",
    "    logits,_ = learn.get_preds(DatasetType.Test)\n",
    "    logits = logits.numpy()\n",
    "    b4_test_logits_list.append(logits)\n",
    "    \n",
    "    np.save(\"../output/stacking/{}_logits_test.npy\".format(m), logits)\n",
    "    print(logits.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### b5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-02T21:31:42.018598Z",
     "start_time": "2019-09-02T21:31:01.873758Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1928, 1)\n"
     ]
    }
   ],
   "source": [
    "b5_test_logits_list = []\n",
    "for m in b5_models:\n",
    "    learn = load_learner(deployment_dir, \"{}.pkl\".format(m))\n",
    "\n",
    "    learn.data.add_test(ImageList.from_df(test_df,\n",
    "                                      '../input/aptos2019-blindness-detection',\n",
    "                                      folder='test_images_ben_preprocessing_sigmaX10',\n",
    "                                      suffix='.png'))\n",
    "    logits,_ = learn.get_preds(DatasetType.Test)\n",
    "    logits = logits.numpy()\n",
    "    b5_test_logits_list.append(logits)\n",
    "    \n",
    "    np.save(\"../output/stacking/{}_logits_test.npy\".format(m), logits)\n",
    "    print(logits.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-22T14:03:21.632429Z",
     "start_time": "2019-08-22T14:03:21.627914Z"
    }
   },
   "source": [
    "# Train Stage 2 model on OOF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:16:39.106658Z",
     "start_time": "2019-09-05T22:16:39.102021Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import make_scorer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:16:39.750943Z",
     "start_time": "2019-09-05T22:16:39.745101Z"
    }
   },
   "outputs": [],
   "source": [
    "def qk_np(y, y_pred):\n",
    "    k = cohen_kappa_score(np.round(y_pred), y, weights='quadratic')\n",
    "    \n",
    "    return k\n",
    "\n",
    "score = make_scorer(qk_np, greater_is_better=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:13:20.014982Z",
     "start_time": "2019-09-05T22:13:19.994165Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(733, 1)\n",
      "(733, 1)\n",
      "(733, 1)\n",
      "(732, 1)\n",
      "(731, 1)\n",
      "(733, 1)\n",
      "(733, 1)\n",
      "(733, 1)\n",
      "(732, 1)\n",
      "(731, 1)\n",
      "(733, 1)\n",
      "(733, 1)\n",
      "(733, 1)\n",
      "(732, 1)\n",
      "(731, 1)\n"
     ]
    }
   ],
   "source": [
    "b3_train_logits_list = []\n",
    "for m in b3_models:\n",
    "    logits = np.load(\"../output/stacking/{}_logits.npy\".format(m))\n",
    "    b3_train_logits_list.append(logits)\n",
    "\n",
    "    print(logits.shape)\n",
    "    \n",
    "b4_train_logits_list = []\n",
    "for m in b4_models:\n",
    "    logits = np.load(\"../output/stacking/{}_logits.npy\".format(m))\n",
    "    b4_train_logits_list.append(logits)\n",
    "\n",
    "    print(logits.shape)\n",
    "\n",
    "b5_train_logits_list = []\n",
    "for m in b5_models:\n",
    "    logits = np.load(\"../output/stacking/{}_logits.npy\".format(m))\n",
    "    b5_train_logits_list.append(logits)\n",
    "\n",
    "    print(logits.shape)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:13:55.854316Z",
     "start_time": "2019-09-05T22:13:55.835799Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3662, 3)\n"
     ]
    }
   ],
   "source": [
    "X_train = np.concatenate([np.concatenate(b3_train_logits_list, axis=0),\n",
    "                         np.concatenate(b4_train_logits_list, axis=0),\n",
    "                         np.concatenate(b5_train_logits_list, axis=0)], axis=1)\n",
    "y_train = []\n",
    "n_fold = 5\n",
    "\n",
    "for i in range(1, n_fold+1):\n",
    "    label_t = df_2019_cv[df_2019_cv[\"is_valid{}\".format(i)]][\"diagnosis\"].tolist()\n",
    "    y_train += label_t\n",
    "    \n",
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-06T20:05:24.080573Z",
     "start_time": "2019-09-06T20:05:24.051047Z"
    }
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-09-06T12:06:58.658Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 16200 candidates, totalling 81000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 64 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  72 tasks      | elapsed:    3.7s\n",
      "[Parallel(n_jobs=-1)]: Done 322 tasks      | elapsed:    6.3s\n",
      "[Parallel(n_jobs=-1)]: Done 672 tasks      | elapsed:   10.4s\n",
      "[Parallel(n_jobs=-1)]: Done 1122 tasks      | elapsed:   15.7s\n",
      "[Parallel(n_jobs=-1)]: Done 1672 tasks      | elapsed:   22.2s\n",
      "[Parallel(n_jobs=-1)]: Done 2322 tasks      | elapsed:   29.8s\n",
      "[Parallel(n_jobs=-1)]: Done 3072 tasks      | elapsed:   38.3s\n",
      "[Parallel(n_jobs=-1)]: Done 3922 tasks      | elapsed:   48.0s\n",
      "[Parallel(n_jobs=-1)]: Done 4872 tasks      | elapsed:   58.9s\n",
      "[Parallel(n_jobs=-1)]: Done 5922 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 7072 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 8322 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 9672 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 11122 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 12672 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 14322 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 16072 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 17922 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 19872 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 21922 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 24072 tasks      | elapsed:  4.7min\n",
      "[Parallel(n_jobs=-1)]: Done 26322 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 28672 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 31122 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=-1)]: Done 33672 tasks      | elapsed:  6.5min\n",
      "[Parallel(n_jobs=-1)]: Done 36322 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done 39072 tasks      | elapsed:  7.6min\n",
      "[Parallel(n_jobs=-1)]: Done 41922 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=-1)]: Done 44872 tasks      | elapsed:  8.7min\n",
      "[Parallel(n_jobs=-1)]: Done 47922 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=-1)]: Done 51072 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 54322 tasks      | elapsed: 10.5min\n"
     ]
    }
   ],
   "source": [
    "estimator = lgb.LGBMRegressor(random_state=SEED)\n",
    "\n",
    "param_grid = {\n",
    "    'max_depth': [3, 5],     \n",
    "#     'max_depth': [5],   \n",
    "#     'learning_rate': [0.05], \n",
    "    'learning_rate': [0.01, 0.05, 0.1],\n",
    "    'feature_fraction': [0.6, 0.7, 0.8, 0.9, 0.95],\n",
    "#     'feature_fraction': [0.7],\n",
    "    'bagging_fraction': [0.6, 0.7, 0.8, 0.9, 0.95],\n",
    "#     'bagging_fraction': [0.7],\n",
    "#     'bagging_freq': [8],\n",
    "    'bagging_freq': [5, 6, 8],\n",
    "    'lambda_l1': [0, 0.1, 0.4],\n",
    "#     'lambda_l1': [0],\n",
    "#     'lambda_l2': [15],\n",
    "    'lambda_l2': [0, 10, 15, 20],\n",
    "#     'cat_smooth': [1],\n",
    "    'cat_smooth': [1, 10, 15],\n",
    "}\n",
    "\n",
    "gbm = GridSearchCV(estimator, param_grid, cv=5, n_jobs=-1, scoring=score, verbose=1)\n",
    "gbm.fit(X_train, y_train)\n",
    "\n",
    "print('Best parameters found by grid search are:', gbm.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-09-06T12:07:00.095Z"
    }
   },
   "outputs": [],
   "source": [
    "gbm.cv_results_   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-09-06T12:07:01.251Z"
    }
   },
   "outputs": [],
   "source": [
    "print(gbm.best_score_, qk_np(y_train, xlf.predict(X_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-09-06T12:07:01.964Z"
    }
   },
   "outputs": [],
   "source": [
    "model_save_name = \"lightgbm-{}\".format(current_time)\n",
    "\n",
    "with open(os.path.join(deployment_dir, model_save_name+\".pkl\"), \"wb\") as f:\n",
    "    pk.dump(gbm.best_estimator_, f)\n",
    "\n",
    "print(model_save_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T09:43:19.360032Z",
     "start_time": "2019-09-03T09:43:19.238200Z"
    }
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T10:39:59.389264Z",
     "start_time": "2019-09-03T10:34:14.040597Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 125 candidates, totalling 625 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=16)]: Using backend LokyBackend with 16 concurrent workers.\n",
      "[Parallel(n_jobs=16)]: Done  18 tasks      | elapsed:   27.9s\n",
      "[Parallel(n_jobs=16)]: Done 168 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=16)]: Done 418 tasks      | elapsed:  4.0min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters found by grid search are: {'colsample_bytree': 0.7, 'learning_rate': 0.1, 'max_delta_step': 2, 'max_depth': 3, 'min_child_weight': 20, 'reg_alpha': 0, 'reg_lambda': 0.6, 'scale_pos_weight': 0.8, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=16)]: Done 625 out of 625 | elapsed:  5.8min finished\n"
     ]
    }
   ],
   "source": [
    "estimator_xgb = xgb.XGBRegressor(n_jobs=8, random_state=SEED)\n",
    "\n",
    "parameters = {\n",
    "              'max_depth': [3],\n",
    "              'learning_rate': [0.1],\n",
    "#               'learning_rate': [0.01, 0.02, 0.05, 0.1, 0.15],\n",
    "              'min_child_weight': [20],\n",
    "#               'min_child_weight': [0, 2, 5, 10, 20],\n",
    "              'max_delta_step': [2],\n",
    "#               'max_delta_step': [0, 0.2, 0.6, 1, 2],\n",
    "              'subsample': [0.8],\n",
    "#               'subsample': [0.6, 0.7, 0.8, 0.85, 0.95],\n",
    "              'colsample_bytree': [0.7],\n",
    "#               'colsample_bytree': [0.5, 0.6, 0.7, 0.8, 0.9],\n",
    "              'reg_alpha': [0],\n",
    "#               'reg_alpha': [0, 0.25, 0.5, 0.75, 1],\n",
    "              'reg_lambda': [0.6],\n",
    "#               'reg_lambda': [0.2, 0.4, 0.6, 0.8, 1],\n",
    "              'scale_pos_weight': [0.8]\n",
    "#               'scale_pos_weight': [0.2, 0.4, 0.6, 0.8, 1]\n",
    "}\n",
    "\n",
    "xlf = GridSearchCV(estimator_xgb, parameters, cv=5, n_jobs=16, scoring=score, verbose=1)\n",
    "xlf.fit(X_train, y_train)\n",
    "\n",
    "print('Best parameters found by grid search are:', xlf.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T10:39:59.447506Z",
     "start_time": "2019-09-03T10:39:59.393698Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([16.576894, 19.437918, 24.421603,  7.953644, ...,  3.725325,  2.692475,  1.895665,  1.726808]),\n",
       " 'std_fit_time': array([5.719804, 7.203137, 5.606267, 4.825887, ..., 0.258053, 0.426859, 0.324031, 0.263813]),\n",
       " 'mean_score_time': array([0.389374, 0.853573, 0.906028, 0.447795, ..., 0.974211, 1.474732, 1.209976, 1.393587]),\n",
       " 'std_score_time': array([0.327546, 0.909945, 0.826804, 0.317137, ..., 0.33777 , 0.426922, 0.727559, 0.281498]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.7, 0.7, 0.7, 0.7, ..., 0.7, 0.7, 0.7, 0.7],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_learning_rate': masked_array(data=[0.1, 0.1, 0.1, 0.1, ..., 0.1, 0.1, 0.1, 0.1],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_max_delta_step': masked_array(data=[2, 2, 2, 2, ..., 2, 2, 2, 2],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 3, ..., 3, 3, 3, 3],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[20, 20, 20, 20, ..., 20, 20, 20, 20],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_alpha': masked_array(data=[0, 0, 0, 0, ..., 1, 1, 1, 1],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.2, 0.2, 0.2, 0.2, ..., 1, 1, 1, 1],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_scale_pos_weight': masked_array(data=[0.2, 0.4, 0.6, 0.8, ..., 0.4, 0.6, 0.8, 1],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.8, 0.8, 0.8, 0.8, ..., 0.8, 0.8, 0.8, 0.8],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.25,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.5,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 0.75,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.2,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.4,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.6,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 0.8,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.2,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.4,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.6,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 0.8,\n",
       "   'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7,\n",
       "   'learning_rate': 0.1,\n",
       "   'max_delta_step': 2,\n",
       "   'max_depth': 3,\n",
       "   'min_child_weight': 20,\n",
       "   'reg_alpha': 1,\n",
       "   'reg_lambda': 1,\n",
       "   'scale_pos_weight': 1,\n",
       "   'subsample': 0.8}],\n",
       " 'split0_test_score': array([0.922279, 0.92716 , 0.927277, 0.928533, ..., 0.926694, 0.927292, 0.927335, 0.929291]),\n",
       " 'split1_test_score': array([0.92659 , 0.933223, 0.932487, 0.927891, ..., 0.929059, 0.933336, 0.929597, 0.925012]),\n",
       " 'split2_test_score': array([0.923173, 0.92633 , 0.925613, 0.926413, ..., 0.925357, 0.926782, 0.923605, 0.920925]),\n",
       " 'split3_test_score': array([0.923646, 0.925544, 0.926207, 0.925394, ..., 0.924209, 0.925105, 0.924492, 0.923461]),\n",
       " 'split4_test_score': array([0.921116, 0.921326, 0.921862, 0.922972, ..., 0.921146, 0.922715, 0.924198, 0.924211]),\n",
       " 'mean_test_score': array([0.923361, 0.926718, 0.926691, 0.926242, ..., 0.925294, 0.927048, 0.925847, 0.924581]),\n",
       " 'std_test_score': array([0.001831, 0.003824, 0.003425, 0.00197 , ..., 0.002628, 0.003528, 0.002275, 0.002726]),\n",
       " 'rank_test_score': array([104,  20,  23,  48, ...,  81,   8,  61,  94], dtype=int32)}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xlf.cv_results_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T10:39:59.481270Z",
     "start_time": "2019-09-03T10:39:59.450665Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9275440917207455 0.9360609597886036\n"
     ]
    }
   ],
   "source": [
    "print(xlf.best_score_, qk_np(y_train, xlf.predict(X_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T10:39:59.496794Z",
     "start_time": "2019-09-03T10:39:59.488512Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost-0903_05-26-03\n"
     ]
    }
   ],
   "source": [
    "model_save_name = \"xgboost-{}\".format(current_time)\n",
    "\n",
    "with open(os.path.join(deployment_dir, model_save_name+\".pkl\"), \"wb\") as f:\n",
    "    pk.dump(xlf.best_estimator_, f)\n",
    "\n",
    "print(model_save_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVR "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:10:25.699155Z",
     "start_time": "2019-09-05T22:10:25.651108Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:15:58.721133Z",
     "start_time": "2019-09-05T22:15:58.071016Z"
    }
   },
   "outputs": [],
   "source": [
    "# svr = SVR(gamma=0.0001, C=100)\n",
    "estimator_svr = SVR()\n",
    "\n",
    "\n",
    "tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],\n",
    "                     'C': [1, 10, 100, 1000]},\n",
    "                    {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]\n",
    "\n",
    "svr = GridSearchCV(estimator_svr, tuned_parameters, cv=5, n_jobs=16, scoring=score, verbose=1)\n",
    "svr.fit(X_train, y_train)\n",
    "\n",
    "print('Best parameters found by grid search are:', svr.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T12:30:02.208829Z",
     "start_time": "2019-09-03T12:30:02.197714Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 0.449111,  0.725864,  0.449926,  0.404292,  0.408877,  0.384097,  0.534327,  0.382056,  0.464625,  1.704576,\n",
       "        10.272197, 88.658682]),\n",
       " 'std_fit_time': array([3.030703e-02, 2.183755e-02, 8.633045e-02, 1.061797e-01, 3.863933e-03, 4.550244e-03, 1.438193e-02, 1.364801e-02,\n",
       "        3.683297e-02, 1.674361e-01, 1.229205e+00, 6.570119e+00]),\n",
       " 'mean_score_time': array([0.037776, 0.053958, 0.035569, 0.032954, 0.035365, 0.034078, 0.027068, 0.026811, 0.008165, 0.009239, 0.008616,\n",
       "        0.009378]),\n",
       " 'std_score_time': array([0.002869, 0.007793, 0.007139, 0.003456, 0.003501, 0.006972, 0.000804, 0.000662, 0.000259, 0.001165, 0.000126,\n",
       "        0.001421]),\n",
       " 'param_C': masked_array(data=[1, 1, 10, 10, 100, 100, 1000, 1000, 1, 10, 100, 1000],\n",
       "              mask=[False, False, False, False, False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_gamma': masked_array(data=[0.001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, --, --, --, --],\n",
       "              mask=[False, False, False, False, False, False, False, False,  True,  True,  True,  True],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_kernel': masked_array(data=['rbf', 'rbf', 'rbf', 'rbf', 'rbf', 'rbf', 'rbf', 'rbf', 'linear', 'linear', 'linear', 'linear'],\n",
       "              mask=[False, False, False, False, False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 1, 'gamma': 0.001, 'kernel': 'rbf'},\n",
       "  {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'},\n",
       "  {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'},\n",
       "  {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'},\n",
       "  {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'},\n",
       "  {'C': 100, 'gamma': 0.0001, 'kernel': 'rbf'},\n",
       "  {'C': 1000, 'gamma': 0.001, 'kernel': 'rbf'},\n",
       "  {'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'},\n",
       "  {'C': 1, 'kernel': 'linear'},\n",
       "  {'C': 10, 'kernel': 'linear'},\n",
       "  {'C': 100, 'kernel': 'linear'},\n",
       "  {'C': 1000, 'kernel': 'linear'}],\n",
       " 'split0_test_score': array([0.92665 , 0.905858, 0.928508, 0.927132, 0.928839, 0.929373, 0.929272, 0.929272, 0.928595, 0.928595, 0.928595,\n",
       "        0.928266]),\n",
       " 'split1_test_score': array([0.928446, 0.897393, 0.929836, 0.927875, 0.929629, 0.92948 , 0.929569, 0.929733, 0.929584, 0.929584, 0.929584,\n",
       "        0.930027]),\n",
       " 'split2_test_score': array([0.923754, 0.890172, 0.922991, 0.923754, 0.923328, 0.924003, 0.924002, 0.922926, 0.923489, 0.923489, 0.923154,\n",
       "        0.923154]),\n",
       " 'split3_test_score': array([0.919894, 0.888756, 0.921598, 0.919757, 0.922968, 0.922159, 0.921929, 0.92313 , 0.921763, 0.921763, 0.922159,\n",
       "        0.922669]),\n",
       " 'split4_test_score': array([0.920999, 0.890466, 0.922773, 0.921331, 0.922241, 0.922822, 0.921731, 0.920889, 0.923292, 0.923292, 0.921944,\n",
       "        0.921484]),\n",
       " 'mean_test_score': array([0.923951, 0.894533, 0.925143, 0.923972, 0.925403, 0.925569, 0.925303, 0.925192, 0.925347, 0.925347, 0.925089,\n",
       "        0.925122]),\n",
       " 'std_test_score': array([0.003247, 0.00641 , 0.003352, 0.003163, 0.003159, 0.003206, 0.003458, 0.003611, 0.003132, 0.003132, 0.003309,\n",
       "        0.003379]),\n",
       " 'rank_test_score': array([11, 12,  7, 10,  2,  1,  5,  6,  3,  3,  9,  8], dtype=int32)}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svr.cv_results_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T12:30:02.394012Z",
     "start_time": "2019-09-03T12:30:02.210477Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9255694040131046 0.9251493380866914\n"
     ]
    }
   ],
   "source": [
    "print(svr.best_score_, qk_np(y_train, svr.predict(X_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-05T22:17:48.658945Z",
     "start_time": "2019-09-05T22:17:48.650747Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svr-0906_06-09-42\n"
     ]
    }
   ],
   "source": [
    "model_save_name = \"svr-{}\".format(current_time)\n",
    "\n",
    "with open(os.path.join(deployment_dir, model_save_name+\".pkl\"), \"wb\") as f:\n",
    "# with open(os.path.join(deployment_dir, \"svr-0903_05-26-03.pkl\"), \"wb\") as f:\n",
    "    pk.dump(svr.best_estimator_, f)    \n",
    "#     pk.dump(svr, f)\n",
    "\n",
    "print(model_save_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-06T16:09:09.194775Z",
     "start_time": "2019-09-06T16:09:09.172957Z"
    }
   },
   "outputs": [],
   "source": [
    "from catboost import CatBoostRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-09-06T08:44:40.279Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1920 candidates, totalling 9600 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=16)]: Using backend LokyBackend with 16 concurrent workers.\n",
      "[Parallel(n_jobs=16)]: Done  18 tasks      | elapsed:    5.8s\n",
      "[Parallel(n_jobs=16)]: Done 168 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=16)]: Done 418 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=16)]: Done 768 tasks      | elapsed:  6.0min\n",
      "/home/ai/anaconda3/lib/python3.6/site-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
      "  \"timeout or by a memory leak.\", UserWarning\n",
      "[Parallel(n_jobs=16)]: Done 1218 tasks      | elapsed: 11.3min\n",
      "[Parallel(n_jobs=16)]: Done 1768 tasks      | elapsed: 17.0min\n",
      "[Parallel(n_jobs=16)]: Done 2418 tasks      | elapsed: 23.5min\n",
      "[Parallel(n_jobs=16)]: Done 3168 tasks      | elapsed: 31.3min\n",
      "[Parallel(n_jobs=16)]: Done 4018 tasks      | elapsed: 40.5min\n",
      "[Parallel(n_jobs=16)]: Done 4968 tasks      | elapsed: 49.1min\n",
      "[Parallel(n_jobs=16)]: Done 6018 tasks      | elapsed: 61.9min\n",
      "[Parallel(n_jobs=16)]: Done 7168 tasks      | elapsed: 73.2min\n"
     ]
    }
   ],
   "source": [
    "estimator_cb = CatBoostRegressor(random_seed=SEED)\n",
    "\n",
    "params = {\n",
    "          'depth':[3,1,2,6,4,5],\n",
    "#           'iterations':[500],\n",
    "          'iterations':[250,500,750,1000],\n",
    "#           'learning_rate':[0.2], \n",
    "          'learning_rate':[0.01,0.1,0.2,0.3], \n",
    "          'l2_leaf_reg':[3,1,5,10],\n",
    "          'border_count':[100,128, 200, 254, 300]\n",
    "         }\n",
    "\n",
    "cb = GridSearchCV(estimator_cb, params, cv=5, n_jobs=16, scoring=score, verbose=1)\n",
    "cb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-06T19:41:32.123763Z",
     "start_time": "2019-09-06T19:41:32.009614Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters found by grid search are: {'border_count': 100, 'depth': 2, 'iterations': 1000, 'l2_leaf_reg': 3, 'learning_rate': 0.01}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 2.073691,  2.13434 ,  2.14068 ,  2.136364, ..., 20.142412, 20.179307, 19.689002, 19.617364]),\n",
       " 'std_fit_time': array([0.03681 , 0.069325, 0.049825, 0.106818, ..., 0.458016, 0.108471, 0.229954, 0.672636]),\n",
       " 'mean_score_time': array([0.017744, 0.0152  , 0.016812, 0.012601, ..., 0.023632, 0.027577, 0.020001, 0.017893]),\n",
       " 'std_score_time': array([0.003141, 0.00561 , 0.002328, 0.004984, ..., 0.003495, 0.00803 , 0.010126, 0.006715]),\n",
       " 'param_border_count': masked_array(data=[100, 100, 100, 100, ..., 300, 300, 300, 300],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_depth': masked_array(data=[3, 3, 3, 3, ..., 5, 5, 5, 5],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_iterations': masked_array(data=[250, 250, 250, 250, ..., 1000, 1000, 1000, 1000],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_l2_leaf_reg': masked_array(data=[3, 3, 3, 3, ..., 10, 10, 10, 10],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_learning_rate': masked_array(data=[0.01, 0.1, 0.2, 0.3, ..., 0.01, 0.1, 0.2, 0.3],\n",
       "              mask=[False, False, False, False, ..., False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 100,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 6,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 4,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 128,\n",
       "   'depth': 5,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 3,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 1,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 2,\n",
       "   'iterations': 1000,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 250,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 5,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 500,\n",
       "   'l2_leaf_reg': 10,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 3,\n",
       "   'learning_rate': 0.3},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.01},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.1},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.2},\n",
       "  {'border_count': 200,\n",
       "   'depth': 6,\n",
       "   'iterations': 750,\n",
       "   'l2_leaf_reg': 1,\n",
       "   'learning_rate': 0.3},\n",
       "  ...],\n",
       " 'split0_test_score': array([0.910979, 0.930141, 0.924152, 0.92528 , ..., 0.930595, 0.92417 , 0.92391 , 0.915974]),\n",
       " 'split1_test_score': array([0.900942, 0.921198, 0.924285, 0.921151, ..., 0.925458, 0.921251, 0.921917, 0.919641]),\n",
       " 'split2_test_score': array([0.899634, 0.920312, 0.92029 , 0.920466, ..., 0.928835, 0.925104, 0.923893, 0.918384]),\n",
       " 'split3_test_score': array([0.899869, 0.922457, 0.919239, 0.913689, ..., 0.923445, 0.923584, 0.916461, 0.913718]),\n",
       " 'split4_test_score': array([0.898955, 0.924063, 0.919855, 0.91821 , ..., 0.921103, 0.918676, 0.914565, 0.918227]),\n",
       " 'mean_test_score': array([0.902078, 0.923635, 0.921566, 0.919761, ..., 0.925888, 0.922557, 0.920151, 0.917189]),\n",
       " 'std_test_score': array([0.004498, 0.003491, 0.002194, 0.003799, ..., 0.003461, 0.00232 , 0.0039  , 0.0021  ]),\n",
       " 'rank_test_score': array([1843,  757, 1205, 1464, ...,  169, 1039, 1400, 1699], dtype=int32)}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Best parameters found by grid search are:', cb.best_params_)\n",
    "cb.cv_results_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-06T19:41:50.906387Z",
     "start_time": "2019-09-06T19:41:50.877805Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.927644233342091 0.9269977028467353\n"
     ]
    }
   ],
   "source": [
    "print(cb.best_score_, qk_np(y_train, cb.predict(X_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-06T19:41:54.490437Z",
     "start_time": "2019-09-06T19:41:54.482030Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cb-0906_06-09-42\n"
     ]
    }
   ],
   "source": [
    "model_save_name = \"cb-{}\".format(current_time)\n",
    "\n",
    "with open(os.path.join(deployment_dir, model_save_name+\".pkl\"), \"wb\") as f:\n",
    "    pk.dump(cb.best_estimator_, f)    \n",
    "\n",
    "print(model_save_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-06T19:46:31.570025Z",
     "start_time": "2019-09-06T19:46:31.549268Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n",
      "(1928, 1)\n"
     ]
    }
   ],
   "source": [
    "b3_test_logits_list = []\n",
    "for m in b3_models:\n",
    "    logits = np.load(\"../output/stacking/{}_logits_test.npy\".format(m))\n",
    "    b3_test_logits_list.append(logits)\n",
    "\n",
    "    print(logits.shape)\n",
    "    \n",
    "b4_test_logits_list = []\n",
    "for m in b4_models:\n",
    "    logits = np.load(\"../output/stacking/{}_logits_test.npy\".format(m))\n",
    "    b4_test_logits_list.append(logits)\n",
    "\n",
    "    print(logits.shape)\n",
    "\n",
    "b5_test_logits_list = []\n",
    "for m in b5_models:\n",
    "    logits = np.load(\"../output/stacking/{}_logits_test.npy\".format(m))\n",
    "    b5_test_logits_list.append(logits)\n",
    "\n",
    "    print(logits.shape)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T17:04:57.344705Z",
     "start_time": "2019-09-03T17:04:57.341799Z"
    }
   },
   "outputs": [],
   "source": [
    "# model_save_name = \"lightgbm-0903_05-26-03\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-04T07:42:15.436232Z",
     "start_time": "2019-09-04T07:42:15.308649Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFV1JREFUeJzt3X+U5XV93/HnKyy/ZA2LYDa4u3HtkZBQaAxMkcQmZ1ZyEn4YoAm0pEQWDumethqxmlOI57TEJG2wJ8QftEfPKkQU4oLEBgJYpcDWmhYqq4QFCWExVBcIKy4srmIUffeP+12dDLM7c++duXd2P8/HOXP2++Pz/X7e89m59zXfz733O6kqJEnt+aFxFyBJGg8DQJIaZQBIUqMMAElqlAEgSY0yACSpUQaA9mlJPpzk95P8XJKHx13P7iR5R5IPjbsOtWXJuAuQRqGq/hdw9Ljr2J2q+k/jrkHt8QpAkhplAGifkuSnk3w+ydeTXA8c1G2fTLJ1SrtLkzzatftikn86Zd9+Sa5I8nSSv0ny5iSVZEm3f2OS30vyF93xn05yxJTjz0jyYJJnu7Y/OWXfJUke7457OMnJ3fbfSXJtt3xQkmuTfK07x+eSLF/wwVNzDADtM5IcAPwZ8FHgZcDHgV/dTfNHgZ8DDgXeCVyb5Mhu378ETgVeAxwPnDXD8f8CuBD4EeAA4Le6Gn4c+BjwVuDlwG3Anyc5IMnRwJuBf1xVLwV+CXhshnOv7epaBRwO/Cvg+bmMgdQPA0D7kpOA/YH3VNV3qupG4HMzNayqj1fVE1X1vaq6HngEOLHb/c+A91bV1qp6Brh8hlP8cVX9dVU9D9xALywA/jlwa1XdXlXfAf4QOBj4WeC7wIHAMUn2r6rHqurRGc79HXpP/K+uqu9W1aaqeq7/4ZD2zADQvuQVwOP19+9w+P9mapjk/CT3dVMszwLHArumcV4BfGVK86+86ATwt1OWvwksnXLs9/usqu91x6+oqi30rgx+B9iWZEOSV8xw7o8CnwI2JHkiyX9Osv+M37E0BANA+5IngRVJMmXbj01vlOSVwAfpTcccXlXLgAeAXcc9CayccsiqPmp4AnjllL7SHf84QFX9SVX9k65NAe+afoLu6uWdVXUMvSuHNwDn91GDNCcGgPYl/wd4AXhLkiVJfoUfTOtMdQi9J9+vAiS5kN4VwC43ABcnWZFkGXBJHzXcAJye5OTut/a3A38H/O8kRyd5fZIDgW/Rm9f/7vQTJFmT5Lgk+wHP0ZsSelE7aVgGgPYZVfVt4FeAC4Bn6M3Hf2KGdl8ErqAXGE8BxwF/MaXJB4FPA/cDX6D3Qu4LzOFJuKoeBn4duBJ4Gvhl4Je72g6k93rC0/SmkH4EeMcMp/lR4EZ6T/4PAf8TuHa2vqV+xT8II+1ZklOBD1TVK2dtLO1FvAKQpklycJLTummkFcBlwH8bd13SfPMKQJomyUvoTbv8BL15+luBi30rpvY1BoAkNcopIElq1KK+G+gRRxxRq1evHvj4b3zjGxxyyCHzV9A8sa7+WFd/rKs/+2JdmzZterqqXj5rw6patF8nnHBCDeOuu+4a6viFYl39sa7+WFd/9sW6gHtrDs+xTgFJUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjFvWtIKTFbPPjO7jg0ltH3u9jl58+8j61b/IKQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVGzBkCSq5NsS/LAlG0vS3J7kke6fw/rtifJ+5JsSXJ/kuOnHLO2a/9IkrUL8+1IkuZqLlcAHwZOmbbtUuCOqjoKuKNbBzgVOKr7Wge8H3qBAVwGvBY4EbhsV2hIksZj1gCoqs8A26dtPhO4plu+BjhryvaPVM/dwLIkRwK/BNxeVdur6hngdl4cKpKkEUpVzd4oWQ3cUlXHduvPVtWyKfufqarDktwCXF5Vn+223wFcAkwCB1XV73fb/z3wfFX94Qx9raN39cDy5ctP2LBhw8Df3M6dO1m6dOnAxy8U6+rPYq1r2/YdPPX86Ps9bsWhe9y/WMfLuvozTF1r1qzZVFUTs7Wb778Ilhm21R62v3hj1XpgPcDExERNTk4OXMzGjRsZ5viFYl39Wax1XXndTVyxefR/VO+x8yb3uH+xjpd19WcUdQ36LqCnuqkdun+3ddu3AqumtFsJPLGH7ZKkMRk0AG4Gdr2TZy1w05Tt53fvBjoJ2FFVTwKfAn4xyWHdi7+/2G2TJI3JrNevST5Gbw7/iCRb6b2b53LghiQXAV8Gzuma3wacBmwBvglcCFBV25P8HvC5rt3vVtX0F5YlSSM0awBU1a/tZtfJM7Qt4E27Oc/VwNV9VSdJWjB+EliSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVFDBUCSf5vkwSQPJPlYkoOSvCrJPUkeSXJ9kgO6tgd261u6/avn4xuQJA1m4ABIsgJ4CzBRVccC+wHnAu8C3l1VRwHPABd1h1wEPFNVrwbe3bWTJI3JsFNAS4CDkywBXgI8CbweuLHbfw1wVrd8ZrdOt//kJBmyf0nSgFJVgx+cXAz8R+B54NPAxcDd3W/5JFkFfLKqjk3yAHBKVW3t9j0KvLaqnp52znXAOoDly5efsGHDhoHr27lzJ0uXLh34+IViXf1ZrHVt276Dp54ffb/HrTh0j/sX63hZV3+GqWvNmjWbqmpitnZLBjo7kOQwer/Vvwp4Fvg4cOoMTXclzEy/7b8ofapqPbAeYGJioiYnJwctkY0bNzLM8QvFuvqzWOu68rqbuGLzwA+hgT123uQe9y/W8bKu/oyirmGmgH4B+Juq+mpVfQf4BPCzwLJuSghgJfBEt7wVWAXQ7T8U2D5E/5KkIQwTAF8GTkrykm4u/2Tgi8BdwNldm7XATd3yzd063f47a5j5J0nSUAYOgKq6h96LuZ8HNnfnWg9cArwtyRbgcOCq7pCrgMO77W8DLh2ibknSkIaawKyqy4DLpm3+EnDiDG2/BZwzTH+SpPnjJ4ElqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkho1VAAkWZbkxiR/leShJD+T5GVJbk/ySPfvYV3bJHlfki1J7k9y/Px8C5KkQQx7BfBe4L9X1U8APwU8BFwK3FFVRwF3dOsApwJHdV/rgPcP2bckaQgDB0CSHwZ+HrgKoKq+XVXPAmcC13TNrgHO6pbPBD5SPXcDy5IcOXDlkqShpKoGOzB5DbAe+CK93/43ARcDj1fVsintnqmqw5LcAlxeVZ/ttt8BXFJV90477zp6VwgsX778hA0bNgxUH8DOnTtZunTpwMcvFOvqz2Kta9v2HTz1/Oj7PW7FoXvcv1jHy7r6M0xda9as2VRVE7O1WzLQ2X9w7PHAb1bVPUneyw+me2aSGba9KH2qaj29YGFiYqImJycHLnDjxo0Mc/xCsa7+LNa6rrzuJq7YPMxDaDCPnTe5x/2Ldbysqz+jqGuY1wC2Alur6p5u/UZ6gfDUrqmd7t9tU9qvmnL8SuCJIfqXJA1h4ACoqr8FvpLk6G7TyfSmg24G1nbb1gI3dcs3A+d37wY6CdhRVU8O2r8kaTjDXr/+JnBdkgOALwEX0guVG5JcBHwZOKdrextwGrAF+GbXVpI0JkMFQFXdB8z0QsPJM7Qt4E3D9CdJmj9+EliSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVFDB0CS/ZJ8Ickt3fqrktyT5JEk1yc5oNt+YLe+pdu/eti+JUmDm48rgIuBh6asvwt4d1UdBTwDXNRtvwh4pqpeDby7aydJGpOhAiDJSuB04EPdeoDXAzd2Ta4BzuqWz+zW6faf3LWXJI1Bqmrwg5MbgT8AXgr8FnABcHf3Wz5JVgGfrKpjkzwAnFJVW7t9jwKvraqnp51zHbAOYPny5Sds2LBh4Pp27tzJ0qVLBz5+oVhXfxZrXdu27+Cp50ff73ErDt3j/sU6XtbVn2HqWrNmzaaqmpit3ZKBzg4keQOwrao2JZnctXmGpjWHfT/YULUeWA8wMTFRk5OT05vM2caNGxnm+IViXf1ZrHVded1NXLF54IfQwB47b3KP+xfreFlXf0ZR1zA/va8DzkhyGnAQ8MPAe4BlSZZU1QvASuCJrv1WYBWwNckS4FBg+xD9S5KGMPBrAFX121W1sqpWA+cCd1bVecBdwNlds7XATd3yzd063f47a5j5J0nSUBbicwCXAG9LsgU4HLiq234VcHi3/W3ApQvQtyRpjuZlArOqNgIbu+UvASfO0OZbwDnz0Z8kaXh+EliSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrU6G9mPkKbH9/BBZfeOvJ+H7v89JH3KUn98gpAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqP26VtBtGb1kLe9ePtxLwx86wxvfyHtfbwCkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0aOACSrEpyV5KHkjyY5OJu+8uS3J7kke7fw7rtSfK+JFuS3J/k+Pn6JiRJ/RvmCuAF4O1V9ZPAScCbkhwDXArcUVVHAXd06wCnAkd1X+uA9w/RtyRpSAN/EKyqngSe7Ja/nuQhYAVwJjDZNbsG2Ahc0m3/SFUVcHeSZUmO7M4jaS8wzIcN/aDh4pPe8/GQJ0lWA58BjgW+XFXLpux7pqoOS3ILcHlVfbbbfgdwSVXdO+1c6+hdIbB8+fITNmzYMHBd27bv4KnnBz58YMetOHSP+3fu3MnSpUvnvd/Nj+8Y6vjlBzPweM32PQ9jocZrWK39fMFwP2P+fPVnmLrWrFmzqaomZms39K0gkiwF/hR4a1U9l2S3TWfY9qL0qar1wHqAiYmJmpycHLi2K6+7iSs2j/5uF4+dN7nH/Rs3bmSY72t3Bv3tape3H/fCwOM12/c8jIUar2G19vMFw/2M+fPVn1HUNdS7gJLsT+/J/7qq+kS3+akkR3b7jwS2ddu3AqumHL4SeGKY/iVJgxvmXUABrgIeqqo/mrLrZmBtt7wWuGnK9vO7dwOdBOxw/l+SxmeY69fXAW8ENie5r9v2DuBy4IYkFwFfBs7p9t0GnAZsAb4JXDhE35KkIQ3zLqDPMvO8PsDJM7Qv4E2D9idJml9+EliSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElq1DB/FF6S9mmrL711bH1/+JRDFrwPrwAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRIw+AJKckeTjJliSXjrp/SVLPSAMgyX7AfwVOBY4Bfi3JMaOsQZLUM+orgBOBLVX1par6NrABOHPENUiSgFTV6DpLzgZOqarf6NbfCLy2qt48pc06YF23ejTw8BBdHgE8PcTxC8W6+mNd/bGu/uyLdb2yql4+W6NR3w00M2z7ewlUVeuB9fPSWXJvVU3Mx7nmk3X1x7r6Y139abmuUU8BbQVWTVlfCTwx4hokSYw+AD4HHJXkVUkOAM4Fbh5xDZIkRjwFVFUvJHkz8ClgP+DqqnpwAbucl6mkBWBd/bGu/lhXf5qta6QvAkuSFg8/CSxJjTIAJKlRe30AzHZriSQHJrm+239PktWLpK4Lknw1yX3d12+MqK6rk2xL8sBu9ifJ+7q6709y/CKpazLJjinj9R9GVNeqJHcleSjJg0kunqHNyMdsjnWNfMySHJTk/yb5y66ud87QZuSPyTnWNa7H5H5JvpDklhn2LexYVdVe+0XvheRHgX8AHAD8JXDMtDb/BvhAt3wucP0iqesC4L+MYcx+HjgeeGA3+08DPknvMxsnAfcskromgVvGMF5HAsd3yy8F/nqG/8uRj9kc6xr5mHVjsLRb3h+4BzhpWptxPCbnUte4HpNvA/5kpv+rhR6rvf0KYC63ljgTuKZbvhE4OclMH0gbdV1jUVWfAbbvocmZwEeq525gWZIjF0FdY1FVT1bV57vlrwMPASumNRv5mM2xrpHrxmBnt7p/9zX9nSYjf0zOsa6RS7ISOB340G6aLOhY7e0BsAL4ypT1rbz4QfD9NlX1ArADOHwR1AXwq92UwY1JVs2wfxzmWvs4/Ex3Cf/JJP9w1J13l98/Te+3x6nGOmZ7qAvGMGbdlMZ9wDbg9qra7XiN8DE5l7pg9I/J9wD/DvjebvYv6Fjt7QEw660l5thmvs2lzz8HVlfVPwL+Bz9I+XEbx3jNxefp3d/kp4ArgT8bZedJlgJ/Cry1qp6bvnuGQ0YyZrPUNZYxq6rvVtVr6H3S/8Qkx05rMpbxmkNdI31MJnkDsK2qNu2p2Qzb5m2s9vYAmMutJb7fJskS4FAWfqph1rqq6mtV9Xfd6geBExa4prlalLfrqKrndl3CV9VtwP5JjhhF30n2p/cke11VfWKGJmMZs9nqGueYdX0+C2wETpm2axyPyVnrGsNj8nXAGUkeozdN/Pok105rs6BjtbcHwFxuLXEzsLZbPhu4s7pXVMZZ17Q54jPozeEuBjcD53fvbDkJ2FFVT467qCQ/umvuM8mJ9H52vzaCfgNcBTxUVX+0m2YjH7O51DWOMUvy8iTLuuWDgV8A/mpas5E/JudS16gfk1X121W1sqpW03uOuLOqfn1aswUdq1HfDXRe1W5uLZHkd4F7q+pmeg+SjybZQi85z10kdb0lyRnAC11dFyx0XQBJPkbv3SFHJNkKXEbvBTGq6gPAbfTe1bIF+CZw4SKp62zgXyd5AXgeOHcEQQ6939LeCGzu5o8B3gH82JTaxjFmc6lrHGN2JHBNen/86YeAG6rqlnE/JudY11gek9ONcqy8FYQkNWpvnwKSJA3IAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmN+v+2rtWGonj+3gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../output/submission/lightgbm-0904_04-09-19-5-fold_avg_logits_test.csv\n"
     ]
    }
   ],
   "source": [
    "b3_test_avg_feats = np.average(b3_test_logits_list, axis=0)\n",
    "b4_test_avg_feats = np.average(b4_test_logits_list, axis=0)\n",
    "b5_test_avg_feats = np.average(b5_test_logits_list, axis=0)\n",
    "\n",
    "\n",
    "X_test = np.concatenate([b3_test_avg_feats, b4_test_avg_feats, b5_test_avg_feats], axis=1)\n",
    "y_pred = gbm.predict(X_test)\n",
    "\n",
    "y_pred = np.round(y_pred)\n",
    "\n",
    "test_df.diagnosis = y_pred.astype(int)\n",
    "\n",
    "test_df.hist()\n",
    "plt.show()\n",
    "\n",
    "submition_filename = \"../output/submission/{}-5-fold_avg_logits_test.csv\".format(model_save_name)\n",
    "test_df.to_csv(submition_filename, index=False)\n",
    "print(submition_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-04T10:30:07.218144Z",
     "start_time": "2019-09-04T10:30:06.977776Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFXlJREFUeJzt3X+0XWV95/H3p4RfGksQNMUkNc6S2mFkqnAHaZ123UhX5YcFpoUZOlQCi07WTLXSwS5JXWuG2h9TnFXqD2aWrihUFGpA6hQKOMogacdOYTRIDUgpwTISoEQMRKNYRb/zx9nR28tN7j3n3HvOTZ73a627svd+nr339zy5+37Ofs6556aqkCS154fGXYAkaTwMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkA2q8l+VCS303y00keGHc9e5Lk7Uk+OO461JYl4y5AGoWq+t/AK8Zdx55U1X8Zdw1qj3cAktQoA0D7lSSvTnJ3kq8nuQ44pNs+mWTblH7rkzzU9ftikn81pe2AJJcneTLJ3yV5c5JKsqRr35Tkd5L8Zbf/p5IcOWX/05Pcl+Tpru8/ndJ2SZJHu/0eSHJSt/23klzTLR+S5JokX+2O8dkkyxd88NQcA0D7jSQHAX8KfAR4IfAx4Bf30P0h4KeBw4B3ANckOapr+3fAKcCrgOOAM2fY/98CFwAvBg4CfqOr4ceAjwK/DrwIuBX4syQHJXkF8GbgX1TVC4DXAw/PcOy1XV2rgCOAfw88M5cxkPphAGh/ciJwIPDuqvpOVd0AfHamjlX1sap6rKq+V1XXAQ8CJ3TN/xp4T1Vtq6qngMtmOMQfVdXfVtUzwPX0wgLg3wC3VNVtVfUd4A+AQ4GfAr4LHAwck+TAqnq4qh6a4djfofeD/+VV9d2q2lxVX+t/OKS9MwC0P3kJ8Gj94084/H8zdUxyXpJ7uimWp4FXAruncV4CPDKl+yPPOQD8/ZTlbwJLp+z7/XNW1fe6/VdU1VZ6dwa/BWxPsjHJS2Y49keATwIbkzyW5L8mOXDGRywNwQDQ/uRxYEWSTNn2o9M7JXkp8AF60zFHVNUy4F5g936PAyun7LKqjxoeA1465Vzp9n8UoKr+uKr+ZdengHdOP0B39/KOqjqG3p3DG4Dz+qhBmhMDQPuTvwKeBd6SZEmSX+AH0zpTPZ/eD9+vACS5gN4dwG7XAxclWZFkGXBJHzVcD5yW5KTuWftbgX8A/k+SVyR5XZKDgW/Rm9f/7vQDJFmT5NgkBwBfozcl9Jx+0rAMAO03qurbwC8A5wNP0ZuP//gM/b4IXE4vMJ4AjgX+ckqXDwCfAr4AfJ7eC7nPMocfwlX1APDLwBXAk8DPAz/f1XYwvdcTnqQ3hfRi4O0zHOZHgBvo/fC/H/hz4JrZzi31K/5BGGnvkpwCvL+qXjprZ2kf4h2ANE2SQ5Oc2k0jrQAuBf7HuOuS5pt3ANI0SZ5Hb9rlx+nN098CXORbMbW/MQAkqVFOAUlSoxb1p4EeeeSRtXr16oH3/8Y3vsHzn//8+StonlhXf6yrP9bVn/2xrs2bNz9ZVS+atWNVLdqv448/voZxxx13DLX/QrGu/lhXf6yrP/tjXcDnag4/Y50CkqRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRi3qj4KQFrMtj+7k/PW3jPy8D1922sjPqf2TdwCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUqFkDIMlVSbYnuXfKthcmuS3Jg92/h3fbk+S9SbYm+UKS46bss7br/2CStQvzcCRJczWXO4APASdP27YeuL2qjgZu79YBTgGO7r7WAe+DXmAAlwKvAU4ALt0dGpKk8Zg1AKrqL4Ad0zafAVzdLV8NnDll+4er505gWZKjgNcDt1XVjqp6CriN54aKJGmEUlWzd0pWAzdX1Su79aeratmU9qeq6vAkNwOXVdVnuu23A5cAk8AhVfW73fb/BDxTVX8ww7nW0bt7YPny5cdv3Lhx4Ae3a9culi5dOvD+C8W6+rNY69q+YydPPDP68x674rC9ti/W8bKu/gxT15o1azZX1cRs/ZYMdPQ9ywzbai/bn7uxagOwAWBiYqImJycHLmbTpk0Ms/9Csa7+LNa6rrj2Ri7fMt+X0OwePndyr+2Ldbysqz+jqGvQdwE90U3t0P27vdu+DVg1pd9K4LG9bJckjcmgAXATsPudPGuBG6dsP697N9CJwM6qehz4JPBzSQ7vXvz9uW6bJGlMZr1/TfJRenP4RybZRu/dPJcB1ye5EPgycHbX/VbgVGAr8E3gAoCq2pHkd4DPdv1+u6qmv7AsSRqhWQOgqn5pD00nzdC3gDft4ThXAVf1VZ0kacH4m8CS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElq1FABkOQ/Jrkvyb1JPprkkCQvS3JXkgeTXJfkoK7vwd361q599Xw8AEnSYAYOgCQrgLcAE1X1SuAA4BzgncC7qupo4Cngwm6XC4GnqurlwLu6fpKkMRl2CmgJcGiSJcDzgMeB1wE3dO1XA2d2y2d063TtJyXJkOeXJA0oVTX4zslFwO8BzwCfAi4C7uye5ZNkFfCJqnplknuBk6tqW9f2EPCaqnpy2jHXAesAli9ffvzGjRsHrm/Xrl0sXbp04P0XinX1Z7HWtX3HTp54ZvTnPXbFYXttX6zjZV39GaauNWvWbK6qidn6LRno6ECSw+k9q38Z8DTwMeCUGbruTpiZnu0/J32qagOwAWBiYqImJycHLZFNmzYxzP4Lxbr6s1jruuLaG7l8y8CX0MAePndyr+2Ldbysqz+jqGuYKaCfBf6uqr5SVd8BPg78FLCsmxICWAk81i1vA1YBdO2HATuGOL8kaQjDBMCXgROTPK+byz8J+CJwB3BW12ctcGO3fFO3Ttf+6Rpm/kmSNJSBA6Cq7qL3Yu7dwJbuWBuAS4CLk2wFjgCu7Ha5Ejii234xsH6IuiVJQxpqArOqLgUunbb5S8AJM/T9FnD2MOeTJM0ffxNYkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjRoqAJIsS3JDkr9Jcn+Sn0zywiS3JXmw+/fwrm+SvDfJ1iRfSHLc/DwESdIghr0DeA/wP6vqx4GfAO4H1gO3V9XRwO3dOsApwNHd1zrgfUOeW5I0hIEDIMkPAz8DXAlQVd+uqqeBM4Cru25XA2d2y2cAH66eO4FlSY4auHJJ0lBSVYPtmLwK2AB8kd6z/83ARcCjVbVsSr+nqurwJDcDl1XVZ7rttwOXVNXnph13Hb07BJYvX378xo0bB6oPYNeuXSxdunTg/ReKdfVnsda1fcdOnnhm9Oc9dsVhe21frONlXf0Zpq41a9ZsrqqJ2fotGejoP9j3OODXququJO/hB9M9M8kM256TPlW1gV6wMDExUZOTkwMXuGnTJobZf6FYV38Wa11XXHsjl28Z5hIazMPnTu61fbGOl3X1ZxR1DfMawDZgW1Xd1a3fQC8Qntg9tdP9u31K/1VT9l8JPDbE+SVJQxg4AKrq74FHkryi23QSvemgm4C13ba1wI3d8k3Aed27gU4EdlbV44OeX5I0nGHvX38NuDbJQcCXgAvohcr1SS4Evgyc3fW9FTgV2Ap8s+srSRqToQKgqu4BZnqh4aQZ+hbwpmHOJ0maP/4msCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkho1dAAkOSDJ55Pc3K2/LMldSR5Mcl2Sg7rtB3frW7v21cOeW5I0uPm4A7gIuH/K+juBd1XV0cBTwIXd9guBp6rq5cC7un6SpDEZKgCSrAROAz7YrQd4HXBD1+Vq4Mxu+Yxuna79pK6/JGkMhr0DeDfwNuB73foRwNNV9Wy3vg1Y0S2vAB4B6Np3dv0lSWOQqhpsx+QNwKlV9atJJoHfAC4A/qqb5iHJKuDWqjo2yX3A66tqW9f2EHBCVX112nHXAesAli9ffvzGjRsHe2TArl27WLp06cD7LxTr6s9irWv7jp088czoz3vsisP22r5Yx8u6+jNMXWvWrNlcVROz9Vsy0NF7XgucnuRU4BDgh+ndESxLsqR7lr8SeKzrvw1YBWxLsgQ4DNgx/aBVtQHYADAxMVGTk5MDF7hp0yaG2X+hWFd/FmtdV1x7I5dvGeYSGszD507utX2xjpd19WcUdQ08BVRVv1lVK6tqNXAO8OmqOhe4Azir67YWuLFbvqlbp2v/dA16+yFJGtpC/B7AJcDFSbbSm+O/stt+JXBEt/1iYP0CnFuSNEfzcv9aVZuATd3yl4ATZujzLeDs+TifJGl4/iawJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEaN/qMMR2jLozs5f/0tIz/vw5edNvJzSlK/vAOQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDVqv/6DMK1ZPeQfv3nrsc8O/Ad0/CM40r5n4DuAJKuS3JHk/iT3Jbmo2/7CJLclebD79/Bue5K8N8nWJF9Ictx8PQhJUv+GuQN4FnhrVd2d5AXA5iS3AecDt1fVZUnWA+uBS4BTgKO7r9cA7+v+lbSPGOYu0zvMxWfgO4Cqeryq7u6Wvw7cD6wAzgCu7rpdDZzZLZ8BfLh67gSWJTlq4MolSUOZlxeBk6wGXg3cBSyvqsehFxLAi7tuK4BHpuy2rdsmSRqDVNVwB0iWAn8O/F5VfTzJ01W1bEr7U1V1eJJbgN+vqs90228H3lZVm6cdbx2wDmD58uXHb9y4ceDatu/YyRPPDLz7wI5dcdhe23ft2sXSpUvn/bxbHt051P7LD2Xg8ZrtMQ9jocZrWK19f8Fw32N+f/VnmLrWrFmzuaomZus31LuAkhwI/AlwbVV9vNv8RJKjqurxbopne7d9G7Bqyu4rgcemH7OqNgAbACYmJmpycnLg+q649kYu3zL6Nzo9fO7kXts3bdrEMI9rTwadX93trcc+O/B4zfaYh7FQ4zWs1r6/YLjvMb+/+jOKuoZ5F1CAK4H7q+oPpzTdBKztltcCN07Zfl73bqATgZ27p4okSaM3zNOX1wJvBLYkuafb9nbgMuD6JBcCXwbO7tpuBU4FtgLfBC4Y4tySpCENHADdXH720HzSDP0LeNOg55MkzS8/CkKSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0a+I/CS9L+bvX6W8Z27g+d/PwFP4d3AJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGjTwAkpyc5IEkW5OsH/X5JUk9Iw2AJAcA/x04BTgG+KUkx4yyBklSz6jvAE4AtlbVl6rq28BG4IwR1yBJAlJVoztZchZwclX9Srf+RuA1VfXmKX3WAeu61VcADwxxyiOBJ4fYf6FYV3+sqz/W1Z/9sa6XVtWLZus06g+Dywzb/lECVdUGYMO8nCz5XFVNzMex5pN19ce6+mNd/Wm5rlFPAW0DVk1ZXwk8NuIaJEmMPgA+Cxyd5GVJDgLOAW4acQ2SJEY8BVRVzyZ5M/BJ4ADgqqq6bwFPOS9TSQvAuvpjXf2xrv40W9dIXwSWJC0e/iawJDXKAJCkRu3zATDbR0skOTjJdV37XUlWL5K6zk/ylST3dF+/MqK6rkqyPcm9e2hPkvd2dX8hyXGLpK7JJDunjNd/HlFdq5LckeT+JPcluWiGPiMfsznWNfIxS3JIkv+b5K+7ut4xQ5+RX5NzrGtc1+QBST6f5OYZ2hZ2rKpqn/2i90LyQ8A/AQ4C/ho4ZlqfXwXe3y2fA1y3SOo6H/hvYxiznwGOA+7dQ/upwCfo/c7GicBdi6SuSeDmMYzXUcBx3fILgL+d4f9y5GM2x7pGPmbdGCztlg8E7gJOnNZnHNfkXOoa1zV5MfDHM/1fLfRY7et3AHP5aIkzgKu75RuAk5LM9Atpo65rLKrqL4Ade+lyBvDh6rkTWJbkqEVQ11hU1eNVdXe3/HXgfmDFtG4jH7M51jVy3Rjs6lYP7L6mv9Nk5NfkHOsauSQrgdOAD+6hy4KO1b4eACuAR6asb+O5F8H3+1TVs8BO4IhFUBfAL3ZTBjckWTVD+zjMtfZx+MnuFv4TSf7ZqE/e3X6/mt6zx6nGOmZ7qQvGMGbdlMY9wHbgtqra43iN8JqcS10w+mvy3cDbgO/toX1Bx2pfD4BZP1pijn3m21zO+WfA6qr658D/4gcpP27jGK+5uJve55v8BHAF8KejPHmSpcCfAL9eVV+b3jzDLiMZs1nqGsuYVdV3q+pV9H7T/4Qkr5zWZSzjNYe6RnpNJnkDsL2qNu+t2wzb5m2s9vUAmMtHS3y/T5IlwGEs/FTDrHVV1Ver6h+61Q8Axy9wTXO1KD+uo6q+tvsWvqpuBQ5McuQozp3kQHo/ZK+tqo/P0GUsYzZbXeMcs+6cTwObgJOnNY3jmpy1rjFck68FTk/yML1p4tcluWZanwUdq309AOby0RI3AWu75bOAT1f3iso465o2R3w6vTncxeAm4LzunS0nAjur6vFxF5XkR3bPfSY5gd737ldHcN4AVwL3V9Uf7qHbyMdsLnWNY8ySvCjJsm75UOBngb+Z1m3k1+Rc6hr1NVlVv1lVK6tqNb2fEZ+uql+e1m1Bx2rUnwY6r2oPHy2R5LeBz1XVTfQuko8k2UovOc9ZJHW9JcnpwLNdXecvdF0AST5K790hRybZBlxK7wUxqur9wK303tWyFfgmcMEiqess4D8keRZ4BjhnBEEOvWdpbwS2dPPHAG8HfnRKbeMYs7nUNY4xOwq4Or0//vRDwPVVdfO4r8k51jWWa3K6UY6VHwUhSY3a16eAJEkDMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSo/4/t2zvdpsEItcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../output/submission/lightgbm-0904_04-09-19-5-fold_logits_avg_test.csv\n"
     ]
    }
   ],
   "source": [
    "# 5 test feature then avg\n",
    "\n",
    "results = []\n",
    "for b3, b4, b5 in zip(b3_test_logits_list, b4_test_logits_list, b5_test_logits_list):\n",
    "    X_test = np.concatenate([b3, b4, b5], axis=1)\n",
    "    res = gbm.predict(X_test)\n",
    "    results.append(res)\n",
    "\n",
    "avg_res_gbm = np.average(results, axis=0)\n",
    "np.save(\"../output/submission/{}-5-fold_logits_avg_test_logits.npy\".format(model_save_name), avg_res_gbm)\n",
    "y_pred = np.round(avg_res_gbm)\n",
    "\n",
    "test_df.diagnosis = y_pred.astype(int)\n",
    "\n",
    "test_df.hist()\n",
    "plt.show()\n",
    "\n",
    "submition_filename = \"../output/submission/{}-5-fold_logits_avg_test.csv\".format(model_save_name)\n",
    "test_df.to_csv(submition_filename, index=False)\n",
    "print(submition_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-04T10:32:07.920482Z",
     "start_time": "2019-09-04T10:32:07.915008Z"
    }
   },
   "outputs": [],
   "source": [
    "model_save_name = \"xgboost-0903_05-26-03\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T10:51:13.945653Z",
     "start_time": "2019-09-03T10:51:13.676091Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFT9JREFUeJzt3X+U5XV93/HnKyy/dA2LoBvc3bj2SE0oNAamSGKTMys5CT8M0ERaUiILh3RPW41YzSkb/ygx6Q/sKfEH7dGzihWFuCCxYQNYpcDUmhYqq8ZFiWExW13YsCK4uorR1Xf/uN+VyTC7M/femXuH/Twf58yZ7/18P9/v5z2fvXde9/u5d+6mqpAktefHxl2AJGk8DABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZADqkJflAkn+b5BeSfGnc9RxIkrcked+461Bblo27AGkUqup/AS8bdx0HUlX/ftw1qD1eAUhSowwAHVKS/GySzyT5VpKbgKO69skkO6f125jk4a7fF5P8o2n7DktyTZLHk/xVktcnqSTLuv1TSf4gyZ91x38iyfHTjj8vyReSfKPr+9PT9l2Z5JHuuC8lObNr/70kN3TbRyW5IcnXu3N8OsnKRZ88NccA0CEjyRHAnwAfAp4PfAT49QN0fxj4BeAY4K3ADUlO6Pb9M+Bs4OXAqcAFsxz/T4HLgBcCRwC/09Xwd4EPA28EXgDcAfxpkiOSvAx4PfAPqup5wK8AO2Y59/qurjXAccA/B56azxxI/TAAdCg5AzgceEdVfb+qbgE+PVvHqvpIVT1aVT+sqpuAh4DTu93/GHhnVe2sqieBq2c5xX+tqr+sqqeAm+mFBcA/AW6vqjur6vvAfwKOBn4e+AFwJHBSksOrakdVPTzLub9P7xf/S6vqB1W1taq+2f90SAdnAOhQ8iLgkfrbn3D4/2brmOSSJJ/rlli+AZwM7F/GeRHw1Wndv/qME8BfT9v+DrB82rE/GrOqftgdv6qqttO7Mvg9YHeSzUleNMu5PwR8HNic5NEk/zHJ4bP+xNIQDAAdSnYBq5JkWttPzuyU5MXAe+ktxxxXVSuAB4D9x+0CVk87ZE0fNTwKvHjaWOmOfwSgqv6oqv5h16eAt808QXf18taqOonelcOrgUv6qEGaFwNAh5L/A+wD3pBkWZJf4+llnemeS++X79cAklxG7wpgv5uBK5KsSrICuLKPGm4Gzk1yZves/c3A3wD/O8nLkrwqyZHAd+mt6/9g5gmSrEtySpLDgG/SWxJ6Rj9pWAaADhlV9T3g14BLgSfprcd/dJZ+XwSuoRcYjwGnAH82rct7gU8Anwc+S++F3H3M45dwVX0J+E3gWuBx4FeBX+1qO5Le6wmP01tCeiHwlllO8xPALfR++T8I/E/ghrnGlvoV/0MY6eCSnA28p6pePGdn6VnEKwBphiRHJzmnW0ZaBVwF/Ldx1yUtNK8ApBmSPIfesstP0Vunvx24wrdi6lBjAEhSo1wCkqRGLelPAz3++ONr7dq1Ax//7W9/m+c+97kLV9ACsa7+WFd/rKs/h2JdW7dufbyqXjBnx6pasl+nnXZaDeOee+4Z6vjFYl39sa7+WFd/DsW6gPtrHr9jXQKSpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGLemPgpCWsm2P7OHSjbePfNwdV5878jF1aPIKQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrUnAGQ5P1Jdid5YFrb85PcmeSh7vuxXXuSvCvJ9iSfT3LqtGPWd/0fSrJ+cX4cSdJ8zecK4APAWTPaNgJ3VdWJwF3dbYCzgRO7rw3Au6EXGMBVwCuA04Gr9oeGJGk85gyAqvok8MSM5vOB67vt64ELprV/sHruBVYkOQH4FeDOqnqiqp4E7uSZoSJJGqFBXwNYWVW7ALrvL+zaVwFfndZvZ9d2oHZJ0pgsW+DzZZa2Okj7M0+QbKC3fMTKlSuZmpoauJi9e/cOdfxisa7+LNW6Vh4Nbz5l38jHnWsulup8WVd/RlHXoAHwWJITqmpXt8Szu2vfCayZ1m818GjXPjmjfWq2E1fVJmATwMTERE1OTs7WbV6mpqYY5vjFYl39Wap1XXvjrVyzbaGfQ81tx8WTB92/VOfLuvoziroGXQLaAux/J8964NZp7Zd07wY6A9jTLRF9HPjlJMd2L/7+ctcmSRqTOZ++JPkwvWfvxyfZSe/dPFcDNye5HPgKcGHX/Q7gHGA78B3gMoCqeiLJHwCf7vr9flXNfGFZkjRCcwZAVf3GAXadOUvfAl53gPO8H3h/X9VJkhaNfwksSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRg0VAEn+VZIvJHkgyYeTHJXkJUnuS/JQkpuSHNH1PbK7vb3bv3YhfgBJ0mAGDoAkq4A3ABNVdTJwGHAR8Dbg7VV1IvAkcHl3yOXAk1X1UuDtXT9J0pgMuwS0DDg6yTLgOcAu4FXALd3+64ELuu3zu9t0+89MkiHHlyQNKFU1+MHJFcC/A54CPgFcAdzbPcsnyRrgY1V1cpIHgLOqame372HgFVX1+IxzbgA2AKxcufK0zZs3D1zf3r17Wb58+cDHLxbr6s9SrWv3E3t47KnRj3vKqmMOun+pzpd19WeYutatW7e1qibm6rdsoLMDSY6l96z+JcA3gI8AZ8/SdX/CzPZs/xnpU1WbgE0AExMTNTk5OWiJTE1NMczxi8W6+rNU67r2xlu5ZtvAD6GB7bh48qD7l+p8WVd/RlHXMEtAvwT8VVV9raq+D3wU+HlgRbckBLAaeLTb3gmsAej2HwM8McT4kqQhDBMAXwHOSPKcbi3/TOCLwD3Aa7o+64Fbu+0t3W26/XfXMOtPkqShDBwAVXUfvRdzPwNs6861CbgSeFOS7cBxwHXdIdcBx3XtbwI2DlG3JGlIQy1gVtVVwFUzmr8MnD5L3+8CFw4zniRp4fiXwJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrUUAGQZEWSW5L8RZIHk/xckucnuTPJQ933Y7u+SfKuJNuTfD7JqQvzI0iSBjHsFcA7gf9eVT8F/AzwILARuKuqTgTu6m4DnA2c2H1tAN495NiSpCEMHABJfhz4ReA6gKr6XlV9AzgfuL7rdj1wQbd9PvDB6rkXWJHkhIErlyQNJVU12IHJy4FNwBfpPfvfClwBPFJVK6b1e7Kqjk1yG3B1VX2qa78LuLKq7p9x3g30rhBYuXLlaZs3bx6oPoC9e/eyfPnygY9fLNbVn6Va1+4n9vDYU6Mf95RVxxx0/1KdL+vqzzB1rVu3bmtVTczVb9lAZ3/62FOB366q+5K8k6eXe2aTWdqekT5VtYlesDAxMVGTk5MDFzg1NcUwxy8W6+rPUq3r2htv5ZptwzyEBrPj4smD7l+q82Vd/RlFXcO8BrAT2FlV93W3b6EXCI/tX9rpvu+e1n/NtONXA48OMb4kaQgDB0BV/TXw1SQv65rOpLcctAVY37WtB27ttrcAl3TvBjoD2FNVuwYdX5I0nGGvX38buDHJEcCXgcvohcrNSS4HvgJc2PW9AzgH2A58p+srSRqToQKgqj4HzPZCw5mz9C3gdcOMJ0laOP4lsCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUqKEDIMlhST6b5Lbu9kuS3JfkoSQ3JTmiaz+yu72927922LElSYNbiCuAK4AHp91+G/D2qjoReBK4vGu/HHiyql4KvL3rJ0kak6ECIMlq4Fzgfd3tAK8Cbum6XA9c0G2f392m239m11+SNAapqsEPTm4B/gPwPOB3gEuBe7tn+SRZA3ysqk5O8gBwVlXt7PY9DLyiqh6fcc4NwAaAlStXnrZ58+aB69u7dy/Lly8f+PjFYl39Wap17X5iD489NfpxT1l1zEH3L9X5sq7+DFPXunXrtlbVxFz9lg10diDJq4HdVbU1yeT+5lm61jz2Pd1QtQnYBDAxMVGTk5Mzu8zb1NQUwxy/WKyrP0u1rmtvvJVrtg38EBrYjosnD7p/qc6XdfVnFHUNc+99JXBeknOAo4AfB94BrEiyrKr2AauBR7v+O4E1wM4ky4BjgCeGGF+SNISBXwOoqt+tqtVVtRa4CLi7qi4G7gFe03VbD9zabW/pbtPtv7uGWX+SJA1lMf4O4ErgTUm2A8cB13Xt1wHHde1vAjYuwtiSpHlakAXMqpoCprrtLwOnz9Lnu8CFCzGeJGl4/iWwJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1avQfZThC2x7Zw6Ubbx/5uDuuPnfkY0pSv7wCkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUqIEDIMmaJPckeTDJF5Jc0bU/P8mdSR7qvh/btSfJu5JsT/L5JKcu1A8hSerfMFcA+4A3V9VPA2cAr0tyErARuKuqTgTu6m4DnA2c2H1tAN49xNiSpCEN/J/CV9UuYFe3/a0kDwKrgPOBya7b9cAUcGXX/sGqKuDeJCuSnNCdRwtg7cbbhzr+zafs49IBz7Hj6nOHGlvPDsPcx7x/LT3p/T4e8iTJWuCTwMnAV6pqxbR9T1bVsUluA66uqk917XcBV1bV/TPOtYHeFQIrV648bfPmzQPXtfuJPTz21MCHD+yUVcccdP/evXtZvnz5go+77ZE9Qx2/8mgGnq+5fuZhLNZ8Dau1+xcMdx/z/tWfYepat27d1qqamKvfwFcA+yVZDvwx8Maq+maSA3adpe0Z6VNVm4BNABMTEzU5OTlwbdfeeCvXbBv6R+zbjosnD7p/amqKYX6uAxn02dV+bz5l38DzNdfPPIzFmq9htXb/guHuY96/+jOKuoZ6F1CSw+n98r+xqj7aNT+W5IRu/wnA7q59J7Bm2uGrgUeHGV+SNLhh3gUU4Drgwar6w2m7tgDru+31wK3T2i/p3g10BrDH9X9JGp9hrl9fCbwW2Jbkc13bW4CrgZuTXA58Bbiw23cHcA6wHfgOcNkQY0uShjTMu4A+xezr+gBnztK/gNcNOp4kaWH5l8CS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkho1zP8JLEmHtLUbbx/b2B8467mLPoZXAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1KiRB0CSs5J8Kcn2JBtHPb4kqWekAZDkMOC/AGcDJwG/keSkUdYgSeoZ9RXA6cD2qvpyVX0P2AycP+IaJElAqmp0gyWvAc6qqt/qbr8WeEVVvX5anw3Ahu7my4AvDTHk8cDjQxy/WKyrP9bVH+vqz6FY14ur6gVzdRr1p4Fmlra/lUBVtQnYtCCDJfdX1cRCnGshWVd/rKs/1tWflusa9RLQTmDNtNurgUdHXIMkidEHwKeBE5O8JMkRwEXAlhHXIElixEtAVbUvyeuBjwOHAe+vqi8s4pALspS0CKyrP9bVH+vqT7N1jfRFYEnS0uFfAktSowwASWrUsz4A5vpoiSRHJrmp239fkrVLpK5Lk3wtyee6r98aUV3vT7I7yQMH2J8k7+rq/nySU5dIXZNJ9kybr38zorrWJLknyYNJvpDkiln6jHzO5lnXyOcsyVFJ/m+SP+/qeussfUb+mJxnXeN6TB6W5LNJbptl3+LOVVU9a7/ovZD8MPB3gCOAPwdOmtHnXwLv6bYvAm5aInVdCvznMczZLwKnAg8cYP85wMfo/c3GGcB9S6SuSeC2MczXCcCp3fbzgL+c5d9y5HM2z7pGPmfdHCzvtg8H7gPOmNFnHI/J+dQ1rsfkm4A/mu3farHn6tl+BTCfj5Y4H7i+274FODPJbH+QNuq6xqKqPgk8cZAu5wMfrJ57gRVJTlgCdY1FVe2qqs90298CHgRWzeg28jmbZ10j183B3u7m4d3XzHeajPwxOc+6Ri7JauBc4H0H6LKoc/VsD4BVwFen3d7JMx8EP+pTVfuAPcBxS6AugF/vlgxuSbJmlv3jMN/ax+Hnukv4jyX5e6MevLv8/ll6zx6nG+ucHaQuGMOcdUsanwN2A3dW1QHna4SPyfnUBaN/TL4D+NfADw+wf1Hn6tkeAHN+tMQ8+yy0+Yz5p8Daqvr7wP/g6ZQft3HM13x8ht7nm/wMcC3wJ6McPMly4I+BN1bVN2funuWQkczZHHWNZc6q6gdV9XJ6f+l/epKTZ3QZy3zNo66RPiaTvBrYXVVbD9ZtlrYFm6tnewDM56MlftQnyTLgGBZ/qWHOuqrq61X1N93N9wKnLXJN87UkP66jqr65/xK+qu4ADk9y/CjGTnI4vV+yN1bVR2fpMpY5m6uucc5ZN+Y3gCngrBm7xvGYnLOuMTwmXwmcl2QHvWXiVyW5YUafRZ2rZ3sAzOejJbYA67vt1wB3V/eKyjjrmrFGfB69NdylYAtwSffOljOAPVW1a9xFJfmJ/WufSU6nd9/9+gjGDXAd8GBV/eEBuo18zuZT1zjmLMkLkqzoto8Gfgn4ixndRv6YnE9do35MVtXvVtXqqlpL73fE3VX1mzO6LepcjfrTQBdUHeCjJZL8PnB/VW2h9yD5UJLt9JLzoiVS1xuSnAfs6+q6dLHrAkjyYXrvDjk+yU7gKnoviFFV7wHuoPeulu3Ad4DLlkhdrwH+RZJ9wFPARSMIcug9S3stsK1bPwZ4C/CT02obx5zNp65xzNkJwPXp/edPPwbcXFW3jfsxOc+6xvKYnGmUc+VHQUhSo57tS0CSpAEZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlR/x8t6tKKKSQDyAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../output/submission/xgboost-0903_05-26-03-5-fold_avg_logits_test.csv\n"
     ]
    }
   ],
   "source": [
    "b3_test_avg_feats = np.average(b3_test_logits_list, axis=0)\n",
    "b4_test_avg_feats = np.average(b4_test_logits_list, axis=0)\n",
    "b5_test_avg_feats = np.average(b5_test_logits_list, axis=0)\n",
    "\n",
    "\n",
    "X_test = np.concatenate([b3_test_avg_feats, b4_test_avg_feats, b5_test_avg_feats], axis=1)\n",
    "y_pred = xlf.predict(X_test)\n",
    "\n",
    "y_pred = np.round(y_pred)\n",
    "\n",
    "test_df.diagnosis = y_pred.astype(int)\n",
    "\n",
    "test_df.hist()\n",
    "plt.show()\n",
    "\n",
    "submition_filename = \"../output/submission/{}-5-fold_avg_logits_test.csv\".format(model_save_name)\n",
    "test_df.to_csv(submition_filename, index=False)\n",
    "print(submition_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-04T10:32:41.938921Z",
     "start_time": "2019-09-04T10:32:41.645027Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFWBJREFUeJzt3X+U5XV93/HnKyy/dA2LYDa4u3HtkZBQaAxMkcQmZ1ZyEkADNMGWlMjCId3TViNWcwrxnJaYpA32hPiD9uhZxYhCXJDY7AaxSoGpNS1UVw2LEsNitrpAWHFhdXWNou/+cb+rk2F2Z+69M/cO+3k+zpmz3/v5fr7fz3s+O995zfdz79xJVSFJas8PjbsASdJ4GACS1CgDQJIaZQBIUqMMAElqlAEgSY0yAHRIS/LeJL+f5OeSfGHc9RxIkjcmefe461Bblo27AGkUqup/ASeNu44Dqar/NO4a1B7vACSpUQaADilJfjrJp5N8PcnNwFFd+2SSndP6XZXkoa7f55P802n7DktybZLHk/xNktckqSTLuv1TSX4vyV90x38syfHTjj8vyeeSPNn1/clp+65M8nB33BeSnNW1/06SG7vto5LcmOSr3Tk+mWTlok+emmMA6JCR5Ajgz4D3A88FPgj86gG6PwT8HHAM8CbgxiQndPv+JXAO8GLgNOCCWY7/F8BlwI8ARwC/1dXw48AHgNcBzwNuB/48yRFJTgJeA/zjqnoO8EvAjlnOvb6raw1wHPCvgH3zmQOpHwaADiVnAocDb62q71TVrcAnZ+tYVR+sqkeq6ntVdTPwIHBGt/ufAW+rqp1V9QRwzSyn+OOq+uuq2gfcQi8sAP458OGquqOqvgP8IXA08LPAd4EjgZOTHF5VO6rqoVnO/R163/hfVFXfraqtVfW1/qdDOjgDQIeS5wMP199/h8P/N1vHJJck+Wy3xPIkcAqwfxnn+cCXp3X/8tNOAH87bfubwPJpx35/zKr6Xnf8qqraTu/O4HeAXUk2JXn+LOd+P/BRYFOSR5L85ySHz/oZS0MwAHQoeRRYlSTT2n5sZqckLwDeRW855riqWgHcD+w/7lFg9bRD1vRRwyPAC6aNle74hwGq6k+q6p90fQp488wTdHcvb6qqk+ndObwCuKSPGqR5MQB0KPk/wFPAa5MsS/Ir/GBZZ7pn0/vm+xWAJJfRuwPY7xbgiiSrkqwAruyjhluAlyc5q/up/Q3A3wH/O8lJSV6W5EjgW/TW9b878wRJ1iU5NclhwNfoLQk9rZ80LANAh4yq+jbwK8ClwBP01uM/NEu/zwPX0guMx4BTgb+Y1uVdwMeA+4DP0Hsi9ynm8U24qr4A/DpwHfA48MvAL3e1HUnv+YTH6S0h/QjwxllO86PArfS++T8A/E/gxrnGlvoV/yCMdHBJzgHeWVUvmLOz9AziHYA0Q5Kjk5zbLSOtAq4G/tu465IWmncA0gxJnkVv2eUn6K3Tfxi4wpdi6lBjAEhSo1wCkqRGLel3Az3++ONr7dq1Ax//jW98g2c/+9kLV9ACsa7+WFd/rKs/h2JdW7dufbyqnjdnx6pash+nn356DePuu+8e6vjFYl39sa7+WFd/DsW6gE/VPL7HugQkSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNWtJvBSEtZdse3sOlV3145OPuuOblIx9ThybvACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNWrOAEjyniS7ktw/re25Se5I8mD377Fde5K8Pcn2JPclOW3aMeu7/g8mWb84n44kab7mcwfwXuDsGW1XAXdW1YnAnd1jgHOAE7uPDcA7oBcYwNXAS4AzgKv3h4YkaTzmDICq+jiwe0bz+cAN3fYNwAXT2t9XPfcAK5KcAPwScEdV7a6qJ4A7eHqoSJJGKFU1d6dkLXBbVZ3SPX6yqlZM2/9EVR2b5Dbgmqr6RNd+J3AlMAkcVVW/37X/e2BfVf3hLGNtoHf3wMqVK0/ftGnTwJ/c3r17Wb58+cDHLxbr6s9SrWvX7j08tm/045666piD7l+q82Vd/RmmrnXr1m2tqom5+i0b6OwHllna6iDtT2+s2ghsBJiYmKjJycmBi5mammKY4xeLdfVnqdZ13U2buXbbQl9Cc9tx8eRB9y/V+bKu/oyirkFfBfRYt7RD9++urn0nsGZav9XAIwdplySNyaABsAXY/0qe9cDmae2XdK8GOhPYU1WPAh8FfjHJsd2Tv7/YtUmSxmTO+9ckH6C3hn98kp30Xs1zDXBLksuBLwGv7LrfDpwLbAe+CVwGUFW7k/we8Mmu3+9W1cwnliVJIzRnAFTVrx1g11mz9C3g1Qc4z3uA9/RVnSRp0fibwJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrUUAGQ5N8m+VyS+5N8IMlRSV6Y5N4kDya5OckRXd8ju8fbu/1rF+ITkCQNZuAASLIKeC0wUVWnAIcBFwFvBt5SVScCTwCXd4dcDjxRVS8C3tL1kySNybBLQMuAo5MsA54FPAq8DLi1238DcEG3fX73mG7/WUky5PiSpAGlqgY/OLkC+I/APuBjwBXAPd1P+SRZA3ykqk5Jcj9wdlXt7PY9BLykqh6fcc4NwAaAlStXnr5p06aB69u7dy/Lly8f+PjFYl39Wap17dq9h8f2jX7cU1cdc9D9S3W+rKs/w9S1bt26rVU1MVe/ZQOdHUhyLL2f6l8IPAl8EDhnlq77E2a2n/aflj5VtRHYCDAxMVGTk5ODlsjU1BTDHL9YrKs/S7Wu627azLXbBr6EBrbj4smD7l+q82Vd/RlFXcMsAf0C8DdV9ZWq+g7wIeBngRXdkhDAauCRbnsnsAag238MsHuI8SVJQxgmAL4EnJnkWd1a/lnA54G7gQu7PuuBzd32lu4x3f67apj1J0nSUAYOgKq6l96TuZ8GtnXn2ghcCbw+yXbgOOD67pDrgeO69tcDVw1RtyRpSEMtYFbV1cDVM5q/CJwxS99vAa8cZjxJ0sLxN4ElqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUqKECIMmKJLcm+askDyT5mSTPTXJHkge7f4/t+ibJ25NsT3JfktMW5lOQJA1i2DuAtwH/vap+Avgp4AHgKuDOqjoRuLN7DHAOcGL3sQF4x5BjS5KGMHAAJPlh4OeB6wGq6ttV9SRwPnBD1+0G4IJu+3zgfdVzD7AiyQkDVy5JGkqqarADkxcDG4HP0/vpfytwBfBwVa2Y1u+Jqjo2yW3ANVX1ia79TuDKqvrUjPNuoHeHwMqVK0/ftGnTQPUB7N27l+XLlw98/GKxrv4s1bp27d7DY/tGP+6pq4456P6lOl/W1Z9h6lq3bt3WqpqYq9+ygc7+g2NPA36zqu5N8jZ+sNwzm8zS9rT0qaqN9IKFiYmJmpycHLjAqakphjl+sVhXf5ZqXdfdtJlrtw1zCQ1mx8WTB92/VOfLuvozirqGeQ5gJ7Czqu7tHt9KLxAe27+00/27a1r/NdOOXw08MsT4kqQhDBwAVfW3wJeTnNQ1nUVvOWgLsL5rWw9s7ra3AJd0rwY6E9hTVY8OOr4kaTjD3r/+JnBTkiOALwKX0QuVW5JcDnwJeGXX93bgXGA78M2uryRpTIYKgKr6LDDbEw1nzdK3gFcPM54kaeH4m8CS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElq1NABkOSwJJ9Jclv3+IVJ7k3yYJKbkxzRtR/ZPd7e7V877NiSpMEtxB3AFcAD0x6/GXhLVZ0IPAFc3rVfDjxRVS8C3tL1kySNyVABkGQ18HLg3d3jAC8Dbu263ABc0G2f3z2m239W11+SNAapqsEPTm4F/gB4DvBbwKXAPd1P+SRZA3ykqk5Jcj9wdlXt7PY9BLykqh6fcc4NwAaAlStXnr5p06aB69u7dy/Lly8f+PjFYl39Wap17dq9h8f2jX7cU1cdc9D9S3W+rKs/w9S1bt26rVU1MVe/ZQOdHUjyCmBXVW1NMrm/eZauNY99P2io2ghsBJiYmKjJycmZXeZtamqKYY5fLNbVn6Va13U3bebabQNfQgPbcfHkQfcv1fmyrv6Moq5hvnpfCpyX5FzgKOCHgbcCK5Isq6qngNXAI13/ncAaYGeSZcAxwO4hxpckDWHg5wCq6reranVVrQUuAu6qqouBu4ELu27rgc3d9pbuMd3+u2qY9SdJ0lAW4/cArgRen2Q7cBxwfdd+PXBc1/564KpFGFuSNE8LsoBZVVPAVLf9ReCMWfp8C3jlQownSRqevwksSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNGv1bGY7Qtof3cOlVHx75uDuuefnIx5SkfnkHIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY06pP8imKSFtXaIv7D3hlOfGvgv9PlX9hbHwHcASdYkuTvJA0k+l+SKrv25Se5I8mD377Fde5K8Pcn2JPclOW2hPglJUv+GWQJ6CnhDVf0kcCbw6iQnA1cBd1bVicCd3WOAc4ATu48NwDuGGFuSNKSBl4Cq6lHg0W7760keAFYB5wOTXbcbgCngyq79fVVVwD1JViQ5oTuPFsAwt+fgLbrUmvS+Hw95kmQt8HHgFOBLVbVi2r4nqurYJLcB11TVJ7r2O4Erq+pTM861gd4dAitXrjx906ZNA9e1a/ceHts38OEDO3XVMQfdv3fvXpYvX77g4257eM9Qx688moHna67PeRiLNV/Dau3rC4b7GvPrqz/D1LVu3bqtVTUxV7+hnwROshz4U+B1VfW1JAfsOkvb09KnqjYCGwEmJiZqcnJy4Nquu2kz124b/fPcOy6ePOj+qakphvm8DmTQn973e8OpTw08X3N9zsNYrPkaVmtfXzDc15hfX/0ZRV1DvQw0yeH0vvnfVFUf6pofS3JCt/8EYFfXvhNYM+3w1cAjw4wvSRrcMK8CCnA98EBV/dG0XVuA9d32emDztPZLulcDnQnscf1fksZnmPvXlwKvArYl+WzX9kbgGuCWJJcDXwJe2e27HTgX2A58E7hsiLElSUMa5lVAn2D2dX2As2bpX8CrBx1PkrSwfCsISWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVHD/FF4STqkrb3qw2Mb+71nP3vRx/AOQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktSokQdAkrOTfCHJ9iRXjXp8SVLPSAMgyWHAfwXOAU4Gfi3JyaOsQZLUM+o7gDOA7VX1xar6NrAJOH/ENUiSgFTV6AZLLgTOrqrf6B6/CnhJVb1mWp8NwIbu4UnAF4YY8njg8SGOXyzW1R/r6o919edQrOsFVfW8uTqN+s3gMkvb30ugqtoIbFyQwZJPVdXEQpxrIVlXf6yrP9bVn5brGvUS0E5gzbTHq4FHRlyDJInRB8AngROTvDDJEcBFwJYR1yBJYsRLQFX1VJLXAB8FDgPeU1WfW8QhF2QpaRFYV3+sqz/W1Z9m6xrpk8CSpKXD3wSWpEYZAJLUqGd8AMz11hJJjkxyc7f/3iRrl0hdlyb5SpLPdh+/MaK63pNkV5L7D7A/Sd7e1X1fktOWSF2TSfZMm6//MKK61iS5O8kDST6X5IpZ+ox8zuZZ18jnLMlRSf5vkr/s6nrTLH1Gfk3Os65xXZOHJflMkttm2be4c1VVz9gPek8kPwT8A+AI4C+Bk2f0+TfAO7vti4Cbl0hdlwL/ZQxz9vPAacD9B9h/LvARer+zcSZw7xKpaxK4bQzzdQJwWrf9HOCvZ/m/HPmczbOukc9ZNwfLu+3DgXuBM2f0Gcc1OZ+6xnVNvh74k9n+rxZ7rp7pdwDzeWuJ84Ebuu1bgbOSzPYLaaOuayyq6uPA7oN0OR94X/XcA6xIcsISqGssqurRqvp0t/114AFg1YxuI5+zedY1ct0c7O0eHt59zHylycivyXnWNXJJVgMvB959gC6LOlfP9ABYBXx52uOdPP0i+H6fqnoK2AMctwTqAvjVbsng1iRrZtk/DvOtfRx+pruF/0iSfzjqwbvb75+m99PjdGOds4PUBWOYs25J47PALuCOqjrgfI3wmpxPXTD6a/KtwL8DvneA/Ys6V8/0AJjzrSXm2WehzWfMPwfWVtU/Av4HP0j5cRvHfM3Hp+m9v8lPAdcBfzbKwZMsB/4UeF1VfW3m7lkOGcmczVHXWOasqr5bVS+m95v+ZyQ5ZUaXsczXPOoa6TWZ5BXArqraerBus7Qt2Fw90wNgPm8t8f0+SZYBx7D4Sw1z1lVVX62qv+sevgs4fZFrmq8l+XYdVfW1/bfwVXU7cHiS40cxdpLD6X2TvamqPjRLl7HM2Vx1jXPOujGfBKaAs2fsGsc1OWddY7gmXwqcl2QHvWXilyW5cUafRZ2rZ3oAzOetJbYA67vtC4G7qntGZZx1zVgjPo/eGu5SsAW4pHtly5nAnqp6dNxFJfnR/WufSc6g97X71RGMG+B64IGq+qMDdBv5nM2nrnHMWZLnJVnRbR8N/ALwVzO6jfyanE9do74mq+q3q2p1Va2l9z3irqr69RndFnWuRv1uoAuqDvDWEkl+F/hUVW2hd5G8P8l2esl50RKp67VJzgOe6uq6dLHrAkjyAXqvDjk+yU7ganpPiFFV7wRup/eqlu3AN4HLlkhdFwL/OslTwD7gohEEOfR+SnsVsK1bPwZ4I/Bj02obx5zNp65xzNkJwA3p/fGnHwJuqarbxn1NzrOusVyTM41yrnwrCElq1DN9CUiSNCADQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXq/wOK8tWGidFDgQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../output/submission/xgboost-0903_05-26-03-5-fold_logits_avg_test.csv\n"
     ]
    }
   ],
   "source": [
    "# 5 test feature then avg\n",
    "results = []\n",
    "for b3, b4, b5 in zip(b3_test_logits_list, b4_test_logits_list, b5_test_logits_list):\n",
    "    X_test = np.concatenate([b3, b4, b5], axis=1)\n",
    "    res = xlf.predict(X_test)\n",
    "    results.append(res)\n",
    "\n",
    "avg_res_xlf = np.average(results, axis=0)\n",
    "np.save(\"../output/submission/{}-5-fold_logits_avg_test_logits.npy\".format(model_save_name), avg_res_xlf)\n",
    "y_pred = np.round(avg_res_xlf)\n",
    "\n",
    "test_df.diagnosis = y_pred.astype(int)\n",
    "\n",
    "test_df.hist()\n",
    "plt.show()\n",
    "\n",
    "submition_filename = \"../output/submission/{}-5-fold_logits_avg_test.csv\".format(model_save_name)\n",
    "test_df.to_csv(submition_filename, index=False)\n",
    "print(submition_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVR "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-04T10:33:00.762429Z",
     "start_time": "2019-09-04T10:33:00.747208Z"
    }
   },
   "outputs": [],
   "source": [
    "model_save_name = \"svr-0903_05-26-03\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-03T14:02:45.516604Z",
     "start_time": "2019-09-03T14:02:45.248042Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFWNJREFUeJzt3X+UpmV93/H3Jyy/dA2LYDa4u3HtkZhQaAxMkcQmZ1ZyEn4YoYm2pEQWDumethqxmFM2ntMSk7TBnhB/0B49qxhRiQsSGwhglQJba1qoLBIWJYTFbHGBsOLC4gpGwW//eO6VyTC7M8/zzDzPsNf7dc6cuX9c9319n2vnns/c1/NjU1VIktrzQ+MuQJI0HgaAJDXKAJCkRhkAktQoA0CSGmUASFKjDADt15J8LMnvJ/m5JPeNu569SfKuJB8Zdx1qy5JxFyCNQlX9L+DV465jb6rqP427BrXHOwBJapQBoP1Kkp9OcmeSbyW5Cjik2z6ZZPuUduuTPNC1+2qSfzpl3wFJLk3yWJK/SfK2JJVkSbd/U5LfS/IX3fGfT3LklOPfmOQrSZ7o2v7klH0XJXmoO+6+JCd3238nySe75UOSfDLJN7tzfCnJ8gUfPDXHANB+I8lBwJ8BnwBeCnwa+NW9NH8A+DngMODdwCeTHNXt+5fAqcBrgOOBM2c4/l8A5wE/AhwE/FZXw48DnwLeAbwMuBH48yQHJXk18DbgH1fVS4BfArbNcO61XV2rgCOAfwU8PZcxkPphAGh/chJwIPC+qvpeVV0DfGmmhlX16ap6uKq+X1VXAfcDJ3a7/xnw/qraXlWPA5fMcIo/rqq/rqqngavphQXAPwduqKqbqup7wB8ChwI/CzwLHAwck+TAqtpWVQ/McO7v0fvF/6qqeraqNlfVk/0Ph7RvBoD2Jy8HHqq//wmH/2+mhknOSXJXN8XyBHAssGca5+XA16c0//rzTgB/O2X5KWDplGN/0GdVfb87fkVVbaV3Z/A7wI4kG5O8fIZzfwL4HLAxycNJ/nOSA2d8xNIQDADtTx4BViTJlG0/Nr1RklcAH6Y3HXNEVS0D7gH2HPcIsHLKIav6qOFh4BVT+kp3/EMAVfUnVfVPujYFvGf6Cbq7l3dX1TH07hzeAJzTRw3SnBgA2p/8H+AZ4O1JliT5FZ6b1pnqxfR++X4DIMl59O4A9rgauCDJiiTLgIv6qOFq4PQkJ3d/tb8T+Dvgfyd5dZLXJzkY+A69ef1np58gyZokxyU5AHiS3pTQ89pJwzIAtN+oqu8CvwKcCzxObz7+MzO0+ypwKb3AeBQ4DviLKU0+DHweuBv4Mr0ncp9hDr+Eq+o+4NeBy4DHgF8Gfrmr7WB6zyc8Rm8K6UeAd81wmh8FrqH3y/9e4H8Cn5ytb6lf8T+EkfYtyanAh6rqFbM2ll5AvAOQpklyaJLTummkFcDFwH8bd13SfPMOQJomyYvoTbv8BL15+huAC3wppvY3BoAkNcopIElq1KL+NNAjjzyyVq9ePfDx3/72t3nxi188fwXNE+vqj3X1x7r6sz/WtXnz5seq6mWzNqyqRft1wgkn1DBuvfXWoY5fKNbVH+vqj3X1Z3+sC7ij5vA71ikgSWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElq1KL+KAhpMdvy0C7OXX/DyPvddsnpI+9T+yfvACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjZo1AJJ8NMmOJPdM2fbSJDclub/7fni3PUk+kGRrkruTHD/lmLVd+/uTrF2YhyNJmqu53AF8DDhl2rb1wM1VdTRwc7cOcCpwdPe1Dvgg9AIDuBh4LXAicPGe0JAkjcesAVBVXwB2Ttt8BnBFt3wFcOaU7R+vntuAZUmOAn4JuKmqdlbV48BNPD9UJEkjlKqavVGyGri+qo7t1p+oqmVT9j9eVYcnuR64pKq+2G2/GbgImAQOqarf77b/e+DpqvrDGfpaR+/ugeXLl5+wcePGgR/c7t27Wbp06cDHLxTr6s9irWvHzl08+vTo+z1uxWH73L9Yx8u6+jNMXWvWrNlcVROztZvv/xQ+M2yrfWx//saqDcAGgImJiZqcnBy4mE2bNjHM8QvFuvqzWOu67MpruXTLfF9Cs9t29uQ+9y/W8bKu/oyirkFfBfRoN7VD931Ht307sGpKu5XAw/vYLkkak0ED4Dpgzyt51gLXTtl+TvdqoJOAXVX1CPA54BeTHN49+fuL3TZJ0pjMev+a5FP05vCPTLKd3qt5LgGuTnI+8CDw5q75jcBpwFbgKeA8gKrameT3gC917X63qqY/sSxJGqFZA6Cqfm0vu06eoW0Bb93LeT4KfLSv6iRJC8Z3AktSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRQwVAkn+b5CtJ7knyqSSHJHllktuT3J/kqiQHdW0P7ta3dvtXz8cDkCQNZuAASLICeDswUVXHAgcAZwHvAd5bVUcDjwPnd4ecDzxeVa8C3tu1kySNybBTQEuAQ5MsAV4EPAK8Hrim238FcGa3fEa3Trf/5CQZsn9J0oBSVYMfnFwA/EfgaeDzwAXAbd1f+SRZBXy2qo5Ncg9wSlVt7/Y9ALy2qh6bds51wDqA5cuXn7Bx48aB69u9ezdLly4d+PiFYl39Wax17di5i0efHn2/x604bJ/7F+t4WVd/hqlrzZo1m6tqYrZ2SwY6O5DkcHp/1b8SeAL4NHDqDE33JMxMf+0/L32qagOwAWBiYqImJycHLZFNmzYxzPELxbr6s1jruuzKa7l0y8CX0MC2nT25z/2Ldbysqz+jqGuYKaBfAP6mqr5RVd8DPgP8LLCsmxICWAk83C1vB1YBdPsPA3YO0b8kaQjDBMCDwElJXtTN5Z8MfBW4FXhT12YtcG23fF23Trf/lhpm/kmSNJSBA6Cqbqf3ZO6dwJbuXBuAi4ALk2wFjgAu7w65HDii234hsH6IuiVJQxpqArOqLgYunrb5a8CJM7T9DvDmYfqTJM0f3wksSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktSooQIgybIk1yT5qyT3JvmZJC9NclOS+7vvh3dtk+QDSbYmuTvJ8fPzECRJgxj2DuD9wH+vqp8Afgq4F1gP3FxVRwM3d+sApwJHd1/rgA8O2bckaQgDB0CSHwZ+HrgcoKq+W1VPAGcAV3TNrgDO7JbPAD5ePbcBy5IcNXDlkqShpKoGOzB5DbAB+Cq9v/43AxcAD1XVsintHq+qw5NcD1xSVV/stt8MXFRVd0w77zp6dwgsX778hI0bNw5UH8Du3btZunTpwMcvFOvqz2Kta8fOXTz69Oj7PW7FYfvcv1jHy7r6M0xda9as2VxVE7O1WzLQ2Z879njgN6vq9iTv57npnplkhm3PS5+q2kAvWJiYmKjJycmBC9y0aRPDHL9QrKs/i7Wuy668lku3DHMJDWbb2ZP73L9Yx8u6+jOKuoZ5DmA7sL2qbu/Wr6EXCI/umdrpvu+Y0n7VlONXAg8P0b8kaQgDB0BV/S3w9SSv7jadTG866DpgbbdtLXBtt3wdcE73aqCTgF1V9cig/UuShjPs/etvAlcmOQj4GnAevVC5Osn5wIPAm7u2NwKnAVuBp7q2kqQxGSoAquouYKYnGk6eoW0Bbx2mP0nS/PGdwJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrU0AGQ5IAkX05yfbf+yiS3J7k/yVVJDuq2H9ytb+32rx62b0nS4ObjDuAC4N4p6+8B3ltVRwOPA+d3288HHq+qVwHv7dpJksZkqABIshI4HfhItx7g9cA1XZMrgDO75TO6dbr9J3ftJUljkKoa/ODkGuAPgJcAvwWcC9zW/ZVPklXAZ6vq2CT3AKdU1fZu3wPAa6vqsWnnXAesA1i+fPkJGzduHLi+3bt3s3Tp0oGPXyjW1Z/FWteOnbt49OnR93vcisP2uX+xjpd19WeYutasWbO5qiZma7dkoLMDSd4A7KiqzUkm92yeoWnNYd9zG6o2ABsAJiYmanJycnqTOdu0aRPDHL9QrKs/i7Wuy668lku3DHwJDWzb2ZP73L9Yx8u6+jOKuob56X0d8MYkpwGHAD8MvA9YlmRJVT0DrAQe7tpvB1YB25MsAQ4Ddg7RvyRpCAM/B1BVv11VK6tqNXAWcEtVnQ3cCrypa7YWuLZbvq5bp9t/Sw0z/yRJGspCvA/gIuDCJFuBI4DLu+2XA0d02y8E1i9A35KkOZqXCcyq2gRs6pa/Bpw4Q5vvAG+ej/4kScPzncCS1CgDQJIaZQBIUqMMAElqlAEgSY0a/dsYR2jLQ7s4d/0NI+932yWnj7xPSeqXdwCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjVoy7gIkvXCsXn/DwMe+87hnOHfA47ddcvrA/WrvBr4DSLIqya1J7k3ylSQXdNtfmuSmJPd33w/vtifJB5JsTXJ3kuPn60FIkvo3zBTQM8A7q+ongZOAtyY5BlgP3FxVRwM3d+sApwJHd1/rgA8O0bckaUgDB0BVPVJVd3bL3wLuBVYAZwBXdM2uAM7sls8APl49twHLkhw1cOWSpKGkqoY/SbIa+AJwLPBgVS2bsu/xqjo8yfXAJVX1xW77zcBFVXXHtHOto3eHwPLly0/YuHHjwHXt2LmLR58e+PCBHbfisH3u3717N0uXLh1RNXNnXf1p8edry0O7Bj52+aEMPF6zPeZhLNafr2HqWrNmzeaqmpit3dBPAidZCvwp8I6qejLJXpvOsO156VNVG4ANABMTEzU5OTlwbZddeS2Xbhn989zbzp7c5/5NmzYxzOPam2GeoAN453HPcukXvz3QsQv5JN1CjdewWvv5AgZ+Ehd6TwIPOl6zPeZhLNafr1HUNdTLQJMcSO+X/5VV9Zlu86N7pna67zu67duBVVMOXwk8PEz/kqTBDfMqoACXA/dW1R9N2XUdsLZbXgtcO2X7Od2rgU4CdlXVI4P2L0kazjD3r68D3gJsSXJXt+1dwCXA1UnOBx4E3tztuxE4DdgKPAWcN0TfkqQhDRwA3ZO5e5vwP3mG9gW8ddD+JEnzy4+CkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrUknEXIEmL1er1N4yt74+d8uIF78M7AElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGjXyAEhySpL7kmxNsn7U/UuSekYaAEkOAP4rcCpwDPBrSY4ZZQ2SpJ5R3wGcCGytqq9V1XeBjcAZI65BkgSkqkbXWfIm4JSq+o1u/S3Aa6vqbVParAPWdauvBu4bossjgceGOH6hWFd/rKs/1tWf/bGuV1TVy2ZrNOrPAsoM2/5eAlXVBmDDvHSW3FFVE/NxrvlkXf2xrv5YV39armvUU0DbgVVT1lcCD4+4BkkSow+ALwFHJ3llkoOAs4DrRlyDJIkRTwFV1TNJ3gZ8DjgA+GhVfWUBu5yXqaQFYF39sa7+WFd/mq1rpE8CS5IWD98JLEmNMgAkqVEv+ACY7aMlkhyc5Kpu/+1JVi+Sus5N8o0kd3VfvzGiuj6aZEeSe/ayP0k+0NV9d5LjF0ldk0l2TRmv/zCiulYluTXJvUm+kuSCGdqMfMzmWNfIxyzJIUn+b5K/7Op69wxtRn5NzrGucV2TByT5cpLrZ9i3sGNVVS/YL3pPJD8A/APgIOAvgWOmtfk3wIe65bOAqxZJXecC/2UMY/bzwPHAPXvZfxrwWXrv2TgJuH2R1DUJXD+G8ToKOL5bfgnw1zP8W458zOZY18jHrBuDpd3ygcDtwEnT2ozjmpxLXeO6Ji8E/mSmf6uFHqsX+h3AXD5a4gzgim75GuDkJDO9IW3UdY1FVX0B2LmPJmcAH6+e24BlSY5aBHWNRVU9UlV3dsvfAu4FVkxrNvIxm2NdI9eNwe5u9cDua/orTUZ+Tc6xrpFLshI4HfjIXpos6Fi90ANgBfD1Kevbef5F8IM2VfUMsAs4YhHUBfCr3ZTBNUlWzbB/HOZa+zj8THcL/9kk/3DUnXe33z9N76/HqcY6ZvuoC8YwZt2Uxl3ADuCmqtrreI3wmpxLXTD6a/J9wL8Dvr+X/Qs6Vi/0AJj1oyXm2Ga+zaXPPwdWV9U/Av4Hz6X8uI1jvObiTnqfb/JTwGXAn42y8yRLgT8F3lFVT07fPcMhIxmzWeoay5hV1bNV9Rp67/Q/Mcmx05qMZbzmUNdIr8kkbwB2VNXmfTWbYdu8jdULPQDm8tESP2iTZAlwGAs/1TBrXVX1zar6u271w8AJC1zTXC3Kj+uoqif33MJX1Y3AgUmOHEXfSQ6k90v2yqr6zAxNxjJms9U1zjHr+nwC2AScMm3XOK7JWesawzX5OuCNSbbRmyZ+fZJPTmuzoGP1Qg+AuXy0xHXA2m75TcAt1T2jMs66ps0Rv5HeHO5icB1wTvfKlpOAXVX1yLiLSvKje+Y+k5xI72f3myPoN8DlwL1V9Ud7aTbyMZtLXeMYsyQvS7KsWz4U+AXgr6Y1G/k1OZe6Rn1NVtVvV9XKqlpN73fELVX169OaLehYjfrTQOdV7eWjJZL8LnBHVV1H7yL5RJKt9JLzrEVS19uTvBF4pqvr3IWuCyDJp+i9OuTIJNuBi+k9IUZVfQi4kd6rWrYCTwHnLZK63gT86yTPAE8DZ40gyKH3V9pbgC3d/DHAu4Afm1LbOMZsLnWNY8yOAq5I7z9/+iHg6qq6ftzX5BzrGss1Od0ox8qPgpCkRr3Qp4AkSQMyACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKj/j8y6s+n64eihAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../output/submission/svr-0903_05-26-03-5-fold_avg_logits_test.csv\n"
     ]
    }
   ],
   "source": [
    "b3_test_avg_feats = np.average(b3_test_logits_list, axis=0)\n",
    "b4_test_avg_feats = np.average(b4_test_logits_list, axis=0)\n",
    "b5_test_avg_feats = np.average(b5_test_logits_list, axis=0)\n",
    "\n",
    "\n",
    "X_test = np.concatenate([b3_test_avg_feats, b4_test_avg_feats, b5_test_avg_feats], axis=1)\n",
    "y_pred = svr.predict(X_test)\n",
    "\n",
    "y_pred = np.round(y_pred)\n",
    "\n",
    "test_df.diagnosis = y_pred.astype(int)\n",
    "\n",
    "test_df.hist()\n",
    "plt.show()\n",
    "\n",
    "submition_filename = \"../output/submission/{}-5-fold_avg_logits_test.csv\".format(model_save_name)\n",
    "test_df.to_csv(submition_filename, index=False)\n",
    "print(submition_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-04T10:33:16.929297Z",
     "start_time": "2019-09-04T10:33:15.995756Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFWNJREFUeJzt3X+UpmV93/H3Jyy/dA2LYDa4u3HtkZhQaAxMkcQmZ1ZyEn4YoYm2pEQWDumethqxmFM2ntMSk7TBnhB/0B49qxhRiQsSGwhglQJba1qoLBIWJYTFbHGBsOLC4gpGwW//eO6VyTC7M8/zzDzPsNf7dc6cuX9c9319n2vnns/c1/NjU1VIktrzQ+MuQJI0HgaAJDXKAJCkRhkAktQoA0CSGmUASFKjDADt15J8LMnvJ/m5JPeNu569SfKuJB8Zdx1qy5JxFyCNQlX9L+DV465jb6rqP427BrXHOwBJapQBoP1Kkp9OcmeSbyW5Cjik2z6ZZPuUduuTPNC1+2qSfzpl3wFJLk3yWJK/SfK2JJVkSbd/U5LfS/IX3fGfT3LklOPfmOQrSZ7o2v7klH0XJXmoO+6+JCd3238nySe75UOSfDLJN7tzfCnJ8gUfPDXHANB+I8lBwJ8BnwBeCnwa+NW9NH8A+DngMODdwCeTHNXt+5fAqcBrgOOBM2c4/l8A5wE/AhwE/FZXw48DnwLeAbwMuBH48yQHJXk18DbgH1fVS4BfArbNcO61XV2rgCOAfwU8PZcxkPphAGh/chJwIPC+qvpeVV0DfGmmhlX16ap6uKq+X1VXAfcDJ3a7/xnw/qraXlWPA5fMcIo/rqq/rqqngavphQXAPwduqKqbqup7wB8ChwI/CzwLHAwck+TAqtpWVQ/McO7v0fvF/6qqeraqNlfVk/0Ph7RvBoD2Jy8HHqq//wmH/2+mhknOSXJXN8XyBHAssGca5+XA16c0//rzTgB/O2X5KWDplGN/0GdVfb87fkVVbaV3Z/A7wI4kG5O8fIZzfwL4HLAxycNJ/nOSA2d8xNIQDADtTx4BViTJlG0/Nr1RklcAH6Y3HXNEVS0D7gH2HPcIsHLKIav6qOFh4BVT+kp3/EMAVfUnVfVPujYFvGf6Cbq7l3dX1TH07hzeAJzTRw3SnBgA2p/8H+AZ4O1JliT5FZ6b1pnqxfR++X4DIMl59O4A9rgauCDJiiTLgIv6qOFq4PQkJ3d/tb8T+Dvgfyd5dZLXJzkY+A69ef1np58gyZokxyU5AHiS3pTQ89pJwzIAtN+oqu8CvwKcCzxObz7+MzO0+ypwKb3AeBQ4DviLKU0+DHweuBv4Mr0ncp9hDr+Eq+o+4NeBy4DHgF8Gfrmr7WB6zyc8Rm8K6UeAd81wmh8FrqH3y/9e4H8Cn5ytb6lf8T+EkfYtyanAh6rqFbM2ll5AvAOQpklyaJLTummkFcDFwH8bd13SfPMOQJomyYvoTbv8BL15+huAC3wppvY3BoAkNcopIElq1KL+NNAjjzyyVq9ePfDx3/72t3nxi188fwXNE+vqj3X1x7r6sz/WtXnz5seq6mWzNqyqRft1wgkn1DBuvfXWoY5fKNbVH+vqj3X1Z3+sC7ij5vA71ikgSWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElq1KL+KAhpMdvy0C7OXX/DyPvddsnpI+9T+yfvACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjZo1AJJ8NMmOJPdM2fbSJDclub/7fni3PUk+kGRrkruTHD/lmLVd+/uTrF2YhyNJmqu53AF8DDhl2rb1wM1VdTRwc7cOcCpwdPe1Dvgg9AIDuBh4LXAicPGe0JAkjcesAVBVXwB2Ttt8BnBFt3wFcOaU7R+vntuAZUmOAn4JuKmqdlbV48BNPD9UJEkjlKqavVGyGri+qo7t1p+oqmVT9j9eVYcnuR64pKq+2G2/GbgImAQOqarf77b/e+DpqvrDGfpaR+/ugeXLl5+wcePGgR/c7t27Wbp06cDHLxTr6s9irWvHzl08+vTo+z1uxWH73L9Yx8u6+jNMXWvWrNlcVROztZvv/xQ+M2yrfWx//saqDcAGgImJiZqcnBy4mE2bNjHM8QvFuvqzWOu67MpruXTLfF9Cs9t29uQ+9y/W8bKu/oyirkFfBfRoN7VD931Ht307sGpKu5XAw/vYLkkak0ED4Dpgzyt51gLXTtl+TvdqoJOAXVX1CPA54BeTHN49+fuL3TZJ0pjMev+a5FP05vCPTLKd3qt5LgGuTnI+8CDw5q75jcBpwFbgKeA8gKrameT3gC917X63qqY/sSxJGqFZA6Cqfm0vu06eoW0Bb93LeT4KfLSv6iRJC8Z3AktSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRQwVAkn+b5CtJ7knyqSSHJHllktuT3J/kqiQHdW0P7ta3dvtXz8cDkCQNZuAASLICeDswUVXHAgcAZwHvAd5bVUcDjwPnd4ecDzxeVa8C3tu1kySNybBTQEuAQ5MsAV4EPAK8Hrim238FcGa3fEa3Trf/5CQZsn9J0oBSVYMfnFwA/EfgaeDzwAXAbd1f+SRZBXy2qo5Ncg9wSlVt7/Y9ALy2qh6bds51wDqA5cuXn7Bx48aB69u9ezdLly4d+PiFYl39Wax17di5i0efHn2/x604bJ/7F+t4WVd/hqlrzZo1m6tqYrZ2SwY6O5DkcHp/1b8SeAL4NHDqDE33JMxMf+0/L32qagOwAWBiYqImJycHLZFNmzYxzPELxbr6s1jruuzKa7l0y8CX0MC2nT25z/2Ldbysqz+jqGuYKaBfAP6mqr5RVd8DPgP8LLCsmxICWAk83C1vB1YBdPsPA3YO0b8kaQjDBMCDwElJXtTN5Z8MfBW4FXhT12YtcG23fF23Trf/lhpm/kmSNJSBA6Cqbqf3ZO6dwJbuXBuAi4ALk2wFjgAu7w65HDii234hsH6IuiVJQxpqArOqLgYunrb5a8CJM7T9DvDmYfqTJM0f3wksSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktSooQIgybIk1yT5qyT3JvmZJC9NclOS+7vvh3dtk+QDSbYmuTvJ8fPzECRJgxj2DuD9wH+vqp8Afgq4F1gP3FxVRwM3d+sApwJHd1/rgA8O2bckaQgDB0CSHwZ+HrgcoKq+W1VPAGcAV3TNrgDO7JbPAD5ePbcBy5IcNXDlkqShpKoGOzB5DbAB+Cq9v/43AxcAD1XVsintHq+qw5NcD1xSVV/stt8MXFRVd0w77zp6dwgsX778hI0bNw5UH8Du3btZunTpwMcvFOvqz2Kta8fOXTz69Oj7PW7FYfvcv1jHy7r6M0xda9as2VxVE7O1WzLQ2Z879njgN6vq9iTv57npnplkhm3PS5+q2kAvWJiYmKjJycmBC9y0aRPDHL9QrKs/i7Wuy668lku3DHMJDWbb2ZP73L9Yx8u6+jOKuoZ5DmA7sL2qbu/Wr6EXCI/umdrpvu+Y0n7VlONXAg8P0b8kaQgDB0BV/S3w9SSv7jadTG866DpgbbdtLXBtt3wdcE73aqCTgF1V9cig/UuShjPs/etvAlcmOQj4GnAevVC5Osn5wIPAm7u2NwKnAVuBp7q2kqQxGSoAquouYKYnGk6eoW0Bbx2mP0nS/PGdwJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrU0AGQ5IAkX05yfbf+yiS3J7k/yVVJDuq2H9ytb+32rx62b0nS4ObjDuAC4N4p6+8B3ltVRwOPA+d3288HHq+qVwHv7dpJksZkqABIshI4HfhItx7g9cA1XZMrgDO75TO6dbr9J3ftJUljkKoa/ODkGuAPgJcAvwWcC9zW/ZVPklXAZ6vq2CT3AKdU1fZu3wPAa6vqsWnnXAesA1i+fPkJGzduHLi+3bt3s3Tp0oGPXyjW1Z/FWteOnbt49OnR93vcisP2uX+xjpd19WeYutasWbO5qiZma7dkoLMDSd4A7KiqzUkm92yeoWnNYd9zG6o2ABsAJiYmanJycnqTOdu0aRPDHL9QrKs/i7Wuy668lku3DHwJDWzb2ZP73L9Yx8u6+jOKuob56X0d8MYkpwGHAD8MvA9YlmRJVT0DrAQe7tpvB1YB25MsAQ4Ddg7RvyRpCAM/B1BVv11VK6tqNXAWcEtVnQ3cCrypa7YWuLZbvq5bp9t/Sw0z/yRJGspCvA/gIuDCJFuBI4DLu+2XA0d02y8E1i9A35KkOZqXCcyq2gRs6pa/Bpw4Q5vvAG+ej/4kScPzncCS1CgDQJIaZQBIUqMMAElqlAEgSY0a/dsYR2jLQ7s4d/0NI+932yWnj7xPSeqXdwCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjVoy7gIkvXCsXn/DwMe+87hnOHfA47ddcvrA/WrvBr4DSLIqya1J7k3ylSQXdNtfmuSmJPd33w/vtifJB5JsTXJ3kuPn60FIkvo3zBTQM8A7q+ongZOAtyY5BlgP3FxVRwM3d+sApwJHd1/rgA8O0bckaUgDB0BVPVJVd3bL3wLuBVYAZwBXdM2uAM7sls8APl49twHLkhw1cOWSpKGkqoY/SbIa+AJwLPBgVS2bsu/xqjo8yfXAJVX1xW77zcBFVXXHtHOto3eHwPLly0/YuHHjwHXt2LmLR58e+PCBHbfisH3u3717N0uXLh1RNXNnXf1p8edry0O7Bj52+aEMPF6zPeZhLNafr2HqWrNmzeaqmpit3dBPAidZCvwp8I6qejLJXpvOsO156VNVG4ANABMTEzU5OTlwbZddeS2Xbhn989zbzp7c5/5NmzYxzOPam2GeoAN453HPcukXvz3QsQv5JN1CjdewWvv5AgZ+Ehd6TwIPOl6zPeZhLNafr1HUNdTLQJMcSO+X/5VV9Zlu86N7pna67zu67duBVVMOXwk8PEz/kqTBDfMqoACXA/dW1R9N2XUdsLZbXgtcO2X7Od2rgU4CdlXVI4P2L0kazjD3r68D3gJsSXJXt+1dwCXA1UnOBx4E3tztuxE4DdgKPAWcN0TfkqQhDRwA3ZO5e5vwP3mG9gW8ddD+JEnzy4+CkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrUknEXIEmL1er1N4yt74+d8uIF78M7AElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGjXyAEhySpL7kmxNsn7U/UuSekYaAEkOAP4rcCpwDPBrSY4ZZQ2SpJ5R3wGcCGytqq9V1XeBjcAZI65BkgSkqkbXWfIm4JSq+o1u/S3Aa6vqbVParAPWdauvBu4bossjgceGOH6hWFd/rKs/1tWf/bGuV1TVy2ZrNOrPAsoM2/5eAlXVBmDDvHSW3FFVE/NxrvlkXf2xrv5YV39armvUU0DbgVVT1lcCD4+4BkkSow+ALwFHJ3llkoOAs4DrRlyDJIkRTwFV1TNJ3gZ8DjgA+GhVfWUBu5yXqaQFYF39sa7+WFd/mq1rpE8CS5IWD98JLEmNMgAkqVEv+ACY7aMlkhyc5Kpu/+1JVi+Sus5N8o0kd3VfvzGiuj6aZEeSe/ayP0k+0NV9d5LjF0ldk0l2TRmv/zCiulYluTXJvUm+kuSCGdqMfMzmWNfIxyzJIUn+b5K/7Op69wxtRn5NzrGucV2TByT5cpLrZ9i3sGNVVS/YL3pPJD8A/APgIOAvgWOmtfk3wIe65bOAqxZJXecC/2UMY/bzwPHAPXvZfxrwWXrv2TgJuH2R1DUJXD+G8ToKOL5bfgnw1zP8W458zOZY18jHrBuDpd3ygcDtwEnT2ozjmpxLXeO6Ji8E/mSmf6uFHqsX+h3AXD5a4gzgim75GuDkJDO9IW3UdY1FVX0B2LmPJmcAH6+e24BlSY5aBHWNRVU9UlV3dsvfAu4FVkxrNvIxm2NdI9eNwe5u9cDua/orTUZ+Tc6xrpFLshI4HfjIXpos6Fi90ANgBfD1Kevbef5F8IM2VfUMsAs4YhHUBfCr3ZTBNUlWzbB/HOZa+zj8THcL/9kk/3DUnXe33z9N76/HqcY6ZvuoC8YwZt2Uxl3ADuCmqtrreI3wmpxLXTD6a/J9wL8Dvr+X/Qs6Vi/0AJj1oyXm2Ga+zaXPPwdWV9U/Av4Hz6X8uI1jvObiTnqfb/JTwGXAn42y8yRLgT8F3lFVT07fPcMhIxmzWeoay5hV1bNV9Rp67/Q/Mcmx05qMZbzmUNdIr8kkbwB2VNXmfTWbYdu8jdULPQDm8tESP2iTZAlwGAs/1TBrXVX1zar6u271w8AJC1zTXC3Kj+uoqif33MJX1Y3AgUmOHEXfSQ6k90v2yqr6zAxNxjJms9U1zjHr+nwC2AScMm3XOK7JWesawzX5OuCNSbbRmyZ+fZJPTmuzoGP1Qg+AuXy0xHXA2m75TcAt1T2jMs66ps0Rv5HeHO5icB1wTvfKlpOAXVX1yLiLSvKje+Y+k5xI72f3myPoN8DlwL1V9Ud7aTbyMZtLXeMYsyQvS7KsWz4U+AXgr6Y1G/k1OZe6Rn1NVtVvV9XKqlpN73fELVX169OaLehYjfrTQOdV7eWjJZL8LnBHVV1H7yL5RJKt9JLzrEVS19uTvBF4pqvr3IWuCyDJp+i9OuTIJNuBi+k9IUZVfQi4kd6rWrYCTwHnLZK63gT86yTPAE8DZ40gyKH3V9pbgC3d/DHAu4Afm1LbOMZsLnWNY8yOAq5I7z9/+iHg6qq6ftzX5BzrGss1Od0ox8qPgpCkRr3Qp4AkSQMyACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKj/j8y6s+n64eihAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../output/submission/svr-0903_05-26-03-5-fold_logits_avg_test.csv\n"
     ]
    }
   ],
   "source": [
    "# 5 test feature then avg\n",
    "\n",
    "results = []\n",
    "for b3, b4, b5 in zip(b3_test_logits_list, b4_test_logits_list, b5_test_logits_list):\n",
    "    X_test = np.concatenate([b3, b4, b5], axis=1)\n",
    "    res = svr.predict(X_test)\n",
    "    results.append(res)\n",
    "\n",
    "avg_res_svr = np.average(results, axis=0)\n",
    "np.save(\"../output/submission/{}-5-fold_logits_avg_test_logits.npy\".format(model_save_name), avg_res_svr)\n",
    "y_pred = np.round(avg_res_svr)\n",
    "\n",
    "test_df.diagnosis = y_pred.astype(int)\n",
    "\n",
    "test_df.hist()\n",
    "plt.show()\n",
    "\n",
    "submition_filename = \"../output/submission/{}-5-fold_logits_avg_test.csv\".format(model_save_name)\n",
    "test_df.to_csv(submition_filename, index=False)\n",
    "print(submition_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-06T19:46:37.450365Z",
     "start_time": "2019-09-06T19:46:37.318254Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFWNJREFUeJzt3X+43nV93/Hnq4RfEksQbIpJatwls3NktXCGtK69TkyvFtAKa3WjoxK46HJt04rTXiP1ujZq2224q9QfbJdeUZxRqQGpa1LEKQMyZzeYBKkBqSVYpgFKxEA0glX0vT/ub+T0cJJz7vs+574P+Twf13Wu8/3x+X4/7/uTfM/rfD/3j5OqQpLUnh8ZdwGSpPEwACSpUQaAJDXKAJCkRhkAktQoA0CSGmUA6LCW5ENJfj/JzyX58rjrOZgkb0vygXHXobYsGXcB0ihU1f8CXjLuOg6mqv7DuGtQe7wDkKRGGQA6rCT56SR3JvlWkmuBY7rtk0l2T2m3Mcn9XbsvJfnHU/YdkeTKJI8m+askb0xSSZZ0+7cn+b0kf9Yd/5kkJ005/jVJ7knyeNf2703Zd1mSB7vjvpxkXbf9d5J8tFs+JslHk3yjO8fnkyxf8MFTcwwAHTaSHAX8CfAR4HnAx4FfPUjz+4GfA44H3g58NMnJ3b5/DpwNvAw4DThvhuP/GXAx8GPAUcBvdTX8XeBjwJuB5wM3An+a5KgkLwHeCPzDqnou8EvAAzOce31X1yrgROBfAE/OZQykfhgAOpycCRwJvKuqvldV1wOfn6lhVX28qh6qqh9U1bXAfcAZ3e5/Ary7qnZX1WPAFTOc4r9W1V9W1ZPAdfTCAuCfAp+sqpuq6nvAHwDHAj8LfB84GnhpkiOr6oGqun+Gc3+P3g/+F1fV96tqR1V9s//hkA7NANDh5AXAg/W3P+Hw/83UMMmFSe7qplgeB04FDkzjvAD42pTmX3vGCeCvpyw/ASydcuwP+6yqH3THr6iqXfTuDH4H2JNkS5IXzHDujwCfBrYkeSjJf0py5IyPWBqCAaDDycPAiiSZsu0npjdK8kLg/fSmY06sqmXA3cCB4x4GVk45ZFUfNTwEvHBKX+mOfxCgqv6oqv5R16aAd0w/QXf38vaqeim9O4dXAxf2UYM0JwaADif/B3gKeFOSJUl+haendaY6jt4P368DJLmY3h3AAdcBlyZZkWQZcFkfNVwHvCrJuu639rcCfwP87yQvSfLKJEcD36E3r//96SdIsjbJmiRHAN+kNyX0jHbSsAwAHTaq6rvArwAXAY/Rm4//xAztvgRcSS8wHgHWAH82pcn7gc8AXwS+QO+J3KeYww/hqvoy8OvAVcCjwC8Dv9zVdjS95xMepTeF9GPA22Y4zY8D19P74X8v8D+Bj87Wt9Sv+AdhpENLcjbwvqp64ayNpWcR7wCkaZIcm+ScbhppBXA58N/GXZc037wDkKZJ8hx60y4/SW+e/pPApb4UU4cbA0CSGuUUkCQ1alF/GuhJJ51Uq1evHvj4b3/72xx33HHzV9A8sa7+WFd/rKs/h2NdO3bseLSqnj9rw6patF+nn356DePWW28d6viFYl39sa7+WFd/Dse6gDtqDj9jnQKSpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGLeqPgpAWs50P7uOijZ8ceb8PXPGqkfepw5N3AJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjZg2AJB9MsifJ3VO2PS/JTUnu676f0G1Pkvck2ZXki0lOm3LM+q79fUnWL8zDkSTN1VzuAD4EnDVt20bg5qo6Bbi5Wwc4Gzil+9oAvBd6gQFcDrwcOAO4/EBoSJLGY9YAqKrPAnunbT4X2NwtbwbOm7L9w9VzG7AsycnALwE3VdXeqnoMuIlnhookaYRSVbM3SlYDN1TVqd3641W1bMr+x6rqhCQ3AFdU1ee67TcDlwGTwDFV9fvd9n8LPFlVfzBDXxvo3T2wfPny07ds2TLwg9u/fz9Lly4d+PiFYl39Wax17dm7j0eeHH2/a1Ycf8j9i3W8rKs/w9S1du3aHVU1MVu7JQOd/eAyw7Y6xPZnbqzaBGwCmJiYqMnJyYGL2b59O8Mcv1Csqz+Lta6rrtnKlTvn+xKa3QMXTB5y/2IdL+vqzyjqGvRVQI90Uzt03/d023cDq6a0Wwk8dIjtkqQxGTQAtgEHXsmzHtg6ZfuF3auBzgT2VdXDwKeBX0xyQvfk7y922yRJYzLr/WuSj9Gbwz8pyW56r+a5ArguySXAV4HXdc1vBM4BdgFPABcDVNXeJL8HfL5r97tVNf2JZUnSCM0aAFX1awfZtW6GtgW84SDn+SDwwb6qkyQtGN8JLEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYNFQBJ/nWSe5LcneRjSY5J8qIktye5L8m1SY7q2h7dre/q9q+ejwcgSRrMwAGQZAXwJmCiqk4FjgDOB94BvLOqTgEeAy7pDrkEeKyqXgy8s2snSRqTYaeAlgDHJlkCPAd4GHglcH23fzNwXrd8brdOt39dkgzZvyRpQKmqwQ9OLgX+PfAk8BngUuC27rd8kqwCPlVVpya5GzirqnZ3++4HXl5Vj0475wZgA8Dy5ctP37Jly8D17d+/n6VLlw58/EKxrv4s1rr27N3HI0+Ovt81K44/5P7FOl7W1Z9h6lq7du2OqpqYrd2Sgc4OJDmB3m/1LwIeBz4OnD1D0wMJM9Nv+89In6raBGwCmJiYqMnJyUFLZPv27Qxz/EKxrv4s1rquumYrV+4c+BIa2AMXTB5y/2IdL+vqzyjqGmYK6BeAv6qqr1fV94BPAD8LLOumhABWAg91y7uBVQDd/uOBvUP0L0kawjAB8FXgzCTP6eby1wFfAm4FXtu1WQ9s7Za3det0+2+pYeafJElDGTgAqup2ek/m3gns7M61CbgMeEuSXcCJwNXdIVcDJ3bb3wJsHKJuSdKQhprArKrLgcunbf4KcMYMbb8DvG6Y/iRJ88d3AktSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRQwVAkmVJrk/yF0nuTfIzSZ6X5KYk93XfT+jaJsl7kuxK8sUkp83PQ5AkDWLYO4B3A/+9qn4S+CngXmAjcHNVnQLc3K0DnA2c0n1tAN47ZN+SpCEMHABJfhT4eeBqgKr6blU9DpwLbO6abQbO65bPBT5cPbcBy5KcPHDlkqShpKoGOzB5GbAJ+BK93/53AJcCD1bVsintHquqE5LcAFxRVZ/rtt8MXFZVd0w77wZ6dwgsX7789C1btgxUH8D+/ftZunTpwMcvFOvqz2Kta8/efTzy5Oj7XbPi+EPuX6zjZV39GaautWvX7qiqidnaLRno7E8fexrwm1V1e5J38/R0z0wyw7ZnpE9VbaIXLExMTNTk5OTABW7fvp1hjl8o1tWfxVrXVdds5cqdw1xCg3nggslD7l+s42Vd/RlFXcM8B7Ab2F1Vt3fr19MLhEcOTO103/dMab9qyvErgYeG6F+SNISBA6Cq/hr4WpKXdJvW0ZsO2gas77atB7Z2y9uAC7tXA50J7KuqhwftX5I0nGHvX38TuCbJUcBXgIvphcp1SS4Bvgq8rmt7I3AOsAt4omsrSRqToQKgqu4CZnqiYd0MbQt4wzD9SZLmj+8ElqRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaNXQAJDkiyReS3NCtvyjJ7UnuS3JtkqO67Ud367u6/auH7VuSNLj5uAO4FLh3yvo7gHdW1SnAY8Al3fZLgMeq6sXAO7t2kqQxGSoAkqwEXgV8oFsP8Erg+q7JZuC8bvncbp1u/7quvSRpDFJVgx+cXA/8R+C5wG8BFwG3db/lk2QV8KmqOjXJ3cBZVbW723c/8PKqenTaOTcAGwCWL19++pYtWwaub//+/SxdunTg4xeKdfVnsda1Z+8+Hnly9P2uWXH8Ifcv1vGyrv4MU9fatWt3VNXEbO2WDHR2IMmrgT1VtSPJ5IHNMzStOex7ekPVJmATwMTERE1OTk5vMmfbt29nmOMXinX1Z7HWddU1W7ly58CX0MAeuGDykPsX63hZV39GUdcw/3tfAbwmyTnAMcCPAu8CliVZUlVPASuBh7r2u4FVwO4kS4Djgb1D9C9JGsLAzwFU1W9X1cqqWg2cD9xSVRcAtwKv7ZqtB7Z2y9u6dbr9t9Qw80+SpKEsxPsALgPekmQXcCJwdbf9auDEbvtbgI0L0LckaY7mZQKzqrYD27vlrwBnzNDmO8Dr5qM/SdLwfCewJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1avQfZThCOx/cx0UbPznyfh+44lUj71OS+uUdgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRg0cAElWJbk1yb1J7klyabf9eUluSnJf9/2EbnuSvCfJriRfTHLafD0ISVL/hvmbwE8Bb62qO5M8F9iR5CbgIuDmqroiyUZgI3AZcDZwSvf1cuC93XfNk9VD/v3jt655auC/oezfQZaefQa+A6iqh6vqzm75W8C9wArgXGBz12wzcF63fC7w4eq5DViW5OSBK5ckDSVVNfxJktXAZ4FTga9W1bIp+x6rqhOS3ABcUVWf67bfDFxWVXdMO9cGYAPA8uXLT9+yZcvAde3Zu49Hnhz48IGtWXH8Iffv37+fpUuXznu/Ox/cN9Txy49l4PGa7TEPY6HGa1it/f8alnX1Z5i61q5du6OqJmZrN8wUEABJlgJ/DLy5qr6Z5KBNZ9j2jPSpqk3AJoCJiYmanJwcuLarrtnKlTuHfoh9e+CCyUPu3759O8M8roMZdPrmgLeueWrg8ZrtMQ9jocZrWK39/4Lhphnfuub7XPm5bw907EJOMS7W/1+jqGuoVwElOZLeD/9rquoT3eZHDkztdN/3dNt3A6umHL4SeGiY/iVJgxvmVUABrgburao/nLJrG7C+W14PbJ2y/cLu1UBnAvuq6uFB+5ckDWeY+9dXAK8Hdia5q9v2NuAK4LoklwBfBV7X7bsROAfYBTwBXDxE35KkIQ0cAN2TuQeb8F83Q/sC3jBof5Kk+eU7gSWpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRw/xJSEk6rK3e+Mmx9f2hs45b8D68A5CkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUqJEHQJKzknw5ya4kG0fdvySpZ6QBkOQI4L8AZwMvBX4tyUtHWYMkqWfUdwBnALuq6itV9V1gC3DuiGuQJAGpqtF1lrwWOKuqfqNbfz3w8qp645Q2G4AN3epLgC8P0eVJwKNDHL9QrKs/1tUf6+rP4VjXC6vq+bM1GvXHQWeGbX8rgapqE7BpXjpL7qiqifk413yyrv5YV3+sqz8t1zXqKaDdwKop6yuBh0ZcgySJ0QfA54FTkrwoyVHA+cC2EdcgSWLEU0BV9VSSNwKfBo4APlhV9yxgl/MylbQArKs/1tUf6+pPs3WN9ElgSdLi4TuBJalRBoAkNepZHwCzfbREkqOTXNvtvz3J6kVS10VJvp7kru7rN0ZU1weT7Ely90H2J8l7urq/mOS0RVLXZJJ9U8br342orlVJbk1yb5J7klw6Q5uRj9kc6xr5mCU5Jsn/TfLnXV1vn6HNyK/JOdY1rmvyiCRfSHLDDPsWdqyq6ln7Re+J5PuBvwMcBfw58NJpbf4V8L5u+Xzg2kVS10XAfx7DmP08cBpw90H2nwN8it57Ns4Ebl8kdU0CN4xhvE4GTuuWnwv85Qz/liMfsznWNfIx68Zgabd8JHA7cOa0NuO4JudS17iuybcAfzTTv9VCj9Wz/Q5gLh8tcS6wuVu+HliXZKY3pI26rrGoqs8Cew/R5Fzgw9VzG7AsycmLoK6xqKqHq+rObvlbwL3AimnNRj5mc6xr5Lox2N+tHtl9TX+lycivyTnWNXJJVgKvAj5wkCYLOlbP9gBYAXxtyvpunnkR/LBNVT0F7ANOXAR1AfxqN2VwfZJVM+wfh7nWPg4/093CfyrJ3x91593t90/T++1xqrGO2SHqgjGMWTelcRewB7ipqg46XiO8JudSF4z+mnwX8G+AHxxk/4KO1bM9AGb9aIk5tplvc+nzT4HVVfUPgP/B0yk/buMYr7m4k97nm/wUcBXwJ6PsPMlS4I+BN1fVN6fvnuGQkYzZLHWNZcyq6vtV9TJ67/Q/I8mp05qMZbzmUNdIr8kkrwb2VNWOQzWbYdu8jdWzPQDm8tESP2yTZAlwPAs/1TBrXVX1jar6m271/cDpC1zTXC3Kj+uoqm8euIWvqhuBI5OcNIq+kxxJ74fsNVX1iRmajGXMZqtrnGPW9fk4sB04a9qucVyTs9Y1hmvyFcBrkjxAb5r4lUk+Oq3Ngo7Vsz0A5vLREtuA9d3ya4FbqntGZZx1TZsjfg29OdzFYBtwYffKljOBfVX18LiLSvLjB+Y+k5xB7//uN0bQb4CrgXur6g8P0mzkYzaXusYxZkmen2RZt3ws8AvAX0xrNvJrci51jfqarKrfrqqVVbWa3s+IW6rq16c1W9CxGvWngc6rOshHSyT5XeCOqtpG7yL5SJJd9JLz/EVS15uSvAZ4qqvrooWuCyDJx+i9OuSkJLuBy+k9IUZVvQ+4kd6rWnYBTwAXL5K6Xgv8yyRPAU8C548gyKH3W9rrgZ3d/DHA24CfmFLbOMZsLnWNY8xOBjan98effgS4rqpuGPc1Oce6xnJNTjfKsfKjICSpUc/2KSBJ0oAMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktSo/w8eHtGfMJ55TQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../output/submission/cb-0906_06-09-42-5-fold_avg_logits_test.csv\n"
     ]
    }
   ],
   "source": [
    "b3_test_avg_feats = np.average(b3_test_logits_list, axis=0)\n",
    "b4_test_avg_feats = np.average(b4_test_logits_list, axis=0)\n",
    "b5_test_avg_feats = np.average(b5_test_logits_list, axis=0)\n",
    "\n",
    "\n",
    "X_test = np.concatenate([b3_test_avg_feats, b4_test_avg_feats, b5_test_avg_feats], axis=1)\n",
    "y_pred = cb.predict(X_test)\n",
    "\n",
    "y_pred = np.round(y_pred)\n",
    "\n",
    "test_df.diagnosis = y_pred.astype(int)\n",
    "\n",
    "test_df.hist()\n",
    "plt.show()\n",
    "\n",
    "submition_filename = \"../output/submission/{}-5-fold_avg_logits_test.csv\".format(model_save_name)\n",
    "test_df.to_csv(submition_filename, index=False)\n",
    "print(submition_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-06T19:46:42.960870Z",
     "start_time": "2019-09-06T19:46:42.824925Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFW9JREFUeJzt3X+0XWV95/H3p4QfSixBsCkmqXGWjB2HTC3cQVqnXTfSVQEtcVqdoUMlsOhkzVQrDnaV1LVmqG1nFs4q9QczS1cURxRqQOoUijjKIBnHTmU0agmYWoLNSIASMSEawWr0O3+cjd5ebnLvOefec27yvF9r3ZW99/Psvb/nyd33c/Zzzj03VYUkqT0/Mu4CJEnjYQBIUqMMAElqlAEgSY0yACSpUQaAJDXKANARLcn7k/xBkp9L8uVx13MwSd6c5L3jrkNtWTLuAqRRqKr/Dbxw3HUcTFX9p3HXoPZ4ByBJjTIAdERJ8tNJPp/km0luBI7rtk8m2TWl38YkD3T9vpTkn09pOyrJ1UkeS/I3SV6fpJIs6dq3JPn9JH/e7f+JJCdP2f/8JPclebzr+4+mtF2R5KFuvy8nObvb/rtJru+Wj0tyfZKvd8f4bJLlCz54ao4BoCNGkmOAPwU+CDwb+DDwKwfp/gDwc8AJwFuA65Oc0rX9a+Bc4MXA6cCrZtj/XwGXAD8GHAP8VlfDPwQ+BLwReA5wO/BnSY5J8kLg9cA/rapnAS8Hds5w7PVdXauAk4B/Azw5lzGQ+mEA6EhyFnA08Paq+m5V3Qx8dqaOVfXhqnq4qr5fVTcC9wNnds3/AnhHVe2qqr3AVTMc4r9V1V9X1ZPATfTCAuBfAh+tqjuq6rvAHwLPAH4W+B5wLPCiJEdX1c6qemCGY3+X3g/+F1TV96pqa1V9o//hkA7NANCR5LnAQ/X3P+Hw/83UMclFSb7YTbE8DpwGPDWN81zgwSndH3zaAeBvpyw/ASydsu8PzllV3+/2X1FVO+jdGfwusDvJ5iTPneHYHwQ+DmxO8nCS/5zk6BkfsTQEA0BHkkeAFUkyZdtPTO+U5HnAe+hNx5xUVcuAe4Gn9nsEWDlll1V91PAw8Lwp50q3/0MAVfXHVfXPuj4FvHX6Abq7l7dU1Yvo3Tm8EriojxqkOTEAdCT5C+AA8IYkS5L8Mj+c1pnqeHo/fL8GkOQSencAT7kJuCzJiiTLgCv6qOEm4BVJzu6etb8J+Dvg/yR5YZKXJTkW+Da9ef3vTT9AkrVJ1iQ5CvgGvSmhp/WThmUA6IhRVd8Bfhm4GNhLbz7+IzP0+xJwNb3AeBRYA/z5lC7vAT4B3AN8gd4LuQeYww/hqvoy8GvANcBjwC8Bv9TVdiy91xMeozeF9GPAm2c4zI8DN9P74b8d+F/A9bOdW+pX/IMw0qElORd4d1U9b9bO0mHEOwBpmiTPSHJeN420ArgS+O/jrkuab94BSNMkeSa9aZefpDdP/1HgMt+KqSONASBJjXIKSJIatag/DfTkk0+u1atXD7z/t771LY4//vj5K2ieWFd/rKs/1tWfI7GurVu3PlZVz5m1Y1Ut2q8zzjijhnHXXXcNtf9Csa7+WFd/rKs/R2JdwOdqDj9jnQKSpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGLeqPgpAWs20P7ePijR8d+Xl3XvWKkZ9TRybvACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1KhZAyDJ+5LsTnLvlG3PTnJHkvu7f0/stifJO5PsSHJPktOn7LO+639/kvUL83AkSXM1lzuA9wPnTNu2Ebizqk4F7uzWAc4FTu2+NgDvgl5gAFcCLwHOBK58KjQkSeMxawBU1aeAPdM2rwOu65avA141ZfsHquczwLIkpwAvB+6oqj1VtRe4g6eHiiRphFJVs3dKVgO3VdVp3frjVbVsSvveqjoxyW3AVVX16W77ncAVwCRwXFX9Qbf93wNPVtUfznCuDfTuHli+fPkZmzdvHvjB7d+/n6VLlw68/0Kxrv4s1rp279nHo0+O/rxrVpxwyPbFOl7W1Z9h6lq7du3WqpqYrd+SgY5+cJlhWx1i+9M3Vm0CNgFMTEzU5OTkwMVs2bKFYfZfKNbVn8Va1zU33MLV2+b7EprdzgsnD9m+WMfLuvoziroGfRfQo93UDt2/u7vtu4BVU/qtBB4+xHZJ0pgMGgC3Ak+9k2c9cMuU7Rd17wY6C9hXVY8AHwd+McmJ3Yu/v9htkySNyaz3r0k+RG8O/+Qku+i9m+cq4KYklwJfBV7Tdb8dOA/YATwBXAJQVXuS/D7w2a7f71XV9BeWJUkjNGsAVNWvHqTp7Bn6FvC6gxznfcD7+qpOkrRg/E1gSWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRQwVAkn+X5L4k9yb5UJLjkjw/yd1J7k9yY5Jjur7Hdus7uvbV8/EAJEmDGTgAkqwA3gBMVNVpwFHABcBbgbdV1anAXuDSbpdLgb1V9QLgbV0/SdKYDDsFtAR4RpIlwDOBR4CXATd37dcBr+qW13XrdO1nJ8mQ55ckDShVNfjOyWXAfwSeBD4BXAZ8pnuWT5JVwMeq6rQk9wLnVNWuru0B4CVV9di0Y24ANgAsX778jM2bNw9c3/79+1m6dOnA+y8U6+rPYq1r9559PPrk6M+7ZsUJh2xfrONlXf0Zpq61a9duraqJ2fotGejoQJIT6T2rfz7wOPBh4NwZuj6VMDM9239a+lTVJmATwMTERE1OTg5aIlu2bGGY/ReKdfVnsdZ1zQ23cPW2gS+hge28cPKQ7Yt1vKyrP6Ooa5gpoF8A/qaqvlZV3wU+AvwssKybEgJYCTzcLe8CVgF07ScAe4Y4vyRpCMMEwFeBs5I8s5vLPxv4EnAX8Oquz3rglm751m6drv2TNcz8kyRpKAMHQFXdTe/F3M8D27pjbQKuAC5PsgM4Cbi22+Va4KRu++XAxiHqliQNaagJzKq6Erhy2uavAGfO0PfbwGuGOZ8kaf74m8CS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElq1FABkGRZkpuT/FWS7Ul+Jsmzk9yR5P7u3xO7vknyziQ7ktyT5PT5eQiSpEEMewfwDuB/VNVPAj8FbAc2AndW1anAnd06wLnAqd3XBuBdQ55bkjSEgQMgyY8CPw9cC1BV36mqx4F1wHVdt+uAV3XL64APVM9ngGVJThm4cknSUFJVg+2YvBjYBHyJ3rP/rcBlwENVtWxKv71VdWKS24CrqurT3fY7gSuq6nPTjruB3h0Cy5cvP2Pz5s0D1Qewf/9+li5dOvD+C8W6+rNY69q9Zx+PPjn6865ZccIh2xfreFlXf4apa+3atVuramK2fksGOvoP9z0d+M2qujvJO/jhdM9MMsO2p6VPVW2iFyxMTEzU5OTkwAVu2bKFYfZfKNbVn8Va1zU33MLV24a5hAaz88LJQ7Yv1vGyrv6Moq5hXgPYBeyqqru79ZvpBcKjT03tdP/untJ/1ZT9VwIPD3F+SdIQBg6Aqvpb4MEkL+w2nU1vOuhWYH23bT1wS7d8K3BR926gs4B9VfXIoOeXJA1n2PvX3wRuSHIM8BXgEnqhclOSS4GvAq/p+t4OnAfsAJ7o+kqSxmSoAKiqLwIzvdBw9gx9C3jdMOeTJM0ffxNYkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWrU0AGQ5KgkX0hyW7f+/CR3J7k/yY1Jjum2H9ut7+jaVw97bknS4ObjDuAyYPuU9bcCb6uqU4G9wKXd9kuBvVX1AuBtXT9J0pgMFQBJVgKvAN7brQd4GXBz1+U64FXd8rpuna797K6/JGkMhr0DeDvw28D3u/WTgMer6kC3vgtY0S2vAB4E6Nr3df0lSWOQqhpsx+SVwHlV9RtJJoHfAi4B/qKb5iHJKuD2qlqT5D7g5VW1q2t7ADizqr4+7bgbgA0Ay5cvP2Pz5s2DPTJg//79LF26dOD9F4p19Wex1rV7zz4efXL0512z4oRDti/W8bKu/gxT19q1a7dW1cRs/ZYMdPSelwLnJzkPOA74UXp3BMuSLOme5a8EHu767wJWAbuSLAFOAPZMP2hVbQI2AUxMTNTk5OTABW7ZsoVh9l8o1tWfxVrXNTfcwtXbhrmEBrPzwslDti/W8bKu/oyiroGngKrqd6pqZVWtBi4APllVFwJ3Aa/uuq0HbumWb+3W6do/WYPefkiShrYQvwdwBXB5kh305viv7bZfC5zUbb8c2LgA55YkzdG83L9W1RZgS7f8FeDMGfp8G3jNfJxPkjQ8fxNYkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqNG/1GGI7TtoX1cvPGjIz/vzqteMfJzSlK/vAOQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRg0cAElWJbkryfYk9yW5rNv+7CR3JLm/+/fEbnuSvDPJjiT3JDl9vh6EJKl/w/xJyAPAm6rq80meBWxNcgdwMXBnVV2VZCOwEbgCOBc4tft6CfCu7l/Nk9VD/vnLN605MPCf0PTPYEqHn4EDoKoeAR7plr+ZZDuwAlgHTHbdrgO20AuAdcAHqqqAzyRZluSU7jiSDgPDPMnwCcbik97P4yEPkqwGPgWcBny1qpZNadtbVScmuQ24qqo+3W2/E7iiqj437VgbgA0Ay5cvP2Pz5s0D17V7zz4efXLg3Qe2ZsUJh2zfv38/S5cunffzbnto31D7L38GA4/XbI95GAs1XsNq7fsLhvse8/urP8PUtXbt2q1VNTFbv2GmgABIshT4E+CNVfWNJAftOsO2p6VPVW0CNgFMTEzU5OTkwLVdc8MtXL1t6IfYt50XTh6yfcuWLQzzuA5m0GdXT3nTmgMDj9dsj3kYCzVew2rt+wuG+x7z+6s/o6hrqHcBJTma3g//G6rqI93mR5Oc0rWfAuzutu8CVk3ZfSXw8DDnlyQNbph3AQW4FtheVX80pelWYH23vB64Zcr2i7p3A50F7HP+X5LGZ5j715cCrwW2Jflit+3NwFXATUkuBb4KvKZrux04D9gBPAFcMsS5JUlDGuZdQJ9m5nl9gLNn6F/A6wY9nyRpfvmbwJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjhvmj8JJ0RFu98aNjO/f7zzl+wc/hHYAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUSMPgCTnJPlykh1JNo76/JKknpEGQJKjgP8KnAu8CPjVJC8aZQ2SpJ5R3wGcCeyoqq9U1XeAzcC6EdcgSQJSVaM7WfJq4Jyq+vVu/bXAS6rq9VP6bAA2dKsvBL48xClPBh4bYv+FYl39sa7+WFd/jsS6nldVz5mt06g/DC4zbPt7CVRVm4BN83Ky5HNVNTEfx5pP1tUf6+qPdfWn5bpGPQW0C1g1ZX0l8PCIa5AkMfoA+CxwapLnJzkGuAC4dcQ1SJIY8RRQVR1I8nrg48BRwPuq6r4FPOW8TCUtAOvqj3X1x7r602xdI30RWJK0ePibwJLUKANAkhp12AfAbB8tkeTYJDd27XcnWb1I6ro4ydeSfLH7+vUR1fW+JLuT3HuQ9iR5Z1f3PUlOXyR1TSbZN2W8/sOI6lqV5K4k25Pcl+SyGfqMfMzmWNfIxyzJcUn+b5K/7Op6ywx9Rn5NzrGucV2TRyX5QpLbZmhb2LGqqsP2i94LyQ8A/wA4BvhL4EXT+vwG8O5u+QLgxkVS18XAfxnDmP08cDpw70HazwM+Ru93Ns4C7l4kdU0Ct41hvE4BTu+WnwX89Qz/lyMfsznWNfIx68Zgabd8NHA3cNa0PuO4JudS17iuycuBP57p/2qhx+pwvwOYy0dLrAOu65ZvBs5OMtMvpI26rrGoqk8Bew7RZR3wger5DLAsySmLoK6xqKpHqurz3fI3ge3AimndRj5mc6xr5Lox2N+tHt19TX+nycivyTnWNXJJVgKvAN57kC4LOlaHewCsAB6csr6Lp18EP+hTVQeAfcBJi6AugF/ppgxuTrJqhvZxmGvt4/Az3S38x5L841GfvLv9/ml6zx6nGuuYHaIuGMOYdVMaXwR2A3dU1UHHa4TX5FzqgtFfk28Hfhv4/kHaF3SsDvcAmPWjJebYZ77N5Zx/Bqyuqn8C/E9+mPLjNo7xmovP0/t8k58CrgH+dJQnT7IU+BPgjVX1jenNM+wykjGbpa6xjFlVfa+qXkzvN/3PTHLatC5jGa851DXSazLJK4HdVbX1UN1m2DZvY3W4B8BcPlriB32SLAFOYOGnGmatq6q+XlV/162+BzhjgWuaq0X5cR1V9Y2nbuGr6nbg6CQnj+LcSY6m90P2hqr6yAxdxjJms9U1zjHrzvk4sAU4Z1rTOK7JWesawzX5UuD8JDvpTRO/LMn10/os6Fgd7gEwl4+WuBVY3y2/Gvhkda+ojLOuaXPE59Obw10MbgUu6t7Zchawr6oeGXdRSX78qbnPJGfS+979+gjOG+BaYHtV/dFBuo18zOZS1zjGLMlzkizrlp8B/ALwV9O6jfyanEtdo74mq+p3qmplVa2m9zPik1X1a9O6LehYjfrTQOdVHeSjJZL8HvC5qrqV3kXywSQ76CXnBYukrjckOR840NV18ULXBZDkQ/TeHXJykl3AlfReEKOq3g3cTu9dLTuAJ4BLFkldrwb+bZIDwJPABSMIcug9S3stsK2bPwZ4M/ATU2obx5jNpa5xjNkpwHXp/fGnHwFuqqrbxn1NzrGusVyT041yrPwoCElq1OE+BSRJGpABIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhr1/wGlbuEw1dARtwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../output/submission/cb-0906_06-09-42-5-fold_logits_avg_test.csv\n"
     ]
    }
   ],
   "source": [
    "# 5 test feature then avg\n",
    "\n",
    "results = []\n",
    "for b3, b4, b5 in zip(b3_test_logits_list, b4_test_logits_list, b5_test_logits_list):\n",
    "    X_test = np.concatenate([b3, b4, b5], axis=1)\n",
    "    res = cb.predict(X_test)\n",
    "    results.append(res)\n",
    "\n",
    "avg_res_svr = np.average(results, axis=0)\n",
    "np.save(\"../output/submission/{}-5-fold_logits_avg_test_logits.npy\".format(model_save_name), avg_res_svr)\n",
    "y_pred = np.round(avg_res_svr)\n",
    "\n",
    "test_df.diagnosis = y_pred.astype(int)\n",
    "\n",
    "test_df.hist()\n",
    "plt.show()\n",
    "\n",
    "submition_filename = \"../output/submission/{}-5-fold_logits_avg_test.csv\".format(model_save_name)\n",
    "test_df.to_csv(submition_filename, index=False)\n",
    "print(submition_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlation Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-04T07:53:26.700102Z",
     "start_time": "2019-09-04T07:53:26.693887Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.      , 0.998509, 0.997369],\n",
       "       [0.998509, 1.      , 0.997143],\n",
       "       [0.997369, 0.997143, 1.      ]])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.corrcoef([avg_res_gbm, avg_res_xlf, avg_res_svr])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "240.65px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}