submission_SML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **SML TEAM**\n",
    "## **Notes**\n",
    "```\n",
    "Requirements: \n",
    "* comet_ml, pip install comet_ml\n",
    "* tensorflow (tested 1.13.1)\n",
    "* scikit-learn\n",
    "* pandas\n",
    "* h5py\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Submission 01  \n",
    "- Overall Mean Square Error: **0.078661**  \n",
    "- Mean Square Error for Engagement level 0: 0.334221  \n",
    "- Mean Square Error for Engagement level 1: 0.083368  \n",
    "- Mean Square Error for Engagement level 2: 0.013338  \n",
    "- Mean Square Error for Engagement level 3: 0.066046\n",
    "\n",
    "### 2. Submission 02\n",
    "- Overall Mean Square Error: **0.091114** \n",
    "- Mean Square Error for Engagement level 0: 0.328931 \n",
    "- Mean Square Error for Engagement level 1: 0.108693 \n",
    "- Mean Square Error for Engagement level 2: 0.027030 \n",
    "- Mean Square Error for Engagement level 3: 0.035344\n",
    "\n",
    "### 3. Submission 03\n",
    "- Overall Mean Square Error: **0.069625** \n",
    "- Mean Square Error for Engagement level 0: 0.268581 \n",
    "- Mean Square Error for Engagement level 1: 0.064421 \n",
    "- Mean Square Error for Engagement level 2: 0.023098 \n",
    "- Mean Square Error for Engagement level 3: 0.064037 \n",
    "\n",
    "### 4. Submission 04  \n",
    "- Overall Mean Square Error: **0.062841** \n",
    "- Mean Square Error for Engagement level 0: 0.220366 \n",
    "- Mean Square Error for Engagement level 1: 0.040467 \n",
    "- Mean Square Error for Engagement level 2: 0.031986 \n",
    "- Mean Square Error for Engagement level 3: 0.102218 \n",
    "\n",
    "### 5. Submission 05\n",
    "- Overall Mean Square Error: **0.059678** \n",
    "- Mean Square Error for Engagement level 0: 0.246069 \n",
    "- Mean Square Error for Engagement level 1: 0.029697 \n",
    "- Mean Square Error for Engagement level 2: 0.022397 \n",
    "- Mean Square Error for Engagement level 3: 0.137770 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version:  1.13.1\n"
     ]
    }
   ],
   "source": [
    "import comet_ml\n",
    "from utils import EngagementDataset\n",
    "import md_config as cfg\n",
    "import os, time, glob\n",
    "import shutil\n",
    "\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
    "import numpy as np\n",
    "import random\n",
    "import joblib\n",
    "\n",
    "seed = 0\n",
    "np.random.seed(seed)\n",
    "random.seed(0)\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.2\n",
    "session = tf.Session(config=config)\n",
    "\n",
    "tf.set_random_seed(seed)\n",
    "\n",
    "import copy\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import CuDNNLSTM, Dense, TimeDistributed, GlobalAveragePooling1D, Activation, Concatenate, \\\n",
    "    InputLayer, PReLU\n",
    "from tensorflow.keras.optimizers import Adam, SGD, RMSprop\n",
    "from tensorflow.keras.callbacks import TensorBoard, ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\n",
    "from tensorflow.keras.activations import relu\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import class_weight\n",
    "import sklearn.utils as skutils\n",
    "import argparse\n",
    "import gc\n",
    "from sklearn.metrics import mean_squared_error, make_scorer\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from sklearn.model_selection import GridSearchCV, PredefinedSplit\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import h5py\n",
    "import pandas as pd\n",
    "\n",
    "print('Tensorflow version: ', tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_model(hparams, model_name):\n",
    "    current_n_lstms = hparams['NUM_LSTM_LAYERS']\n",
    "    current_lstm_units = hparams['LSTM_UNITS']\n",
    "    current_n_denses = hparams['NUM_DENSE_LAYERS']\n",
    "    current_dense_units = hparams['DENSE_UNITS']\n",
    "    current_dropout_rates = hparams['DROPOUT_RATES']\n",
    "    current_time_step = hparams['TIME_STEP']\n",
    "    current_input_units = hparams['INPUT_UNITS']\n",
    "    current_densen_act = hparams['ACTIVATION_F']\n",
    "\n",
    "    model = Sequential()\n",
    "    if hparams['FC1'][1] > 0:\n",
    "        model.add(TimeDistributed(Dense(hparams['FC1'][1], activation='relu'),\n",
    "                                  input_shape=(current_time_step, hparams['FC1'][0])))\n",
    "\n",
    "    model.add(\n",
    "        CuDNNLSTM(current_lstm_units[0], return_sequences=True, input_shape=(current_time_step, current_input_units),\n",
    "                  stateful=False))\n",
    "\n",
    "    if current_n_lstms > 1:\n",
    "        for idx in range(1, current_n_lstms):\n",
    "            model.add(CuDNNLSTM(current_lstm_units[idx], return_sequences=True))\n",
    "\n",
    "    for idx in range(current_n_denses):\n",
    "        model.add(TimeDistributed(Dense(current_dense_units[idx], activation='relu')))\n",
    "        # model.add(TimeDistributed(Dropout(0.3)))\n",
    "\n",
    "    model.add(TimeDistributed(Dense(1, activation=current_densen_act)))\n",
    "    model.add(GlobalAveragePooling1D())\n",
    "\n",
    "    return model\n",
    "\n",
    "def model_objective(hparams, train_set, val_set=None, test_set=None, model_name='', logdir='', best_score=1e+5):\n",
    "    experiment = comet_ml.Experiment(api_key='uG1BcicYOr83KvLjFEZQMrWVg',\n",
    "                                     project_name='Engagement_V6_{}'.format(model_name),\n",
    "                                     auto_param_logging=True, auto_output_logging=None, log_env_gpu=False,\n",
    "                                     log_env_host=False, log_env_details=True)\n",
    "    print(hparams)\n",
    "    tf.keras.backend.set_session(session)\n",
    "    tf.keras.backend.clear_session()  # Reset state\n",
    "    # tf.reset_default_graph()\n",
    "    np.random.seed(seed)\n",
    "    random.seed(0)\n",
    "    tf.random.set_random_seed(seed)\n",
    "\n",
    "    # sess = tf.Session(graph=tf.get_default_graph(), config=config)\n",
    "    # keras.backend.tensorflow_backend.set_session(sess)\n",
    "\n",
    "    if val_set is None:\n",
    "        X_train, X_val, y_train, y_val = train_test_split(train_set[0], train_set[1], test_size=0.2, random_state=42)\n",
    "    else:\n",
    "        X_train, y_train = train_set\n",
    "        X_val, y_val = val_set\n",
    "\n",
    "    batch_size = hparams['BATCH_SIZE']\n",
    "    epochs = hparams['EPOCHS']\n",
    "\n",
    "    # Determine INPUT UNITS, TIME_STEP\n",
    "    hparams['INPUT_UNITS'] = hparams['FC1'] if hparams['FC1'] > 0 else X_train.shape[2]\n",
    "    hparams['TIME_STEP'] = X_train.shape[1]\n",
    "\n",
    "    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=1.0 / 10, patience=5, min_lr=1e-6)\n",
    "\n",
    "    ckpt = ModelCheckpoint('{}/{}/{}_best_weight.h5'.format(logdir, model_name, experiment.get_key()[:10]),\n",
    "                           monitor='val_loss', save_best_only=True,\n",
    "                           save_weights_only=True, verbose=0)\n",
    "    ely_stop = EarlyStopping(monitor='val_loss', patience=10, min_delta=1e-6)\n",
    "\n",
    "    if hparams['CLSW'] == 1:\n",
    "        cls_weight = class_weight.compute_class_weight('balanced', np.unique(y_train), y_train)\n",
    "    else:\n",
    "        cls_weight = None\n",
    "\n",
    "    if hparams['optimizer'] == 'adam':\n",
    "        optim = Adam(hparams['learning_rate'])\n",
    "    elif hparams['optimizer'] == 'sgd':\n",
    "        optim = SGD(hparams['learning_rate'])\n",
    "    else:\n",
    "        optim = RMSprop(hparams['learning_rate'])\n",
    "\n",
    "    os.makedirs('{}/{}'.format(logdir, model_name), exist_ok=True)\n",
    "    current_model = define_model(hparams, model_name=ft_type)\n",
    "    # current_model.build()\n",
    "    current_model.compile(optimizer=optim, loss=keras.losses.MeanSquaredError(),\n",
    "                          metrics=[keras.metrics.mean_squared_error])\n",
    "\n",
    "    # Training\n",
    "    with experiment.train():\n",
    "        history = current_model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs,\n",
    "                                    validation_data=(X_val, y_val),\n",
    "                                    shuffle=True, callbacks=[ckpt, reduce_lr, ely_stop], verbose=0,\n",
    "                                    class_weight=cls_weight)\n",
    "\n",
    "        # Plot history\n",
    "        fig = plt.figure()\n",
    "        plt.plot(history.history['loss'], label='train')\n",
    "        plt.plot(history.history['val_loss'], label='test')\n",
    "        plt.legend()\n",
    "        fig.savefig('{}/{}/{}.png'.format(logdir, model_name, experiment.get_key()[:10]))\n",
    "        experiment.log_figure(figure_name=experiment.get_key()[:10], figure=fig)\n",
    "        plt.close()\n",
    "\n",
    "    # current_model.summary()\n",
    "    with experiment.test():\n",
    "        # Evaluation\n",
    "        current_model.load_weights('{}/{}/{}_best_weight.h5'.format(logdir, model_name, experiment.get_key()[:10]))\n",
    "        ret = current_model.evaluate(test_set[0], test_set[1], batch_size=batch_size, verbose=1)\n",
    "        y_pred = current_model.predict(test_set[0], batch_size=batch_size)\n",
    "\n",
    "        experiment.log_metrics({'MSE': ret[0], 'Min_Predict': np.min(y_pred), 'Max_Preidct': np.max(y_pred)})\n",
    "        hparams['MSE_score'] = ret\n",
    "        hparams['n_epochs_trained'] = len(history.history['loss'])\n",
    "        with open('{}/{}/{}.txt'.format(logdir, model_name, experiment.get_key()[:10]), 'w') as f:\n",
    "            f.write(str(hparams))\n",
    "\n",
    "    del current_model\n",
    "    # current_model = None\n",
    "    gc.collect()\n",
    "    print(hparams)\n",
    "    tf.keras.backend.clear_session()  # Reset state\n",
    "\n",
    "    experiment.end()\n",
    "    print('Completed experiment')\n",
    "    del experiment\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data loading, util functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(hparams, ft_type):\n",
    "    current_set = {}\n",
    "    current_scores = {}\n",
    "    tmp = np.load(\n",
    "        './dataset_npz/{}_{}/{}_feat_{}.npz'.format(hparams['n_segments'], hparams['alpha'], ft_type, 'Train'),\n",
    "        allow_pickle=True)\n",
    "    current_set['Train'], current_scores['Train'] = tmp['ft'].item()['Train'], tmp['sc'].item()['Train']\n",
    "\n",
    "    tmp2 = np.load(\n",
    "        './dataset_npz/{}_{}/{}_feat_{}.npz'.format(hparams['n_segments'], hparams['alpha'], ft_type, 'Validation'),\n",
    "        allow_pickle=True)\n",
    "    current_set['Validation'], current_scores['Validation'] = tmp2['ft'].item()['Validation'], tmp2['sc'].item()[\n",
    "        'Validation']\n",
    "\n",
    "    tmp3 = np.load(\n",
    "        './dataset_npz/{}_{}/{}_feat_{}.npz'.format(hparams['n_segments'], hparams['alpha'], ft_type, 'Test'),\n",
    "        allow_pickle=True)\n",
    "    current_set['Test'], current_scores['Test'] = tmp3['ft'].item()['Test'], tmp3['sc'].item()['Test']\n",
    "\n",
    "    return current_set, current_scores\n",
    "\n",
    "def set_keras_session():\n",
    "    tf.keras.backend.set_session(session)\n",
    "    tf.keras.backend.clear_session()  # Reset state\n",
    "    # tf.reset_default_graph()\n",
    "    np.random.seed(seed)\n",
    "    random.seed(0)\n",
    "    tf.random.set_random_seed(seed)\n",
    "\n",
    "\n",
    "def reset_keras_session():\n",
    "    tf.keras.backend.clear_session()\n",
    "\n",
    "\n",
    "def write_txt(ld_names, ld_predict, txt_path='./submission_01/'):\n",
    "    \"\"\"\n",
    "    Write predicted results into txt\n",
    "    :param ld_names:\n",
    "    :param ld_predict:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    os.makedirs(txt_path, exist_ok=True)\n",
    "    for idx in range(ld_predict.size):\n",
    "        current_name = ld_names[idx]\n",
    "        current_predict = ld_predict[idx]\n",
    "        with open(txt_path + current_name + '.txt', 'w') as f:\n",
    "            f.write(str(current_predict))\n",
    "            \n",
    "def load_weights_to_model(current_model, hparams):\n",
    "    \"\"\" Only apply to the LSTM model in this file, for other models, try to change :v\"\"\"\n",
    "    f = h5py.File('./models/{}_{}_models_{}_{}_0_epochs{}_best_weight.h5'.format(hparams['model_path'], ft_type,\n",
    "                                                                                 hparams['n_segments'],\n",
    "                                                                                 hparams['alpha'],\n",
    "                                                                                 hparams['EPOCHS']), 'r')\n",
    "    print(list(f.keys()))\n",
    "\n",
    "    tmp2 = current_model.layers[6].get_weights()\n",
    "\n",
    "    current_model.layers[0].set_weights([f['time_distributed_2']['time_distributed_2']['kernel:0'].value,\n",
    "                                         f['time_distributed_2']['time_distributed_2']['bias:0'].value])\n",
    "    current_model.layers[1].set_weights(\n",
    "        [f['cu_dnnlstm']['cu_dnnlstm']['kernel:0'].value, f['cu_dnnlstm']['cu_dnnlstm']['recurrent_kernel:0'].value,\n",
    "         f['cu_dnnlstm']['cu_dnnlstm']['bias:0'].value])\n",
    "    current_model.layers[2].set_weights([f['cu_dnnlstm_1']['cu_dnnlstm_1']['kernel:0'].value,\n",
    "                                         f['cu_dnnlstm_1']['cu_dnnlstm_1']['recurrent_kernel:0'].value,\n",
    "                                         f['cu_dnnlstm_1']['cu_dnnlstm_1']['bias:0'].value])\n",
    "    current_model.layers[3].set_weights([f['time_distributed']['time_distributed']['kernel:0'].value,\n",
    "                                         f['time_distributed']['time_distributed']['bias:0'].value])\n",
    "    current_model.layers[4].set_weights([f['time_distributed_1']['time_distributed_1']['kernel:0'].value,\n",
    "                                         f['time_distributed_1']['time_distributed_1']['bias:0'].value])\n",
    "    current_model.layers[5].set_weights([f['time_distributed_3']['time_distributed_3']['kernel:0'].value,\n",
    "                                         f['time_distributed_3']['time_distributed_3']['bias:0'].value])\n",
    "\n",
    "    f.close()\n",
    "    return current_model\n",
    "\n",
    "def gridsearch_mse_clsw(y_true, y_pred):\n",
    "    \"\"\" MSe with class weighting \"\"\"\n",
    "    clsw = class_weight.compute_class_weight('balanced', np.unique(y_true), y_true)\n",
    "    sw = class_weight.compute_sample_weight('balanced', y_true)\n",
    "    return mean_squared_error(y_true, y_pred, sample_weight=sw)\n",
    "\n",
    "def clf_predict(ypredict, yscores, clf, scaler, svm_experiment=None, write_path=None, verbose=0):\n",
    "    xtrain = scaler.transform(ypredict['Train'])\n",
    "    xval = scaler.transform(ypredict['Valid'])\n",
    "    xtest = scaler.transform(ypredict['Test'])\n",
    "    ypredict_fn = dict()\n",
    "\n",
    "    if svm_experiment is not None:\n",
    "        svm_experiment.log_dataset_hash(ypredict)\n",
    "\n",
    "    ypredict_fn['Train'] = clf.predict(xtrain)\n",
    "    ypredict_fn['Test'] = clf.predict(xtest)\n",
    "    ypredict_fn['Valid'] = clf.predict(xval)\n",
    "\n",
    "    for dt in ['Train', 'Valid', 'Test']:\n",
    "        ypredict_fn[dt][ypredict_fn[dt] >= 1.0] = 1.0\n",
    "        ypredict_fn[dt][ypredict_fn[dt] <= 0.0] = 0.0\n",
    "        print(dt)\n",
    "        if dt != 'Test':\n",
    "            if svm_experiment is not None:\n",
    "                svm_experiment.log_metrics({dt + '_Min': np.min(ypredict_fn[dt]), dt + '_Max': np.max(ypredict_fn[dt]),\n",
    "                                            dt + '_MSE': mean_squared_error(yscores[dt], ypredict_fn[dt])})\n",
    "            print('{}. Min {}. Max {}. MSE {}'.format(dt, np.min(ypredict_fn[dt]), np.max(ypredict_fn[dt]),\n",
    "                                                      mean_squared_error(yscores[dt], ypredict_fn[dt])))\n",
    "        else:\n",
    "            if svm_experiment is not None:\n",
    "                svm_experiment.log_metrics({dt + '_Min': np.min(ypredict_fn[dt]), dt + '_Max': np.max(ypredict_fn[dt])})\n",
    "            print('{}. Min {}. Max {}'.format(dt, np.min(ypredict_fn[dt]), np.max(ypredict_fn[dt])))\n",
    "            if write_path:\n",
    "                write_txt(yscores[dt], ypredict_fn[dt], write_path)\n",
    "\n",
    "    return ypredict_fn, mean_squared_error(yscores['Train'], ypredict_fn['Train']), mean_squared_error(yscores['Valid'],\n",
    "                                                                                                       ypredict_fn[\n",
    "                                                                                                           'Valid'])\n",
    "\n",
    "def avg_ensemble(ypredict, yscores, md_index, write_path=None):\n",
    "    \"\"\"\n",
    "    AVG ensemble\n",
    "    :param ypredicts:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    n_models = len(md_index)\n",
    "    print(\"Number of models to be average: {}\".format(n_models))\n",
    "    wgt_arr = np.ones(n_models, dtype=np.float64) / n_models\n",
    "    ypredict_fn = dict()\n",
    "    for dt in ['Train', 'Valid', 'Test']:\n",
    "        ypredict_fn[dt] = np.average(ypredict[dt][md_index, :], axis=0, weights=wgt_arr).flatten()\n",
    "        if dt != 'Test':\n",
    "            print('{}:\\nMin {}. Max {}. MSE {}'.format(dt, np.min(ypredict_fn[dt]), np.max(ypredict_fn[dt]),\n",
    "                                                      mean_squared_error(yscores[dt], ypredict_fn[dt])))\n",
    "        else:\n",
    "            print('{}:\\nMin {}. Max {}'.format(dt, np.min(ypredict_fn[dt]), np.max(ypredict_fn[dt])))\n",
    "            if write_path is not None:\n",
    "                if write_path:\n",
    "                    write_txt(yscores[dt], ypredict_fn[dt], write_path)\n",
    "\n",
    "    return ypredict_fn, mean_squared_error(yscores['Train'], ypredict_fn['Train']), mean_squared_error(yscores['Valid'],\n",
    "                                                                                                       ypredict_fn[\n",
    "                                                                                                           'Valid'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_predictions(verbose=0):\n",
    "    ld_cfg = cfg.md_cfg\n",
    "    ret_predict = []\n",
    "    ret_test_predict = []\n",
    "    ytrue = []\n",
    "    ytruep = None\n",
    "    log_dir = './tmp'\n",
    "\n",
    "    ypredict = {'Train': [], 'Valid': [], 'Test': []}\n",
    "    yscores = {}\n",
    "\n",
    "    for idx in range(0, len(ld_cfg)):\n",
    "        hparams = copy.deepcopy(ld_cfg[idx])\n",
    "        hparams['optimizer'] = 'adam'\n",
    "        hparams['ACTIVATION_F'] = 'tanh'\n",
    "        hparams['CLSW'] = 1\n",
    "\n",
    "        # gc.collect()\n",
    "        if 'VGG' in hparams['NAME']:\n",
    "            ft_type = 'vgg2'\n",
    "        elif 'OF' in hparams['NAME']:\n",
    "            ft_type = 'of'\n",
    "        else:\n",
    "            ft_type = 'au'\n",
    "\n",
    "        current_set, current_scores = load_data(hparams, ft_type)\n",
    "\n",
    "        # hparams['FC1'] = [current_set['Train'].shape[2], fc1]\n",
    "        hparams['INPUT_UNITS'] = hparams['FC1'][1] if hparams['FC1'][1] > 0 else current_set['Train'].shape[2]\n",
    "        hparams['TIME_STEP'] = current_set['Train'].shape[1]\n",
    "\n",
    "        set_keras_session()\n",
    "        if hparams['optimizer'] == 'adam':\n",
    "            optim = Adam(hparams['learning_rate'])\n",
    "        elif hparams['optimizer'] == 'sgd':\n",
    "            optim = SGD(hparams['learning_rate'])\n",
    "        else:\n",
    "            optim = RMSprop(hparams['learning_rate'])\n",
    "\n",
    "        current_model = define_model(hparams, hparams['NAME'])\n",
    "\n",
    "        # Fit model :v\n",
    "        # model_objective(hparams, (current_set['Train'], current_scores['Train']),\n",
    "        #                 val_set=(current_set['Validation'], current_scores['Validation']),\n",
    "        #                 test_set=(current_set['Validation'], current_scores['Validation']), model_name=ft_type,\n",
    "        #                 logdir=logdir)\n",
    "\n",
    "        if hparams['optimizer'] == 'adam':\n",
    "            optim = Adam(hparams['learning_rate'])\n",
    "        elif hparams['optimizer'] == 'sgd':\n",
    "            optim = SGD(hparams['learning_rate'])\n",
    "        else:\n",
    "            optim = RMSprop(hparams['learning_rate'])\n",
    "\n",
    "        current_model.compile(optimizer=optim, loss=keras.losses.MeanSquaredError(),\n",
    "                              metrics=[keras.metrics.mean_squared_error])\n",
    "        current_model.build()\n",
    "\n",
    "        # Load weight from old old model :v\n",
    "        # current_model = load_weights_to_model(current_model, hparams)\n",
    "        # current_model.save_weights(\n",
    "        #     './models/{}_{}_models_{}_{}_0_epochs{}_best_weight.h5'.format(hparams['model_path'], ft_type,\n",
    "        #                                                                    hparams['n_segments'], hparams['alpha'],\n",
    "        #                                                                    hparams['EPOCHS']))\n",
    "\n",
    "        if verbose:\n",
    "            print(\"Load weight {}_{}\".format(hparams['NAME'], ft_type))\n",
    "        current_model.load_weights(\n",
    "            './models/{}_{}_models_{}_{}_0_epochs{}_best_weight.h5'.format(hparams['model_path'], ft_type,\n",
    "                                                                           hparams['n_segments'], hparams['alpha'],\n",
    "                                                                           hparams['EPOCHS']))\n",
    "        current_model.evaluate(current_set['Validation'], current_scores['Validation'],\n",
    "                               batch_size=hparams['BATCH_SIZE'], verbose=verbose)\n",
    "\n",
    "        current_train_predict = current_model.predict(current_set['Train'], batch_size=hparams['BATCH_SIZE'])\n",
    "        current_predict = current_model.predict(current_set['Validation'], batch_size=hparams['BATCH_SIZE'])\n",
    "        current_test_predict = current_model.predict(current_set['Test'], batch_size=hparams['BATCH_SIZE'])\n",
    "\n",
    "        if verbose:\n",
    "            print('Validation: Min {}. Max {}.'.format(np.min(current_predict), np.max(current_predict)))\n",
    "            print('Test: Min {}. Max {}.'.format(np.min(current_test_predict), np.max(current_test_predict)))\n",
    "\n",
    "        current_model = None\n",
    "        reset_keras_session()\n",
    "\n",
    "        current_predict[current_predict <= 0] = 0.0\n",
    "        current_predict[current_predict >= 1] = 1.0\n",
    "\n",
    "        current_test_predict[current_test_predict <= 0] = 0.0\n",
    "        current_test_predict[current_test_predict >= 1] = 1.0\n",
    "\n",
    "        ypredict['Train'].append(current_train_predict.flatten())\n",
    "        yscores['Train'] = current_scores['Train']\n",
    "\n",
    "        ypredict['Valid'].append(current_predict.flatten())\n",
    "        yscores['Valid'] = current_scores['Validation']\n",
    "\n",
    "        ypredict['Test'].append(current_test_predict.flatten())\n",
    "        yscores['Test'] = current_scores['Test']\n",
    "\n",
    "        ret_predict.append(current_predict.flatten())\n",
    "        ret_test_predict.append(current_test_predict.flatten())\n",
    "        \n",
    "        if verbose:\n",
    "            print('sklearn MSE: ', mean_squared_error(current_scores['Validation'].flatten(), current_predict.flatten()))\n",
    "\n",
    "        # ytrue = current_scores['Validation']\n",
    "        # yval_true = current_scores['Validation']\n",
    "        #if ytruep is not None:\n",
    "        #    print('DIFF: ', np.sum(ytrue != ytruep))\n",
    "\n",
    "        # ytruep = ytrue\n",
    "        \n",
    "    ypredict['Train'] = np.array(ypredict['Train'])\n",
    "    ypredict['Valid'] = np.array(ypredict['Valid'])\n",
    "    ypredict['Test'] = np.array(ypredict['Test'])\n",
    "    \n",
    "    return ypredict, yscores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\miniconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From c:\\miniconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\keras\\utils\\losses_utils.py:170: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Num train 146, val 48\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl0VfW5//H3wyBRgSJDuIFcDLUoYziGEKBSCiJT64AKlRR/DUON2mtV0Cq23qWyWJUCt+L14hCn5rZccBYu4EAZtFeqkSEIghRQFDAFRCGgSBPy/P7IIQ0QyElykpPsfF5rnXX2/p49PNniJzvfs/d3m7sjIiJ1X4NYFyAiItGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIB0agmd9a6dWtPSkqqyV2KiNR5a9as+cLd25S3XI0GelJSEqtXr67JXYqI1Hlm9mkky0XU5WJmk8zsQzPbaGbzzCzOzDqa2XtmttXMnjOzs6pWskgwbdmyhVAoVPJq3rw5s2fPJjc3l759+xIKhUhNTSUnJyfWpUodZ+UNzmVm7YH/A7q6+xEzex5YAvwIeNnd55vZ48B6d3/sTNtKTU11naFLfXbs2DHat2/Pe++9xw033MCkSZMYMWIES5YsYcaMGaxcuTLWJUotZGZr3D21vOUi/VK0EXC2mTUCzgHygEuBF8OfZwMjK1OoSH2ybNkyLrjgAs4//3zMjPz8fAAOHjxIu3btYlyd1HXl9qG7+24zmwV8BhwB3gTWAAfcvTC82C6gfVnrm1kmkAnQoUOHaNQsUmfNnz+f9PR0AGbPns2wYcO48847KSoqYtWqVdW234KCAnbt2sW3335bbfuQqouLiyMxMZHGjRtXav1IulzOA14CrgMOAC+E5+9z9++Fl/lXYIm79zjTttTlIvXZP/7xD9q1a8eHH35I27ZtufXWW/nhD3/Itddey/PPP09WVhZ//vOfq2Xfn3zyCc2aNaNVq1aYWbXsQ6rG3dm/fz+HDh2iY8eOJ3wWzS6Xy4BP3H2fuxcALwPfB1qEu2AAEoHPK1a+SP3y2muvkZKSQtu2bQHIzs7mmmuuAWD06NHV+qXot99+qzCv5cyMVq1aVemvqEgC/TOgr5mdY8X/GgYDm4AVwKjwMhnAgkpXIVIPzJs3r6S7BaBdu3a89dZbACxfvpxOnTpV6/4V5rVfVf8bRdKH/p6ZvQisBQqBdUAWsBiYb2bTwm1PV6kSkQD75ptvWLp0KU888URJ25NPPsltt91GYWEhcXFxZGVlxbBCCYKIbixy9/uA+05q/hhIi3pFIgF0zjnnsH///hPa+vfvz5o1a2JST9KUxVHd3o7pPz7j5/v372fw4MEA/P3vf6dhw4a0aVN842NOTg5nnVX+bSzjx49nypQpXHTRRVUvOMq2bdvGqFGjyM3NjWkdNXqnqEh9Eu3QhPKDs7Zq1apVSdjdf//9NG3alDvvvPOEZdwdd6dBg7J7gp999tlqr7Ou0+BcIhIz27Zto3v37tx0002kpKSQl5dHZmYmqampdOvWjalTp5Ys279/f3JzcyksLKRFixZMmTKFnj170q9fP/bu3XvKtpcvX07Pnj0JhUKkpKTw9ddfk5+fz6WXXkpKSgrJycksWrTohDomTJhAt27d+NnPfsYbb7zB97//fS688MKSIUvuvfdeMjIyGDRoEJ06deKZZ545Zb+FhYVMnjyZtLQ0kpOTeeqppwDYvXs3/fv3JxQK0b1792q5TFVn6CISU5s2beLZZ5/l8ccfB2D69Om0bNmSwsJCBg0axKhRo+jatesJ6xw8eJAf/vCHTJ8+ncmTJ/PMM88wZcqUE5aZOXMmWVlZ9OnTh8OHDxMXF0dRURELFiygWbNm7N27l0suuYTLL78cKB6i4fnnn6dz586kpKTQpEkTVq1axUsvvcT06dN58cXi+yg3bNjAqlWryM/PJyUlhR//+MS/mrKysoiPjycnJ4ejR4/St29fhg4dyrx587jiiiu4++67OXbsGEeOHIn6sdQZuojE1AUXXEDv3r1L5ufNm0dKSgopKSls3ryZTZs2nbLO2WefzYgRIwDo1asXO3bsOGWZSy65hNtvv51HHnmE/Px8GjZsiLtz9913k5yczNChQ9m5cydffPEFAN/73vfo2rUrDRo0oGvXrlx22WUA9OjR44Ttjxw5kri4OOLj4xkwYADvv//+Cft98803efbZZwmFQvTp04cDBw6wdetWevfuzVNPPcUDDzzAxo0badq0aVUP3Sl0hi4iMXXuueeWTG/dupWHH36YnJwcWrRowfXXX1/mddmlv0Rt2LAhhYWFpyxz7733cuWVV7J48WJ69+7NypUreeuttzh48CBr166lUaNGJCYmlmy/SZMmJes2aNCgZL5BgwYnbP/kSwtPnnd3Hn300ZIvgUtbuXIlixcvZuzYsdxzzz2MHTv2jMemonSGLiK1Rn5+Ps2aNaN58+bk5eXxxhtvVHpb27dvJzk5mXvuuYeLL76YLVu2cPDgQeLj42nUqBFLly5l9+7dFd7uq6++ytGjR/niiy/4y1/+QmrqiTdwDhs2jEcffbTkl8CWLVs4cuQIn376Kf/yL/9CZmYm48aNY926dZX+2U5HZ+gi9VBtvVomJSWFrl270r17d7773e9yySWXVHpbs2bN4i9/+QsNGjQo6WJJS0vjiiuuIDU1lZSUlErdzNW7d29GjBjBzp07eeCBB2jbti2HDh0q+fzGG2/ks88+IxQKARAfH8+CBQtYtmwZv//972ncuDFNmzblT3/6U6V/ttMpdyyXaNJYLlKf1KbLFjdv3kyXLl2iXE39c++999K6dWtuv/32attHWf+toj18roiI1HLqchERidC0adNiXcIZ6QxdRCQgFOgiIgGhQBcRCQgFuohIQOhLUZH66P7vRHl7B8/48cCBA7nnnnsYNmxYSdvs2bP529/+xqOPPnra9Zo2bcrhw4f5/PPPufXWW0vGUzl527NmzTrlBp9Fixbx7//+7xQVFVFQUMBtt93GjTfeWMEfLHKnG0WyJinQRaTapaenM3/+/BMCff78+cycOTOi9du1a1dmmJ9OQUEBmZmZ5OTkkJiYyNGjR8sc7yVo1OUiItVu1KhRLFq0iKNHjwKwY8cOPv/8c/r378/hw4cZPHgwKSkp9OjRgwULTn2a5Y4dO+jevTsAR44cYcyYMSQnJ3PdddeVOWrhoUOHKCwspFWrVkDxOC3HH4zxv//7v/Tp04eLL76Yyy67jD179gDFZ9gZGRkMHTqUpKQkXn75Ze666y569OjB8OHDKSgoACApKYm7776btLQ00tLS2LZt2yn73759O8OHD6dXr1784Ac/4KOPPgLghRdeoHv37vTs2ZMBAwZU9bCeotxAN7OLzCy31CvfzG43s5ZmttTMtobfz4t6dSISCK1atSItLY3XX38dKD47v+666zAz4uLieOWVV1i7di0rVqzgjjvu4Ex3sD/22GOcc845fPDBB/zmN78p86lPLVu25Morr+T8888nPT2duXPnUlRUBBSPq/7uu++ybt06xowZw4wZM0rW2759O4sXL2bBggVcf/31DBo0iA0bNnD22WezePE/7/xt3rw5OTk53HLLLWXeNZqZmckjjzzCmjVrmDVrFr/4xS8AmDp1Km+88Qbr169n4cKFlTuYZ1BuoLv7FncPuXsI6AV8A7wCTAGWuXsnYFl4XkSkTMe7XaA40I8/MNvd+fWvf01ycjKXXXYZu3fvLjlrLsvbb7/N9ddfD0BycjLJycllLvfUU0+xbNky0tLSmDVrFhMmTABg165dDBs2jB49ejBz5kw+/PDDknVGjBhB48aN6dGjB8eOHWP48OHAqUPoHq89PT2dv/71ryfs9/Dhw6xatYrRo0cTCoW48cYbycvLA4qH9B03bhxPPvkkx44di/jYRaqiXS6Dge3u/ilwFZAdbs8GRkazMBEJlpEjR7Js2TLWrl3LkSNHSElJAWDu3Lns27ePNWvWkJubS9u2bcscMre0k4esPZ0ePXowadIkli5dyksvvQTAL3/5S2655RY2bNjAE088ccK+Sg+Z27hx45L9nGkI3ZNrKSoqokWLFuTm5pa8Nm/eDMDjjz/OtGnT2LlzJ6FQ6JTnzFZVRQN9DDAvPN3W3fMAwu/x0SxMRIKladOmDBw4kAkTJpSc4QIlQ9o2btyYFStW8Omnn55xOwMGDGDu3LkAbNy4kQ8++OCUZQ4fPszKlStL5nNzczn//PNL9te+fXsAsrOzT1k3Es8991zJe79+/U74rHnz5nTs2JEXXngBKP4LZP369UBxl06fPn2YOnUqrVu3ZufOnZXa/+lEfJWLmZ0FXAncU5EdmFkmkAnQoUOHChUnItWknMsMq0t6ejrXXHNNSdcLwNixY0uGtA2FQnTu3PmM27j55psZP348ycnJhEIh0tLSTlnG3ZkxYwY33ngjZ599Nueeey5/+MMfgOIvP0ePHk379u3p27cvn3zySYV/jqNHj9KnTx+KioqYN2/eKZ/PnTuXm2++mWnTplFQUMCYMWPo2bMnv/rVr9i6dSvuzuDBg+nZs2eF930mEQ+fa2ZXAf/m7kPD81uAge6eZ2YJwEp3v+hM29DwuVKfaPjcYEpKSmL16tW0bt26WrZfU8PnpvPP7haAhUBGeDoDOPVaIxERqTERdbmY2TnAEKD0bVbTgefNbCLwGTA6+uWJiNQutfkGpYgC3d2/AVqd1Laf4qteRKQOcPeIrw6R2KjqE+R0p6hIPRAXF8f+/furHBhSfdyd/fv3ExcXV+ltaCwXkXogMTGRXbt2sW/fvliXImcQFxdHYmJipddXoIvUA40bN6Zjx46xLkOqmbpcREQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJiIgC3cxamNmLZvaRmW02s35m1tLMlprZ1vD7edVdrIiInF6kZ+gPA6+7e2egJ7AZmAIsc/dOwLLwvIiIxEi5gW5mzYEBwNMA7v4Pdz8AXAVkhxfLBkZWV5EiIlK+SM7QvwvsA541s3Vm9pSZnQu0dfc8gPB7fDXWKSIi5Ygk0BsBKcBj7n4x8DUV6F4xs0wzW21mq/WAWhGR6hNJoO8Cdrn7e+H5FykO+D1mlgAQft9b1srunuXuqe6e2qZNm2jULCIiZSg30N3978BOM7so3DQY2AQsBDLCbRnAgmqpUEREItIowuV+Ccw1s7OAj4HxFP8yeN7MJgKfAaOrp0QREYlERIHu7rlAahkfDY5uOSIiUlm6U1REJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkICJ6YpGZ7QAOAceAQndPNbOWwHNAErAD+Im7f1U9ZYqISHkqcoY+yN1D7n78UXRTgGXu3glYFp4XEZEYqUqXy1VAdng6GxhZ9XJERKSyIg10B940szVmlhlua+vueQDh9/jqKFBERCITUR86cIm7f25m8cBSM/so0h2EfwFkAnTo0KESJYqISCQiOkN398/D73uBV4A0YI+ZJQCE3/eeZt0sd09199Q2bdpEp2oRETlFuYFuZueaWbPj08BQYCOwEMgIL5YBLKiuIkVEpHyRdLm0BV4xs+PL/4+7v25m7wPPm9lE4DNgdPWVKSIi5Sk30N39Y6BnGe37gcHVUZSIiFSc7hQVEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQEQc6GbW0MzWmdmi8HxHM3vPzLaa2XNmdlb1lSkiIuWpyBn6bcDmUvO/Ax5y907AV8DEaBYmIiIVE1Ggm1ki8GPgqfC8AZcCL4YXyQZGVkeBIiISmUjP0GcDdwFF4flWwAF3LwzP7wLaR7k2ERGpgHID3cwuB/a6+5rSzWUs6qdZP9PMVpvZ6n379lWyTBERKU8kZ+iXAFea2Q5gPsVdLbOBFmbWKLxMIvB5WSu7e5a7p7p7aps2baJQsoiIlKXcQHf3e9w90d2TgDHAcncfC6wARoUXywAWVFuVIiJSrqpch343MNnMtlHcp/50dEoSEZHKaFT+Iv/k7iuBleHpj4G06JckIiKVoTtF5QTffvstaWlp9OzZk27dunHfffcBMG7cODp27EgoFCIUCpGbmxvjSkXkZBU6Q5fga9KkCcuXL6dp06YUFBTQv39/RowYAcDMmTMZNWpUOVsQkVjRGbqcwMxo2rQpAAUFBRQUFFB8H5mI1HYKdDnFsWPHCIVCxMfHM2TIEPr06QPAb37zG5KTk5k0aRJHjx6NcZUicjIFupyiYcOG5ObmsmvXLnJycti4cSMPPvggH330Ee+//z5ffvklv/vd72JdpoicRIEup9WiRQsGDhzI66+/TkJCAmZGkyZNGD9+PDk5ObEuT0ROokCXE+zbt48DBw4AcOTIEf785z/TuXNn8vLyAHB3Xn31Vbp37x7LMkWkDLrKRU6Ql5dHRkYGx44do6ioiJ/85CdcfvnlXHrppezbtw93JxQK8fjjj8e6VBE5ibmXOaZWtUhNTfXVq1fX2P6kYpKmLI76NndM/3HUt1lX6HhKtJjZGndPLW85dbmIiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgCg30M0szsxyzGy9mX1oZg+E2zua2XtmttXMnjOzs6q/XBEROZ1IztCPApe6e08gBAw3s77A74CH3L0T8BUwsfrKFBEptnPnTgYNGkSXLl3o1q0bDz/8MAC5ubn07duXUChEampqvRxArtxA92KHw7ONwy8HLgVeDLdnAyOrpUIRkVIaNWrEf/zHf7B582beffdd5syZw6ZNm7jrrru47777yM3NZerUqdx1112xLrXGRTQ4l5k1BNYA3wPmANuBA+5eGF5kF9C+WioUESklISGBhIQEAJo1a0aXLl3YvXs3ZkZ+fj4ABw8epF27drEsMyYiCnR3PwaEzKwF8ArQpazFylrXzDKBTIAOHTpUskwRkVPt2LGDdevW0adPH2bPns2wYcO48847KSoqYtWqVbEur8ZV6CoXdz8ArAT6Ai3M7PgvhETg89Osk+Xuqe6e2qZNm6rUKiJS4vDhw1x77bXMnj2b5s2b89hjj/HQQw+xc+dOHnroISZOrH9f60VylUub8Jk5ZnY2cBmwGVgBHH8EfAawoLqKFBEpraCggGuvvZaxY8dyzTXXAJCdnV0yPXr0aH0pehoJwAoz+wB4H1jq7ouAu4HJZrYNaAU8XX1liogUc3cmTpxIly5dmDx5ckl7u3bteOuttwBYvnw5nTp1ilWJMVNuH7q7fwBcXEb7x0BadRQlInI677zzDn/84x/p0aMHoVAIgN/+9rc8+eST3HbbbRQWFhIXF0dWVlaMK615egSdiNQJpZ8Adf7di8gv9dkv3nbgIAyZCsVTXPvC3+GFMz81KmhPgNKt/yIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXaQOmjBhAvHx8XTv3r2kbf369fTr148ePXpwxRVXkJ+ff4YtSBAp0EXqoHHjxvH666+f0Pbzn/+c6dOns2HDBq6++mpmzpwZo+okViJ5SPS/mtkKM9tsZh+a2W3h9pZmttTMtobfz6v+ckUEYMCAAbRs2fKEti1btjBgwAAAhgwZwksvvRSL0iSGIjlDLwTucPcuQF/g38ysKzAFWObunYBl4XkRiZHu3buzcOFCAF544QV27twZ44qkppUb6O6e5+5rw9OHgM1Ae+AqIDu8WDYwsrqKFJHyPfPMM8yZM4devXpx6NAhzjrrrFiXJDWsQg+JNrMk4GLgPaCtu+dBceibWXzUqxORiHXu3Jk333wTgL/97W8sXnzmByRL8ET8paiZNQVeAm5394i/PjezTDNbbWar9+3bV5kaRSQCe/fuBaCoqIhp06Zx0003xbgiqWkRBbqZNaY4zOe6+8vh5j1mlhD+PAHYW9a67p7l7qnuntqmTZto1CxS76Wnp9OvXz+2bNlCYmIiTz/9NPPmzePCCy+kc+fOtGvXjvHjx8e6TKlh5Xa5mJkBTwOb3f33pT5aCGQA08PvC6qlQhH5p/u/A8C8i4CLAM4BDsHOyQDc9tPjCz4GDzwW4TYPRrdGiZlI+tAvAf4fsMHMcsNtv6Y4yJ83s4nAZ8Do6ilRREQiUW6gu/v/AXaajwdHtxwREaks3SkqIvVeWUMp3H///bRv355QKEQoFGLJkiUxrDAyCnQRqffKGkoBYNKkSeTm5pKbm8uPfvSjGFRWMQp0Ean3yhpKoS5SoIuInMZ//dd/kZyczIQJE/jqq69iXU65FOgiImW4+eab2b59O7m5uSQkJHDHHXfEuqRyKdBFRMrQtm1bGjZsSIMGDbjhhhvIycmJdUnlUqCLiJQhLy+vZPqVV1454QqY2qpCg3OJiARReno6K1eu5IsvviAxMZEHHniAlStXkpubi5mRlJTEE088Eesyy6VAF5H66wxDKUy8ALjg+IKfwhOdI9xm7IZSUJeLiEhAKNBFRAJCgS4iEhAKdBGRgFCgS40oa/CjL7/8kiFDhtCpUyeGDBlSJ+7EE6nNFOhSI8oa/Gj69OkMHjyYrVu3MnjwYKZPnx6j6kSCQYEuNaKswY8WLFhARkYGABkZGbz66quxKE0kMBToEjN79uwhISEBgISEhJKHHItI5SjQRUQCotxAN7NnzGyvmW0s1dbSzJaa2dbw+3nVW6YEUdu2bUvGy8jLyyM+Pj7GFYnUbZGcof8BGH5S2xRgmbt3ApaF50Uq5MorryQ7OxuA7OxsrrrqqhhXJFK3lRvo7v428OVJzVcB2eHpbGBklOuSgElPT6dfv35s2bKFxMREnn76aaZMmcLSpUvp1KkTS5cuZcoUnReIVEVlB+dq6+55AO6eZ2an/VvZzDKBTIAOHTpUcndSZ51h8CMemcyyHxxfcC/8Z8cItxm7wY9EarNq/1LU3bPcPdXdU9u0aVPduxMRqbcqG+h7zCwBIPyu681ERGKssoG+EMgIT2cAC6JTjoiIVFYkly3OA/4KXGRmu8xsIjAdGGJmW4Eh4XkREYmhcr8Udff003w0OMq1iIhIFQTiEXRJSUk0a9aMhg0b0qhRI1avXh3rkkREalwgAh1gxYoVtG7dOtZliIjEjMZyOYNjx45x8cUXc/nll8e6FBGRcgUi0M2MoUOH0qtXL7KysqK23YcffpguXbpEbXsiItUpEIH+zjvvsHbtWl577TXmzJnD22+/XeVt7tq1i8WLF/Pzn/88ChWKiFS/QAR6u3btAIiPj+fqq68mJyenytu8/fbbmTFjBg0aBOIQiUg9UOfT6uuvv+bQoUMl02+++eYJz62sjEWLFhEfH0+vXr2iUaKISI2o81e57Nmzh6uvvhqAwsJCfvrTnzJ8+Mmj/VbMO++8w8KFC1myZAnffvst+fn5XH/99fzpT3+KRskiItWizgf6pVmbYcRvS+afPARPTllcpW3umP4gDz74IAArV65k1qxZCnMRqfXqfJeLiIgUq/Nn6NUiPIY3wEBgYOqJbZXbpsbwFpHqpTN0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAVCnQzWy4mW0xs21mNiVaRYmISMVVOtDNrCEwBxgBdAXSzaxrtAoTEZGKqcoZehqwzd0/dvd/APOBq6JTloiIVFRVAr09sLPU/K5wm4iIxEBVbv23Mtr8lIXMMoHM8OxhM9tShX3WCIPWwBdR3egDZR2u4NOxjC4dz+iqQ8fz/EgWqkqg7wL+tdR8IvD5yQu5exYQvefC1QAzW+3uqbGuIwh0LKNLxzO6gnY8q9Ll8j7Qycw6mtlZwBhgYXTKEhGRiqr0Gbq7F5rZLcAbQEPgGXf/MGqViYhIhVRp+Fx3XwIsiVIttUmd6iKq5XQso0vHM7oCdTzN/ZTvMUVEpA7Srf8iIgFRbwO9vGELzKyJmT0X/vw9M0uq+SprtwiO4U1mtsHMcs3s/47fSWxmaeG2XDNbb2ZX13z1tU8kQ2mY2U/MbJOZfWhm/1OqvYOZvWlmm8OfJ9VU3bWdmT1jZnvNbONpPjcz+8/wcf/AzFJqusaocfd696L4S9ztwHeBs4D1QNeTlvkF8Hh4egzwXKzrrk2vCI9h81LTVwKvh6fPARqFpxOAvcfn6+srwuPZCVgHnBeejy/12UpgSHi6KXBOrH+m2vICBgApwMbTfP4j4DWK763pC7wX65or+6qvZ+iRDFtwFZAdnn4RGGxm9fcOjFOVewzdPb/U7LmEbzxz92/cvTDcHkcZN6TVQ5H8m7wBmOPuXwG4+16A8F8+jdx9abj9sLt/U3Ol127u/jbw5RkWuQr4by/2LtDCzBJqprroqq+BHsmwBSXLhMPnINCqRqqrGyIa+sHM/s3MtgMzgFtLtfcxsw+BDcBNpQK+vorkeF4IXGhm75jZu2Y2vFT7ATN72czWmdnM8OB5EpnADGNSXwM9kmELIhraoB6L6Pi4+xx3vwC4G7i3VPt77t4N6A3cY2Zx1VZp3RDJ8WxEcbfLQCAdeMrMWoTbfwDcSfHx/C4wrroKDaDA/L9eXwM9kmELSpYxs0bAdzjzn231TURDP5QyHxh5cqO7bwa+BrpHtbq6J9J/kwvcvcDdPwG2UBzwu4B14e6aQuBVivuMJTIV/bdca9XXQI9k2IKFQEZ4ehSw3MPfoAgQwTE0s06lZn8MbA23dwz/ksTMzgcuAnbURNG1WCT/Jl8FBgGYWWuKu1o+Dq97npm1CS93KbCpRqoOhoXAz8JXu/QFDrp7XqyLqowq3SlaV/lphi0ws6lEgAMSAAAArElEQVTAandfCDwN/NHMtlF8Zj4mdhXXPhEew1vM7DKgAPiKf/6C7A9MMbMCoAj4hbtHd8S7OibC4/kGMNTMNgHHgF+5+34AM7sTWBb+4n4N8GRMfpBayMzmUdxN1drMdgH3AY0B3P1xiu92/xGwDfgGGB+bSqtOd4qKiAREfe1yEREJHAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgHx/wHMVQAvWvTJogAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# import pandas as pd\n",
    "ypredict, yscores = get_predictions(verbose=0)\n",
    "eg_level, train_cnt = np.unique(yscores['Train'], return_counts=True)\n",
    "eg_level, val_cnt = np.unique(yscores['Valid'], return_counts=True)\n",
    "df = pd.DataFrame({'Train samples':train_cnt, 'Valid Samples': val_cnt}, index=eg_level)\n",
    "ax = df.plot.bar(rot=0)\n",
    "n_trains = np.sum(train_cnt)\n",
    "n_vals = np.sum(val_cnt)\n",
    "print('Num train {}, val {}'.format(n_trains, n_vals))\n",
    "for p in ax.patches:\n",
    "    ax.annotate(str(p.get_height()), (p.get_x() * 1.005, p.get_height() * 1.005))\n",
    "    \n",
    "# print(67*4/48)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission 01 \n",
    "```\n",
    "Average 3 model\n",
    "```\n",
    "- Overall Mean Square Error: **0.078661**  \n",
    "- Mean Square Error for Engagement level 0: 0.334221  \n",
    "- Mean Square Error for Engagement level 1: 0.083368  \n",
    "- Mean Square Error for Engagement level 2: 0.013338  \n",
    "- Mean Square Error for Engagement level 3: 0.066046"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of models to be average: 3\n",
      "Train:\n",
      "Min -0.005037905027469. Max 0.9521716237068176. MSE 0.0046861650825803115\n",
      "Valid:\n",
      "Min 0.3349830359220505. Max 0.8595088322957356. MSE 0.049722080986325856\n",
      "Test:\n",
      "Min 0.25669900079568225. Max 0.8801403840382893\n",
      "Exact ^^. Completed.\n"
     ]
    }
   ],
   "source": [
    "def submission_01_ck():\n",
    "    ypredict, yscores = get_predictions()\n",
    "    ld_cfg = cfg.md_cfg\n",
    "    # Submission 01, average of model 0, 1, 2\n",
    "    md_index = [0, 1, 2]\n",
    "    ypredict_avg_fn, train_mse, val_mse = avg_ensemble(copy.deepcopy(ypredict), copy.deepcopy(yscores), md_index = [0, 1, 2],\n",
    "                                                           write_path=None)\n",
    "    \n",
    "    # Write submission to files\n",
    "    write_txt(yscores['Test'], ypredict_avg_fn['Test'], txt_path='./submission_01/')\n",
    "    # Load submission 01 to check\n",
    "    yx = []\n",
    "    yx_names = []\n",
    "    ldx = os.listdir('./submission_01/')\n",
    "    for idx in range(len(ldx)):\n",
    "        current_name = ldx[idx]\n",
    "        if current_name.endswith('.txt'):\n",
    "            with open('submission_01/' + current_name, 'r') as f:\n",
    "                yx.append(float(f.readline()))\n",
    "                yx_names.append(current_name[:-4])\n",
    "                \n",
    "    ck_names = np.array([yx_names, yscores['Test']]).T\n",
    "    ck_values = np.array([yx, ypredict_avg_fn['Test']]).T\n",
    "    \n",
    "    ckn = np.count_nonzero(ck_names[:, 0] != ck_names[:, 1])\n",
    "    ckv = np.count_nonzero(ck_values[:, 0] != ck_values[:, 1])\n",
    "    if ckn == 0 and ckv == 0:\n",
    "        print(\"Exact ^^. Completed.\")\n",
    "    else:\n",
    "        print('Failed, please check again.')\n",
    "    \n",
    "    return ypredict_avg_fn\n",
    "\n",
    "_ = submission_01_ck()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission 02\n",
    "```SVM ensemble 4 models```\n",
    "- Overall Mean Square Error: **0.091114** \n",
    "- Mean Square Error for Engagement level 0: 0.328931 \n",
    "- Mean Square Error for Engagement level 1: 0.108693 \n",
    "- Mean Square Error for Engagement level 2: 0.027030 \n",
    "- Mean Square Error for Engagement level 3: 0.035344"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train\n",
      "Train. Min 0.127007890896647. Max 0.9427618711407286. MSE 0.008219488234086704\n",
      "Valid\n",
      "Valid. Min 0.19037981562931455. Max 0.9173414134357303. MSE 0.03723437079793676\n",
      "Test\n",
      "Test. Min 0.1632684008307948. Max 0.9245766071797195\n",
      "Exact ^^. Completed.\n"
     ]
    }
   ],
   "source": [
    "def submission_02_ck():\n",
    "    ypredict, yscores = get_predictions()\n",
    "    ld_cfg = cfg.md_cfg\n",
    "    svm_exper_key = '3c4e56dd6f8a4fb7b76d6f7d970e48a2'  # \n",
    "    \n",
    "    # Submission 02, SVM of 0, 1, 2, 3\n",
    "    ypredict['Train'] = ypredict['Train'].T\n",
    "    ypredict['Valid'] = ypredict['Valid'].T\n",
    "    ypredict['Test'] = ypredict['Test'].T\n",
    "\n",
    "    scaler = joblib.load('./sklearn_model/{}_sc.pkl'.format(svm_exper_key))\n",
    "    clf = joblib.load('./sklearn_model/{}_clf.pkl'.format(svm_exper_key))\n",
    "\n",
    "    ypredict_svm_fn, train_svm_mse, val_svm_mse = clf_predict(ypredict, yscores, clf, scaler, svm_experiment=None, write_path=None, verbose=0)\n",
    "            \n",
    "    # Write submission to files\n",
    "    write_txt(yscores['Test'], ypredict_svm_fn['Test'], txt_path='./submission_02/')\n",
    "    \n",
    "    # Load submission 02 to check\n",
    "    yx = []\n",
    "    yx_names = []\n",
    "    ldx = os.listdir('./submission_02/')\n",
    "    for idx in range(len(ldx)):\n",
    "        current_name = ldx[idx]\n",
    "        if current_name.endswith('.txt'):\n",
    "            with open('submission_02/' + current_name, 'r') as f:\n",
    "                yx.append(float(f.readline()))\n",
    "                yx_names.append(current_name[:-4])\n",
    "            \n",
    "    ck_names = np.array([yx_names, yscores['Test']]).T\n",
    "    ck_values = np.array([yx, ypredict_svm_fn['Test']]).T\n",
    "    \n",
    "    ckn = np.count_nonzero(ck_names[:, 0] != ck_names[:, 1])\n",
    "    ckv = np.count_nonzero(ck_values[:, 0] != ck_values[:, 1])\n",
    "    if ckn == 0 and ckv == 0:\n",
    "        print(\"Exact ^^. Completed.\")\n",
    "    else:\n",
    "        print('Failed, please check again.') \n",
    "    \n",
    "    return ypredict_svm_fn\n",
    "\n",
    "_ = submission_02_ck()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission 03\n",
    "```Average 2 models```\n",
    "- Overall Mean Square Error: **0.069625** \n",
    "- Mean Square Error for Engagement level 0: 0.268581 \n",
    "- Mean Square Error for Engagement level 1: 0.064421 \n",
    "- Mean Square Error for Engagement level 2: 0.023098 \n",
    "- Mean Square Error for Engagement level 3: 0.064037 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of models to be average: 2\n",
      "Train:\n",
      "Min -0.03020166844362393. Max 0.9489489793777466. MSE 0.005174875540833624\n",
      "Valid:\n",
      "Min 0.2734040692448616. Max 0.8653900027275085. MSE 0.051754981281858514\n",
      "Test:\n",
      "Min 0.20518453419208527. Max 0.8657699525356293\n",
      "Exact ^^. Completed.\n"
     ]
    }
   ],
   "source": [
    "def submission_03_ck():\n",
    "    ypredict, yscores = get_predictions()\n",
    "    ld_cfg = cfg.md_cfg\n",
    "    # Submission 01, average of model 0, 2\n",
    "    md_index = [0, 2]\n",
    "    ypredict_avg_fn, train_mse, val_mse = avg_ensemble(copy.deepcopy(ypredict), copy.deepcopy(yscores), md_index = md_index,\n",
    "                                                           write_path=None)\n",
    "    \n",
    "    # Write submission to files\n",
    "    write_txt(yscores['Test'], ypredict_avg_fn['Test'], txt_path='./submission_03/')\n",
    "    # Load submission 03 to check\n",
    "    yx = []\n",
    "    yx_names = []\n",
    "    ldx = os.listdir('./submission_03/')\n",
    "    for idx in range(len(ldx)):\n",
    "        current_name = ldx[idx]\n",
    "        if current_name.endswith('.txt'):\n",
    "            with open('submission_03/' + current_name, 'r') as f:\n",
    "                yx.append(float(f.readline()))\n",
    "                yx_names.append(current_name[:-4])\n",
    "            \n",
    "    ck_names = np.array([yx_names, yscores['Test']]).T\n",
    "    ck_values = np.array([yx, ypredict_avg_fn['Test']]).T\n",
    "    \n",
    "    ckn = np.count_nonzero(ck_names[:, 0] != ck_names[:, 1])\n",
    "    ckv = np.count_nonzero(ck_values[:, 0] != ck_values[:, 1])\n",
    "    if ckn == 0 and ckv == 0:\n",
    "        print(\"Exact ^^. Completed.\")\n",
    "    else:\n",
    "        print('Failed, please check again.')  \n",
    "    \n",
    "    return ypredict_avg_fn\n",
    "_ = submission_03_ck()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}