MlpExperiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 22311,
     "status": "ok",
     "timestamp": 1693527126975,
     "user": {
      "displayName": "bartosz azaniux",
      "userId": "12656151146140381527"
     },
     "user_tz": -60
    },
    "id": "hG3fkgsQBBCL",
    "outputId": "902e6808-21ae-4d46-fb4a-8ceb6bf74e5e",
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Drive version:\n",
    "# !pip uninstall TabularExperimentTrackerClient --y\n",
    "# !pip install git+https://github.com/DanielWarfield1/TabularExperimentTrackerClient\n",
    "# !pip uninstall NeuralNetworksTrainingPackage --y\n",
    "# !pip install git+https://github.com/Bartosz-G/NeuralNetworksTrainingPackage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%pip uninstall NeuralNetworksTrainingPackage --y\n",
    "%pip uninstall TabularExperimentTrackerClient --y\n",
    "%pip install git+https://github.com/DanielWarfield1/TabularExperimentTrackerClient\n",
    "%pip install git+https://github.com/Bartosz-G/NeuralNetworksTrainingPackage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 5858,
     "status": "ok",
     "timestamp": 1693527132821,
     "user": {
      "displayName": "bartosz azaniux",
      "userId": "12656151146140381527"
     },
     "user_tz": -60
    },
    "id": "jguLhhgFBSkE",
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import torch\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 43635,
     "status": "ok",
     "timestamp": 1693527176450,
     "user": {
      "displayName": "bartosz azaniux",
      "userId": "12656151146140381527"
     },
     "user_tz": -60
    },
    "id": "xsHc3zlSCO7J",
    "outputId": "121cd4b8-d138-4d2f-a0dc-d0de85c48f65",
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from TabularExperimentTrackerClient.ExperimentClient import ExperimentClient\n",
    "\n",
    "path =  '../creds/'\n",
    "creds_orch_file = \"creds-orch.txt\"\n",
    "creds_openml_file = \"creds-openml.txt\"\n",
    "\n",
    "\n",
    "\n",
    "with open(os.path.join(path, creds_orch_file), 'r') as file:\n",
    "    lines = file.readlines()\n",
    "    orchname = lines[0].strip()\n",
    "    orchsecret = lines[1].strip()\n",
    "\n",
    "with open (os.path.join(path, creds_openml_file), \"r\") as myfile:\n",
    "    openMLAPIKey = myfile.read()\n",
    "\n",
    "ex = ExperimentClient(verbose = True)\n",
    "\n",
    "\n",
    "ex.define_orch_cred(orchname, orchsecret)\n",
    "ex.define_opml_cred(openMLAPIKey)\n",
    "\n",
    "# Colab version\n",
    "# ex.define_opml_cred_drive('/My Drive/research/non-homogenous-data/creds/creds-openml.txt')\n",
    "# ex.define_orch_cred_drive('bart', '/My Drive//research/non-homogenous-data/creds/creds-colab.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RBYDfcuJE-bP",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 1. Defining the experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 19,
     "status": "ok",
     "timestamp": 1693527176452,
     "user": {
      "displayName": "bartosz azaniux",
      "userId": "12656151146140381527"
     },
     "user_tz": -60
    },
    "id": "lyMEiU5ODM2e",
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "experiment_name = 'Mlp_experiment_1'\n",
    "\n",
    "\n",
    "# Mlp parameters\n",
    "depth = {'distribution': 'categorical', 'values':[2,3,4,5,6,7,8,9,10,11,12]}\n",
    "hidden_dim = {'distribution': 'categorical', 'values':[32, 64, 128, 256]}\n",
    "seed = {'distribution': 'constant', 'value': 42}\n",
    "regularize = {'distribution': 'categorical', 'values':[None, None, 0.25, 0.5, 'bn', 'bn']} # float implies dropout\n",
    "embd_size = {'distribution': 'categorical', 'values':[None, None, None, 'sqrt', 64, 128, 256]} # 'sqrt implies embedding is sqrt smaller than the number of categories\n",
    "optimizer = {'distribution': 'categorical', 'values': ['SGD', 'SGD', 'SGD', 'SGD', 'Adam']}\n",
    "batch_size = {'distribution': 'categorical', 'values':[64, 128, 256, 512, 1024]}\n",
    "epochs = {'distribution': 'constant', 'value': 60}\n",
    "lr = {'distribution': 'log_uniform', 'min':1e-5, 'max':1e-2}\n",
    "momentum = {'distribution': 'categorical', 'values': [0.5, 0.9]}\n",
    "no_cuda = {'distribution': 'constant', 'value': False}\n",
    "lr_step_size = {'distribution': 'constant', 'value': 20}\n",
    "gamma = {'distribution': 'constant', 'value': 0.1}\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 15,
     "status": "ok",
     "timestamp": 1693527176453,
     "user": {
      "displayName": "bartosz azaniux",
      "userId": "12656151146140381527"
     },
     "user_tz": -60
    },
    "id": "3atPjhTUEn05",
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#============================================================\n",
    "# MLP on datasets containing categorical variables\n",
    "#============================================================\n",
    "Mlp_relu_cat_space = {\n",
    "    'depth': depth,\n",
    "    'seed': seed,\n",
    "    'regularize':regularize,\n",
    "    'embd_size':embd_size,\n",
    "    'activation': {'distribution': 'constant', 'value': 'relu'},\n",
    "    'hidden_dim': hidden_dim,\n",
    "    'optimizer': optimizer,\n",
    "    'batch_size': batch_size,\n",
    "    'epochs': epochs,\n",
    "    'lr': lr,\n",
    "    'momentum': momentum,\n",
    "    'no_cuda': no_cuda,\n",
    "    'lr_step_size': lr_step_size,\n",
    "    'gamma': gamma\n",
    "    }\n",
    "\n",
    "Mlp_sigm_cat_space = {\n",
    "    'depth': depth,\n",
    "    'seed': seed,\n",
    "    'regularize':regularize,\n",
    "    'embd_size':embd_size,\n",
    "    'activation': {'distribution': 'constant', 'value': 'sigmoid'},\n",
    "    'hidden_dim': hidden_dim,\n",
    "    'optimizer': optimizer,\n",
    "    'batch_size': batch_size,\n",
    "    'epochs': epochs,\n",
    "    'lr': lr,\n",
    "    'momentum': momentum,\n",
    "    'no_cuda': no_cuda,\n",
    "    'lr_step_size': lr_step_size,\n",
    "    'gamma': gamma\n",
    "}\n",
    "\n",
    "#============================================================\n",
    "# MLP on continous predictors\n",
    "#============================================================\n",
    "\n",
    "Mlp_relu_cont_space = {\n",
    "    'depth': depth,\n",
    "    'seed': seed,\n",
    "    'regularize':regularize,\n",
    "    'embd_size': {'distribution': 'constant', 'value': None},\n",
    "    'activation': {'distribution': 'constant', 'value': 'relu'},\n",
    "    'hidden_dim': hidden_dim,\n",
    "    'optimizer': optimizer,\n",
    "    'batch_size': batch_size,\n",
    "    'epochs': epochs,\n",
    "    'lr': lr,\n",
    "    'momentum': momentum,\n",
    "    'no_cuda': no_cuda,\n",
    "    'lr_step_size': lr_step_size,\n",
    "    'gamma': gamma\n",
    "}\n",
    "\n",
    "\n",
    "Mlp_sigm_cont_space = {\n",
    "    'depth': depth,\n",
    "    'seed': seed,\n",
    "    'regularize':regularize,\n",
    "    'embd_size': {'distribution': 'constant', 'value': None},\n",
    "    'activation': {'distribution': 'constant', 'value': 'sigmoid'},\n",
    "    'hidden_dim': hidden_dim,\n",
    "    'optimizer': optimizer,\n",
    "    'batch_size': batch_size,\n",
    "    'epochs': epochs,\n",
    "    'lr': lr,\n",
    "    'momentum': momentum,\n",
    "    'no_cuda': no_cuda,\n",
    "    'lr_step_size': lr_step_size,\n",
    "    'gamma': gamma\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 13,
     "status": "ok",
     "timestamp": 1693527176453,
     "user": {
      "displayName": "bartosz azaniux",
      "userId": "12656151146140381527"
     },
     "user_tz": -60
    },
    "id": "5UI-B-EbEJUh",
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_groups = {\n",
    "    'Mlp_relu_cat':{'model':'Mlp_relu_cat', 'hype':Mlp_relu_cat_space},\n",
    "    'Mlp_sigm_cat':{'model':'Mlp_sigm_cat', 'hype':Mlp_sigm_cat_space},\n",
    "    'Mlp_relu_cont':{'model':'Mlp_relu_cont', 'hype':Mlp_relu_cont_space},\n",
    "    'Mlp_sigm_cont':{'model':'Mlp_sigm_cont', 'hype':Mlp_sigm_cont_space}\n",
    "}\n",
    "\n",
    "ex.def_model_groups(model_groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ex.def_data_groups_opml()\n",
    "print(f'automatically defined data groups: {ex.data_groups.keys()}')\n",
    "\n",
    "categorical_models = [k for k in model_groups.keys() if '_cat' in k]\n",
    "continous_models = [k for k in model_groups.keys() if '_cont' in k]\n",
    "\n",
    "\n",
    "applications = {'opml_reg_purnum_group': continous_models,\n",
    "                'opml_reg_numcat_group': categorical_models,\n",
    "                'opml_class_purnum_group': continous_models,\n",
    "                'opml_class_numcat_group': categorical_models}\n",
    "\n",
    "ex.def_applications(applications)\n",
    "ex.reg_experiment(experiment_name)\n",
    "\n",
    "\n",
    "# Required to distinguish between classification or regression tasks\n",
    "regression_tasks = ex.__dict__['data_groups']['opml_reg_purnum_group'] + ex.__dict__['data_groups']['opml_reg_numcat_group']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "exp_info = ex.experiment_info()\n",
    "successful_runs = exp_info['successful_runs']\n",
    "required_runs = exp_info['required_runs']\n",
    "print('successful runs: {}'.format(successful_runs))\n",
    "print('total required runs: {}'.format(required_runs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 2. General Data pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from NeuralNetworksTrainingPackage.event_handler import dataPreProcessingEventEmitter\n",
    "from NeuralNetworksTrainingPackage.dataprocessing.basic_pre_processing import filterCardinality, quantileTransform, truncateData ,balancedTruncateData, oneHotEncodePredictors, oneHotEncodeTargets, toDataFrame, splitTrainValTest, balancedSplitTrainValTest\n",
    "\n",
    "n_sample = 20000\n",
    "split = [0.5, 0.25, 0.25]\n",
    "quantile_transform_distribution='normal'\n",
    "\n",
    "\n",
    "data_pre_processing = dataPreProcessingEventEmitter()\n",
    "\n",
    "filter_cardinality = filterCardinality(transform = 'all')\n",
    "truncate_data = truncateData(n = n_sample, transform = 'all')\n",
    "balanced_truncate_data = balancedTruncateData(n = n_sample, transform = 'all') # Ensures balance of classes\n",
    "one_hot_encode_predictors = oneHotEncodePredictors(transform = 'all')\n",
    "one_hot_encode_targets = oneHotEncodeTargets(transform = 'all')\n",
    "to_data_frame = toDataFrame(transform = 'all')\n",
    "split_train_val_test = splitTrainValTest(split = split)\n",
    "balanced_split_train_val_test = balancedSplitTrainValTest(split = split)\n",
    "quantile_transform = quantileTransform(output_distribution = quantile_transform_distribution, transform = 'all')\n",
    "\n",
    "\n",
    "# Transformations will be called in the order they're added to data_pre_processing\n",
    "data_pre_processing.add_pre_processing_step('regression', filter_cardinality)\n",
    "data_pre_processing.add_pre_processing_step('regression', truncate_data)\n",
    "data_pre_processing.add_pre_processing_step('regression', one_hot_encode_predictors)\n",
    "data_pre_processing.add_pre_processing_step('regression', to_data_frame)\n",
    "data_pre_processing.add_pre_processing_step('regression', split_train_val_test)\n",
    "data_pre_processing.add_pre_processing_step('regression', quantile_transform)\n",
    "\n",
    "\n",
    "data_pre_processing.add_pre_processing_step('classification', filter_cardinality)\n",
    "data_pre_processing.add_pre_processing_step('classification', balanced_truncate_data)\n",
    "data_pre_processing.add_pre_processing_step('classification', one_hot_encode_predictors)\n",
    "data_pre_processing.add_pre_processing_step('classification', one_hot_encode_targets)\n",
    "data_pre_processing.add_pre_processing_step('classification', to_data_frame)\n",
    "data_pre_processing.add_pre_processing_step('classification', balanced_split_train_val_test)\n",
    "data_pre_processing.add_pre_processing_step('classification', quantile_transform)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 3. Model Specific Data pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from NeuralNetworksTrainingPackage.dataprocessing.basic_pre_processing import CustomCategoricalSplitDataset, toPyTorchDatasets\n",
    "\n",
    "pytorch_data_class = CustomCategoricalSplitDataset\n",
    "to_pytorch_datasets = toPyTorchDatasets(wrapper = pytorch_data_class)\n",
    "\n",
    "\n",
    "# Transformations will be called after general pre-processing steps, and in order they're added\n",
    "data_pre_processing.add_pre_processing_step('Mlp_relu_cat', to_pytorch_datasets)\n",
    "\n",
    "data_pre_processing.add_pre_processing_step('Mlp_sigm_cat', to_pytorch_datasets)\n",
    "\n",
    "data_pre_processing.add_pre_processing_step('Mlp_relu_cont', to_pytorch_datasets)\n",
    "\n",
    "data_pre_processing.add_pre_processing_step('Mlp_sigm_cont', to_pytorch_datasets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 4. Model Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from metrics.MlpMetrics import MlpMetricsClassification, MlpMetricsRegression\n",
    "\n",
    "mlp_metrics_regression = MlpMetricsRegression()\n",
    "mlp_metrics_classification = MlpMetricsClassification()\n",
    "\n",
    "mlp_metrics = {'regression': mlp_metrics_regression,\n",
    "               'classification': mlp_metrics_classification}\n",
    "\n",
    "metric_model_pairs = {\n",
    "    'Mlp_relu_cat': mlp_metrics,\n",
    "    'Mlp_sigm_cat': mlp_metrics,\n",
    "    'Mlp_relu_cont': mlp_metrics,\n",
    "    'Mlp_sigm_cont': mlp_metrics,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 5. Model Training routine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from models.MlpNetwork import init_MlpNetwork\n",
    "from training.MlpTrain import MlpTrainingRoutine\n",
    "\n",
    "mlp_model_and_training = {'model_init':init_MlpNetwork, 'training_routine': MlpTrainingRoutine}\n",
    "\n",
    "model_training_pairs = {\n",
    "    'Mlp_relu_cat': mlp_model_and_training,\n",
    "    'Mlp_sigm_cat': mlp_model_and_training,\n",
    "    'Mlp_relu_cont': mlp_model_and_training,\n",
    "    'Mlp_sigm_cont': mlp_model_and_training,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 6. Main Experiment loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch_models = ('Mlp_relu_cat', 'Mlp_sigm_cat', 'Mlp_relu_cont', 'Mlp_sigm_cont')\n",
    "\n",
    "sklearn_models = ('no_sklearn_models_in_this_training')\n",
    "\n",
    "\n",
    "for i in range(7080):\n",
    "    print(f'==== Begin run:{i} ====')\n",
    "    # run_info = ex.begin_run_sticky()\n",
    "    run_info = ex.begin_run()\n",
    "\n",
    "    hyperparameters = run_info['hyp']\n",
    "    model_name = run_info['model']\n",
    "\n",
    "    if hyperparameters.get('task') is None:\n",
    "        if run_info['task'] in regression_tasks:\n",
    "            hyperparameters['task'] = 'regression'\n",
    "        else:\n",
    "            hyperparameters['task'] = 'classification'\n",
    "    task = hyperparameters['task']\n",
    "    seed = hyperparameters['seed']\n",
    "\n",
    "    # Debugging:\n",
    "    # hyperparameters['embd_size'] = 64\n",
    "\n",
    "    torch.manual_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "\n",
    "    print('---- Loading datasets ----')\n",
    "    X, y, categorical_indicator, attribute_names = ex.opml_load_task(run_info['mtpair_task'])\n",
    "\n",
    "    # Pre-processing\n",
    "    data_pre_processing.set_seed_for_all(seed)\n",
    "    data_pre_processing.set_dataset(X, y, categorical_indicator, attribute_names)\n",
    "    data_pre_processing.apply(task)\n",
    "    data_pre_processing.apply(model_name)\n",
    "    train_data, val_data, test_data = data_pre_processing.get_train_val_test()\n",
    "\n",
    "\n",
    "    # Getting appropriate metrics\n",
    "    metrics_calculator = metric_model_pairs[model_name][task]\n",
    "\n",
    "\n",
    "    match model_name:\n",
    "        case _ if model_name in sklearn_models:\n",
    "            pass\n",
    "\n",
    "        case _ if model_name in torch_models:\n",
    "            # hyperparameters will be updated with {'input_dim': num_columns_X, 'output_dim':num_columns_Y}\n",
    "            hyperparameters.update(train_data.get_dims())\n",
    "\n",
    "            train_batch_size = hyperparameters['batch_size']\n",
    "            train_dataloader = torch.utils.data.DataLoader(train_data, batch_size=train_batch_size,shuffle= True)\n",
    "            val_dataloader = torch.utils.data.DataLoader(val_data,batch_size=len(val_data),shuffle= True)\n",
    "            test_dataloader = torch.utils.data.DataLoader(test_data,batch_size=len(test_data),shuffle= True)\n",
    "\n",
    "\n",
    "            init_model = model_training_pairs[model_name]['model_init']\n",
    "            TrainingRoutine = model_training_pairs[model_name]['training_routine']\n",
    "\n",
    "\n",
    "            model = init_model(**hyperparameters)\n",
    "            training_routine = TrainingRoutine(**hyperparameters)\n",
    "            \n",
    "            training_routine.set_optimizer_scheduler(model)\n",
    "\n",
    "            start_epoch = 1  # start from epoch 1 or last checkpoint epoch\n",
    "            total_epochs = hyperparameters['epochs']\n",
    "            start_time = time.time()\n",
    "\n",
    "            for epoch in range(start_epoch, total_epochs + start_epoch):\n",
    "                print(f\"----{epoch}th training epoch ----\")\n",
    "                epoch_metrics = {}\n",
    "\n",
    "                training_routine.scheduler_step(epoch)\n",
    "                train_loss = training_routine.train(model, train_dataloader)\n",
    "\n",
    "\n",
    "                if train_loss is None:\n",
    "                    print('---Stopping training due to loss being nan!---')\n",
    "                    epoch_metrics = {'train_loss': train_loss, 'epoch': epoch}\n",
    "                    ex.update_run(epoch_metrics)\n",
    "                    break\n",
    "\n",
    "                if epoch == total_epochs:\n",
    "                    continue\n",
    "\n",
    "                epoch_metrics.update(train_loss)\n",
    "                epoch_metrics.update({'epoch': epoch})\n",
    "                ex.update_run(epoch_metrics)\n",
    "                # print(epoch_metrics)\n",
    "\n",
    "            else:\n",
    "                final_metrics = {}\n",
    "                training_time = time.time()-start_time\n",
    "\n",
    "                train_metrics = metrics_calculator.get_metrics(model, train_dataloader, hyperparameters, 'train')\n",
    "                val_all_metrics = metrics_calculator.get_all(model, val_dataloader, hyperparameters, 'val')\n",
    "                test_all_metrics = metrics_calculator.get_all(model, test_dataloader, hyperparameters, 'test')\n",
    "\n",
    "                final_metrics.update(train_loss)\n",
    "                final_metrics.update(train_metrics)\n",
    "                final_metrics.update(val_all_metrics)\n",
    "                final_metrics.update(test_all_metrics)\n",
    "                final_metrics.update({'epoch': epoch})\n",
    "                final_metrics.update({'epoch_time': training_time})\n",
    "\n",
    "                ex.update_run(final_metrics)\n",
    "                print(final_metrics)\n",
    "\n",
    "    ex.end_run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Checking whether the code works as intended"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# torch_models = ('LCN_reg', 'LCN_cls', 'LLN_reg', 'LLN_cls')\n",
    "\n",
    "# sklearn_models = ('no_sklearn_models_in_this_training')\n",
    "\n",
    "\n",
    "# for i in range(1):\n",
    "#     print(f'==== Begin run:{i} ====')\n",
    "#     run_info = ex.begin_run()\n",
    "\n",
    "#     hyperparameters = run_info['hyp']\n",
    "#     model_name = run_info['model']\n",
    "\n",
    "#     task = hyperparameters['task']\n",
    "#     seed = hyperparameters['seed']\n",
    "\n",
    "\n",
    "\n",
    "#     torch.manual_seed(seed)\n",
    "#     np.random.seed(seed)\n",
    "\n",
    "\n",
    "#     print('---- Loading datasets ----')\n",
    "#     X, y, categorical_indicator, attribute_names = ex.opml_load_task(run_info['mtpair_task'])\n",
    "\n",
    "#     # Pre-processing\n",
    "#     data_pre_processing.set_seed_for_all(seed)\n",
    "#     data_pre_processing.set_dataset(X, y, categorical_indicator, attribute_names)\n",
    "#     data_pre_processing.apply(task)\n",
    "#     data_pre_processing.apply(model_name)\n",
    "#     train_data, val_data, test_data = data_pre_processing.get_train_val_test()\n",
    "\n",
    "\n",
    "#     # Getting appropriate metrics\n",
    "#     metrics_calculator = metric_model_pairs[model_name][task]\n",
    "\n",
    "\n",
    "#     match model_name:\n",
    "#         case _ if model_name in sklearn_models:\n",
    "#             pass\n",
    "\n",
    "#         case _ if model_name in torch_models:\n",
    "#             # hyperparameters will be updated with {'input_dim': num_columns_X, 'output_dim':num_columns_Y}\n",
    "#             hyperparameters.update(train_data.get_dims())\n",
    "\n",
    "#             train_batch_size = hyperparameters['batch_size']\n",
    "#             train_dataloader = torch.utils.data.DataLoader(train_data, batch_size=train_batch_size,shuffle= True)\n",
    "#             val_dataloader = torch.utils.data.DataLoader(val_data,batch_size=len(val_data),shuffle= True)\n",
    "#             test_dataloader = torch.utils.data.DataLoader(test_data,batch_size=len(test_data),shuffle= True)\n",
    "\n",
    "\n",
    "#             init_model = model_training_pairs[model_name]['model_init']\n",
    "#             TrainingRoutine = model_training_pairs[model_name]['training_routine']\n",
    "\n",
    "\n",
    "#             model = init_model(**hyperparameters)\n",
    "#             training_routine = TrainingRoutine(**hyperparameters)\n",
    "            \n",
    "#             training_routine.set_optimizer_scheduler(model)\n",
    "\n",
    "#             start_epoch = 1  # start from epoch 1 or last checkpoint epoch\n",
    "#             total_epochs = hyperparameters['epochs']\n",
    "#             start_time = time.time()\n",
    "\n",
    "#             for epoch in range(start_epoch, total_epochs + start_epoch):\n",
    "#                 print(f\"----{epoch}th training epoch ----\")\n",
    "#                 epoch_metrics = {}\n",
    "\n",
    "#                 training_routine.scheduler_step(epoch)\n",
    "#                 train_loss = training_routine.train(model, train_dataloader)\n",
    "\n",
    "\n",
    "#                 if train_loss is None:\n",
    "#                     print('---Stopping training due to loss being nan!---')\n",
    "#                     epoch_metrics = {'train_loss': train_loss, 'epoch': epoch}\n",
    "#                     ex.update_run(epoch_metrics)\n",
    "#                     break\n",
    "\n",
    "#                 if epoch == total_epochs:\n",
    "#                     continue\n",
    "\n",
    "#                 epoch_metrics.update(train_loss)\n",
    "#                 epoch_metrics.update({'epoch': epoch})\n",
    "#                 ex.update_run(epoch_metrics)\n",
    "#                 print(epoch_metrics)\n",
    "\n",
    "#             else:\n",
    "#                 final_metrics = {}\n",
    "#                 training_time = time.time()-start_time\n",
    "\n",
    "#                 train_metrics = metrics_calculator.get_metrics(model, train_dataloader, hyperparameters, 'train')\n",
    "#                 val_all_metrics = metrics_calculator.get_all(model, val_dataloader, hyperparameters, 'val')\n",
    "#                 test_all_metrics = metrics_calculator.get_all(model, test_dataloader, hyperparameters, 'test')\n",
    "\n",
    "#                 final_metrics.update(train_loss)\n",
    "#                 final_metrics.update(train_metrics)\n",
    "#                 final_metrics.update(val_all_metrics)\n",
    "#                 final_metrics.update(test_all_metrics)\n",
    "#                 final_metrics.update({'epoch': epoch})\n",
    "#                 final_metrics.update({'epoch_time': training_time})\n",
    "\n",
    "#                 ex.update_run(final_metrics)\n",
    "#                 print(final_metrics)\n",
    "\n",
    "#     ex.end_run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# torch_models = ('LCN_reg', 'LCN_cls', 'LLN_reg', 'LLN_cls')\n",
    "\n",
    "# sklearn_models = ('no_sklearn_models_in_this_training')\n",
    "\n",
    "\n",
    "# for i in range(1):\n",
    "#     print(f'==== Begin run:{i} ====')\n",
    "# #     run_info = ex.begin_run()\n",
    "\n",
    "#     hyperparameters = run_info['hyp']\n",
    "#     model_name = run_info['model']\n",
    "\n",
    "#     task = hyperparameters['task']\n",
    "#     seed = hyperparameters['seed']\n",
    "\n",
    "\n",
    "\n",
    "#     torch.manual_seed(seed)\n",
    "#     np.random.seed(seed)\n",
    "\n",
    "\n",
    "#     print('---- Loading datasets ----')\n",
    "#     X, y, categorical_indicator, attribute_names = ex.opml_load_task(run_info['mtpair_task'])\n",
    "\n",
    "#     # Pre-processing\n",
    "#     data_pre_processing.set_seed_for_all(seed)\n",
    "#     data_pre_processing.set_dataset(X, y, categorical_indicator, attribute_names)\n",
    "#     data_pre_processing.apply(task)\n",
    "#     data_pre_processing.apply(model_name)\n",
    "#     train_data, val_data, test_data = data_pre_processing.get_train_val_test()\n",
    "\n",
    "\n",
    "#     # Getting appropriate metrics\n",
    "#     metrics_calculator = metric_model_pairs[model_name][task]\n",
    "\n",
    "\n",
    "#     match model_name:\n",
    "#         case _ if model_name in sklearn_models:\n",
    "#             pass\n",
    "\n",
    "#         case _ if model_name in torch_models:\n",
    "#             # hyperparameters will be updated with {'input_dim': num_columns_X, 'output_dim':num_columns_Y}\n",
    "#             hyperparameters.update(train_data.get_dims())\n",
    "\n",
    "#             train_batch_size = hyperparameters['batch_size']\n",
    "#             train_dataloader = torch.utils.data.DataLoader(train_data, batch_size=train_batch_size,shuffle= True)\n",
    "#             val_dataloader = torch.utils.data.DataLoader(val_data,batch_size=len(val_data),shuffle= True)\n",
    "#             test_dataloader = torch.utils.data.DataLoader(test_data,batch_size=len(test_data),shuffle= True)\n",
    "\n",
    "\n",
    "#             init_model = model_training_pairs[model_name]['model_init']\n",
    "#             TrainingRoutine = model_training_pairs[model_name]['training_routine']\n",
    "\n",
    "\n",
    "#             model = init_model(**hyperparameters)\n",
    "#             training_routine = TrainingRoutine(**hyperparameters)\n",
    "            \n",
    "#             training_routine.set_optimizer_scheduler(model)\n",
    "\n",
    "#             start_epoch = 1  # start from epoch 1 or last checkpoint epoch\n",
    "#             total_epochs = hyperparameters['epochs']\n",
    "#             start_time = time.time()\n",
    "\n",
    "#             for epoch in range(start_epoch, total_epochs + start_epoch):\n",
    "#                 print(f\"----{epoch}th training epoch ----\")\n",
    "#                 epoch_metrics = {}\n",
    "\n",
    "#                 training_routine.scheduler_step(epoch)\n",
    "#                 train_loss = training_routine.train(model, train_dataloader)\n",
    "\n",
    "\n",
    "#                 if train_loss is None:\n",
    "#                     print('---Stopping training due to loss being nan!---')\n",
    "#                     epoch_metrics = {'train_loss': train_loss, 'epoch': epoch}\n",
    "#                     print(epoch_metrics)\n",
    "#                     # ex.update_run(epoch_metrics)\n",
    "#                     break\n",
    "\n",
    "#                 if epoch == total_epochs:\n",
    "#                     continue\n",
    "\n",
    "#                 epoch_metrics.update(train_loss)\n",
    "#                 epoch_metrics.update({'epoch': epoch})\n",
    "#                 # ex.update_run(epoch_metrics)\n",
    "#                 print(epoch_metrics)\n",
    "\n",
    "#             else:\n",
    "#                 final_metrics = {}\n",
    "#                 training_time = time.time()-start_time\n",
    "\n",
    "#                 train_metrics = metrics_calculator.get_metrics(model, train_dataloader, hyperparameters, 'train')\n",
    "#                 val_all_metrics = metrics_calculator.get_all(model, val_dataloader, hyperparameters, 'val')\n",
    "#                 test_all_metrics = metrics_calculator.get_all(model, test_dataloader, hyperparameters, 'test')\n",
    "\n",
    "#                 final_metrics.update(train_loss)\n",
    "#                 final_metrics.update(train_metrics)\n",
    "#                 final_metrics.update(val_all_metrics)\n",
    "#                 final_metrics.update(test_all_metrics)\n",
    "#                 final_metrics.update({'epoch': epoch})\n",
    "#                 final_metrics.update({'epoch_time': training_time})\n",
    "\n",
    "#                 # ex.update_run(final_metrics)\n",
    "#                 print(final_metrics)\n",
    "\n",
    "#     # ex.end_run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# class oneHotEncodeTargets():\n",
    "#     def __init__(self, transform = 'all'):\n",
    "#         self.parent = None\n",
    "#         self.transform = transform\n",
    "\n",
    "#     def apply(self, X, y, categorical_indicator, attribute_names):\n",
    "        \n",
    "#         y = pd.get_dummies(y, dtype=int)\n",
    "\n",
    "#         # if isinstance(y, pd.DataFrame):\n",
    "#         #     is_categorical = any(y[col].dtype.name == 'category' for col in y.columns)\n",
    "#         #     if is_categorical:\n",
    "#         #         y = pd.get_dummies(y)\n",
    "#         # \n",
    "#         # if isinstance(y, pd.Series):\n",
    "#         #     is_categorical = y.dtype.name == 'category'\n",
    "#         # \n",
    "#         #     if is_categorical:\n",
    "#         #         y = pd.get_dummies(y)\n",
    "\n",
    "#         return X, y, categorical_indicator, attribute_names\n",
    "    \n",
    "# one_hot_encode_targets = oneHotEncodeTargets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#     hyperparameters = run_info['hyp']\n",
    "#     model_name = run_info['model']\n",
    "\n",
    "#     task = hyperparameters['task']\n",
    "#     seed = hyperparameters['seed']\n",
    "\n",
    "\n",
    "\n",
    "#     torch.manual_seed(seed)\n",
    "#     np.random.seed(seed)\n",
    "\n",
    "\n",
    "#     print('---- Loading datasets ----')\n",
    "#     X, y, categorical_indicator, attribute_names = ex.opml_load_task(run_info['mtpair_task'])\n",
    "#     print(f'dtype: {y.dtype}')\n",
    "    \n",
    "#     X_o, y_o, categorical_indicator_o, attribute_names_0 = one_hot_encode_targets.apply(X, y, categorical_indicator, attribute_names)\n",
    "\n",
    "#     print(y_o)\n",
    "    \n",
    "#     # Pre-processing\n",
    "# #     data_pre_processing.set_seed_for_all(seed)\n",
    "# #     data_pre_processing.set_dataset(X, y, categorical_indicator, attribute_names)\n",
    "# #     data_pre_processing.apply(task)\n",
    "# #     data_pre_processing.apply(model_name)\n",
    "# #     train_data, val_data, test_data = data_pre_processing.get_train_val_test()\n",
    "    \n",
    "# #     print(train_data.Y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# data_pre_processing.events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(next(iter(train_dataloader))[0].shape)\n",
    "print(next(iter(train_dataloader))[1].shape)\n",
    "print(next(iter(train_dataloader))[2].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "#X_c, y_c, categorical_indicator_c, attribute_names_c = deepcopy(X), deepcopy(y), deepcopy(categorical_indicator), deepcopy(attribute_names)\n",
    "X, y, categorical_indicator, attribute_names = deepcopy(X_c), deepcopy(y_c), deepcopy(categorical_indicator_c), deepcopy(attribute_names_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# from NeuralNetworksTrainingPackage.event_handler import dataPreProcessingEventEmitter\n",
    "# from NeuralNetworksTrainingPackage.dataprocessing.basic_pre_processing import filterCardinality, quantileTransform, truncateData ,balancedTruncateData, oneHotEncodePredictors, toDataFrame, splitTrainValTest, balancedSplitTrainValTest\n",
    "\n",
    "# n_sample = 20000\n",
    "# split = [0.5, 0.25, 0.25]\n",
    "# quantile_transform_distribution='normal'\n",
    "\n",
    "# class oneHotEncodeTargets():\n",
    "#     def __init__(self, transform = 'all'):\n",
    "#         self.parent = None\n",
    "#         self.transform = transform\n",
    "\n",
    "#     def apply(self, X, y, categorical_indicator, attribute_names):\n",
    "        \n",
    "#         y = pd.get_dummies(y, dtype=int)\n",
    "\n",
    "#         # if isinstance(y, pd.DataFrame):\n",
    "#         #     is_categorical = any(y[col].dtype.name == 'category' for col in y.columns)\n",
    "#         #     if is_categorical:\n",
    "#         #         y = pd.get_dummies(y)\n",
    "#         # \n",
    "#         # if isinstance(y, pd.Series):\n",
    "#         #     is_categorical = y.dtype.name == 'category'\n",
    "#         # \n",
    "#         #     if is_categorical:\n",
    "#         #         y = pd.get_dummies(y)\n",
    "\n",
    "#         return X, y, categorical_indicator, attribute_names\n",
    "\n",
    "\n",
    "# data_pre_processing = dataPreProcessingEventEmitter()\n",
    "\n",
    "# filter_cardinality = filterCardinality(transform = 'all')\n",
    "# truncate_data = truncateData(n = n_sample, transform = 'all')\n",
    "# balanced_truncate_data = balancedTruncateData(n = n_sample, transform = 'all') # Ensures balance of classes\n",
    "# one_hot_encode_predictors = oneHotEncodePredictors(transform = 'all')\n",
    "# one_hot_encode_targets = oneHotEncodeTargets(transform = 'all')\n",
    "# to_data_frame = toDataFrame(transform = 'all')\n",
    "# split_train_val_test = splitTrainValTest(split = split)\n",
    "# balanced_split_train_val_test = balancedSplitTrainValTest(split = split)\n",
    "# quantile_transform = quantileTransform(output_distribution = quantile_transform_distribution, transform = 'all')\n",
    "\n",
    "\n",
    "# # Transformations will be called in the order they're added to data_pre_processing\n",
    "# data_pre_processing.add_pre_processing_step('regression', filter_cardinality)\n",
    "# data_pre_processing.add_pre_processing_step('regression', truncate_data)\n",
    "# data_pre_processing.add_pre_processing_step('regression', one_hot_encode_predictors)\n",
    "# data_pre_processing.add_pre_processing_step('regression', to_data_frame)\n",
    "# data_pre_processing.add_pre_processing_step('regression', split_train_val_test)\n",
    "# data_pre_processing.add_pre_processing_step('regression', quantile_transform)\n",
    "\n",
    "\n",
    "# data_pre_processing.add_pre_processing_step('classification', filter_cardinality)\n",
    "# data_pre_processing.add_pre_processing_step('classification', balanced_truncate_data)\n",
    "# data_pre_processing.add_pre_processing_step('classification', one_hot_encode_predictors)\n",
    "# data_pre_processing.add_pre_processing_step('classification', one_hot_encode_targets)\n",
    "# data_pre_processing.add_pre_processing_step('classification', to_data_frame)\n",
    "# data_pre_processing.add_pre_processing_step('classification', balanced_split_train_val_test)\n",
    "# data_pre_processing.add_pre_processing_step('classification', quantile_transform)\n",
    "\n",
    "# from NeuralNetworksTrainingPackage.dataprocessing.basic_pre_processing import CustomDataset, toPyTorchDatasets\n",
    "\n",
    "# to_pytorch_datasets = toPyTorchDatasets(wrapper = CustomDataset)\n",
    "\n",
    "# # Transformations will be called after general pre-processing steps, and in order they're added\n",
    "# data_pre_processing.add_pre_processing_step('LCN_reg', to_pytorch_datasets)\n",
    "\n",
    "# data_pre_processing.add_pre_processing_step('LCN_cls', to_pytorch_datasets)\n",
    "\n",
    "# data_pre_processing.add_pre_processing_step('LLN_reg', to_pytorch_datasets)\n",
    "\n",
    "# data_pre_processing.add_pre_processing_step('LLN_cls', to_pytorch_datasets)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "data_pre_processing.set_seed_for_all(seed)\n",
    "data_pre_processing.set_dataset(X, y, categorical_indicator, attribute_names)\n",
    "data_pre_processing.apply('classification')\n",
    "data_pre_processing.apply(model_name)\n",
    "train_data, val_data, test_data = data_pre_processing.get_train_val_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_dataloader = torch.utils.data.DataLoader(train_data, batch_size=train_batch_size,shuffle= True)\n",
    "val_dataloader = torch.utils.data.DataLoader(val_data,batch_size=len(val_data),shuffle= True)\n",
    "test_dataloader = torch.utils.data.DataLoader(test_data,batch_size=len(test_data),shuffle= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "X, y, categorical_indicator, attribute_names = train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print(X.shape)\n",
    "print(X_c.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "non_categorical_columns = [attr for attr, is_cat in zip(attribute_names, categorical_indicator) if not is_cat]\n",
    "print(X[non_categorical_columns].mean())\n",
    "print(X[non_categorical_columns].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "non_categorical_columns = [attr for attr, is_cat in zip(attribute_names_c, categorical_indicator_c) if not is_cat]\n",
    "print(X_c[non_categorical_columns].mean())\n",
    "print(X_c[non_categorical_columns].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print(X.loc[:, categorical_indicator].nunique())\n",
    "print(X_c.loc[:, categorical_indicator_c].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "val_data = data_pre_processing.get('val')\n",
    "test_data = data_pre_processing.get('test')\n",
    "print(val_data[1].value_counts())\n",
    "print(test_data[1].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "type(train_data.Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for data, target in train_dataloader:\n",
    "    print(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "'336-361072' in regression_tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "run_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "ex.__dict__['data_groups']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "'336-361072' in regression_tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ex.__dict__['data_groups']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "colab": {
   "authorship_tag": "ABX9TyO6d7scFvmxtcd+gePzcxnN",
   "provenance": []
  },
  "instance_type": "ml.g4dn.xlarge",
  "kernelspec": {
   "display_name": "Python 3 (PyTorch 2.0.0 Python 3.10 GPU Optimized)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-2:712779665605:image/pytorch-2.0.0-gpu-py310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}