notebooks/DataPreprocessing-pH.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MODIFY THE PATH OF Microbiome-Factorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ROOT = \"/Users/earmingol/Dropbox/Universidad/UCSanDiego/Lab_Knight/Microbiome-Factorization/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "otus_filename = ROOT + \"/data/pH_study/OTUs_table.tsv\"\n",
    "metadata_filename = ROOT + \"/data/pH_study/metadata.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "OTUs = pd.read_csv(otus_filename, sep='\\t', index_col='#OTU ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>103.CA2</th>\n",
       "      <th>103.CO3</th>\n",
       "      <th>103.SR3</th>\n",
       "      <th>103.IE2</th>\n",
       "      <th>103.BP1</th>\n",
       "      <th>103.VC2</th>\n",
       "      <th>103.SA2</th>\n",
       "      <th>103.GB2</th>\n",
       "      <th>103.CO2</th>\n",
       "      <th>103.KP1</th>\n",
       "      <th>...</th>\n",
       "      <th>103.LQ1</th>\n",
       "      <th>103.HI1</th>\n",
       "      <th>103.RT1</th>\n",
       "      <th>103.HI2</th>\n",
       "      <th>103.DF1</th>\n",
       "      <th>103.CF3</th>\n",
       "      <th>103.AR1</th>\n",
       "      <th>103.TL1</th>\n",
       "      <th>103.HI4</th>\n",
       "      <th>103.BB1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>#OTU ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1124701</th>\n",
       "      <td>15.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244336</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>973124</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>809489</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533625</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 89 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         103.CA2  103.CO3  103.SR3  103.IE2  103.BP1  103.VC2  103.SA2  \\\n",
       "#OTU ID                                                                  \n",
       "1124701     15.0     14.0      1.0      8.0     13.0      7.0      6.0   \n",
       "244336       0.0      0.0      0.0      1.0      0.0      0.0      0.0   \n",
       "973124       0.0      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "809489       0.0      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "533625       0.0      0.0      0.0      0.0      0.0      0.0      0.0   \n",
       "\n",
       "         103.GB2  103.CO2  103.KP1  ...  103.LQ1  103.HI1  103.RT1  103.HI2  \\\n",
       "#OTU ID                             ...                                       \n",
       "1124701      3.0      2.0      2.0  ...      0.0      0.0      0.0      0.0   \n",
       "244336       0.0      0.0      0.0  ...      0.0      0.0      0.0      0.0   \n",
       "973124       0.0      0.0      1.0  ...      0.0      0.0      0.0      0.0   \n",
       "809489       1.0      0.0      0.0  ...      0.0      0.0      0.0      0.0   \n",
       "533625       0.0      0.0      0.0  ...      0.0      0.0      0.0      0.0   \n",
       "\n",
       "         103.DF1  103.CF3  103.AR1  103.TL1  103.HI4  103.BB1  \n",
       "#OTU ID                                                        \n",
       "1124701      0.0      0.0      0.0      0.0      0.0      0.0  \n",
       "244336       0.0      0.0      0.0      0.0      0.0      0.0  \n",
       "973124       0.0      0.0      0.0      0.0      0.0      0.0  \n",
       "809489       0.0      0.0      0.0      0.0      0.0      0.0  \n",
       "533625       0.0      0.0      0.0      0.0      0.0      0.0  \n",
       "\n",
       "[5 rows x 89 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "OTUs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7396, 89)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "OTUs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Metadata = pd.read_csv(metadata_filename, sep='\\t', index_col='#SampleID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "Metadata['PhClasses'] = Metadata['ph'].apply(lambda x: 1 if (x < 7.0) else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BarcodeSequence</th>\n",
       "      <th>LinkerPrimerSequence</th>\n",
       "      <th>barcode_read_group_tag</th>\n",
       "      <th>experiment_alias</th>\n",
       "      <th>experiment_center</th>\n",
       "      <th>experiment_design_description</th>\n",
       "      <th>experiment_title</th>\n",
       "      <th>instrument_name</th>\n",
       "      <th>key_seq</th>\n",
       "      <th>library_construction_protocol</th>\n",
       "      <th>...</th>\n",
       "      <th>silt_clay</th>\n",
       "      <th>soil_moisture_deficit</th>\n",
       "      <th>soil_type</th>\n",
       "      <th>specific_location</th>\n",
       "      <th>texture</th>\n",
       "      <th>title</th>\n",
       "      <th>tot_org_carb</th>\n",
       "      <th>tot_org_nitro</th>\n",
       "      <th>Description</th>\n",
       "      <th>PhClasses</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>#SampleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>103.BZ1</th>\n",
       "      <td>ACACATGTCTAC</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>BZ1_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>80</td>\n",
       "      <td>132.6</td>\n",
       "      <td>gelisol</td>\n",
       "      <td>Bonanza Creek LTER, AK USA</td>\n",
       "      <td>silt loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>30.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103.CR1</th>\n",
       "      <td>ACCACATACATC</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>CR1_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>160.1</td>\n",
       "      <td>mollisol</td>\n",
       "      <td>Coffey Ranch, TX USA</td>\n",
       "      <td>loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>28.3</td>\n",
       "      <td>10.9</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103.GB2</th>\n",
       "      <td>ACCTGTCTCTCT</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>GB2_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>65</td>\n",
       "      <td>-112.3</td>\n",
       "      <td>mollisol</td>\n",
       "      <td>Great Basin Experimental Range, UT USA</td>\n",
       "      <td>loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>68.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103.GB3</th>\n",
       "      <td>ACGACGTCTTAG</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>GB3_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>76</td>\n",
       "      <td>-112.3</td>\n",
       "      <td>mollisol</td>\n",
       "      <td>Great Basin Experimental Range, UT USA</td>\n",
       "      <td>clay loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>57.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103.GB1</th>\n",
       "      <td>ACCTCGATCAGA</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>GB1_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>77</td>\n",
       "      <td>-112.3</td>\n",
       "      <td>mollisol</td>\n",
       "      <td>Great Basin Experimental Range, UT USA</td>\n",
       "      <td>clay loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>28.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          BarcodeSequence   LinkerPrimerSequence barcode_read_group_tag  \\\n",
       "#SampleID                                                                 \n",
       "103.BZ1      ACACATGTCTAC  CATGCTGCCTCCCGTAGGAGT                 BZ1_V2   \n",
       "103.CR1      ACCACATACATC  CATGCTGCCTCCCGTAGGAGT                 CR1_V2   \n",
       "103.GB2      ACCTGTCTCTCT  CATGCTGCCTCCCGTAGGAGT                 GB2_V2   \n",
       "103.GB3      ACGACGTCTTAG  CATGCTGCCTCCCGTAGGAGT                 GB3_V2   \n",
       "103.GB1      ACCTCGATCAGA  CATGCTGCCTCCCGTAGGAGT                 GB1_V2   \n",
       "\n",
       "          experiment_alias experiment_center experiment_design_description  \\\n",
       "#SampleID                                                                    \n",
       "103.BZ1    lauber_88_soils              CCME        bacterial biogeography   \n",
       "103.CR1    lauber_88_soils              CCME        bacterial biogeography   \n",
       "103.GB2    lauber_88_soils              CCME        bacterial biogeography   \n",
       "103.GB3    lauber_88_soils              CCME        bacterial biogeography   \n",
       "103.GB1    lauber_88_soils              CCME        bacterial biogeography   \n",
       "\n",
       "                                            experiment_title instrument_name  \\\n",
       "#SampleID                                                                      \n",
       "103.BZ1    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "103.CR1    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "103.GB2    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "103.GB3    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "103.GB1    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "\n",
       "          key_seq  library_construction_protocol  ... silt_clay  \\\n",
       "#SampleID                                         ...             \n",
       "103.BZ1      TCAG                       19502440  ...        80   \n",
       "103.CR1      TCAG                       19502440  ...        64   \n",
       "103.GB2      TCAG                       19502440  ...        65   \n",
       "103.GB3      TCAG                       19502440  ...        76   \n",
       "103.GB1      TCAG                       19502440  ...        77   \n",
       "\n",
       "          soil_moisture_deficit soil_type  \\\n",
       "#SampleID                                   \n",
       "103.BZ1                   132.6   gelisol   \n",
       "103.CR1                   160.1  mollisol   \n",
       "103.GB2                  -112.3  mollisol   \n",
       "103.GB3                  -112.3  mollisol   \n",
       "103.GB1                  -112.3  mollisol   \n",
       "\n",
       "                                specific_location    texture  \\\n",
       "#SampleID                                                      \n",
       "103.BZ1                Bonanza Creek LTER, AK USA  silt loam   \n",
       "103.CR1                      Coffey Ranch, TX USA       loam   \n",
       "103.GB2    Great Basin Experimental Range, UT USA       loam   \n",
       "103.GB3    Great Basin Experimental Range, UT USA  clay loam   \n",
       "103.GB1    Great Basin Experimental Range, UT USA  clay loam   \n",
       "\n",
       "                                                       title  tot_org_carb  \\\n",
       "#SampleID                                                                    \n",
       "103.BZ1    Pyrosequencing-Based Assessment of Soil pH as ...          30.3   \n",
       "103.CR1    Pyrosequencing-Based Assessment of Soil pH as ...          28.3   \n",
       "103.GB2    Pyrosequencing-Based Assessment of Soil pH as ...          68.9   \n",
       "103.GB3    Pyrosequencing-Based Assessment of Soil pH as ...          57.1   \n",
       "103.GB1    Pyrosequencing-Based Assessment of Soil pH as ...          28.2   \n",
       "\n",
       "          tot_org_nitro      Description PhClasses  \n",
       "#SampleID                                           \n",
       "103.BZ1             0.0  soil metagenome         1  \n",
       "103.CR1            10.9  soil metagenome         0  \n",
       "103.GB2             0.0  soil metagenome         0  \n",
       "103.GB3             0.0  soil metagenome         0  \n",
       "103.GB1             0.0  soil metagenome         1  \n",
       "\n",
       "[5 rows x 59 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metadata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(89, 59)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metadata.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data filtering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Visualize distribution**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x116679ba8>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE7RJREFUeJzt3W+MXXd95/H3tzERTgYyhMBdrxOtqbDSP4ww9VWWbrR0JoYKGkT8AFpQWjlVpNkHXcosXm3dPlhUqdUGaVOm0lYrWQmsV6UZ0pAoEWHZRm5u/0glxRPcHYLJGrJusOPatLUNQ6NFbr99MMfVxJ7JOffOufeOf32/pNG958zvd87Ho+gzJ7+5957ITCRJV78fGncASVI7LHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSIbaM8mQ33XRT7tixY6C53//+97n++uvbDTQE5myXOds18pzPP7/yeOutfU3z5/lKi4uLf52Zb6odmJkj+9q9e3cO6umnnx547iiZs13mbNfIc/7UT6189cmf5ysBR7JBx7rkIkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhRjpW/8laZx2HHiy1ePtn7rIPQ2OeeK+O1s973q8QpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRCNCj0i/kNEPBcRX4uIhyLitRHxloh4JiKOR8TnIuLaYYeVJK2vttAjYjvwy0A3M98GXAN8GPgk8KnM3AmcA+4dZlBJ0qtruuSyBdgaEVuA64DTwB3AI9X3DwF7248nSWqqttAz8xTwX4EXWSnyC8AicD4zL1bDTgLbhxVSklQvVm4o/SoDIt4AfB74OeA88PvV9icy863VmFuAL2bm1BrzZ4FZgE6ns3thYWGgoMvLy0xMTAw0d5TM2S5ztmvUOXfNzQFwdH6+r3nDyrl06kKrx+tshTMv14+b2n7Dhs4zMzOzmJndunFNPpzr3cD/y8zvAETEo8C/ASYjYkt1lX4z8NJakzPzIHAQoNvt5vT0dLN/wWV6vR6Dzh0lc7bLnO0aec7JSYC+zzmsnE0+SKsf+6cucv9SfY2euHu61fOup8ka+ovAOyPiuogIYA/wdeBp4IPVmH3A48OJKElqoska+jOs/PHzWWCpmnMQ+BXg4xHxTeCNwINDzClJqtHo89Az8xPAJy7b/QJwW+uJJEkD8Z2iklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RC1BZ6RNwaEUdXfX03IuYi4saIeCoijlePbxhFYEnS2prcgu75zNyVmbuA3cDfAY8BB4DDmbkTOFxtS5LGpN8llz3AtzLzL4G7gEPV/kPA3jaDSZL6E5nZfHDEp4FnM/O/RcT5zJxc9b1zmXnFsktEzAKzAJ1OZ/fCwsJAQZeXl5mYmBho7iiZs13mbNeoc+6amwPg6Px8X/OGlXPp1IVWj9fZCmderh83tf2GDZ1nZmZmMTO7deMaF3pEXAu8BPx4Zp5pWuirdbvdPHLkSKPzXa7X6zE9PT3Q3FEyZ7vM2a6R57x0rl6vr2nDyrnjwJOtHm//1EXuX9pSO+7EfXdu6DwR0ajQ+1lyeR8rV+dnqu0zEbGtOtk24Gz/MSVJbemn0D8CPLRq+wlgX/V8H/B4W6EkSf1rVOgRcR3wHuDRVbvvA94TEcer793XfjxJUlP1iz9AZv4d8MbL9v0NK696kSRtAr5TVJIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEI0vcHFZEQ8EhHfiIhjEfGTEXFjRDwVEcerx1e9n6gkabiaXqH/NvClzPwR4O3AMeAAcDgzdwKHq21J0pjUFnpEvB54F/AgQGb+IDPPA3cBh6phh4C9wwopSarX5Ar9h4HvAJ+JiK9GxAMRcT3QyczTANXjm4eYU5JUIzLz1QdEdIEvA7dn5jMR8dvAd4GPZubkqnHnMvOKdfSImAVmATqdzu6FhYWBgi4vLzMxMTHQ3FEyZ7vM2a5R59w1NwfA0fn5vuYNK+fSqQutHq+zFc68XD9uavsNGzrPzMzMYmZ268Y1KfR/AXw5M3dU2/+WlfXytwLTmXk6IrYBvcy89dWO1e1288iRIw3/Ca/U6/WYnp4eaO4ombNd5mzXyHNeOlev19e0YeXcceDJVo+3f+oi9y9tqR134r47N3SeiGhU6LVLLpn5V8C3I+JSWe8Bvg48Aeyr9u0DHh8wqySpBfW/WlZ8FPhsRFwLvAD8Iiu/DB6OiHuBF4EPDSeiJKmJRoWemUeBtS7397QbR5I0KN8pKkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEafR56RJwAvgf8PXAxM7sRcSPwOWAHcAL42cw8N5yYkqQ6/Vyhz2TmrlX3tTsAHM7MncDhaluSNCYbWXK5CzhUPT8E7N14HEnSoJoWegJ/EBGLETFb7etk5mmA6vHNwwgoSWomMrN+UMS/zMyXIuLNwFOs3DT6icycXDXmXGa+YY25s8AsQKfT2b2wsDBQ0OXlZSYmJgaaO0rmbJc52zXqnLvm5gA4Oj/f17xh5Vw6daHV43W2wpmX68dNbb9hQ+eZmZlZXLXcva6mN4l+qXo8GxGPAbcBZyJiW2aejohtwNl15h4EDgJ0u92cnp5u+E94pV6vx6BzR8mc7TJnu0aec3Llmq/fcw4r5z0Hnmz1ePunLnL/Un2Nnrh7utXzrqd2ySUiro+I1116Dvw08DXgCWBfNWwf8PiwQkqS6jW5Qu8Aj0XEpfG/l5lfioivAA9HxL3Ai8CHhhdTklSnttAz8wXg7Wvs/xtgzzBCSZL65ztFJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFaFzoEXFNRHw1Ir5Qbb8lIp6JiOMR8bmIuHZ4MSVJdfq5Qv8YcGzV9ieBT2XmTuAccG+bwSRJ/WlU6BFxM3An8EC1HcAdwCPVkEPA3mEElCQ1E5lZPyjiEeC/AK8D/iNwD/DlzHxr9f1bgP+VmW9bY+4sMAvQ6XR2LywsDBR0eXmZiYmJgeaOkjnbZc52jTrnrrk5AI7Oz/c1b1g5l05daPV4na1w5uX6cVPbb9jQeWZmZhYzs1s3rvYm0RHxfuBsZi5GxPSl3WsMXfM3Q2YeBA4CdLvdnJ6eXmtYrV6vx6BzR8mc7TJnu0aec3ISoO9zDivnPQeebPV4+6cucv9SbY1y4u7pVs+7nvokcDvwgYj4GeC1wOuBeWAyIrZk5kXgZuCl4cWUJNWpXUPPzF/NzJszcwfwYeAPM/Nu4Gngg9WwfcDjQ0spSaq1kdeh/wrw8Yj4JvBG4MF2IkmSBtFkyeWfZGYP6FXPXwBuaz+SJGkQvlNUkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQtQWekS8NiL+PCL+IiKei4hfr/a/JSKeiYjjEfG5iLh2+HElSetpcoX+/4E7MvPtwC7gvRHxTuCTwKcycydwDrh3eDElSXWa3FM0M3O52nxN9ZXAHcAj1f5DwN6hJJQkNdJoDT0iromIo8BZ4CngW8D5zLxYDTkJbB9ORElSE5GZzQdHTAKPAf8Z+ExmvrXafwvwxcycWmPOLDAL0Ol0di8sLAwUdHl5mYmJiYHmjpI522XOdo065665OQCOzs/3NW9YOZdOXWj1eJ2tcObl+nFT22/Y0HlmZmYWM7NbN67fm0Sfj4ge8E5gMiK2VFfpNwMvrTPnIHAQoNvt5vT0dD+n/Ce9Xo9B546SOdtlznaNPOfkJEDf5xxWznsOPNnq8fZPXeT+pfoaPXH3dKvnXU+TV7m8qboyJyK2Au8GjgFPAx+shu0DHh9WSElSvSZX6NuAQxFxDSu/AB7OzC9ExNeBhYj4DeCrwINDzClJqlFb6Jn5f4B3rLH/BeC2YYSSJPXPd4pKUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUiCZ3LLolIp6OiGMR8VxEfKzaf2NEPBURx6vHNww/riRpPU2u0C8C+zPzR1m5l+gvRcSPAQeAw5m5EzhcbUuSxqS20DPzdGY+Wz3/Hiv3E90O3AUcqoYdAvYOK6QkqV5fa+gRsYOV29E9A3Qy8zSslD7w5rbDSZKai8xsNjBiAvgj4Dcz89GIOJ+Zk6u+fy4zr1hHj4hZYBag0+nsXlhYGCjo8vIyExMTA80dJXO2y5ztGnXOXXNzABydn+9r3rByLp260OrxOlvhzMv146a237Ch88zMzCxmZrduXO1NogEi4jXA54HPZuaj1e4zEbEtM09HxDbg7FpzM/MgcBCg2+3m9PR0k1NeodfrMejcUTJnu8zZrpHnnFy55uv3nMPKec+BJ1s93v6pi9y/VF+jJ+6ebvW862nyKpcAHgSOZeZvrfrWE8C+6vk+4PH240mSmmpyhX478AvAUkQcrfb9GnAf8HBE3Au8CHxoOBElSU3UFnpm/ikQ63x7T7txJEmD8p2iklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCNLkF3acj4mxEfG3Vvhsj4qmIOF49XnFzaEnSaDW5Qv8fwHsv23cAOJyZO4HD1bYkaYxqCz0z/xj428t23wUcqp4fAva2nEuS1KfIzPpBETuAL2Tm26rt85k5uer75zJzzWWXiJgFZgE6nc7uhYWFgYIuLy8zMTEx0NxRMme7zNmuUefcNTcHwNH5+b7mDSvn0qkLrR6vsxXOvFw/bmr7DRs6z8zMzGJmduvG1d4keqMy8yBwEKDb7eb09PRAx+n1egw6d5TM2S5ztmvkOSdXrvv6Peewct5z4MlWj7d/6iL3L9XX6Im7p1s973oGfZXLmYjYBlA9nm0vkiRpEIMW+hPAvur5PuDxduJIkgbV5GWLDwF/BtwaEScj4l7gPuA9EXEceE+1LUkao9rFn8z8yDrf2tNyFknSBvhOUUkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRBD/zx0SbrcjprPJd8/dbH1zy7/58ArdEkqhIUuSYWw0CWpEBsq9Ih4b0Q8HxHfjIgDbYWSJPVv4D+KRsQ1wO+wcseik8BXIuKJzPx6W+FWWzp1YSx/JDlx350jP6ckDWIjV+i3Ad/MzBcy8wfAAnBXO7EkSf3aSKFvB769avtktU+SNAYbeR16rLEvrxgUMQvMVpvLEfH8gOe7CfjrAecOLD7Z95Sx5ByAOdtlzlcTa9XF+n75Kvl5Ns05QI9c7l81GbSRQj8J3LJq+2bgpcsHZeZB4OAGzgNARBzJzO5GjzNs5myXOdtlznZttpwbWXL5CrAzIt4SEdcCHwaeaCeWJKlfA1+hZ+bFiPj3wP8GrgE+nZnPtZZMktSXDX2WS2Z+EfhiS1nqbHjZZkTM2S5ztsuc7dpUOSPzir9jSpKuQr71X5IKcVUU+tXwEQMR8emIOBsRXxt3llcTEbdExNMRcSwinouIj40701oi4rUR8ecR8RdVzl8fd6b1RMQ1EfHViPjCuLOsJyJORMRSRByNiCPjzrOeiJiMiEci4hvVf6M/Oe5Ml4uIW6uf46Wv70bE3LhzwVWw5FJ9xMD/ZdVHDAAfGdZHDAwqIt4FLAP/MzPfNu4864mIbcC2zHw2Il4HLAJ7N+HPM4DrM3M5Il4D/Cnwscz88pijXSEiPg50gddn5vvHnWctEXEC6Gbmpn5td0QcAv4kMx+oXj13XWaeH3eu9VT9dAr415n5l+POczVcoV8VHzGQmX8M/O24c9TJzNOZ+Wz1/HvAMTbhO3xzxXK1+Zrqa9NdfUTEzcCdwAPjznK1i4jXA+8CHgTIzB9s5jKv7AG+tRnKHK6OQvcjBoYkInYA7wCeGW+StVVLGUeBs8BTmbkZc84D/wn4h3EHqZHAH0TEYvXu7c3oh4HvAJ+plrAeiIjrxx2qxoeBh8Yd4pKrodAbfcSA+hMRE8DngbnM/O6486wlM/8+M3ex8i7k2yJiUy1lRcT7gbOZuTjuLA3cnpk/AbwP+KVqiXCz2QL8BPDfM/MdwPeBTfk3M4BqSegDwO+PO8slV0OhN/qIATVXrUl/HvhsZj467jx1qv/t7gHvHXOUy90OfKBan14A7oiI3x1vpLVl5kvV41ngMVaWMjebk8DJVf8n9ggrBb9ZvQ94NjPPjDvIJVdDofsRAy2q/tj4IHAsM39r3HnWExFviojJ6vlW4N3AN8ab6pUy81cz8+bM3MHKf5d/mJk/P+ZYV4iI66s/gFMtYfw0sOlejZWZfwV8OyJurXbtATbVH+sv8xE20XILbPCdoqNwtXzEQEQ8BEwDN0XESeATmfngeFOt6XbgF4Clan0a4Neqd/1uJtuAQ9WrCH4IeDgzN+3LAje5DvDYyu9ytgC/l5lfGm+kdX0U+Gx18fYC8ItjzrOmiLiOlVfe/btxZ1lt079sUZLUzNWw5CJJasBCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEP8Id9PvnZvRPUgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Log-transform data\n",
    "OTUs.sum(axis=0).apply(np.log).hist(bins=10)\n",
    "# Filter\n",
    "count_filter = 300\n",
    "plt.axvline(x=np.log(count_filter), color='red')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Filter Samples in both OTUs and Metada tables**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "OTUs = OTUs.loc[:, (OTUs.sum(axis=0) >= count_filter)]\n",
    "OTUs = OTUs.loc[(OTUs.mean(axis=1) >= 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(52, 88)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "OTUs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "Metadata = Metadata.loc[OTUs.columns, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(88, 59)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metadata.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "Metadata.index.name = \"#SampleID\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BarcodeSequence</th>\n",
       "      <th>LinkerPrimerSequence</th>\n",
       "      <th>barcode_read_group_tag</th>\n",
       "      <th>experiment_alias</th>\n",
       "      <th>experiment_center</th>\n",
       "      <th>experiment_design_description</th>\n",
       "      <th>experiment_title</th>\n",
       "      <th>instrument_name</th>\n",
       "      <th>key_seq</th>\n",
       "      <th>library_construction_protocol</th>\n",
       "      <th>...</th>\n",
       "      <th>silt_clay</th>\n",
       "      <th>soil_moisture_deficit</th>\n",
       "      <th>soil_type</th>\n",
       "      <th>specific_location</th>\n",
       "      <th>texture</th>\n",
       "      <th>title</th>\n",
       "      <th>tot_org_carb</th>\n",
       "      <th>tot_org_nitro</th>\n",
       "      <th>Description</th>\n",
       "      <th>PhClasses</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>#SampleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>103.CA2</th>\n",
       "      <td>ACACGGTGTCTA</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>CA2_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>74</td>\n",
       "      <td>198.1</td>\n",
       "      <td>alfisol</td>\n",
       "      <td>Cedar Mtn. AZ, USA</td>\n",
       "      <td>silt loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>21.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103.CO3</th>\n",
       "      <td>ACATTCAGCGCA</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>CO3_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>24</td>\n",
       "      <td>230.6</td>\n",
       "      <td>mollisol</td>\n",
       "      <td>Shortgrass Steppe LTER, CO USA</td>\n",
       "      <td>sandy loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>8.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103.SR3</th>\n",
       "      <td>AGCTCCATACAG</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>SR3_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>61</td>\n",
       "      <td>323.8</td>\n",
       "      <td>mollisol</td>\n",
       "      <td>Sedgwick Reserve, CA USA</td>\n",
       "      <td>loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103.IE2</th>\n",
       "      <td>ACGTCTGTAGCA</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>IE2_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>49</td>\n",
       "      <td>-609.9</td>\n",
       "      <td>inceptisol</td>\n",
       "      <td>Institute for Ecosystem Studies, NY USA</td>\n",
       "      <td>sandy loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>40.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103.BP1</th>\n",
       "      <td>ACACACTATGGC</td>\n",
       "      <td>CATGCTGCCTCCCGTAGGAGT</td>\n",
       "      <td>BP1_V2</td>\n",
       "      <td>lauber_88_soils</td>\n",
       "      <td>CCME</td>\n",
       "      <td>bacterial biogeography</td>\n",
       "      <td>Pyrosequencing_based_assessment_of_soil_pH_as_...</td>\n",
       "      <td>Engencore</td>\n",
       "      <td>TCAG</td>\n",
       "      <td>19502440</td>\n",
       "      <td>...</td>\n",
       "      <td>65</td>\n",
       "      <td>79.2</td>\n",
       "      <td>entisol</td>\n",
       "      <td>Badlands National Park, SD USA</td>\n",
       "      <td>silt loam</td>\n",
       "      <td>Pyrosequencing-Based Assessment of Soil pH as ...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>soil metagenome</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          BarcodeSequence   LinkerPrimerSequence barcode_read_group_tag  \\\n",
       "#SampleID                                                                 \n",
       "103.CA2      ACACGGTGTCTA  CATGCTGCCTCCCGTAGGAGT                 CA2_V2   \n",
       "103.CO3      ACATTCAGCGCA  CATGCTGCCTCCCGTAGGAGT                 CO3_V2   \n",
       "103.SR3      AGCTCCATACAG  CATGCTGCCTCCCGTAGGAGT                 SR3_V2   \n",
       "103.IE2      ACGTCTGTAGCA  CATGCTGCCTCCCGTAGGAGT                 IE2_V2   \n",
       "103.BP1      ACACACTATGGC  CATGCTGCCTCCCGTAGGAGT                 BP1_V2   \n",
       "\n",
       "          experiment_alias experiment_center experiment_design_description  \\\n",
       "#SampleID                                                                    \n",
       "103.CA2    lauber_88_soils              CCME        bacterial biogeography   \n",
       "103.CO3    lauber_88_soils              CCME        bacterial biogeography   \n",
       "103.SR3    lauber_88_soils              CCME        bacterial biogeography   \n",
       "103.IE2    lauber_88_soils              CCME        bacterial biogeography   \n",
       "103.BP1    lauber_88_soils              CCME        bacterial biogeography   \n",
       "\n",
       "                                            experiment_title instrument_name  \\\n",
       "#SampleID                                                                      \n",
       "103.CA2    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "103.CO3    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "103.SR3    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "103.IE2    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "103.BP1    Pyrosequencing_based_assessment_of_soil_pH_as_...       Engencore   \n",
       "\n",
       "          key_seq  library_construction_protocol  ... silt_clay  \\\n",
       "#SampleID                                         ...             \n",
       "103.CA2      TCAG                       19502440  ...        74   \n",
       "103.CO3      TCAG                       19502440  ...        24   \n",
       "103.SR3      TCAG                       19502440  ...        61   \n",
       "103.IE2      TCAG                       19502440  ...        49   \n",
       "103.BP1      TCAG                       19502440  ...        65   \n",
       "\n",
       "          soil_moisture_deficit   soil_type  \\\n",
       "#SampleID                                     \n",
       "103.CA2                   198.1     alfisol   \n",
       "103.CO3                   230.6    mollisol   \n",
       "103.SR3                   323.8    mollisol   \n",
       "103.IE2                  -609.9  inceptisol   \n",
       "103.BP1                    79.2     entisol   \n",
       "\n",
       "                                 specific_location     texture  \\\n",
       "#SampleID                                                        \n",
       "103.CA2                         Cedar Mtn. AZ, USA   silt loam   \n",
       "103.CO3             Shortgrass Steppe LTER, CO USA  sandy loam   \n",
       "103.SR3                   Sedgwick Reserve, CA USA        loam   \n",
       "103.IE2    Institute for Ecosystem Studies, NY USA  sandy loam   \n",
       "103.BP1             Badlands National Park, SD USA   silt loam   \n",
       "\n",
       "                                                       title  tot_org_carb  \\\n",
       "#SampleID                                                                    \n",
       "103.CA2    Pyrosequencing-Based Assessment of Soil pH as ...          21.5   \n",
       "103.CO3    Pyrosequencing-Based Assessment of Soil pH as ...           8.2   \n",
       "103.SR3    Pyrosequencing-Based Assessment of Soil pH as ...          33.0   \n",
       "103.IE2    Pyrosequencing-Based Assessment of Soil pH as ...          40.7   \n",
       "103.BP1    Pyrosequencing-Based Assessment of Soil pH as ...          31.0   \n",
       "\n",
       "          tot_org_nitro      Description PhClasses  \n",
       "#SampleID                                           \n",
       "103.CA2             0.0  soil metagenome         0  \n",
       "103.CO3             0.0  soil metagenome         1  \n",
       "103.SR3             0.0  soil metagenome         1  \n",
       "103.IE2             0.0  soil metagenome         1  \n",
       "103.BP1             0.0  soil metagenome         0  \n",
       "\n",
       "[5 rows x 59 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Metadata.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "OTUs.to_csv(ROOT + \"/outputs/filtered_OTUs_table.tsv\", sep='\\t')\n",
    "Metadata.to_csv(ROOT + \"/outputs/filtered_metadata.txt\", sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "subsampling_percentage=0.6\n",
    "samples = list(Metadata.index)\n",
    "last_item = int(len(samples) * subsampling_percentage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(8)\n",
    "random.shuffle(samples)\n",
    "train_samples = samples[:last_item]\n",
    "test_samples = samples[last_item:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save train\n",
    "OTUs.loc[:, train_samples].to_csv(ROOT + \"/outputs/train_filtered_OTUs_table.tsv\", sep='\\t')\n",
    "Metadata.loc[train_samples, :].to_csv(ROOT + \"/outputs/train_filtered_metadata.txt\", sep='\\t')\n",
    "\n",
    "# Save test\n",
    "OTUs.loc[:, test_samples].to_csv(ROOT + \"/outputs/test_filtered_OTUs_table.tsv\", sep='\\t')\n",
    "Metadata.loc[test_samples,:].to_csv(ROOT + \"/outputs/test_filtered_metadata.txt\", sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}