BERT_Classification_Task_Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_I0wuC-cns9k"
   },
   "source": [
    "**Loading the Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9JeBp3MYk4h-"
   },
   "outputs": [],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "jp3dsYS1iga0"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anshj\\anaconda3\\envs\\torch-gpu\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizer, BertForMaskedLM, BertModel\n",
    "from torch.utils.data import DataLoader, Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "1_SHio5_io_G"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "#Loading the Domain Specific Trained Model \n",
    "model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n",
    "model.load_state_dict(torch.load('Model_Domain_weigths'))\n",
    "bert_for_masked_lm_weights = model.state_dict()\n",
    "\n",
    "Task_model = BertModel.from_pretrained('bert-base-uncased')\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Copy the weights of the BertModel layers from the BertForMaskedLM \n",
    "bert_model_dict = Task_model.state_dict()\n",
    "for key in bert_for_masked_lm_weights.keys():\n",
    "    if key in bert_model_dict:\n",
    "        bert_model_dict[key] = bert_for_masked_lm_weights[key]\n",
    "\n",
    "# Load the weights into the BertModel object\n",
    "Task_model.load_state_dict(bert_model_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KNhlyMernoYa"
   },
   "source": [
    "**Dataset Preparation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WjfdOsqilDt1"
   },
   "outputs": [],
   "source": [
    "#Loading the datasets\n",
    "!unzip 'cosmetics.csv.zip'\n",
    "!unzip 'sephora_website_dataset.csv.zip'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "S7FpchzCl-KT"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df1 = pd.read_csv('cosmetics.csv')\n",
    "df2 = pd.read_csv('sephora_website_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lU33-oC0mLfD"
   },
   "outputs": [],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qglfHaecmMht"
   },
   "outputs": [],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "lTe74f5amNTM"
   },
   "outputs": [],
   "source": [
    "df2.rename(columns = {'ingredients':'Ingredients', 'category':'Label'}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "XIEkVbqPnOWH"
   },
   "outputs": [],
   "source": [
    "df2 = df2[['Ingredients', 'Label']]\n",
    "df1 = df1[['Ingredients', 'Label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "1GoAa_vanU9C"
   },
   "outputs": [],
   "source": [
    "dataset = pd.concat([df1, df2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "MvyshbjNgghH"
   },
   "outputs": [],
   "source": [
    "dataset.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k5yZDilJngoq"
   },
   "outputs": [],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RvNx2cYcniXh"
   },
   "outputs": [],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UI8BvG5OAb-f"
   },
   "outputs": [],
   "source": [
    "dataset['Label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4EQrjHTwAfQh"
   },
   "outputs": [],
   "source": [
    "#Have to preprocess/combine some labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "_zDg0arlAAkK"
   },
   "outputs": [],
   "source": [
    "num_labels = len(dataset['Label'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "WgbR7zPLYgrF"
   },
   "outputs": [],
   "source": [
    "# Preprocessing \n",
    "# 1) Converting labels into numberical values\n",
    "# 2) Data Split\n",
    "# 3) Tokenizing - includes padding and truncation\n",
    "\n",
    "labels = pd.get_dummies(dataset['Label']).values\n",
    "\n",
    "train_data = dataset[:9000]\n",
    "train_labels= labels[:9000]\n",
    "train_data.reset_index(inplace=True)\n",
    "\n",
    "test_data= dataset[9000:]\n",
    "test_labels= labels[9000:]\n",
    "test_data.reset_index(inplace=True)\n",
    "\n",
    "\n",
    "class Ingredients(Dataset):\n",
    "  def __init__(self, data, labels, tokenizer):\n",
    "    self.data = data\n",
    "    self.labels = labels\n",
    "    self.tokenizer = tokenizer\n",
    "  def __getitem__(self, index):\n",
    "    text = dataset.loc[index, 'Ingredients']\n",
    "    input = self.tokenizer(text, return_tensors='pt', add_special_tokens=True, max_length=512, padding='max_length', truncation=True)\n",
    "    input_ids = input.input_ids.squeeze()\n",
    "    attention_mask = input.attention_mask.squeeze()\n",
    "    label = self.labels[index]\n",
    "    return {'input_ids' : input_ids, 'attention_mask': attention_mask, 'labels' : label}\n",
    "  def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "train_dataset = Ingredients(train_data, train_labels, tokenizer)\n",
    "train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
    "\n",
    "test_dataset = Ingredients(test_data, test_labels, tokenizer)\n",
    "test_loader = DataLoader(test_dataset, batch_size=8, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8BIAbCkugDih"
   },
   "outputs": [],
   "source": [
    "print(dataset.loc[0]['Ingredients'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ttU-Hdfpf5GA"
   },
   "outputs": [],
   "source": [
    "train_dataset.__getitem__(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ynUVzPRW-JIf"
   },
   "source": [
    "**Model Architecture Changes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "wMWCfSWE_05v"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "wStF83wf-Nux"
   },
   "outputs": [],
   "source": [
    "class BertClassifier(nn.Module):\n",
    "    def __init__(self, bert_model, num_classes):\n",
    "        super(BertClassifier, self).__init__()\n",
    "        self.bert = bert_model\n",
    "        self.dropout = nn.Dropout(0.3)\n",
    "        self.classifier = nn.Linear(768, num_classes)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        _, outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask,return_dict=False)\n",
    "        \n",
    "        '''\n",
    "        The first variable, which we named _ in the code above, contains the embedding vectors of all of the tokens in a sequence.\n",
    "The second variable, which we named outputs, contains the embedding vector of [CLS] token. For a text classification task, it is enough to use this embedding as an input for our classifier.\n",
    "We then pass the outputs variable into a linear layer. At the end of the linear layer, we have a vector of size 149, each corresponds to a category of our labels\n",
    "        '''\n",
    "        \n",
    "        pooled_output = outputs\n",
    "        dropout_output = self.dropout(pooled_output)\n",
    "        logits = self.classifier(dropout_output)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "pTRFF1Jt_vwh"
   },
   "outputs": [],
   "source": [
    "model = BertClassifier(Task_model,num_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "EU2-YycXApmA"
   },
   "outputs": [],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XsOE3qx-pRkK"
   },
   "source": [
    "**Model Training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "8FBduAJseYL9"
   },
   "outputs": [],
   "source": [
    "from torch.optim import Adam\n",
    "# Set up the optimizer\n",
    "optimizer = Adam(model.parameters(), lr=2e-5)\n",
    "\n",
    "# Define the loss function\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertClassifier(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.3, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=149, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "-U_wx-aHeRls"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertClassifier(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.3, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=149, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jTdkzEr-eUY1",
    "outputId": "359c02b7-8910-4897-a2cc-d08ed7f68763"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0: 100%|███████████████████████████████████████████████████████████| 1125/1125 [03:54<00:00,  4.79it/s, loss=3.1]\n",
      "Epoch 1: 100%|██████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.84it/s, loss=2.98]\n",
      "Epoch 2: 100%|██████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.85it/s, loss=1.75]\n",
      "Epoch 3: 100%|██████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.84it/s, loss=1.97]\n",
      "Epoch 4: 100%|██████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.84it/s, loss=1.59]\n",
      "Epoch 5: 100%|██████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.83it/s, loss=1.63]\n",
      "Epoch 6: 100%|██████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.84it/s, loss=1.49]\n",
      "Epoch 7: 100%|██████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.84it/s, loss=2.37]\n",
      "Epoch 8: 100%|█████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.84it/s, loss=0.729]\n",
      "Epoch 9: 100%|██████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.84it/s, loss=1.21]\n",
      "Epoch 10: 100%|████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.83it/s, loss=0.783]\n",
      "Epoch 11: 100%|█████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.83it/s, loss=1.31]\n",
      "Epoch 12: 100%|████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.83it/s, loss=0.757]\n",
      "Epoch 13: 100%|█████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.83it/s, loss=1.06]\n",
      "Epoch 14: 100%|█████████████████████████████████████████████████████████| 1125/1125 [03:52<00:00,  4.83it/s, loss=1.67]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epochs = 15\n",
    "for epoch in range(epochs):\n",
    "    loop = tqdm(train_loader, leave = True)\n",
    "    for batch in loop:\n",
    "        # Move the inputs and labels to the device\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels']\n",
    "        labels = torch.argmax(labels, dim=1).to(device)\n",
    "#         labels = labels.long()\n",
    "        \n",
    "        # Zero the gradients\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward pass\n",
    "        outputs = model(input_ids=input_ids, attention_mask = attention_mask)\n",
    "        \n",
    "        # Compute the loss\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward pass and update the parameters\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        loop.set_description(f'Epoch {epoch}') \n",
    "        loop.set_postfix(loss=loss.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GjmfN1MWe91m"
   },
   "source": [
    "**Model Evaluation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "jwQrqv6kfAUO"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 205/205 [00:15<00:00, 12.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.18%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model on the test data\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    loop = tqdm(test_loader, leave = True)\n",
    "    for batch in loop:\n",
    "        # Move the inputs and labels to the device\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        labels = torch.argmax(labels, dim=1).to(device)\n",
    "        \n",
    "        # Forward pass\n",
    "        outputs = model(input_ids = input_ids, attention_mask=attention_mask)\n",
    "        \n",
    "        # Predict the labels\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        \n",
    "        # Compute the accuracy\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "    \n",
    "    accuracy = 100 * correct / total\n",
    "    print(f'Test accuracy: {accuracy:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MsPSvYc4wNvG"
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'Model_Task_weights')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python (torch-gpu)",
   "language": "python",
   "name": "torch-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}