BERT_Task_Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_I0wuC-cns9k"
   },
   "source": [
    "**Loading the Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9JeBp3MYk4h-"
   },
   "outputs": [],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "jp3dsYS1iga0"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anshj\\anaconda3\\envs\\torch-gpu\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizer, BertForMaskedLM\n",
    "from torch.utils.data import DataLoader, Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "1_SHio5_io_G"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "#Loading the Domain Specific Trained Model \n",
    "model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n",
    "model.load_state_dict(torch.load('Model_Domain_weigths'))\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KNhlyMernoYa"
   },
   "source": [
    "**Dataset Preparation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WjfdOsqilDt1"
   },
   "outputs": [],
   "source": [
    "#Loading the datasets\n",
    "!unzip 'cosmetics.csv.zip'\n",
    "!unzip 'sephora_website_dataset.csv.zip'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "S7FpchzCl-KT"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df1 = pd.read_csv('cosmetics.csv')\n",
    "df2 = pd.read_csv('sephora_website_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lU33-oC0mLfD"
   },
   "outputs": [],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qglfHaecmMht"
   },
   "outputs": [],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "lTe74f5amNTM"
   },
   "outputs": [],
   "source": [
    "df2.rename(columns = {'ingredients':'Ingredients', 'category':'Label'}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "XIEkVbqPnOWH"
   },
   "outputs": [],
   "source": [
    "df2 = df2[['Ingredients', 'Label']]\n",
    "df1 = df1[['Ingredients', 'Label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "1GoAa_vanU9C"
   },
   "outputs": [],
   "source": [
    "dataset = pd.concat([df1, df2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "MvyshbjNgghH"
   },
   "outputs": [],
   "source": [
    "dataset.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k5yZDilJngoq"
   },
   "outputs": [],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RvNx2cYcniXh"
   },
   "outputs": [],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UI8BvG5OAb-f"
   },
   "outputs": [],
   "source": [
    "dataset['Label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4EQrjHTwAfQh"
   },
   "outputs": [],
   "source": [
    "#Have to preprocess/combine some labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "_zDg0arlAAkK"
   },
   "outputs": [],
   "source": [
    "num_labels = len(dataset['Label'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "WgbR7zPLYgrF"
   },
   "outputs": [],
   "source": [
    "# Preprocessing \n",
    "# 1) Converting labels into numberical values\n",
    "# 2) Data Split\n",
    "# 3) Tokenizing - includes padding and truncation\n",
    "\n",
    "labels = pd.get_dummies(dataset['Label']).values\n",
    "\n",
    "train_data = dataset[:9000]\n",
    "train_labels= labels[:9000]\n",
    "train_data.reset_index(inplace=True)\n",
    "\n",
    "test_data= dataset[9000:]\n",
    "test_labels= labels[9000:]\n",
    "test_data.reset_index(inplace=True)\n",
    "\n",
    "\n",
    "class Ingredients(Dataset):\n",
    "  def __init__(self, data, labels, tokenizer):\n",
    "    self.data = data\n",
    "    self.labels = labels\n",
    "    self.tokenizer = tokenizer\n",
    "  def __getitem__(self, index):\n",
    "    text = dataset.loc[index, 'Ingredients']\n",
    "    input = self.tokenizer(text, return_tensors='pt', add_special_tokens=True, max_length=512, padding='max_length', truncation=True)\n",
    "    input_ids = input.input_ids.squeeze()\n",
    "    attention_mask = input.attention_mask.squeeze()\n",
    "    label = self.labels[index]\n",
    "    return {'input_ids' : input_ids, 'attention_mask': attention_mask, 'labels' : label}\n",
    "  def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "train_dataset = Ingredients(train_data, train_labels, tokenizer)\n",
    "train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
    "\n",
    "test_dataset = Ingredients(test_data, test_labels, tokenizer)\n",
    "test_loader = DataLoader(test_dataset, batch_size=8, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8BIAbCkugDih"
   },
   "outputs": [],
   "source": [
    "print(dataset.loc[0]['Ingredients'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ttU-Hdfpf5GA"
   },
   "outputs": [],
   "source": [
    "train_dataset.__getitem__(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ynUVzPRW-JIf"
   },
   "source": [
    "**Model Architecture Changes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "wMWCfSWE_05v"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "wStF83wf-Nux"
   },
   "outputs": [],
   "source": [
    "class BertClassifier(nn.Module):\n",
    "    def __init__(self, bert_model, num_classes):\n",
    "        super(BertClassifier, self).__init__()\n",
    "        self.bert = bert_model\n",
    "        self.classifier = nn.Linear(30522, num_classes)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        pooled_output = outputs.logits[:, 0, :]\n",
    "        logits = self.classifier(pooled_output)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "pTRFF1Jt_vwh"
   },
   "outputs": [],
   "source": [
    "model = BertClassifier(model,num_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "EU2-YycXApmA"
   },
   "outputs": [],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XsOE3qx-pRkK"
   },
   "source": [
    "**Model Training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "8FBduAJseYL9"
   },
   "outputs": [],
   "source": [
    "from torch.optim import Adam\n",
    "# Set up the optimizer\n",
    "optimizer = Adam(model.parameters(), lr=2e-5)\n",
    "\n",
    "# Define the loss function\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertClassifier(\n",
       "  (bert): BertForMaskedLM(\n",
       "    (bert): BertModel(\n",
       "      (embeddings): BertEmbeddings(\n",
       "        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "        (position_embeddings): Embedding(512, 768)\n",
       "        (token_type_embeddings): Embedding(2, 768)\n",
       "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (encoder): BertEncoder(\n",
       "        (layer): ModuleList(\n",
       "          (0-11): 12 x BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cls): BertOnlyMLMHead(\n",
       "      (predictions): BertLMPredictionHead(\n",
       "        (transform): BertPredictionHeadTransform(\n",
       "          (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "          (transform_act_fn): GELUActivation()\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        )\n",
       "        (decoder): Linear(in_features=768, out_features=30522, bias=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): Linear(in_features=30522, out_features=149, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "-U_wx-aHeRls"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertClassifier(\n",
       "  (bert): BertForMaskedLM(\n",
       "    (bert): BertModel(\n",
       "      (embeddings): BertEmbeddings(\n",
       "        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "        (position_embeddings): Embedding(512, 768)\n",
       "        (token_type_embeddings): Embedding(2, 768)\n",
       "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (encoder): BertEncoder(\n",
       "        (layer): ModuleList(\n",
       "          (0-11): 12 x BertLayer(\n",
       "            (attention): BertAttention(\n",
       "              (self): BertSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): BertSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): BertIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): BertOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (cls): BertOnlyMLMHead(\n",
       "      (predictions): BertLMPredictionHead(\n",
       "        (transform): BertPredictionHeadTransform(\n",
       "          (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "          (transform_act_fn): GELUActivation()\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        )\n",
       "        (decoder): Linear(in_features=768, out_features=30522, bias=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): Linear(in_features=30522, out_features=149, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jTdkzEr-eUY1",
    "outputId": "359c02b7-8910-4897-a2cc-d08ed7f68763"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0: 100%|██████████████████████████████████████████████████████████| 1125/1125 [04:28<00:00,  4.19it/s, loss=2.36]\n",
      "Epoch 1: 100%|██████████████████████████████████████████████████████████| 1125/1125 [04:27<00:00,  4.21it/s, loss=1.54]\n",
      "Epoch 2: 100%|███████████████████████████████████████████████████████████| 1125/1125 [04:26<00:00,  4.22it/s, loss=2.6]\n",
      "Epoch 3: 100%|██████████████████████████████████████████████████████████| 1125/1125 [04:26<00:00,  4.22it/s, loss=2.24]\n",
      "Epoch 4: 100%|██████████████████████████████████████████████████████████| 1125/1125 [04:26<00:00,  4.22it/s, loss=2.72]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epochs = 5\n",
    "for epoch in range(epochs):\n",
    "    loop = tqdm(train_loader, leave = True)\n",
    "    for batch in loop:\n",
    "        # Move the inputs and labels to the device\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels']\n",
    "        labels = torch.argmax(labels, dim=1).to(device)\n",
    "#         labels = labels.long()\n",
    "        \n",
    "        # Zero the gradients\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward pass\n",
    "        outputs = model(input_ids=input_ids, attention_mask = attention_mask)\n",
    "        \n",
    "        # Compute the loss\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward pass and update the parameters\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        loop.set_description(f'Epoch {epoch}') \n",
    "        loop.set_postfix(loss=loss.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GjmfN1MWe91m"
   },
   "source": [
    "**Model Evaluation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "jwQrqv6kfAUO"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 205/205 [00:18<00:00, 11.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.12%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model on the test data\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    loop = tqdm(test_loader, leave = True)\n",
    "    for batch in loop:\n",
    "        # Move the inputs and labels to the device\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        labels = torch.argmax(labels, dim=1).to(device)\n",
    "        \n",
    "        # Forward pass\n",
    "        outputs = model(input_ids = input_ids, attention_mask=attention_mask)\n",
    "        \n",
    "        # Predict the labels\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        \n",
    "        # Compute the accuracy\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "    \n",
    "    accuracy = 100 * correct / total\n",
    "    print(f'Test accuracy: {accuracy:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MsPSvYc4wNvG"
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'Model_Task_weights')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python (torch-gpu)",
   "language": "python",
   "name": "torch-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}