OpenNMT_Capstone_Training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "OpenNMT Capstone Training.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "V_P9hZPqPPZ0"
      ],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y8JSFMr5dd8C",
        "outputId": "8d7da73e-5f4e-48b9-c048-7b489ece1a21"
      },
      "source": [
        "!rm -rf sample_data/\n",
        "!pip install OpenNMT-tf\n",
        "!pip install gdown\n",
        "!pip install sacremoses\n",
        "import opennmt\n",
        "import os\n",
        "import tensorflow as tf\n",
        "import tensorflow_addons as tfa\n",
        "import tensorflow.keras.backend as K\n",
        "import numpy as np\n",
        "import sacrebleu\n",
        "import pyonmttok\n",
        "from opennmt.utils import checkpoint as checkpoint_util\n",
        "from pyonmttok import SentencePieceTokenizer\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive',force_remount=True)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting OpenNMT-tf\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/83/30/525a1b22667dfb387bd91ac880fea3f6f534997a332b876f982c527e28fd/OpenNMT_tf-2.19.0-py3-none-any.whl (154kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 3.8MB/s \n",
            "\u001b[?25hCollecting pyyaml<5.5,>=5.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7a/a5/393c087efdc78091afa2af9f1378762f9821c9c1d7a22c5753fb5ac5f97a/PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636kB)\n",
            "\u001b[K     |████████████████████████████████| 645kB 4.8MB/s \n",
            "\u001b[?25hCollecting rouge<2,>=1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/43/cc/e18e33be20971ff73a056ebdb023476b5a545e744e3fc22acd8c758f1e0d/rouge-1.0.0-py3-none-any.whl\n",
            "Collecting sacrebleu<1.6,>=1.5.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/57/0c7ca4e31a126189dab99c19951910bd081dea5bbd25f24b77107750eae7/sacrebleu-1.5.1-py3-none-any.whl (54kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 4.7MB/s \n",
            "\u001b[?25hCollecting tensorflow-addons<0.14,>=0.13\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/66/4b/e893d194e626c24b3df2253066aa418f46a432fdb68250cde14bf9bb0700/tensorflow_addons-0.13.0-cp37-cp37m-manylinux2010_x86_64.whl (679kB)\n",
            "\u001b[K     |████████████████████████████████| 686kB 5.8MB/s \n",
            "\u001b[?25hCollecting pyonmttok<2,>=1.25.0; platform_system == \"Linux\" or platform_system == \"Darwin\"\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/3c/26c6c699462fdf90d137cbd862a54ad4d873a90dd99d068acf75f40a4745/pyonmttok-1.26.1-cp37-cp37m-manylinux1_x86_64.whl (14.3MB)\n",
            "\u001b[K     |████████████████████████████████| 14.3MB 439kB/s \n",
            "\u001b[?25hCollecting ctranslate2<2,>=1.18.1; platform_system == \"Linux\" or platform_system == \"Darwin\"\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/18/15/b6dd092ad363ae13343e084ff6764caa24ea33702b388d5012dc7e69ebd4/ctranslate2-1.20.1-cp37-cp37m-manylinux2014_x86_64.whl (76.8MB)\n",
            "\u001b[K     |████████████████████████████████| 76.8MB 70kB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from rouge<2,>=1.0->OpenNMT-tf) (1.15.0)\n",
            "Collecting portalocker==2.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/89/a6/3814b7107e0788040870e8825eebf214d72166adf656ba7d4bf14759a06a/portalocker-2.0.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: typeguard>=2.7 in /usr/local/lib/python3.7/dist-packages (from tensorflow-addons<0.14,>=0.13->OpenNMT-tf) (2.7.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from ctranslate2<2,>=1.18.1; platform_system == \"Linux\" or platform_system == \"Darwin\"->OpenNMT-tf) (1.19.5)\n",
            "Installing collected packages: pyyaml, rouge, portalocker, sacrebleu, tensorflow-addons, pyonmttok, ctranslate2, OpenNMT-tf\n",
            "  Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed OpenNMT-tf-2.19.0 ctranslate2-1.20.1 portalocker-2.0.0 pyonmttok-1.26.1 pyyaml-5.4.1 rouge-1.0.0 sacrebleu-1.5.1 tensorflow-addons-0.13.0\n",
            "Requirement already satisfied: gdown in /usr/local/lib/python3.7/dist-packages (3.6.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from gdown) (4.41.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from gdown) (2.23.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from gdown) (1.15.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->gdown) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->gdown) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->gdown) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->gdown) (2020.12.5)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 3.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sacremoses) (4.41.1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2019.12.20)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.0.1)\n",
            "Installing collected packages: sacremoses\n",
            "Successfully installed sacremoses-0.0.45\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0bYaTz9YDpqd"
      },
      "source": [
        "def split_data(data, train_size, val_size, test_size):\n",
        "  if train_size + val_size + test_size != 1.0:\n",
        "    raise Exception(\"Train, validation, and test sizes must add up to 1.\") \n",
        "  \n",
        "  train_mark = int(len(data) * train_size)\n",
        "  val_mark = train_mark + int(len(data) * val_size)\n",
        "\n",
        "  train_data = data[0:train_mark]\n",
        "  val_data = data[train_mark:val_mark]\n",
        "  test_data = data[val_mark:]\n",
        "\n",
        "  return train_data, val_data, test_data\n",
        "\n",
        "def save_data(data, data_folder_name, filename):\n",
        "  with open(os.path.join(data_folder_name, filename), mode=\"w\") as f:\n",
        "    for line in data:\n",
        "      if line.strip():\n",
        "        f.write(line)\n",
        "\n",
        "def count_weights(model):\n",
        "  trainable_count = np.sum([K.count_params(w) for w in model.trainable_weights])\n",
        "  non_trainable_count = np.sum([K.count_params(w) for w in model.non_trainable_weights])\n",
        "\n",
        "  print('Total params: {:,}'.format(trainable_count + non_trainable_count))\n",
        "  print('Trainable params: {:,}'.format(trainable_count))\n",
        "  print('Non-trainable params: {:,}'.format(non_trainable_count))\n",
        "\n",
        "def display_weights(model):\n",
        "  for layer in model.encoder.layers:\n",
        "    print(f\"===== LAYER: {layer.name} =====\")\n",
        "    if layer.get_weights() != []:\n",
        "        weights = layer.get_weights()[0]\n",
        "        biases = layer.get_weights()[1]\n",
        "        print(\"weights:\")\n",
        "        print(weights)\n",
        "        print(\"biases:\")\n",
        "        print(biases)\n",
        "    else:\n",
        "        print(\"weights: \", [])\n",
        "\n",
        "def compute_scores(runner, features_filename, labels_filename, pred_filename, include_ppl=False, include_ter=False):\n",
        "  runner.infer(features_filename, pred_filename)\n",
        "\n",
        "  dot_idx = pred_filename.index('.')\n",
        "  base_pred_name = pred_filename[0:dot_idx]\n",
        "  dot_idx = labels_filename.index('.')\n",
        "  base_model_name = labels_filename[0:dot_idx]\n",
        "  pred_filename = detokenize_data(base_pred_name, base_model_name)\n",
        "  detokenized_labels_filename = detokenize_data(base_model_name, base_model_name)\n",
        "  preds = []\n",
        "  truth = []\n",
        "  with open(pred_filename) as f:\n",
        "    preds = f.readlines()\n",
        "\n",
        "  with open(detokenized_labels_filename) as f:\n",
        "    truth = f.readlines()\n",
        "\n",
        "  scores = dict()\n",
        "  if include_ppl:\n",
        "    scores = runner.evaluate(\n",
        "        features_file=features_filename,\n",
        "        labels_file=labels_filename)\n",
        "  \n",
        "  bleu = sacrebleu.corpus_bleu(preds, [truth])\n",
        "  scores.update({'bleu': bleu.score})\n",
        "  if include_ter:\n",
        "    ter = sacrebleu.corpus_ter(preds, [truth])\n",
        "    scores.update({'ter': ter.score})\n",
        "  \n",
        "  return scores\n",
        "\n",
        "def tokenize_data(save_folder_name, basename):\n",
        "  tokenize_sub_data(save_folder_name, basename, \"train\")\n",
        "  tokenize_sub_data(save_folder_name, basename, \"test\")\n",
        "  tokenize_sub_data(save_folder_name, basename, \"val\")\n",
        "\n",
        "def tokenize_sub_data(save_folder_name, basename, set_type):\n",
        "  model_path = os.path.join(\"sentencepiece_models\", f\"{basename}.model\")\n",
        "  vocabulary_path = os.path.join(\"sentencepiece_models\", f\"{basename}.vocab\")\n",
        "  tokenizer = SentencePieceTokenizer(model_path=model_path,\n",
        "                                     vocabulary_path=vocabulary_path,)\n",
        "  \n",
        "  with open(os.path.join(f\"{save_folder_name}_raw\", f\"{basename}_{set_type}.raw\")) as f:\n",
        "    with open(os.path.join(save_folder_name, f\"{basename}_{set_type}.tok\"), mode=\"w\") as fout:\n",
        "      for line in f.readlines():\n",
        "        if line.strip():\n",
        "          fout.write(\" \".join(tokenizer.tokenize(line)[0]) + \"\\n\")\n",
        "\n",
        "\n",
        "def detokenize_data(tokenized_basename, model_basename):\n",
        "  model_path = os.path.join(\"sentencepiece_models\", model_basename + \".model\")\n",
        "  vocabulary_path = os.path.join(\"sentencepiece_models\", f\"{model_basename}.vocab\")\n",
        "  tokenizer = SentencePieceTokenizer(model_path=model_path,\n",
        "                                     vocabulary_path=vocabulary_path,)\n",
        "  \n",
        "  with open(f\"{tokenized_basename}.tok\") as f:\n",
        "    with open(f\"{tokenized_basename}.txt\", mode=\"w\") as fout:\n",
        "      for line in f.readlines():\n",
        "        fout.write(tokenizer.detokenize(line.strip().split(\" \")) + \"\\n\")\n",
        "\n",
        "  return f\"{tokenized_basename}.txt\""
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120
        },
        "id": "XSafXG7K5aVX",
        "outputId": "f6d9fb0b-8b67-43fb-e3dc-46dfdc1a9cc3"
      },
      "source": [
        "#!cp '/content/gdrive/My Drive/capstone-data-final/src_pvt_data.zip' '/content/'\n",
        "#!cp '/content/gdrive/My Drive/capstone-data-final/pvt_tgt_data.zip' '/content/'\n",
        "!cp '/content/gdrive/My Drive/capstone-data-final/src_tgt_data.zip' '/content/'\n",
        "!cp '/content/gdrive/My Drive/capstone-data-final/sentencepiece_models.zip' '/content/'  \n",
        "!cp '/content/gdrive/My Drive/capstone-models/src_tgt_model.zip' '/content/'  \n",
        "#!cp '/content/gdrive/My Drive/capstone-models/pvt_tgt_model.zip' '/content/'  \n",
        "#!cp '/content/gdrive/My Drive/capstone-models/src_pvt_model.zip' '/content/'  \n",
        "#!cp '/content/gdrive/My Drive/capstone-models/baseline_model.zip' '/content/'  \n",
        "\n",
        "\"\"\"\n",
        "!cp '/content/gdrive/My Drive/capstone-data-final/es_it.zip' '/content/'\n",
        "!cp '/content/gdrive/My Drive/capstone-data-final/es_ca.zip' '/content/'\n",
        "!cp '/content/gdrive/My Drive/capstone-data-final/ca_it.zip' '/content/'\n",
        "\n",
        "!cp '/content/gdrive/My Drive/capstone-models/src_pvt_model.zip' '/content/'  \n",
        "!cp '/content/gdrive/My Drive/capstone-models/pvt_tgt_model.zip' '/content/'  \n",
        "!cp '/content/gdrive/My Drive/capstone-models/src_tgt_model.zip' '/content/'  \n",
        "!cp '/content/gdrive/My Drive/capstone-models/baseline_model.zip' '/content/'  \n",
        "\"\"\""
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"\\n!cp '/content/gdrive/My Drive/capstone-data-final/es_it.zip' '/content/'\\n!cp '/content/gdrive/My Drive/capstone-data-final/es_ca.zip' '/content/'\\n!cp '/content/gdrive/My Drive/capstone-data-final/ca_it.zip' '/content/'\\n\\n!cp '/content/gdrive/My Drive/capstone-models/src_pvt_model.zip' '/content/'  \\n!cp '/content/gdrive/My Drive/capstone-models/pvt_tgt_model.zip' '/content/'  \\n!cp '/content/gdrive/My Drive/capstone-models/src_tgt_model.zip' '/content/'  \\n!cp '/content/gdrive/My Drive/capstone-models/baseline_model.zip' '/content/'  \\n\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "PnEa0G8HV3mP",
        "outputId": "f6cda066-0917-41cc-f73c-a9623844c5a9"
      },
      "source": [
        "!unzip src_tgt_data.zip\n",
        "#!unzip pvt_tgt_data.zip\n",
        "#!unzip src_pvt_data.zip\n",
        "!unzip sentencepiece_models.zip\n",
        "#!unzip src_pvt_model.zip\n",
        "#!unzip pvt_tgt_model.zip\n",
        "!unzip src_tgt_model.zip\n",
        "\n",
        "\"\"\"\n",
        "!unzip es_ca.zip\n",
        "!unzip es_it.zip\n",
        "!unzip ca_it.zip\n",
        "\n",
        "!unzip src_pvt_model.zip\n",
        "!unzip pvt_tgt_model.zip\n",
        "!unzip src_tgt_model.zip\n",
        "!unzip baseline_model.zip\n",
        "\n",
        "!mkdir sentencepiece_models\n",
        "\"\"\""
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  src_tgt_data.zip\n",
            "   creating: src_tgt_data/\n",
            "  inflating: src_tgt_data/src_tgt_train.tok  \n",
            "   creating: src_tgt_data/.ipynb_checkpoints/\n",
            "  inflating: src_tgt_data/tgt_src_val.tok  \n",
            "  inflating: src_tgt_data/tgt_src_train.tok  \n",
            "  inflating: src_tgt_data/tgt_src_test.tok  \n",
            "  inflating: src_tgt_data/src_tgt_val.tok  \n",
            "  inflating: src_tgt_data/src_tgt_test.tok  \n",
            "Archive:  sentencepiece_models.zip\n",
            "   creating: sentencepiece_models/\n",
            "  inflating: sentencepiece_models/src_tgt.model  \n",
            "  inflating: sentencepiece_models/tgt.vocab  \n",
            "  inflating: sentencepiece_models/tgt_src.vocab  \n",
            "  inflating: sentencepiece_models/pvt_tgt.vocab  \n",
            "  inflating: sentencepiece_models/src.vocab  \n",
            "  inflating: sentencepiece_models/pvt_tgt.model  \n",
            "  inflating: sentencepiece_models/pvt_src.vocab  \n",
            "  inflating: sentencepiece_models/src.model  \n",
            "  inflating: sentencepiece_models/tgt.model  \n",
            "  inflating: sentencepiece_models/pvt_src.model  \n",
            "  inflating: sentencepiece_models/src_tgt.vocab  \n",
            "  inflating: sentencepiece_models/tgt_src.model  \n",
            "Archive:  src_tgt_model.zip\n",
            "   creating: src_tgt_model/\n",
            "  inflating: src_tgt_model/checkpoint  \n",
            "  inflating: src_tgt_model/ckpt-4500.data-00000-of-00001  \n",
            "   creating: src_tgt_model/.ipynb_checkpoints/\n",
            "  inflating: src_tgt_model/events.out.tfevents.1622053019.c787f24b4b08.71.261162.v2  \n",
            "  inflating: src_tgt_model/ckpt-4000.index  \n",
            "  inflating: src_tgt_model/ckpt-4500.index  \n",
            "   creating: src_tgt_model/eval/\n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622058283.c787f24b4b08.71.435988.v2  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622053019.c787f24b4b08.71.260855.v2  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622054366.c787f24b4b08.71.342662.v2  \n",
            "  inflating: src_tgt_model/events.out.tfevents.1622058283.c787f24b4b08.71.436295.v2  \n",
            "  inflating: src_tgt_model/events.out.tfevents.1622054366.c787f24b4b08.71.342969.v2  \n",
            "  inflating: src_tgt_model/ckpt-4000.data-00000-of-00001  \n",
            "  inflating: src_tgt_model/events.out.tfevents.1622067352.c787f24b4b08.3666.349.v2  \n",
            "  inflating: src_tgt_model/events.out.tfevents.1622067693.c787f24b4b08.3666.24159.v2  \n",
            "  inflating: src_tgt_model/ckpt-5000.index  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622067523.c787f24b4b08.3666.598.v2  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622067693.c787f24b4b08.3666.23852.v2  \n",
            "  inflating: src_tgt_model/eval/predictions.txt.5000  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622067351.c787f24b4b08.3666.42.v2  \n",
            "  inflating: src_tgt_model/events.out.tfevents.1622067523.c787f24b4b08.3666.905.v2  \n",
            "  inflating: src_tgt_model/model_examples_inputter_labels_inputter_embedding.txt  \n",
            "  inflating: src_tgt_model/ckpt-5000.data-00000-of-00001  \n",
            "  inflating: src_tgt_model/projector_config.pbtxt  \n",
            "  inflating: src_tgt_model/model_examples_inputter_features_inputter_embedding.txt  \n",
            "  inflating: src_tgt_model/ckpt-10000.index  \n",
            "  inflating: src_tgt_model/ckpt-9500.index  \n",
            "  inflating: src_tgt_model/ckpt-10000.data-00000-of-00001  \n",
            "  inflating: src_tgt_model/events.out.tfevents.1622138173.500f116e85a5.70.32176.v2  \n",
            "  inflating: src_tgt_model/ckpt-9500.data-00000-of-00001  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622149604.500f116e85a5.70.266069.v2  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622138172.500f116e85a5.70.31869.v2  \n",
            "  inflating: src_tgt_model/eval/predictions.txt.10000  \n",
            "  inflating: src_tgt_model/ckpt-20500.index  \n",
            "  inflating: src_tgt_model/ckpt-20500.data-00000-of-00001  \n",
            "  inflating: src_tgt_model/events.out.tfevents.1622573928.b5a0d7eb7b16.71.28490.v2  \n",
            "  inflating: src_tgt_model/ckpt-21000.data-00000-of-00001  \n",
            "  inflating: src_tgt_model/ckpt-21000.index  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622573927.b5a0d7eb7b16.71.28179.v2  \n",
            "  inflating: src_tgt_model/ckpt-25000.data-00000-of-00001  \n",
            "  inflating: src_tgt_model/ckpt-25000.index  \n",
            "  inflating: src_tgt_model/ckpt-24500.index  \n",
            "  inflating: src_tgt_model/events.out.tfevents.1622621037.b5a0d7eb7b16.71.646515.v2  \n",
            "  inflating: src_tgt_model/ckpt-24500.data-00000-of-00001  \n",
            "  inflating: src_tgt_model/eval/predictions.txt.25000  \n",
            "  inflating: src_tgt_model/eval/events.out.tfevents.1622621037.b5a0d7eb7b16.71.646204.v2  \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\n!unzip es_ca.zip\\n!unzip es_it.zip\\n!unzip ca_it.zip\\n\\n!unzip src_pvt_model.zip\\n!unzip pvt_tgt_model.zip\\n!unzip src_tgt_model.zip\\n!unzip baseline_model.zip\\n\\n!mkdir sentencepiece_models\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i5b-2BgLedXC"
      },
      "source": [
        "# Build vocab (uses SentencePiece)\n",
        "# source = catalan   (ca)\n",
        "# pivot  = spanish   (es)\n",
        "# target = italian  (it)\n",
        "\n",
        "!onmt-build-vocab --sentencepiece model_type=bpe --size 32000 --save_vocab /content/sentencepiece_models/src /content/src_pvt_data/src_train.raw\n",
        "!onmt-build-vocab --sentencepiece model_type=bpe --size 32000 --save_vocab /content/sentencepiece_models/pvt_src /content/src_pvt_data/pvt_src_train.raw\n",
        "\n",
        "!onmt-build-vocab --sentencepiece model_type=bpe --size 32000 --save_vocab /content/sentencepiece_models/tgt /content/pvt_tgt_data/tgt_train.raw\n",
        "!onmt-build-vocab --sentencepiece model_type=bpe --size 32000 --save_vocab /content/sentencepiece_models/pvt_tgt /content/pvt_tgt_data/pvt_tgt_train.raw\n",
        "\n",
        "!onmt-build-vocab --sentencepiece model_type=bpe --size 32000 --save_vocab /content/sentencepiece_models/src_tgt /content/src_tgt_data/src_tgt_train.raw\n",
        "!onmt-build-vocab --sentencepiece model_type=bpe --size 32000 --save_vocab /content/sentencepiece_models/tgt_src /content/src_tgt_data/tgt_src_train.raw"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7L6vvAY_ow3g"
      },
      "source": [
        "!mv /content/sentencepiece_models/src.vocab /content/src_pvt_data/src_vocab.txt\n",
        "!mv /content/sentencepiece_models/pvt_src.vocab /content/src_pvt_data/pvt_src_vocab.txt\n",
        "\n",
        "!mv /content/sentencepiece_models/tgt.vocab /content/pvt_tgt_data/tgt_vocab.txt\n",
        "!mv /content/sentencepiece_models/pvt_tgt.vocab /content/pvt_tgt_data/pvt_tgt_vocab.txt\n",
        "\n",
        "!mv /content/sentencepiece_models/src_tgt.vocab /content/src_tgt_data/src_tgt_vocab.txt\n",
        "!mv /content/sentencepiece_models/tgt_src.vocab /content/src_tgt_data/tgt_src_vocab.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kk7pKHsaho23"
      },
      "source": [
        "config_src_pvt = {\n",
        "    \"model_dir\": \"/content/src_pvt_model/\",\n",
        "    \"data\": {\n",
        "        \"train_features_file\": \"/content/src_pvt_data/src_train.tok/\",\n",
        "        \"train_labels_file\": \"/content/src_pvt_data/pvt_src_train.tok/\",\n",
        "        \"eval_features_file\": \"/content/src_pvt_data/src_val.tok/\",\n",
        "        \"eval_labels_file\": \"/content/src_pvt_data/pvt_src_val.tok/\",\n",
        "        \"source_vocabulary\": \"/content/sentencepiece_models/src.vocab/\",\n",
        "        \"target_vocabulary\": \"/content/sentencepiece_models/pvt_src.vocab/\",\n",
        "    },\n",
        "    \"train\": {\n",
        "        \"max_step\": 25000,\n",
        "        \"save_checkpoints_steps\": 500,\n",
        "        \"keep_checkpoint_max\": 2,\n",
        "    },\n",
        "    \"eval\": {\n",
        "        \"save_eval_predictions\": True,\n",
        "        \"steps\": 50000,\n",
        "        \"max_exports_to_keep\": 2,\n",
        "        \"early_stopping\": {\n",
        "            \"metric\": \"loss\",\n",
        "            \"min_improvement\": 0.1,\n",
        "            \"steps\": 100,\n",
        "        },\n",
        "    }\n",
        "}\n",
        "\n",
        "config_pvt_tgt = {\n",
        "    \"model_dir\": \"/content/pvt_tgt_model/\",\n",
        "    \"data\": {\n",
        "        \"train_features_file\": \"/content/pvt_tgt_data/pvt_tgt_train.tok/\",\n",
        "        \"train_labels_file\": \"/content/pvt_tgt_data/tgt_train.tok/\",\n",
        "        \"eval_features_file\": \"/content/pvt_tgt_data/pvt_tgt_val.tok/\",\n",
        "        \"eval_labels_file\": \"/content/pvt_tgt_data/tgt_val.tok/\",\n",
        "        \"source_vocabulary\": \"/content/sentencepiece_models/pvt_tgt.vocab/\",\n",
        "        \"target_vocabulary\": \"/content/sentencepiece_models/tgt.vocab/\",\n",
        "    },\n",
        "    \"train\": {\n",
        "        \"max_step\": 25000,\n",
        "        \"save_checkpoints_steps\": 500,\n",
        "        \"keep_checkpoint_max\": 2,\n",
        "    },\n",
        "    \"eval\": {\n",
        "        \"save_eval_predictions\": True,\n",
        "        \"steps\": 50000,\n",
        "        \"max_exports_to_keep\": 2,\n",
        "        \"early_stopping\": {\n",
        "            \"metric\": \"loss\",\n",
        "            \"min_improvement\": 0.1,\n",
        "            \"steps\": 100,\n",
        "        },\n",
        "    }\n",
        "}\n",
        "\n",
        "config_src_tgt = {\n",
        "    \"model_dir\": \"/content/src_tgt_model/\",\n",
        "    \"data\": {\n",
        "        \"train_features_file\": \"/content/src_tgt_data/src_tgt_train.tok\",\n",
        "        \"train_labels_file\": \"/content/src_tgt_data/tgt_src_train.tok\",\n",
        "        \"eval_features_file\": \"/content/src_tgt_data/src_tgt_val.tok\",\n",
        "        \"eval_labels_file\": \"/content/src_tgt_data/tgt_src_val.tok\",\n",
        "        \"source_vocabulary\": \"/content/sentencepiece_models/src_tgt.vocab\",\n",
        "        \"target_vocabulary\": \"/content/sentencepiece_models/tgt_src.vocab\",\n",
        "    },\n",
        "    \"train\": {\n",
        "        \"max_step\": 25000,\n",
        "        \"save_checkpoints_steps\": 500,\n",
        "        \"keep_checkpoint_max\": 2,\n",
        "    },\n",
        "    \"eval\": {\n",
        "        \"save_eval_predictions\": True,\n",
        "        \"steps\": 50000,\n",
        "        \"max_exports_to_keep\": 2,\n",
        "        \"early_stopping\": {\n",
        "            \"metric\": \"loss\",\n",
        "            \"min_improvement\": 0.1,\n",
        "            \"steps\": 100,\n",
        "        },\n",
        "    }\n",
        "}\n",
        "\n",
        "config_baseline = {\n",
        "    \"model_dir\": \"/content/baseline_model/\",\n",
        "    \"data\": {\n",
        "        \"train_features_file\": \"/content/src_tgt_data/src_tgt_train.tok/\",\n",
        "        \"train_labels_file\": \"/content/src_tgt_data/tgt_src_train.tok/\",\n",
        "        \"eval_features_file\": \"/content/src_tgt_data/src_tgt_val.tok/\",\n",
        "        \"eval_labels_file\": \"/content/src_tgt_data/tgt_src_val.tok/\",\n",
        "        \"source_vocabulary\": \"/content/sentencepiece_models/src_tgt.vocab/\",\n",
        "        \"target_vocabulary\": \"/content/sentencepiece_models/tgt_src.vocab/\",\n",
        "    },\n",
        "    \"train\": {\n",
        "        \"max_step\": 25000,\n",
        "        \"save_checkpoints_steps\": 500,\n",
        "        \"keep_checkpoint_max\": 2,\n",
        "    },\n",
        "    \"eval\": {\n",
        "        \"save_eval_predictions\": True,\n",
        "        \"steps\": 50000,\n",
        "        \"max_exports_to_keep\": 2,\n",
        "        \"early_stopping\": {\n",
        "            \"metric\": \"loss\",\n",
        "            \"min_improvement\": 0.1,\n",
        "            \"steps\": 100,\n",
        "        },\n",
        "    }\n",
        "}"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KM2gDhAv_Ahd"
      },
      "source": [
        "learning_rate = opennmt.schedules.NoamDecay(scale=2.0, model_dim=512, warmup_steps=8000)\n",
        "optimizer = tfa.optimizers.LazyAdam(learning_rate)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSbSfQRMi73X"
      },
      "source": [
        "# Training source-pivot model\n",
        "src_pvt_model = opennmt.models.TransformerBase()\n",
        "src_pvt_runner = opennmt.Runner(src_pvt_model, config_src_pvt, auto_config=True)\n",
        "sp_config = src_pvt_runner._finalize_config(training=True)\n",
        "\n",
        "#src_pvt_runner.train(num_devices=1, with_eval=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hTTJTGDTuU7p"
      },
      "source": [
        "!zip -r src_pvt_model.zip src_pvt_model/\n",
        "!cp src_pvt_model.zip '/content/gdrive/My Drive/capstone-models/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0rv_Ldfhi-an"
      },
      "source": [
        "# Training pivot-target model\n",
        "pvt_tgt_model = opennmt.models.TransformerBase()\n",
        "pvt_tgt_runner = opennmt.Runner(pvt_tgt_model, config_pvt_tgt, auto_config=True)\n",
        "pt_config = pvt_tgt_runner._finalize_config(training=True)\n",
        "\n",
        "#pvt_tgt_runner.train(num_devices=1, with_eval=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ARXn7e3uIyV"
      },
      "source": [
        "!zip -r pvt_tgt_model.zip pvt_tgt_model/\n",
        "!cp pvt_tgt_model.zip '/content/gdrive/My Drive/capstone-models/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BJMmqNhz4FZm"
      },
      "source": [
        "!head -n 5000 src_tgt_data/src_tgt_val.tok > src_tgt.tok\n",
        "!head -n 5000 src_tgt_data/tgt_src_val.tok > tgt_src.tok"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avfouFM041p-"
      },
      "source": [
        "def specific_detokenize(model_basename, tokenized_basename):\n",
        "  model_path = os.path.join(\"sentencepiece_models\", model_basename + \".model\")\n",
        "  vocabulary_path = os.path.join(\"sentencepiece_models\", f\"{model_basename}.vocab\")\n",
        "  detokenizer = SentencePieceTokenizer(model_path=model_path,\n",
        "                                     vocabulary_path=vocabulary_path,)\n",
        "  \n",
        "  with open(f\"{tokenized_basename}.tok\") as f:\n",
        "    with open(f\"{tokenized_basename}.txt\", mode=\"w\") as fout:\n",
        "      for line in f.readlines():\n",
        "        fout.write(detokenizer.detokenize(line.strip().split(\" \")) + \"\\n\")\n",
        "\n",
        "  return f\"{tokenized_basename}.txt\"\n",
        "\n",
        "def specific_tokenize(input_file, basename):\n",
        "  model_path = os.path.join(\"sentencepiece_models\", f\"{basename}.model\")\n",
        "  vocabulary_path = os.path.join(\"sentencepiece_models\", f\"{basename}.vocab\")\n",
        "  tokenizer = SentencePieceTokenizer(model_path=model_path,\n",
        "                                     vocabulary_path=vocabulary_path,)\n",
        "  \n",
        "  with open(os.path.join(f\"{input_file}\")) as f:\n",
        "    with open(os.path.join(f\"{basename}.tok\"), mode=\"w\") as fout:\n",
        "      for line in f.readlines():\n",
        "        if line.strip():\n",
        "          fout.write(\" \".join(tokenizer.tokenize(line)[0]) + \"\\n\")"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "J7kTzUGY531m",
        "outputId": "54a9a4a3-94c3-4582-af17-1d2f33566f67"
      },
      "source": [
        "#specific_tokenize(\"src_tgt_TRUTH.txt\",\"src\")\n",
        "#src_pvt_runner.infer(\"src.tok\", \"pvt_src.tok\")\n",
        "#specific_detokenize(\"pvt_src\", \"pvt_src\")\n",
        "\n",
        "#specific_tokenize(\"pvt_src.txt\",\"pvt_tgt\")\n",
        "#pvt_tgt_runner.infer(\"pvt_tgt.tok\", \"tgt.tok\")\n",
        "#specific_detokenize(\"tgt\", \"tgt\")"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'tgt.txt'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZsFacZ93DMY"
      },
      "source": [
        "pred_filename = \"tgt.txt\"\n",
        "truth_filename = \"tgt_src_TRUTH.txt\"\n",
        "\n",
        "with open(pred_filename) as f:\n",
        "  preds = f.readlines()\n",
        "\n",
        "with open(truth_filename) as f:\n",
        "  truth = f.readlines()\n",
        "\n",
        "scores = dict()\n",
        "\n",
        "bleu = sacrebleu.corpus_bleu(preds, [truth])\n",
        "ter = sacrebleu.corpus_ter(preds, [truth])\n",
        "scores.update({'bleu': bleu.score})\n",
        "scores.update({'ter': ter.score})  "
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t4M5NxBt9-SN",
        "outputId": "4eb15a7e-2b52-40dd-ff6e-cca7ee8a3980"
      },
      "source": [
        "print(scores)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'bleu': 21.133085989899232, 'ter': 0.7513867399244266}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E9UqsqCbk5PB"
      },
      "source": [
        "!head -n 5000 pvt_tgt_data/pvt_tgt_val.tok > pvt_tgt.tok\n",
        "!head -n 5000 pvt_tgt_data/tgt_val.tok > tgt.tok\n",
        "\n",
        "scores = compute_scores(pvt_tgt_runner, \"pvt_tgt.tok\", \"tgt.tok\", \"pred.tok\")\n",
        "print(f\"============ Baseline Pivot-Target NMT Evaluation ============\\n {scores}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vSAiz8rKJ0bw"
      },
      "source": [
        "# Restore both models weights\n",
        "src_pvt_model.initialize(data_config=sp_config['data'], params=sp_config['params'])\n",
        "src_pvt_model.create_variables(optimizer=optimizer)\n",
        "\n",
        "pvt_tgt_model.initialize(data_config=pt_config['data'], params=pt_config['params'])\n",
        "pvt_tgt_model.create_variables(optimizer=optimizer)\n",
        "\n",
        "checkpoint_path = sp_config['model_dir']\n",
        "checkpoint = checkpoint_util.Checkpoint.from_config(sp_config, src_pvt_model, optimizer=optimizer)\n",
        "checkpoint.restore(checkpoint_path=checkpoint_path, weights_only=True)\n",
        "\n",
        "checkpoint_path = pt_config['model_dir']\n",
        "checkpoint = checkpoint_util.Checkpoint.from_config(pt_config, pvt_tgt_model, optimizer=optimizer)\n",
        "checkpoint.restore(checkpoint_path=checkpoint_path, weights_only=True)\n",
        "\n",
        "count_weights(src_pvt_model)\n",
        "count_weights(pvt_tgt_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "utqcEQnWXNXk"
      },
      "source": [
        "# Transfer weights to src_tgt_model\n",
        "src_tgt_model = opennmt.models.TransformerBase()\n",
        "src_tgt_runner = opennmt.Runner(src_tgt_model, config_src_tgt, auto_config=True)\n",
        "st_config = src_tgt_runner._finalize_config(training=True)\n",
        "\n",
        "src_tgt_model.initialize(data_config=st_config['data'], params=st_config['params'])\n",
        "src_tgt_model.create_variables(optimizer=optimizer)\n",
        "\n",
        "src_tgt_model.encoder = src_pvt_model.encoder\n",
        "src_tgt_model.decoder = pvt_tgt_model.decoder\n",
        "\n",
        "new_checkpoint = checkpoint_util.Checkpoint.from_config(st_config, src_tgt_model, optimizer=optimizer)\n",
        "new_checkpoint.save()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9P6AnVrZHRBQ",
        "outputId": "ed753dad-35b4-462a-8465-7275ddb4457d"
      },
      "source": [
        "# Training source-target model (using pretrained models)\n",
        "src_tgt_model = opennmt.models.TransformerBase()\n",
        "src_tgt_runner = opennmt.Runner(src_tgt_model, config_src_tgt, auto_config=True)\n",
        "st_config = src_tgt_runner._finalize_config(training=True)\n",
        "#src_tgt_runner.train(num_devices=1, with_eval=True)\n",
        "\n",
        "#!zip -r src_tgt_model.zip src_tgt_model/\n",
        "#!cp src_tgt_model.zip '/content/gdrive/My Drive/capstone-models/'"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Using OpenNMT-tf version 2.19.0\n",
            "INFO:tensorflow:Using model:\n",
            "(model): TransformerBase(\n",
            "  (examples_inputter): SequenceToSequenceInputter(\n",
            "    (features_inputter): WordEmbedder()\n",
            "    (labels_inputter): WordEmbedder()\n",
            "    (inputters): ListWrapper(\n",
            "      (0): WordEmbedder()\n",
            "      (1): WordEmbedder()\n",
            "    )\n",
            "  )\n",
            "  (encoder): SelfAttentionEncoder(\n",
            "    (position_encoder): SinusoidalPositionEncoder(\n",
            "      (reducer): SumReducer()\n",
            "    )\n",
            "    (layer_norm): LayerNorm()\n",
            "    (layers): ListWrapper(\n",
            "      (0): SelfAttentionEncoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (1): SelfAttentionEncoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (2): SelfAttentionEncoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (3): SelfAttentionEncoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (4): SelfAttentionEncoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (5): SelfAttentionEncoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (decoder): SelfAttentionDecoder(\n",
            "    (position_encoder): SinusoidalPositionEncoder(\n",
            "      (reducer): SumReducer()\n",
            "    )\n",
            "    (layer_norm): LayerNorm()\n",
            "    (layers): ListWrapper(\n",
            "      (0): SelfAttentionDecoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (attention): ListWrapper(\n",
            "          (0): TransformerLayerWrapper(\n",
            "            (layer): MultiHeadAttention(\n",
            "              (linear_queries): Dense(512)\n",
            "              (linear_keys): Dense(512)\n",
            "              (linear_values): Dense(512)\n",
            "              (linear_output): Dense(512)\n",
            "            )\n",
            "            (input_layer_norm): LayerNorm()\n",
            "          )\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (1): SelfAttentionDecoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (attention): ListWrapper(\n",
            "          (0): TransformerLayerWrapper(\n",
            "            (layer): MultiHeadAttention(\n",
            "              (linear_queries): Dense(512)\n",
            "              (linear_keys): Dense(512)\n",
            "              (linear_values): Dense(512)\n",
            "              (linear_output): Dense(512)\n",
            "            )\n",
            "            (input_layer_norm): LayerNorm()\n",
            "          )\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (2): SelfAttentionDecoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (attention): ListWrapper(\n",
            "          (0): TransformerLayerWrapper(\n",
            "            (layer): MultiHeadAttention(\n",
            "              (linear_queries): Dense(512)\n",
            "              (linear_keys): Dense(512)\n",
            "              (linear_values): Dense(512)\n",
            "              (linear_output): Dense(512)\n",
            "            )\n",
            "            (input_layer_norm): LayerNorm()\n",
            "          )\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (3): SelfAttentionDecoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (attention): ListWrapper(\n",
            "          (0): TransformerLayerWrapper(\n",
            "            (layer): MultiHeadAttention(\n",
            "              (linear_queries): Dense(512)\n",
            "              (linear_keys): Dense(512)\n",
            "              (linear_values): Dense(512)\n",
            "              (linear_output): Dense(512)\n",
            "            )\n",
            "            (input_layer_norm): LayerNorm()\n",
            "          )\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (4): SelfAttentionDecoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (attention): ListWrapper(\n",
            "          (0): TransformerLayerWrapper(\n",
            "            (layer): MultiHeadAttention(\n",
            "              (linear_queries): Dense(512)\n",
            "              (linear_keys): Dense(512)\n",
            "              (linear_values): Dense(512)\n",
            "              (linear_output): Dense(512)\n",
            "            )\n",
            "            (input_layer_norm): LayerNorm()\n",
            "          )\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "      (5): SelfAttentionDecoderLayer(\n",
            "        (self_attention): TransformerLayerWrapper(\n",
            "          (layer): MultiHeadAttention(\n",
            "            (linear_queries): Dense(512)\n",
            "            (linear_keys): Dense(512)\n",
            "            (linear_values): Dense(512)\n",
            "            (linear_output): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "        (attention): ListWrapper(\n",
            "          (0): TransformerLayerWrapper(\n",
            "            (layer): MultiHeadAttention(\n",
            "              (linear_queries): Dense(512)\n",
            "              (linear_keys): Dense(512)\n",
            "              (linear_values): Dense(512)\n",
            "              (linear_output): Dense(512)\n",
            "            )\n",
            "            (input_layer_norm): LayerNorm()\n",
            "          )\n",
            "        )\n",
            "        (ffn): TransformerLayerWrapper(\n",
            "          (layer): FeedForwardNetwork(\n",
            "            (inner): Dense(2048)\n",
            "            (outer): Dense(512)\n",
            "          )\n",
            "          (input_layer_norm): LayerNorm()\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            ")\n",
            "\n",
            "INFO:tensorflow:Using parameters:\n",
            "data:\n",
            "  eval_features_file: /content/src_tgt_data/src_tgt_val.tok\n",
            "  eval_labels_file: /content/src_tgt_data/tgt_src_val.tok\n",
            "  source_vocabulary: /content/sentencepiece_models/src_tgt.vocab\n",
            "  target_vocabulary: /content/sentencepiece_models/tgt_src.vocab\n",
            "  train_features_file: /content/src_tgt_data/src_tgt_train.tok\n",
            "  train_labels_file: /content/src_tgt_data/tgt_src_train.tok\n",
            "eval:\n",
            "  batch_size: 32\n",
            "  batch_type: examples\n",
            "  early_stopping:\n",
            "    metric: loss\n",
            "    min_improvement: 0.1\n",
            "    steps: 100\n",
            "  length_bucket_width: 5\n",
            "  max_exports_to_keep: 2\n",
            "  save_eval_predictions: true\n",
            "  steps: 50000\n",
            "infer:\n",
            "  batch_size: 32\n",
            "  batch_type: examples\n",
            "  length_bucket_width: 5\n",
            "model_dir: /content/src_tgt_model/\n",
            "params:\n",
            "  average_loss_in_time: true\n",
            "  beam_width: 4\n",
            "  decay_params:\n",
            "    model_dim: 512\n",
            "    warmup_steps: 8000\n",
            "  decay_type: NoamDecay\n",
            "  label_smoothing: 0.1\n",
            "  learning_rate: 2.0\n",
            "  num_hypotheses: 1\n",
            "  optimizer: LazyAdam\n",
            "  optimizer_params:\n",
            "    beta_1: 0.9\n",
            "    beta_2: 0.998\n",
            "score:\n",
            "  batch_size: 64\n",
            "  batch_type: examples\n",
            "  length_bucket_width: 5\n",
            "train:\n",
            "  average_last_checkpoints: 8\n",
            "  batch_size: 3072\n",
            "  batch_type: tokens\n",
            "  effective_batch_size: 25000\n",
            "  keep_checkpoint_max: 2\n",
            "  length_bucket_width: 1\n",
            "  max_step: 25000\n",
            "  maximum_features_length: 100\n",
            "  maximum_labels_length: 100\n",
            "  sample_buffer_size: -1\n",
            "  save_checkpoints_steps: 500\n",
            "  save_summary_steps: 100\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "52D3Pq9wu8FT"
      },
      "source": [
        "!zip -r src_tgt_model.zip src_tgt_model/\n",
        "!cp src_tgt_model.zip '/content/gdrive/My Drive/capstone-models/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "875R9siBLNMW"
      },
      "source": [
        "!head -n 5000 src_tgt_data/src_tgt_val.tok > src_tgt.tok\n",
        "!head -n 5000 src_tgt_data/tgt_src_val.tok > tgt_src.tok\n",
        "\n",
        "scores = compute_scores(src_tgt_runner, \"src_tgt.tok\", \"tgt_src.tok\", \"pred.tok\", True, True)\n",
        "print(f\"============ Baseline Source-Target NMT Evaluation ============\\n {scores}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r7H76tXv2mr_"
      },
      "source": [
        "# Training source-target model (using no models)\n",
        "baseline_model = opennmt.models.TransformerBase()\n",
        "baseline_runner = opennmt.Runner(baseline_model, config_baseline, auto_config=True)\n",
        "\n",
        "baseline_runner.train(num_devices=1, with_eval=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZAJLUhoba--0"
      },
      "source": [
        "!zip -r baseline_model.zip baseline_model/\n",
        "!cp baseline_model.zip '/content/gdrive/My Drive/capstone-models/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j_8BpmvshfnN"
      },
      "source": [
        "!head -n 5000 src_tgt_data/src_tgt_val.tok > src_tgt.tok\n",
        "!head -n 5000 src_tgt_data/tgt_src_val.tok > tgt_src.tok\n",
        "\n",
        "scores = compute_scores(baseline_runner, \"src_tgt.tok\", \"tgt_src.tok\", \"pred.tok\", True, True)\n",
        "print(f\"============ Baseline Source-Target NMT Evaluation ============\\n {scores}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zt1u_WGoDp1L"
      },
      "source": [
        "#!zip -r src_pvt_model.zip src_pvt_model/\n",
        "#!zip -r pvt_tgt_model.zip pvt_tgt_model/\n",
        "#!zip -r src_tgt_model.zip src_tgt_model/\n",
        "#!zip -r baseline_model.zip baseline_model/\n",
        "\n",
        "#!cp '/content/gdrive/My Drive/capstone-models/src_pvt_model.zip' .\n",
        "#!cp pvt_tgt_model.zip '/content/gdrive/My Drive/capstone-models/'\n",
        "#!cp src_tgt_model.zip '/content/gdrive/My Drive/capstone-models/'\n",
        "#!cp baseline_model.zip '/content/gdrive/My Drive/capstone-models/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gr3pMMdoZUIk"
      },
      "source": [
        "!zip -r src_pvt_data.zip src_pvt_data\n",
        "!zip -r pvt_tgt_data.zip pvt_tgt_data\n",
        "!zip -r src_tgt_data.zip src_tgt_data\n",
        "\n",
        "!cp src_pvt_data.zip '/content/gdrive/My Drive/capstone-data-final/'\n",
        "!cp pvt_tgt_data.zip '/content/gdrive/My Drive/capstone-data-final/'\n",
        "!cp src_tgt_data.zip '/content/gdrive/My Drive/capstone-data-final/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LzgD5BPpNQAF"
      },
      "source": [
        "# Compute scores\n",
        "baseline_scores = compute_scores(\n",
        "    runner=baseline_runner,\n",
        "    features_filename=\"/content/src_tgt_data/src_tgt_test.txt\",\n",
        "    labels_filename=\"/content/src_tgt_data/tgt_src_test.txt\",\n",
        "    pred_filename=\"/content/baseline_pred.txt\")\n",
        "\n",
        "pivot_based_tl_scores = compute_scores(\n",
        "    runner=src_tgt_runner,\n",
        "    features_filename=\"/content/src_tgt_data/src_tgt_test.txt\",\n",
        "    labels_filename=\"/content/src_tgt_data/tgt_src_test.txt\",\n",
        "    pred_filename=\"/content/src_to_tgt_pred.txt\")\n",
        "\n",
        "print(f\"============ Baseline Source-Target NMT Evaluation ============\\n {baseline_scores}\")\n",
        "print(f\"============ Pretrain Source-Target NMT Evaluation ============\\n {pivot_based_tl_scores}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tXsu6Nst-Bat"
      },
      "source": [
        "!rm -rf src_pvt_model*\n",
        "!rm -rf pvt_tgt_model*\n",
        "!rm -rf src_tgt_model*\n",
        "!rm -rf baseline_model*"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V_P9hZPqPPZ0"
      },
      "source": [
        "# Old Stuff"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "06xCb_h0S9fH"
      },
      "source": [
        "!zip -r src_pvt_model.pt.zip src_pvt_model.pt/\n",
        "!zip -r pvt_tgt_model.pt.zip pvt_tgt_model.pt/\n",
        "!zip -r src_tgt_model.pt.zip src_tgt_model.pt/\n",
        "\n",
        "from google.colab import files\n",
        "files.download('src_pvt_model.pt.zip') \n",
        "files.download('pvt_tgt_model.pt.zip') \n",
        "files.download('src_tgt_model.pt.zip') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ty7Kx9DqpBHT"
      },
      "source": [
        "# Export models\n",
        "src_pvt_runner.export(\"src_pvt_saved\")\n",
        "pvt_tgt_runner.export(\"pvt_tgt_saved\")\n",
        "src_tgt_runner.export(\"src_tgt_saved\")\n",
        "baseline_runner.export(\"baseline_saved\")"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}