score_cam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "score-cam.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "dMX3mJc0FoHA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "7d70442f-f654-43ce-c610-1c7c9a73ea94"
      },
      "source": [
        "!pip install opencv-python==4.1.0.25"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting opencv-python==4.1.0.25\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7b/d2/a2dbf83d4553ca6b3701d91d75e42fe50aea97acdc00652dca515749fb5d/opencv_python-4.1.0.25-cp36-cp36m-manylinux1_x86_64.whl (26.6MB)\n",
            "\u001b[K     |████████████████████████████████| 26.6MB 104kB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from opencv-python==4.1.0.25) (1.18.5)\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: opencv-python\n",
            "  Found existing installation: opencv-python 4.1.2.30\n",
            "    Uninstalling opencv-python-4.1.2.30:\n",
            "      Successfully uninstalled opencv-python-4.1.2.30\n",
            "Successfully installed opencv-python-4.1.0.25\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PqFt5u5o7YDh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "gpus = tf.config.list_physical_devices('GPU')\n",
        "for gpu in gpus:\n",
        "    tf.config.experimental.set_memory_growth(gpu, True)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lMFldewP70cl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "73c49f7a-6997-4f0a-b729-3ca20942a4e6"
      },
      "source": [
        "from tensorflow.keras.applications.vgg16 import preprocess_input\n",
        "import numpy as np\n",
        "\n",
        "IMAGE_PATH = \"./cat.jpg\"\n",
        "\n",
        "model = tf.keras.applications.VGG16(weights=\"imagenet\", include_top=True)\n",
        "\n",
        "input_shape = (224, 224)\n",
        "\n",
        "img = tf.keras.preprocessing.image.load_img(IMAGE_PATH, target_size=input_shape)\n",
        "img = tf.keras.preprocessing.image.img_to_array(img)\n",
        "img = np.expand_dims(img, axis=0)\n",
        "\n",
        "data = (preprocess_input(img), None)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5\n",
            "553467904/553467096 [==============================] - 3s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4X3V4FkP8Ci7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def resize_activations(enhanced_model_output, input_shape):\n",
        "    \"\"\"Utility function to resize a given tensor\n",
        "\n",
        "    Args:\n",
        "        enhanced_model_output (tf.Tensor): 4D-Tensor with shape (batch_size, H, W, K)\n",
        "        input_shape (Tuple[int, int]): shape of the input, e.g. (224, 224)\n",
        "\n",
        "    Returns\n",
        "        tensor (tf.Tensor): 4D-Tensor with shape (batch_size, K, H, W)\n",
        "    \"\"\"\n",
        "\n",
        "    resized_activations = list()\n",
        "\n",
        "    for j in range(enhanced_model_output.shape[0]):\n",
        "        acts = list()\n",
        "        for i in range(enhanced_model_output.shape[-1]):\n",
        "            acts.append(\n",
        "                transform.resize(enhanced_model_output[j, ..., i], input_shape, preserve_range=True)\n",
        "            )\n",
        "        resized_activations.append(np.array(acts))\n",
        "\n",
        "    return tf.convert_to_tensor(np.array(resized_activations), dtype=tf.float32)\n",
        "\n",
        "def normalize_activations(tensor):\n",
        "    \"\"\"Utility function to normalize a given tensor\n",
        "\n",
        "    Args:\n",
        "        tensor (tf.Tensor): 4D-Tensor with shape (batch_size, K, H, W)\n",
        "\n",
        "    Returns:\n",
        "        tf.Tensor: 4D-Tensor with shape (batch_size, K, H, W)\n",
        "    \"\"\"\n",
        "\n",
        "    tensors = list()\n",
        "\n",
        "    # goes through each image\n",
        "    for i in range(tensor.shape[0]):\n",
        "        flattened = tf.reshape(tensor[i], (tensor[i].shape[0], -1))\n",
        "\n",
        "        max_a = tf.math.reduce_max(flattened, axis=1)\n",
        "        min_a = tf.math.reduce_min(flattened, axis=1)\n",
        "\n",
        "        diffs = tf.where(max_a > min_a, max_a - min_a, 1)\n",
        "\n",
        "        normalized_tensor = (tensor[i] - tf.reshape(min_a, (-1, 1, 1))) / tf.reshape(diffs, (-1, 1, 1))\n",
        "\n",
        "        tensors.append(normalized_tensor)\n",
        "\n",
        "    return tf.stack(tensors, axis=0)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qGMnun6-8ZCD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_last_convolutional_layer_name(model):\n",
        "    \"\"\"\n",
        "    Search for the last convolutional layer to perform Score-CAM, as stated\n",
        "    in section 4.1 in the original paper.\n",
        "\n",
        "    Args:\n",
        "        model (tf.keras.Model): tf.keras model to inspect\n",
        "\n",
        "    Returns:\n",
        "        str: Name of the target layer\n",
        "    \"\"\"\n",
        "    for layer in reversed(model.layers):\n",
        "        # Select closest 4D layer to the end of the network.\n",
        "        if len(layer.output_shape) == 4 and layer.name.count('conv') > 0:\n",
        "            return layer.name\n",
        "\n",
        "    raise ValueError(\n",
        "        \"Model does not seem to contain 4D layer. Score-CAM cannot be applied.\"\n",
        "    )"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rzacyss-J0-L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from skimage import transform"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v8274WYH8_Jh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "images, _ = data\n",
        "batch_size = images.shape[0]\n",
        "\n",
        "# according to section 4.1 of paper, we need the last convolutional layer\n",
        "layer_name = get_last_convolutional_layer_name(model)\n",
        "\n",
        "# normalize feature maps, calculate masks and compute the\n",
        "# output score\n",
        "# weights, maps = get_filters(\n",
        "#     model, images, layer_name, 281, input_shape\n",
        "# )\n",
        "\n",
        "conv_model = tf.keras.Model(\n",
        "    inputs=model.input,\n",
        "    outputs=model.get_layer(layer_name).output\n",
        ")\n",
        "\n",
        "softmax_model = tf.keras.models.Model(\n",
        "    [model.inputs], [model.outputs]\n",
        ")\n",
        "\n",
        "inputs = tf.cast(images, tf.float32)\n",
        "\n",
        "conv_output = conv_model.predict(inputs)\n",
        "resized_conv_output = resize_activations(conv_output, input_shape)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gd6EUUPXaAi-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "normalized_maps = normalize_activations(resized_conv_output) # shape (batch_size, K, H, W)\n",
        "shape = normalized_maps.shape"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fMh6U53phOuf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "normalized_maps = tf.reshape(normalized_maps, (shape[1], shape[2], shape[3], shape[0]))"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GL_sNZXgkBD5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# (512, 224, 224, 1) * (512, 224, 224, 3)\n",
        "masked_images = tf.math.multiply(normalized_maps, tf.tile(inputs, (normalized_maps.shape[0], 1, 1, 1)))"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J2VJzIr8ZAWS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classes_activation_scale = softmax_model.predict(masked_images)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6WtMotcC9R2G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# return the output only for the given class\n",
        "weights = classes_activation_scale[0][:, 281] # shape (K,)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7xebjZbL99yc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weights = weights.reshape((-1, 1, 1, batch_size)) # shape (K, 1, 1, 1)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4_kYYeFl3kr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "313a0ea7-e1df-48cc-f128-78315619fd15"
      },
      "source": [
        "weights.shape"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(512, 1, 1, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PKRkz76rl5Ax",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a3fdbc41-160f-4cc4-ba67-e5f246bfe547"
      },
      "source": [
        "normalized_maps.shape"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([512, 224, 224, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AaEyymgbmh4d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cam = tf.math.multiply(weights, normalized_maps)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D54ICLjoA3uM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# relu\n",
        "cam = tf.math.reduce_max(cam, axis=0)\n",
        "relu_cam = tf.where(cam > 0, cam, 0)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8hCt2X-xJkDX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d781e898-f169-472d-9f3e-515a3d3db9de"
      },
      "source": [
        "relu_cam.shape"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([224, 224, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ChJFbcrg4H1B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "relu_cam = tf.reshape(relu_cam, (relu_cam.shape[2], relu_cam.shape[0], relu_cam.shape[1]))"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l7To1qLOS2Dg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "33df867d-1931-4bd7-bdc9-7b6cc68e8816"
      },
      "source": [
        "relu_cam.shape"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([1, 224, 224])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JjbqJg7a4kxX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pathlib import Path\n",
        "import cv2\n",
        "\n",
        "def save_rgb(image, output_dir, output_name):\n",
        "    \"\"\"\n",
        "    Save a 3D Numpy array (H, W, 3) as an image.\n",
        "\n",
        "    Args:\n",
        "        image (numpy.ndarray): Image to save\n",
        "        output_dir (str): Output directory\n",
        "        output_name (str): Output name\n",
        "    \"\"\"\n",
        "    Path.mkdir(Path(output_dir), parents=True, exist_ok=True)\n",
        "\n",
        "    cv2.imwrite(\n",
        "        str(Path(output_dir) / output_name), cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
        "    )"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "92Rzcnzm3q8e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def image_to_uint_255(image):\n",
        "    \"\"\"\n",
        "    Convert float images to int 0-255 images.\n",
        "    Args:\n",
        "        image (numpy.ndarray): Input image. Can be either [0, 255], [0, 1], [-1, 1]\n",
        "    Returns:\n",
        "        numpy.ndarray:\n",
        "    \"\"\"\n",
        "    if image.dtype == np.uint8:\n",
        "        return image\n",
        "\n",
        "    if image.max() > 1:\n",
        "        return image.astype(\"uint8\")\n",
        "\n",
        "    if image.min() < 0:\n",
        "        image = (image + 1.0) / 2.0\n",
        "\n",
        "    return (image * 255).astype(\"uint8\")\n",
        "\n",
        "def heatmap_display(\n",
        "    heatmap, original_image, colormap=cv2.COLORMAP_JET, image_weight=0.7\n",
        "):\n",
        "    \"\"\"\n",
        "    Apply a heatmap (as an np.ndarray) on top of an original image.\n",
        "\n",
        "    Args:\n",
        "        heatmap (numpy.ndarray): Array corresponding to the heatmap\n",
        "        original_image (numpy.ndarray): Image on which we apply the heatmap\n",
        "        colormap (int): OpenCV Colormap to use for heatmap visualization\n",
        "        image_weight (float): An optional `float` value in range [0,1] indicating the weight of\n",
        "            the input image to be overlaying the calculated attribution maps. Defaults to `0.7`\n",
        "\n",
        "    Returns:\n",
        "        np.ndarray: Original image with heatmap applied\n",
        "    \"\"\"\n",
        "    heatmap = cv2.resize(heatmap, (original_image.shape[1], original_image.shape[0]))\n",
        "\n",
        "    image = image_to_uint_255(original_image)\n",
        "\n",
        "    heatmap = (heatmap - np.min(heatmap)) / (heatmap.max() - heatmap.min())\n",
        "\n",
        "    heatmap = cv2.applyColorMap(\n",
        "        cv2.cvtColor((heatmap * 255).astype(\"uint8\"), cv2.COLOR_GRAY2BGR), colormap\n",
        "    )\n",
        "\n",
        "    output = cv2.addWeighted(\n",
        "        cv2.cvtColor(image, cv2.COLOR_RGB2BGR), image_weight, heatmap, 1, 0\n",
        "    )\n",
        "\n",
        "    return cv2.cvtColor(output, cv2.COLOR_BGR2RGB)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IH8Q3uEqGcrr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "colormap = cv2.COLORMAP_JET\n",
        "image_weight = 0.7\n",
        "\n",
        "heatmaps = np.array(\n",
        "    [\n",
        "    # not showing the actual image if image_weight=0\n",
        "    heatmap_display(_cam.numpy(), image, colormap, image_weight)\n",
        "    for _cam, image in zip(relu_cam, img)\n",
        "    ]\n",
        ")"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DdwVjdhu4wzS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "save_rgb(heatmaps[0], \".\", \"score_cam.png\")"
      ],
      "execution_count": 24,
      "outputs": []
    }
  ]
}