code.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Image_outpainting_updated_likhitha.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "W3-uQQdg8-mt",
        "colab_type": "code",
        "outputId": "dc463663-6260-493d-d844-99d5c731faf8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "80m5JJbqNSVs",
        "colab_type": "code",
        "outputId": "18e9cba6-7bc1-402f-d356-476556e6453b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install imageio"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: imageio in /usr/local/lib/python2.7/dist-packages (2.4.1)\n",
            "Requirement already satisfied: futures in /usr/local/lib/python2.7/dist-packages (from imageio) (3.2.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python2.7/dist-packages (from imageio) (1.14.6)\n",
            "Requirement already satisfied: enum34 in /usr/local/lib/python2.7/dist-packages (from imageio) (1.1.6)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python2.7/dist-packages (from imageio) (4.0.0)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python2.7/dist-packages (from pillow->imageio) (0.46)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "A88lzdnfNgi3",
        "colab_type": "code",
        "outputId": "91872768-05b7-44c2-ac24-206bb2090f12",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 797
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install tensorlayer"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorlayer in /usr/local/lib/python2.7/dist-packages (1.11.1)\n",
            "Requirement already satisfied: requests<2.21,>=2.19 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (2.20.1)\n",
            "Requirement already satisfied: scikit-image<0.15,>=0.14 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (0.14.1)\n",
            "Requirement already satisfied: numpy<1.16,>=1.14 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (1.14.6)\n",
            "Requirement already satisfied: lxml<4.3,>=4.2 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (4.2.5)\n",
            "Requirement already satisfied: matplotlib<3.1,>=2.2 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (2.2.3)\n",
            "Requirement already satisfied: scikit-learn<0.21,>=0.19 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (0.19.2)\n",
            "Requirement already satisfied: scipy<1.2,>=1.1 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (1.1.0)\n",
            "Requirement already satisfied: progressbar2<3.39,>=3.38 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (3.38.0)\n",
            "Requirement already satisfied: wrapt<1.11,>=1.10 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (1.10.11)\n",
            "Requirement already satisfied: imageio<2.5,>=2.3 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (2.4.1)\n",
            "Requirement already satisfied: tqdm<4.29,>=4.23 in /usr/local/lib/python2.7/dist-packages (from tensorlayer) (4.28.1)\n",
            "Requirement already satisfied: idna<2.8,>=2.5 in /usr/local/lib/python2.7/dist-packages (from requests<2.21,>=2.19->tensorlayer) (2.6)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python2.7/dist-packages (from requests<2.21,>=2.19->tensorlayer) (1.22)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python2.7/dist-packages (from requests<2.21,>=2.19->tensorlayer) (2018.11.29)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python2.7/dist-packages (from requests<2.21,>=2.19->tensorlayer) (3.0.4)\n",
            "Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python2.7/dist-packages (from scikit-image<0.15,>=0.14->tensorlayer) (1.0.1)\n",
            "Requirement already satisfied: cloudpickle>=0.2.1 in /usr/local/lib/python2.7/dist-packages (from scikit-image<0.15,>=0.14->tensorlayer) (0.6.1)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python2.7/dist-packages (from scikit-image<0.15,>=0.14->tensorlayer) (1.11.0)\n",
            "Requirement already satisfied: networkx>=1.8 in /usr/local/lib/python2.7/dist-packages (from scikit-image<0.15,>=0.14->tensorlayer) (2.2)\n",
            "Requirement already satisfied: dask[array]>=0.9.0 in /usr/local/lib/python2.7/dist-packages (from scikit-image<0.15,>=0.14->tensorlayer) (1.0.0)\n",
            "Collecting pillow>=4.3.0 (from scikit-image<0.15,>=0.14->tensorlayer)\n",
            "  Using cached https://files.pythonhosted.org/packages/9a/f6/3b3c82c5c75cae471e02fb584136168d732e17ae9db2d21c5dc82f9790f8/Pillow-5.3.0-cp27-cp27mu-manylinux1_x86_64.whl\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python2.7/dist-packages (from matplotlib<3.1,>=2.2->tensorlayer) (0.10.0)\n",
            "Requirement already satisfied: backports.functools-lru-cache in /usr/local/lib/python2.7/dist-packages (from matplotlib<3.1,>=2.2->tensorlayer) (1.5)\n",
            "Requirement already satisfied: subprocess32 in /usr/local/lib/python2.7/dist-packages (from matplotlib<3.1,>=2.2->tensorlayer) (3.5.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python2.7/dist-packages (from matplotlib<3.1,>=2.2->tensorlayer) (1.0.1)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python2.7/dist-packages (from matplotlib<3.1,>=2.2->tensorlayer) (2018.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python2.7/dist-packages (from matplotlib<3.1,>=2.2->tensorlayer) (2.5.3)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python2.7/dist-packages (from matplotlib<3.1,>=2.2->tensorlayer) (2.3.0)\n",
            "Requirement already satisfied: python-utils>=2.3.0 in /usr/local/lib/python2.7/dist-packages (from progressbar2<3.39,>=3.38->tensorlayer) (2.3.0)\n",
            "Requirement already satisfied: futures in /usr/local/lib/python2.7/dist-packages (from imageio<2.5,>=2.3->tensorlayer) (3.2.0)\n",
            "Requirement already satisfied: enum34 in /usr/local/lib/python2.7/dist-packages (from imageio<2.5,>=2.3->tensorlayer) (1.1.6)\n",
            "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python2.7/dist-packages (from networkx>=1.8->scikit-image<0.15,>=0.14->tensorlayer) (4.3.0)\n",
            "Requirement already satisfied: toolz>=0.7.3; extra == \"array\" in /usr/local/lib/python2.7/dist-packages (from dask[array]>=0.9.0->scikit-image<0.15,>=0.14->tensorlayer) (0.9.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python2.7/dist-packages (from kiwisolver>=1.0.1->matplotlib<3.1,>=2.2->tensorlayer) (40.6.2)\n",
            "Installing collected packages: pillow\n",
            "  Found existing installation: Pillow 4.0.0\n",
            "    Uninstalling Pillow-4.0.0:\n",
            "      Successfully uninstalled Pillow-4.0.0\n",
            "Successfully installed pillow-5.3.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "1x22h6npR-8G",
        "colab_type": "code",
        "outputId": "a254c444-357e-4ddf-84c6-1784fc814e62",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 352
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install Pillow==4.0.0\n",
        "!pip install PIL\n",
        "!pip install image\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting Pillow==4.0.0\n",
            "  Using cached https://files.pythonhosted.org/packages/89/99/0e3522a9764fe371bf9f7729404b1ef7d9c4fc49cbe5f1761c6e07812345/Pillow-4.0.0-cp27-cp27mu-manylinux1_x86_64.whl\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python2.7/dist-packages (from Pillow==4.0.0) (0.46)\n",
            "\u001b[31mscikit-image 0.14.1 has requirement pillow>=4.3.0, but you'll have pillow 4.0.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: Pillow\n",
            "  Found existing installation: Pillow 5.3.0\n",
            "    Uninstalling Pillow-5.3.0:\n",
            "      Successfully uninstalled Pillow-5.3.0\n",
            "Successfully installed Pillow-4.0.0\n",
            "Collecting PIL\n",
            "\u001b[31m  Could not find a version that satisfies the requirement PIL (from versions: )\u001b[0m\n",
            "\u001b[31mNo matching distribution found for PIL\u001b[0m\n",
            "Requirement already satisfied: image in /usr/local/lib/python2.7/dist-packages (1.5.27)\n",
            "Requirement already satisfied: django in /usr/local/lib/python2.7/dist-packages (from image) (1.11.17)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python2.7/dist-packages (from image) (4.0.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python2.7/dist-packages (from django->image) (2018.7)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python2.7/dist-packages (from pillow->image) (0.46)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "r0sbofpl7ihv",
        "colab_type": "code",
        "outputId": "0bc15f88-6ab9-4d98-cb1c-800572184c06",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1240
        }
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.pyplot import imshow\n",
        "import scipy.misc\n",
        "import matplotlib.pyplot as plt\n",
        "import cv2\n",
        "import os\n",
        "import re\n",
        "import imageio\n",
        "import pickle\n",
        "from PIL import Image\n",
        "IMAGE_SZ = 128 \n",
        "PATCH1_SIZE=64\n",
        "NUMBER1=128//64\n",
        "PATCH2_SIZE=8\n",
        "NUMBER2=128//8\n",
        "Label_size=32\n",
        "half_label_size=8\n",
        "\n",
        "\n",
        "#loads image and crops it into 128X128\n",
        "def load_city_image(image):\n",
        "    im = Image.open(image).convert('RGB')\n",
        "    width, height = im.size\n",
        "    print(im.size)\n",
        "    left = (width - IMAGE_SZ) / 2\n",
        "    top = (height - IMAGE_SZ) / 2\n",
        "    im = im.crop((left, top, left + IMAGE_SZ, top + IMAGE_SZ))\n",
        "    pix = np.array(im)\n",
        "    assert pix.shape == (IMAGE_SZ, IMAGE_SZ, 3)\n",
        "    return im\n",
        "\n",
        "#Global patch:  centred crop 128X64, \n",
        "def Patch_0(im,Label_size):\n",
        "    a=[]\n",
        "    im=im.crop((32,0,96,IMAGE_SZ))\n",
        "    for i in range(Label_size):\n",
        "                a.append(np.array(im))\n",
        "    pix=np.array(im)\n",
        "    return a\n",
        "\n",
        "def Patch(image,PATCH_SIZE,NUMBER,Label_size):\n",
        "    count= Label_size//(2*NUMBER)\n",
        "    print(type(image))\n",
        "    a=[]\n",
        "    for i in range(2):\n",
        "        for j in range(NUMBER):\n",
        "            left=32+i*64\n",
        "            top=(PATCH_SIZE/2)+j*PATCH_SIZE\n",
        "            if i==0:\n",
        "                im = image.crop((left, top-PATCH_SIZE/2, left + PATCH_SIZE/2, top + PATCH_SIZE/2))\n",
        "            elif i==1:\n",
        "                im = image.crop((left-PATCH_SIZE/2, top-PATCH_SIZE/2, left , top + PATCH_SIZE/2))\n",
        "            for k in range(count):\n",
        "                a.append(np.array(im)) \n",
        "    return a\n",
        "\n",
        "def Patch_label(image,PATCH_SIZE,NUMBER,Label_size):\n",
        "    count= Label_size//(2*NUMBER)\n",
        "    print(type(image))\n",
        "    a=[]\n",
        "    for i in range(2):\n",
        "        for j in range(NUMBER):\n",
        "            left=32+i*64\n",
        "            top=(PATCH_SIZE/2)+j*PATCH_SIZE\n",
        "            im = image.crop((left-PATCH_SIZE/2, top-PATCH_SIZE/2, left + PATCH_SIZE/2, top + PATCH_SIZE/2))\n",
        "            for k in range(count):\n",
        "                pix=np.array(im)\n",
        "                a.append(pix) \n",
        "    return a\n",
        "\n",
        "#if training\n",
        "DATA=\"/content/drive/My Drive/NN&RL/coast\"\n",
        "#if testing\n",
        "#DATA=\"/content/drive/My Drive/NN&RL/test\"\n",
        "os.chdir(DATA)\n",
        "features1=[]\n",
        "features2=[]\n",
        "labels=[]\n",
        "count=0\n",
        "for img in os.listdir(DATA):\n",
        "  if os.path.isfile(img):\n",
        "    if count<90  :\n",
        "      print(img)\n",
        "      orig_img=load_city_image(img)\n",
        "      for i in range(32):\n",
        "        features1.append(Patch_0(orig_img,Label_size)[i])\n",
        "        features2.append(Patch(orig_img,PATCH1_SIZE,NUMBER1,Label_size)[i])\n",
        "        labels.append(Patch_label(orig_img,PATCH2_SIZE,NUMBER2,Label_size)[i])\n",
        "    count=count+1\n",
        "labels=np.array(labels)\n",
        "  \n",
        "\n",
        "def pre_processing(images):\n",
        "    newImages = []\n",
        "\n",
        "    for image in images:\n",
        "        newImage = skimage.transform.resize(image, (224, 224), mode='constant')\n",
        "        newImages.append(newImage)\n",
        "        \n",
        "    return newImages\n",
        "        \n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Copy of bea1.jpg\n",
            "(256, 256)\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n",
            "<class 'PIL.Image.Image'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "2zZvXtww7ih9",
        "colab_type": "code",
        "outputId": "2664e1fa-2d6f-4df6-e7c1-6477610a1dc8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "np.shape(labels)\n",
        "\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(32, 8, 8, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "metadata": {
        "id": "0wHz8OOs7iiI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "d4929e4d-cc38-4df5-d32d-0eab2e46b122"
      },
      "cell_type": "code",
      "source": [
        "from __future__ import absolute_import\n",
        "from __future__ import division\n",
        "from __future__ import print_function\n",
        "import tensorflow as tf\n",
        "import tensorlayer as tl\n",
        "import skimage.transform\n",
        "\n",
        "features1_post = np.array(pre_processing(features1))\n",
        "features2_post = np.array(pre_processing(features2))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python2.7/dist-packages/skimage/transform/_warps.py:110: UserWarning: Anti-aliasing will be enabled by default in skimage 0.15 to avoid aliasing artifacts when down-sampling images.\n",
            "  warn(\"Anti-aliasing will be enabled by default in skimage 0.15 to \"\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "5WvaIfdH7iiQ",
        "colab_type": "code",
        "outputId": "c5fa52c6-11c4-475a-bb70-30b8ce64be78",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "#if training\n",
        "labels = np.reshape(labels, (2880,192))\n",
        "\n",
        "#if testing\n",
        "#labels = np.reshape(labels, (32,192))\n",
        "labels = labels/255\n",
        "print(labels.dtype)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "iIR58XQET9jp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YtxtOD_47iiY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "def cnn_model_fn(features1, features2, labels,features_orig,n,restore=True):\n",
        "\n",
        "#         input_layer = tf.reshape(features[\"x\"], [-1, 28, 28, 1])\n",
        "    epochs=11\n",
        "    batch_size=32\n",
        "    num_batches= int(n/batch_size)\n",
        "\n",
        "    #if restore==True: batch_size=1\n",
        "    \n",
        "    X1 = tf.placeholder(tf.float32, [batch_size, 224, 224, 3])\n",
        "    X2 = tf.placeholder(tf.float32, [batch_size, 224, 224, 3])\n",
        "    y=tf.placeholder(tf.float32, [batch_size, 192])\n",
        "    # get VGG without the last layer\n",
        "    \n",
        "    vgg1 = tl.models.VGG16(X1, end_with='fc2_relu')\n",
        "    \n",
        "    vgg2 = tl.models.VGG16(X2, end_with='fc2_relu', reuse = True)\n",
        "    \n",
        "     #     restore pre-trained VGG parameters`\n",
        "    sess1 = tf.InteractiveSession()\n",
        "    vgg1.restore_params(sess1)\n",
        "    vgg_concat = tf.concat([vgg1.outputs, vgg2.outputs],1)\n",
        "    \n",
        "    # add one more layer\n",
        "    net = tl.layers.InputLayer(vgg_concat, name='input')\n",
        "    prediction = tl.layers.DenseLayer(net, 192, act=tf.nn.sigmoid, name='out')\n",
        "    \n",
        "    print((prediction.outputs).get_shape())\n",
        "    \n",
        "    saver = tf.train.Saver()\n",
        "    if restore==False:\n",
        "        loss_vec = []\n",
        "        for i in range(0,batch_size):\n",
        "            loss=0.0\n",
        "            loss = tf.losses.mean_squared_error(labels[i], prediction.outputs[i])\n",
        "            loss_vec.append(loss)\n",
        "\n",
        "    #     loss=0.0\n",
        "    #         for j in range(0, 128):\n",
        "    # #             print(label)\n",
        "    #             loss += tf.losses.mean_squared_error(labels[i][j], prediction.outputs[i][j])\n",
        "        loss_op = tf.reduce_mean( loss_vec  , name='MSE' ) \n",
        "        print(loss_op)\n",
        "        optimizer = tf.train.AdamOptimizer(learning_rate=0.0001)\n",
        "        train_op = optimizer.minimize(loss_op)\n",
        "\n",
        "\n",
        "    #     x_axis = list(range(0, epochs))\n",
        "        acc_mean, loss_mean = [], []\n",
        "\n",
        "      \n",
        "        init=tf.global_variables_initializer()\n",
        "        with tf.Session() as sess:\n",
        "            init.run()\n",
        "            saver.restore(sess, \"/content/drive/My Drive/NN&RL/test/model_dir_l_20/model.ckpt\")\n",
        "            for i in range(epochs):\n",
        "\n",
        "                acc, los=[], []\n",
        "                for j in range(num_batches):\n",
        "\n",
        "                    x1_arr = features1[(j*batch_size):((j+1)*batch_size),:,:,:]\n",
        "                    x2_arr = features2[(j*batch_size):((j+1)*batch_size),:,:,:]\n",
        "                    y_arr = labels[(j*batch_size):((j+1)*batch_size),:]\n",
        "                    p,l, _ = sess.run([prediction.outputs, loss_op, train_op], feed_dict={X1: x1_arr, X2: x2_arr, y: y_arr})\n",
        "    #                 print(\"pred:\" , p)\n",
        "                    los.append(l)\n",
        "    # #                 acc_train1 = accuracy1.eval(feed_dict={X: X_batch, y: y_batch})\n",
        "    #                 print(\"Train accuracy-level1:\", acc_train)\n",
        "    #                 acc1.append(acc_train)\n",
        "    #             acc_mean.append(sum(acc) / len(acc))\n",
        "                loss_mean.append(sum(los) / len(los))\n",
        "                print(\"epoch:\",i)\n",
        "                print(loss_mean)\n",
        "                if(i==10):\n",
        "                  saver.save(sess, \"/content/drive/My Drive/NN&RL/test/model_dir_l_30/model.ckpt\")\n",
        "    else:\n",
        "        #tf.reset_default_graph()\n",
        "        loss_vec = []\n",
        "        for i in range(0,batch_size):\n",
        "            loss=0.0\n",
        "            loss = tf.losses.mean_squared_error(labels[i], prediction.outputs[i])\n",
        "            loss_vec.append(loss)\n",
        "\n",
        "    #     loss=0.0\n",
        "    #         for j in range(0, 128):\n",
        "    # #             print(label)\n",
        "    #             loss += tf.losses.mean_squared_error(labels[i][j], prediction.outputs[i][j])\n",
        "        loss_op = tf.reduce_mean( loss_vec  , name='MSE' ) \n",
        "        with tf.Session() as sess:\n",
        "            saver.restore(sess, \"/content/drive/My Drive/NN&RL/test/model_dir_l_30/model.ckpt\")\n",
        "            print(\"Model restored.\")\n",
        "            result=[]\n",
        "            for i in range(n):\n",
        "                x1_arr = features1[i:(i+batch_size),:,:,:]\n",
        "                x2_arr = features2[i:(i+batch_size),:,:,:]\n",
        "\n",
        "                p,l = sess.run([prediction.outputs,loss_op], feed_dict={X1: x1_arr, X2: x2_arr})\n",
        "                #print(p[0].shape)\n",
        "                #print(p.shape)\n",
        "                #print(l)\n",
        "                #print(len(p))\n",
        "                #temp=np.\n",
        "                #for i in range(len(p)):\n",
        "                #  temp[i,:,:]= p[i]\n",
        "   \n",
        "                #p=np.array(p)\n",
        "                #p=np.transpose(p,(1, 2,0))\n",
        "                #print(p)\n",
        "                #print(\"pre:\",p)\n",
        "            \n",
        "            temp=np.reshape(p,(32,8,8,3))\n",
        "            temp=temp*255\n",
        "            full_image=np.ones((128,80,3))\n",
        "            #orig=(tf.image.rgb_to_grayscale(features_orig[0][:,:,:])).eval()\n",
        "            full_image[:,8:72,:]=features_orig[0][:,:,:]\n",
        "            for k in range(2):\n",
        "                for i in range(0,128,8):\n",
        "                    a2=temp[k*16+(i//8),:,:,:]\n",
        "                    #print(a)\n",
        "                    print(a2.shape)\n",
        "                    full_image[i:i+8,0+k*72:8+k*72,:]=temp[k*16+(i//8),:,:,:]\n",
        "                   \n",
        "            final_result=Image.fromarray(np.uint8(full_image))\n",
        "            final_result.save(\"/content/drive/My Drive/NN&RL/final_img.png\",\"PNG\")\n",
        "            print(\"Saved image\")\n",
        "            "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "scrolled": true,
        "id": "pRHy3ANv7iie",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2148
        },
        "outputId": "8f0350ab-fc8b-4666-e3f5-bfb001bbe01c"
      },
      "cell_type": "code",
      "source": [
        "#if training\n",
        "cnn_model_fn(features1_post, features2_post, labels,features1,2880,False)\n",
        "#if testing\n",
        "#cnn_model_fn(features1_post, features2_post, labels,features1,1,True)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[TL] InputLayer  vgg16/input: (32, 224, 224, 3)\n",
            "[TL] Conv2d vgg16/conv1_1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv1_2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool1: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] Conv2d vgg16/conv2_1: n_filter: 128 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv2_2: n_filter: 128 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool2: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] Conv2d vgg16/conv3_1: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv3_2: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv3_3: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool3: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] Conv2d vgg16/conv4_1: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv4_2: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv4_3: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool4: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] Conv2d vgg16/conv5_1: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv5_2: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv5_3: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool5: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] FlattenLayer vgg16/flatten: 25088\n",
            "[TL] DenseLayer  vgg16/fc1_relu: 4096 relu\n",
            "[TL] DenseLayer  vgg16/fc2_relu: 4096 relu\n",
            "[TL] InputLayer  vgg16/input: (32, 224, 224, 3)\n",
            "[TL] Conv2d vgg16/conv1_1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv1_2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool1: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] Conv2d vgg16/conv2_1: n_filter: 128 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv2_2: n_filter: 128 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool2: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] Conv2d vgg16/conv3_1: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv3_2: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv3_3: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool3: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] Conv2d vgg16/conv4_1: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv4_2: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv4_3: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool4: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] Conv2d vgg16/conv5_1: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv5_2: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] Conv2d vgg16/conv5_3: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
            "[TL] MaxPool2d vgg16/pool5: filter_size: (2, 2) strides: (2, 2) padding: SAME\n",
            "[TL] FlattenLayer vgg16/flatten: 25088\n",
            "[TL] DenseLayer  vgg16/fc1_relu: 4096 relu\n",
            "[TL] DenseLayer  vgg16/fc2_relu: 4096 relu\n",
            "[TL] Restore pre-trained parameters\n",
            "[TL]   Loading params (3, 3, 3, 64)\n",
            "[TL]   Loading params (64,)\n",
            "[TL]   Loading params (3, 3, 64, 64)\n",
            "[TL]   Loading params (64,)\n",
            "[TL]   Loading params (3, 3, 64, 128)\n",
            "[TL]   Loading params (128,)\n",
            "[TL]   Loading params (3, 3, 128, 128)\n",
            "[TL]   Loading params (128,)\n",
            "[TL]   Loading params (3, 3, 128, 256)\n",
            "[TL]   Loading params (256,)\n",
            "[TL]   Loading params (3, 3, 256, 256)\n",
            "[TL]   Loading params (256,)\n",
            "[TL]   Loading params (3, 3, 256, 256)\n",
            "[TL]   Loading params (256,)\n",
            "[TL]   Loading params (3, 3, 256, 512)\n",
            "[TL]   Loading params (512,)\n",
            "[TL]   Loading params (3, 3, 512, 512)\n",
            "[TL]   Loading params (512,)\n",
            "[TL]   Loading params (3, 3, 512, 512)\n",
            "[TL]   Loading params (512,)\n",
            "[TL]   Loading params (3, 3, 512, 512)\n",
            "[TL]   Loading params (512,)\n",
            "[TL]   Loading params (3, 3, 512, 512)\n",
            "[TL]   Loading params (512,)\n",
            "[TL]   Loading params (3, 3, 512, 512)\n",
            "[TL]   Loading params (512,)\n",
            "[TL]   Loading params (25088, 4096)\n",
            "[TL]   Loading params (4096,)\n",
            "[TL]   Loading params (4096, 4096)\n",
            "[TL]   Loading params (4096,)\n",
            "[TL] InputLayer  input: (32, 8192)\n",
            "[TL] DenseLayer  out: 192 sigmoid\n",
            "(32, 192)\n",
            "INFO:tensorflow:Restoring parameters from /content/drive/My Drive/NN&RL/test/model_dir_l_30/model.ckpt\n",
            "Model restored.\n",
            "(32, 192)\n",
            "0.06626297\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "(8, 8, 3)\n",
            "Saved image\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}