Demo for CNN

%% Cell type:code id: tags:

%% Cell type:markdown id: tags:

``` python

# !git clone https://github.com/cocodataset/cocoapi.git

# !python cocoapi/PythonAPI/setup.py install

# !pip install -r requirements.txt

# CNN example Notebook

from __future__ import print_function

from ipywidgets import interact, interactive, fixed, interact_manual

import ipywidgets as widgets

import os.path

```

%% Cell type:markdown id: tags:

## Imports

%% Cell type:code id: tags:

``` python

%matplotlib inline

from __future__ import print_function

from ipywidgets import interact, interactive, fixed, interact_manual

import ipywidgets as widgets

import os.path

import pickle

import numpy as np

from pycocotools.coco import COCO

import skimage.io as io

import matplotlib.pyplot as plt

import matplotlib.image as mpimg

from IPython.display import display

import numpy as np

import tensorflow as tf

```

%% Output

    /proj/mahoni/anaconda3/envs/py36/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.

      from ._conv import register_converters as _register_converters

%% Cell type:markdown id: tags:

## Load Data

%% Cell type:code id: tags:

``` python

path = '/softpro/ss18/caption/cnn_exercise/good_models/fourth_model_all_objects_different_hyperparameters/'

coco = COCO('/softpro/ss18/caption/cocoapi/annotations/val/instances_val2017.json')

coco_caps = COCO('/softpro/ss18/caption/cocoapi/annotations/val/captions_val2017.json')

imgIds = coco.getImgIds()

```

%% Output

    loading annotations into memory...

    Done (t=0.76s)

    Done (t=0.83s)

    creating index...

    index created!

    loading annotations into memory...

    Done (t=0.05s)

    Done (t=0.22s)

    creating index...

    index created!

%% Cell type:markdown id: tags:

## Create Example Cases

%% Cell type:code id: tags:

``` python

from enum import Enum

class Pictures(Enum):

    bear = 205776

    cat = 89271

    child = 235836

    rockstar = 297353

    random = imgIds[np.random.randint(0, len(imgIds))]

```

%% Cell type:code id: tags:

``` python

images = [Pictures.bear, Pictures.cat, Pictures.child, Pictures.rockstar, Pictures.random]

```

%% Cell type:code id: tags:

``` python

from IPython.display import display

```

%% Cell type:code id: tags:

``` python

def browse_images(images):

    def view_image(i):

        img = coco.loadImgs(i.value)[0]

        annIds = coco_caps.getAnnIds(imgIds=img['id'])

        anns = coco_caps.loadAnns(annIds)

        file_name = '/softpro/ss18/caption/cocoapi/val/' + img['file_name']

        I = mpimg.imread(file_name)

        coco_caps.showAnns(anns)

        plt.title('Title: %s' % i.name)

        plt.imshow(I)

        plt.axis('off')

        plt.show()

    view_image(images)

```

%% Cell type:code id: tags:

``` python

w = interactive(browse_images, images=images)

```

%% Cell type:markdown id: tags:

## choose Image

%% Cell type:code id: tags:

``` python

w

```

%% Output

%% Cell type:markdown id: tags:

## Populate Interface with coco data

%% Cell type:code id: tags:

``` python

img = coco.loadImgs(w.kwargs['images'].value)[0]

annIds = coco_caps.getAnnIds(imgIds=img['id'])

anns = coco_caps.loadAnns(annIds)

img['captions'] = anns

print(img)

```

%% Output

    {'license': 3, 'file_name': '000000205776.jpg', 'coco_url': 'http://images.cocodataset.org/val2017/000000205776.jpg', 'height': 425, 'width': 640, 'date_captured': '2013-11-18 10:05:28', 'flickr_url': 'http://farm6.staticflickr.com/5350/6918177128_2f33d14e67_z.jpg', 'id': 205776, 'captions': [{'image_id': 205776, 'id': 537617, 'caption': 'A bear laying in a canvas type net'}, {'image_id': 205776, 'id': 537659, 'caption': 'A big brown bear asleep in a hammock.'}, {'image_id': 205776, 'id': 540317, 'caption': 'a brown bear is lying asleep in a hammock'}, {'image_id': 205776, 'id': 541925, 'caption': 'A bear sleeps in a hammock made of heavy canvas.'}, {'image_id': 205776, 'id': 543986, 'caption': 'A large bear laying down on a leather hammock.'}]}

    {'license': 6, 'file_name': '000000089271.jpg', 'coco_url': 'http://images.cocodataset.org/val2017/000000089271.jpg', 'height': 524, 'width': 640, 'date_captured': '2013-11-17 03:19:00', 'flickr_url': 'http://farm5.staticflickr.com/4122/4873790839_1f8aa7d6b2_z.jpg', 'id': 89271, 'captions': [{'image_id': 89271, 'id': 715892, 'caption': "A cat wearing a hat while resting it's paws on top of a chair."}, {'image_id': 89271, 'id': 723659, 'caption': 'Cat wearing a baseball cap with ears sticking out. '}, {'image_id': 89271, 'id': 723749, 'caption': 'Tabby cat with green eyes wearing a hat'}, {'image_id': 89271, 'id': 724229, 'caption': 'A cat peeks over a chair while wearing a hat'}, {'image_id': 89271, 'id': 730640, 'caption': 'A cat is wearing an orange and brown hat.'}]}

%% Cell type:code id: tags:

%% Cell type:markdown id: tags:

``` python

import numpy as np

import tensorflow as tf

```

## Run Classifier on Image

%% Output

%% Cell type:markdown id: tags:

    /proj/mahoni/anaconda3/envs/py36/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.

      from ._conv import register_converters as _register_converters

### Load graph from trained model

%% Cell type:code id: tags:

``` python

def load_graph(model_file):

  graph = tf.Graph()

  graph_def = tf.GraphDef()

  with open(model_file, "rb") as f:

    graph_def.ParseFromString(f.read())

  with graph.as_default():

    tf.import_graph_def(graph_def)

  return graph

```

%% Cell type:markdown id: tags:

### Process image

%% Cell type:code id: tags:

``` python

def read_tensor_from_image_file(file_name,

                                input_height=299,

                                input_width=299,

                                input_mean=0,

                                input_std=255):

  input_name = "file_reader"

  output_name = "normalized"

  file_reader = tf.read_file(file_name, input_name)

  if file_name.endswith(".png"):

    image_reader = tf.image.decode_png(

        file_reader, channels=3, name="png_reader")

  elif file_name.endswith(".gif"):

    image_reader = tf.squeeze(

        tf.image.decode_gif(file_reader, name="gif_reader"))

  elif file_name.endswith(".bmp"):

    image_reader = tf.image.decode_bmp(file_reader, name="bmp_reader")

  else:

    image_reader = tf.image.decode_jpeg(

        file_reader, channels=3, name="jpeg_reader")

  float_caster = tf.cast(image_reader, tf.float32)

  dims_expander = tf.expand_dims(float_caster, 0)

  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])

  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])

  sess = tf.Session()

  result = sess.run(normalized)

  return result

```

%% Cell type:code id: tags:

``` python

def load_labels(label_file):

  label = []

  proto_as_ascii_lines = tf.gfile.GFile(label_file).readlines()

  for l in proto_as_ascii_lines:

    label.append(l.rstrip())

  return label

```

%% Cell type:markdown id: tags:

### input data (graph, input-layer, output-layer, image)

%% Cell type:code id: tags:

``` python

file_name = '/softpro/ss18/caption/cocoapi/val/' + img['file_name']

label_file = path + 'output_labels.txt'

model_file = path + 'output_graph.pb'

input_layer = 'Placeholder'

output_layer= 'module_apply_default/hub_output/feature_vector/SpatialSqueeze'

input_name = "import/" + input_layer

output_name = "import/" + output_layer

graph = load_graph(model_file)

input_operation = graph.get_operation_by_name(input_name)

output_operation = graph.get_operation_by_name(output_name)

print(file_name)

```

%% Output

    /softpro/ss18/caption/cocoapi/val/000000205776.jpg

    /softpro/ss18/caption/cocoapi/val/000000089271.jpg

%% Cell type:markdown id: tags:

## Populate interface with extracted features (2048 dim. vector)

%% Cell type:code id: tags:

``` python

img['captions'] = anns

t = read_tensor_from_image_file(

        file_name)

with tf.Session(graph=graph) as sess:

        results = sess.run(output_operation.outputs[0], {

            input_operation.outputs[0]: t

        })

img['vector'] = np.squeeze(results)

interface = list()

interface.append(img)

```

%% Cell type:markdown id: tags:

## Print single example

%% Cell type:code id: tags:

``` python

import pprint

pprint.pprint(img, indent=4)

```

%% Output

    {   'captions': [   {   'caption': 'A bear laying in a canvas type net',

                            'id': 537617,

                            'image_id': 205776},

                        {   'caption': 'A big brown bear asleep in a hammock.',

                            'id': 537659,

                            'image_id': 205776},

                        {   'caption': 'a brown bear is lying asleep in a hammock',

                            'id': 540317,

                            'image_id': 205776},

                        {   'caption': 'A bear sleeps in a hammock made of heavy '

                                       'canvas.',

                            'id': 541925,

                            'image_id': 205776},

                        {   'caption': 'A large bear laying down on a leather '

                                       'hammock.',

                            'id': 543986,

                            'image_id': 205776}],

        'coco_url': 'http://images.cocodataset.org/val2017/000000205776.jpg',

        'date_captured': '2013-11-18 10:05:28',

        'file_name': '000000205776.jpg',

        'flickr_url': 'http://farm6.staticflickr.com/5350/6918177128_2f33d14e67_z.jpg',

        'height': 425,

        'id': 205776,

        'license': 3,

        'vector': array([0.5091342 , 0.28544462, 0.3536299 , ..., 0.14337565, 0.13621491,

           0.36187065], dtype=float32),

    {   'captions': [   {   'caption': "A cat wearing a hat while resting it's "

                                       'paws on top of a chair.',

                            'id': 715892,

                            'image_id': 89271},

                        {   'caption': 'Cat wearing a baseball cap with ears '

                                       'sticking out. ',

                            'id': 723659,

                            'image_id': 89271},

                        {   'caption': 'Tabby cat with green eyes wearing a hat',

                            'id': 723749,

                            'image_id': 89271},

                        {   'caption': 'A cat peeks over a chair while wearing a '

                                       'hat',

                            'id': 724229,

                            'image_id': 89271},

                        {   'caption': 'A cat is wearing an orange and brown hat.',

                            'id': 730640,

                            'image_id': 89271}],

        'coco_url': 'http://images.cocodataset.org/val2017/000000089271.jpg',

        'date_captured': '2013-11-17 03:19:00',

        'file_name': '000000089271.jpg',

        'flickr_url': 'http://farm5.staticflickr.com/4122/4873790839_1f8aa7d6b2_z.jpg',

        'height': 524,

        'id': 89271,

        'license': 6,

        'vector': array([0.2896427 , 0.24722002, 0.06824438, ..., 0.66454995, 0.46067858,

           0.28921682], dtype=float32),

        'width': 640}

%% Cell type:markdown id: tags:

## Save interface (needed for LSTM)

%% Cell type:code id: tags:

``` python

with open('example.pickle', 'wb') as handle:

    pickle.dump(img, handle, protocol=pickle.HIGHEST_PROTOCOL)

```

%% Cell type:code id: tags:

``` python

```