add example vectors

%% Cell type:markdown id: tags:

# CNN example Notebook

%% Cell type:markdown id: tags:

## Imports

%% Cell type:code id: tags:

``` python

%matplotlib inline

from __future__ import print_function

from ipywidgets import interact, interactive, fixed, interact_manual

import ipywidgets as widgets

import os.path

import pickle

import numpy as np

from pycocotools.coco import COCO

import skimage.io as io

import matplotlib.pyplot as plt

import matplotlib.image as mpimg

from IPython.display import display

import numpy as np

import tensorflow as tf

```

%% Output

    /proj/mahoni/anaconda3/envs/py36/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.

      from ._conv import register_converters as _register_converters

%% Cell type:markdown id: tags:

## Load Data

%% Cell type:code id: tags:

``` python

path = '/softpro/ss18/caption/cnn_exercise/good_models/fourth_model_all_objects_different_hyperparameters/'

coco = COCO('/softpro/ss18/caption/cocoapi/annotations/val/instances_val2017.json')

coco_caps = COCO('/softpro/ss18/caption/cocoapi/annotations/val/captions_val2017.json')

imgIds = coco.getImgIds()

```

%% Output

    loading annotations into memory...

    Done (t=1.00s)

    creating index...

    index created!

    loading annotations into memory...

    Done (t=0.25s)

    Done (t=0.24s)

    creating index...

    index created!

%% Cell type:markdown id: tags:

## Create Example Cases

%% Cell type:code id: tags:

``` python

def browse_images(images):

    def view_image(i):

        img = coco.loadImgs(i.value)[0]

        annIds = coco_caps.getAnnIds(imgIds=img['id'])

        anns = coco_caps.loadAnns(annIds)

        file_name = '/softpro/ss18/caption/cocoapi/val/' + img['file_name']

        I = mpimg.imread(file_name)

        coco_caps.showAnns(anns)

        plt.title('Title: %s' % i.name)

        plt.imshow(I)

        plt.axis('off')

        plt.show()

    view_image(images)

```

%% Cell type:code id: tags:

``` python

from enum import Enum

class Pictures(Enum):

    cat = 213445

    food = 17714

    skater = 257084

    bathroom = 306733

    random = imgIds[np.random.randint(0, len(imgIds))]

images = [Pictures.cat, Pictures.food, Pictures.skater, Pictures.bathroom, Pictures.random]

```

%% Cell type:code id: tags:

``` python

w = interactive(browse_images, images=images)

```

%% Cell type:markdown id: tags:

## choose Image

%% Cell type:code id: tags:

``` python

w

```

%% Output

%% Cell type:markdown id: tags:

## Populate Interface with coco data

%% Cell type:code id: tags:

``` python

img = coco.loadImgs(w.kwargs['images'].value)[0]

annIds = coco_caps.getAnnIds(imgIds=img['id'])

anns = coco_caps.loadAnns(annIds)

img['captions'] = anns

print(img)

```

%% Output

    {'license': 1, 'file_name': '000000213445.jpg', 'coco_url': 'http://images.cocodataset.org/val2017/000000213445.jpg', 'height': 500, 'width': 408, 'date_captured': '2013-11-15 17:17:16', 'flickr_url': 'http://farm5.staticflickr.com/4007/4463515326_b08b55025f_z.jpg', 'id': 213445, 'captions': [{'image_id': 213445, 'id': 631848, 'caption': 'a big cat sitting in a little bowl '}, {'image_id': 213445, 'id': 633591, 'caption': 'A cat is sitting in a small bowl on the table.'}, {'image_id': 213445, 'id': 635193, 'caption': 'A cat sitting in a bowl on a table.'}, {'image_id': 213445, 'id': 637341, 'caption': 'The cat is sitting inside of a bowl on a table.'}, {'image_id': 213445, 'id': 637632, 'caption': 'A cutting sitting upright in a pottery bowl on a coffee table'}]}

    {'license': 3, 'file_name': '000000306733.jpg', 'coco_url': 'http://images.cocodataset.org/val2017/000000306733.jpg', 'height': 426, 'width': 640, 'date_captured': '2013-11-15 04:56:04', 'flickr_url': 'http://farm6.staticflickr.com/5217/5538841372_569150936c_z.jpg', 'id': 306733, 'captions': [{'image_id': 306733, 'id': 472765, 'caption': 'Bathroom with sink, shower, towels, curtain, and more with window'}, {'image_id': 306733, 'id': 473629, 'caption': 'A picture of a bathroom with a fern plant near the sink and a photo of a city above the toilet. '}, {'image_id': 306733, 'id': 474730, 'caption': 'the mirror, sink and bathtub area of a white bathroom'}, {'image_id': 306733, 'id': 474751, 'caption': 'Well decorated bathroom in a small yellow tinted area.'}, {'image_id': 306733, 'id': 820003, 'caption': 'A bathroom with a sink, towel rack and shower stall.'}]}

%% Cell type:markdown id: tags:

## Run Classifier on Image

%% Cell type:markdown id: tags:

### Load graph from trained model

%% Cell type:code id: tags:

``` python

def load_graph(model_file):

  graph = tf.Graph()

  graph_def = tf.GraphDef()

  with open(model_file, "rb") as f:

    graph_def.ParseFromString(f.read())

  with graph.as_default():

    tf.import_graph_def(graph_def)

  return graph

```

%% Cell type:markdown id: tags:

### Process image

%% Cell type:code id: tags:

``` python

def read_tensor_from_image_file(file_name,

                                input_height=299,

                                input_width=299,

                                input_mean=0,

                                input_std=255):

  input_name = "file_reader"

  output_name = "normalized"

  file_reader = tf.read_file(file_name, input_name)

  if file_name.endswith(".png"):

    image_reader = tf.image.decode_png(

        file_reader, channels=3, name="png_reader")

  elif file_name.endswith(".gif"):

    image_reader = tf.squeeze(

        tf.image.decode_gif(file_reader, name="gif_reader"))

  elif file_name.endswith(".bmp"):

    image_reader = tf.image.decode_bmp(file_reader, name="bmp_reader")

  else:

    image_reader = tf.image.decode_jpeg(

        file_reader, channels=3, name="jpeg_reader")

  float_caster = tf.cast(image_reader, tf.float32)

  dims_expander = tf.expand_dims(float_caster, 0)

  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])

  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])

  sess = tf.Session()

  result = sess.run(normalized)

  return result

```

%% Cell type:code id: tags:

``` python

def load_labels(label_file):

  label = []

  proto_as_ascii_lines = tf.gfile.GFile(label_file).readlines()

  for l in proto_as_ascii_lines:

    label.append(l.rstrip())

  return label

```

%% Cell type:markdown id: tags:

### input data (graph, input-layer, output-layer, image)

%% Cell type:code id: tags:

``` python

file_name = '/softpro/ss18/caption/cocoapi/val/' + img['file_name']

label_file = path + 'output_labels.txt'

model_file = path + 'output_graph.pb'

input_layer = 'Placeholder'

output_layer= 'module_apply_default/hub_output/feature_vector/SpatialSqueeze'

input_name = "import/" + input_layer

output_name = "import/" + output_layer

graph = load_graph(model_file)

input_operation = graph.get_operation_by_name(input_name)

output_operation = graph.get_operation_by_name(output_name)

print(file_name)

```

%% Output

    /softpro/ss18/caption/cocoapi/val/000000213445.jpg

    /softpro/ss18/caption/cocoapi/val/000000306733.jpg

%% Cell type:markdown id: tags:

## Populate interface with extracted features (2048 dim. vector)

%% Cell type:code id: tags:

``` python

img['captions'] = anns

t = read_tensor_from_image_file(

        file_name)

with tf.Session(graph=graph) as sess:

        results = sess.run(output_operation.outputs[0], {

            input_operation.outputs[0]: t

        })

img['vector'] = np.squeeze(results)

interface = list()

interface.append(img)

```

%% Cell type:markdown id: tags:

## Print single example

%% Cell type:code id: tags:

``` python

import pprint

pprint.pprint(img, indent=4)

```

%% Output

    {   'captions': [   {   'caption': 'a big cat sitting in a little bowl ',

                            'id': 631848,

                            'image_id': 213445},

                        {   'caption': 'A cat is sitting in a small bowl on the '

                                       'table.',

                            'id': 633591,

                            'image_id': 213445},

                        {   'caption': 'A cat sitting in a bowl on a table.',

                            'id': 635193,

                            'image_id': 213445},

                        {   'caption': 'The cat is sitting inside of a bowl on a '

                                       'table.',

                            'id': 637341,

                            'image_id': 213445},

                        {   'caption': 'A cutting sitting upright in a pottery '

                                       'bowl on a coffee table',

                            'id': 637632,

                            'image_id': 213445}],

        'coco_url': 'http://images.cocodataset.org/val2017/000000213445.jpg',

        'date_captured': '2013-11-15 17:17:16',

        'file_name': '000000213445.jpg',

        'flickr_url': 'http://farm5.staticflickr.com/4007/4463515326_b08b55025f_z.jpg',

        'height': 500,

        'id': 213445,

        'license': 1,

        'vector': array([0.46763384, 0.00917622, 0.27030513, ..., 0.32595807, 0.13369742,

           0.00646778], dtype=float32),

        'width': 408}

    {   'captions': [   {   'caption': 'Bathroom with sink, shower, towels, '

                                       'curtain, and more with window',

                            'id': 472765,

                            'image_id': 306733},

                        {   'caption': 'A picture of a bathroom with a fern plant '

                                       'near the sink and a photo of a city above '

                                       'the toilet. ',

                            'id': 473629,

                            'image_id': 306733},

                        {   'caption': 'the mirror, sink and bathtub area of a '

                                       'white bathroom',

                            'id': 474730,

                            'image_id': 306733},

                        {   'caption': 'Well decorated bathroom in a small yellow '

                                       'tinted area.',

                            'id': 474751,

                            'image_id': 306733},

                        {   'caption': 'A bathroom with a sink, towel rack and '

                                       'shower stall.',

                            'id': 820003,

                            'image_id': 306733}],

        'coco_url': 'http://images.cocodataset.org/val2017/000000306733.jpg',

        'date_captured': '2013-11-15 04:56:04',

        'file_name': '000000306733.jpg',

        'flickr_url': 'http://farm6.staticflickr.com/5217/5538841372_569150936c_z.jpg',

        'height': 426,

        'id': 306733,

        'license': 3,

        'vector': array([0.4651293 , 0.38331467, 0.8267945 , ..., 0.26268384, 0.9879284 ,

           0.599533  ], dtype=float32),

        'width': 640}

%% Cell type:markdown id: tags:

## Save interface (needed for LSTM)

%% Cell type:code id: tags:

``` python

with open('example.pickle', 'wb') as handle:

    pickle.dump(img, handle)

with open('bathroom_singlelabel.pickle', 'wb') as handle:

    pickle.dump([img], handle)

```

%% Cell type:code id: tags:

``` python

```