Merge branch 'master' of https://gitlab.cl.uni-heidelberg.de/jmrs/caption

%% Cell type:code id: tags:

``` python

# !git clone https://github.com/cocodataset/cocoapi.git

# !python cocoapi/PythonAPI/setup.py install

# !pip install -r requirements.txt

from __future__ import print_function

from ipywidgets import interact, interactive, fixed, interact_manual

import ipywidgets as widgets

import os.path

```

%% Cell type:code id: tags:

``` python

%matplotlib inline

import pickle

import numpy as np

from pycocotools.coco import COCO

import skimage.io as io

import matplotlib.pyplot as plt

import matplotlib.image as mpimg

```

%% Cell type:code id: tags:

``` python

path = '/softpro/ss18/caption/cnn_exercise/good_models/fourth_model_all_objects_different_hyperparameters/'

coco = COCO('/softpro/ss18/caption/cocoapi/annotations/val/instances_val2017.json')

coco_caps = COCO('/softpro/ss18/caption/cocoapi/annotations/val/captions_val2017.json')

imgIds = coco.getImgIds()

```

%% Output

    loading annotations into memory...

    Done (t=0.76s)

    creating index...

    index created!

    loading annotations into memory...

    Done (t=0.05s)

    creating index...

    index created!

%% Cell type:code id: tags:

``` python

from enum import Enum

class Pictures(Enum):

    bear = 205776

    cat = 89271

    child = 235836

    rockstar = 297353

    random = imgIds[np.random.randint(0, len(imgIds))]

```

%% Cell type:code id: tags:

``` python

images = [Pictures.bear, Pictures.cat, Pictures.child, Pictures.rockstar, Pictures.random]

```

%% Cell type:code id: tags:

``` python

from IPython.display import display

```

%% Cell type:code id: tags:

``` python

def browse_images(images):

    def view_image(i):

        img = coco.loadImgs(i.value)[0]

        annIds = coco_caps.getAnnIds(imgIds=img['id'])

        anns = coco_caps.loadAnns(annIds)

        file_name = '/softpro/ss18/caption/cocoapi/val/' + img['file_name']

        I = mpimg.imread(file_name)

        coco_caps.showAnns(anns)

        plt.title('Title: %s' % i.name)

        plt.imshow(I)

        plt.axis('off')

        plt.show()

    view_image(images)

```

%% Cell type:code id: tags:

``` python

w = interactive(browse_images, images=images)

```

%% Cell type:code id: tags:

``` python

w

```

%% Output

%% Cell type:code id: tags:

``` python

img = coco.loadImgs(w.kwargs['images'].value)[0]

annIds = coco_caps.getAnnIds(imgIds=img['id'])

anns = coco_caps.loadAnns(annIds)

img['captions'] = anns

print(img)

```

%% Output

    {'license': 3, 'file_name': '000000205776.jpg', 'coco_url': 'http://images.cocodataset.org/val2017/000000205776.jpg', 'height': 425, 'width': 640, 'date_captured': '2013-11-18 10:05:28', 'flickr_url': 'http://farm6.staticflickr.com/5350/6918177128_2f33d14e67_z.jpg', 'id': 205776, 'captions': [{'image_id': 205776, 'id': 537617, 'caption': 'A bear laying in a canvas type net'}, {'image_id': 205776, 'id': 537659, 'caption': 'A big brown bear asleep in a hammock.'}, {'image_id': 205776, 'id': 540317, 'caption': 'a brown bear is lying asleep in a hammock'}, {'image_id': 205776, 'id': 541925, 'caption': 'A bear sleeps in a hammock made of heavy canvas.'}, {'image_id': 205776, 'id': 543986, 'caption': 'A large bear laying down on a leather hammock.'}]}

%% Cell type:code id: tags:

``` python

import numpy as np

import tensorflow as tf

```

%% Output

    /proj/mahoni/anaconda3/envs/py36/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.

      from ._conv import register_converters as _register_converters

%% Cell type:code id: tags:

``` python

def load_graph(model_file):

  graph = tf.Graph()

  graph_def = tf.GraphDef()

  with open(model_file, "rb") as f:

    graph_def.ParseFromString(f.read())

  with graph.as_default():

    tf.import_graph_def(graph_def)

  return graph

```

%% Cell type:code id: tags:

``` python

def read_tensor_from_image_file(file_name,

                                input_height=299,

                                input_width=299,

                                input_mean=0,

                                input_std=255):

  input_name = "file_reader"

  output_name = "normalized"

  file_reader = tf.read_file(file_name, input_name)

  if file_name.endswith(".png"):

    image_reader = tf.image.decode_png(

        file_reader, channels=3, name="png_reader")

  elif file_name.endswith(".gif"):

    image_reader = tf.squeeze(

        tf.image.decode_gif(file_reader, name="gif_reader"))

  elif file_name.endswith(".bmp"):

    image_reader = tf.image.decode_bmp(file_reader, name="bmp_reader")

  else:

    image_reader = tf.image.decode_jpeg(

        file_reader, channels=3, name="jpeg_reader")

  float_caster = tf.cast(image_reader, tf.float32)

  dims_expander = tf.expand_dims(float_caster, 0)

  resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])

  normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])

  sess = tf.Session()

  result = sess.run(normalized)

  return result

```

%% Cell type:code id: tags:

``` python

def load_labels(label_file):

  label = []

  proto_as_ascii_lines = tf.gfile.GFile(label_file).readlines()

  for l in proto_as_ascii_lines:

    label.append(l.rstrip())

  return label

```

%% Cell type:code id: tags:

``` python

file_name = '/softpro/ss18/caption/cocoapi/val/' + img['file_name']

label_file = path + 'output_labels.txt'

model_file = path + 'output_graph.pb'

input_layer = 'Placeholder'

output_layer= 'module_apply_default/hub_output/feature_vector/SpatialSqueeze'

input_name = "import/" + input_layer

output_name = "import/" + output_layer

graph = load_graph(model_file)

input_operation = graph.get_operation_by_name(input_name)

output_operation = graph.get_operation_by_name(output_name)

print(file_name)

```

%% Output

    /softpro/ss18/caption/cocoapi/val/000000205776.jpg

%% Cell type:code id: tags:

``` python

img['captions'] = anns

t = read_tensor_from_image_file(

        file_name)

with tf.Session(graph=graph) as sess:

        results = sess.run(output_operation.outputs[0], {

            input_operation.outputs[0]: t

        })

img['vector'] = np.squeeze(results)

```

%% Cell type:code id: tags:

``` python

import pprint

pprint.pprint(img, indent=4)

```

%% Output

    {   'captions': [   {   'caption': 'A bear laying in a canvas type net',

                            'id': 537617,

                            'image_id': 205776},

                        {   'caption': 'A big brown bear asleep in a hammock.',

                            'id': 537659,

                            'image_id': 205776},

                        {   'caption': 'a brown bear is lying asleep in a hammock',

                            'id': 540317,

                            'image_id': 205776},

                        {   'caption': 'A bear sleeps in a hammock made of heavy '

                                       'canvas.',

                            'id': 541925,

                            'image_id': 205776},

                        {   'caption': 'A large bear laying down on a leather '

                                       'hammock.',

                            'id': 543986,

                            'image_id': 205776}],

        'coco_url': 'http://images.cocodataset.org/val2017/000000205776.jpg',

        'date_captured': '2013-11-18 10:05:28',

        'file_name': '000000205776.jpg',

        'flickr_url': 'http://farm6.staticflickr.com/5350/6918177128_2f33d14e67_z.jpg',

        'height': 425,

        'id': 205776,

        'license': 3,

        'vector': array([0.5091342 , 0.28544462, 0.3536299 , ..., 0.14337565, 0.13621491,

           0.36187065], dtype=float32),

        'width': 640}

%% Cell type:code id: tags:

``` python

with open('example.pickle', 'wb') as handle:

    pickle.dump(img, handle, protocol=pickle.HIGHEST_PROTOCOL)

```

%% Cell type:code id: tags:

``` python

```

tensorflow==1.8.0

pycocotools==2.0

scikit-image==0.13.1

matplotlib==2.1.1