In this small post I decided to play more with SSD. My goal was to count all objects per all classes that Network was able to detect. Also, it was interesting how class distribution changes per different Neural Network’s certainty.

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import os
%matplotlib inline
plt.rcParams['figure.figsize'] = (20, 20)
plt.rcParams.update({'font.size': 22})

import os
import operator
from collections import Counter

caffe_root = '/home/veronika/materials/cv/detection/caffe/examples/'  
# this file is expected to be in {caffe_root}/examples
import sys
#sys.path.append("/home/veronika/materials/cv/detection/caffe/build/tools/caffe")
import sys
sys.path.append('/home/veronika/materials/cv/detection/caffe/python')
sys.path.remove('/home/veronika/caffe/python')

import caffe

from google.protobuf import text_format
from caffe.proto import caffe_pb2

# load PASCAL VOC labels
labelmap_file = '/home/veronika/materials/cv/detection/git/labelmap_voc.prototxt'
file = open(labelmap_file, 'r')
labelmap = caffe_pb2.LabelMap()
text_format.Merge(str(file.read()), labelmap)
/usr/local/lib/python2.7/dist-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')





<caffe.proto.caffe_pb2.LabelMap at 0x7f1ecb828938>
def get_labelname(labelmap, labels):
    num_labels = len(labelmap.item)
    labelnames = []
    if type(labels) is not list:
        labels = [labels]
    for label in labels:
        found = False
        for i in xrange(0, num_labels):
            if label == labelmap.item[i].label:
                found = True
                labelnames.append(labelmap.item[i].display_name)
                break
        assert found == True
    return labelnames
caffe.set_device(0)
caffe.set_mode_gpu()
model_def = '/home/veronika/materials/cv/detection/models_trained/VGGNet/deploy.prototxt'
model_weights = '/home/veronika/materials/cv/detection/models_trained/VGGNet/VGG_VOC0712_SSD_300x300_iter_60000.caffemodel'

net = caffe.Net(model_def,      # defines the structure of the model
                model_weights,  # contains the trained weights
                caffe.TEST)     # use test mode (e.g., don't perform dropout)
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2, 0, 1))
transformer.set_mean('data', np.array([104,117,123])) # mean pixel
transformer.set_raw_scale('data', 255)  # the reference model operates on images in [0,255] range instead of [0,1]
transformer.set_channel_swap('data', (2,1,0))  # the reference model has channels in BGR order instead of RGB
labels = pd.read_csv("/home/veronika/materials/cv/cv_organizer/mydata/test_labels.csv",
                    sep = " ", header = None)

The following function is actually detects all objects that have higher probability that conf_level for all images that are in test set.

def get_object_descr(path_to_test, conf_level):
    descr_total = {}
    for i in range(labels[0].shape[0]):
        #path_to_test = path_to_img
        image = caffe.io.load_image(os.path.join(path_to_test, labels[0][i]))
        transformed_image = transformer.preprocess('data', image)
        net.blobs['data'].data[...] = transformed_image

        # Forward pass.
        detections = net.forward()['detection_out']
    
        det_label = detections[0,0,:,1]
        det_conf = detections[0,0,:,2]

        # Get detections with confidence higher than 0.6.
        top_indices = [i for i, conf in enumerate(det_conf) if conf >= conf_level]

        top_conf = det_conf[top_indices]
        top_label_indices = det_label[top_indices].tolist()
        top_labels = get_labelname(labelmap, top_label_indices)
        #descr = Counter(dict((i,top_labels.count(i)) for i in set(top_labels)))
        descr = Counter(top_labels)
        descr_total = descr + Counter(descr_total)
        print("Done!")
    return(descr_total)
path_to_test = "/home/veronika/materials/cv/cv_organizer/mydata/test/"
object_desc_02 = get_object_descr(path_to_test, 0.2)
object_desc_04 = get_object_descr(path_to_test, 0.4)
object_desc_06 = get_object_descr(path_to_test, 0.6)
object_desc_08 = get_object_descr(path_to_test, 0.8)

This is a distribution per classes with 80% of model’s certainty.

object_desc_08
Counter({u'aeroplane': 8,
         u'bicycle': 54,
         u'bird': 6,
         u'boat': 31,
         u'bottle': 3,
         u'bus': 5,
         u'car': 62,
         u'cat': 1,
         u'chair': 71,
         u'diningtable': 19,
         u'dog': 16,
         u'horse': 49,
         u'motorbike': 9,
         u'person': 2158,
         u'pottedplant': 30,
         u'sheep': 1,
         u'sofa': 14,
         u'train': 6,
         u'tvmonitor': 18})
object_desc_02_df = pd.DataFrame(object_desc_02.items(), columns=['ClassType', 'Prob02'])
object_desc_04_df = pd.DataFrame(object_desc_04.items(), columns=['ClassType', 'Prob04'])
object_desc_06_df = pd.DataFrame(object_desc_06.items(), columns=['ClassType', 'Prob06'])
#object_desc_08_df = pd.DataFrame(object_desc_08.items(), columns=['ClassType', 'Prob08'])

The following table was actually my goal. I can see how many objects CNN found per class with different probability levels.

totaldata = pd.merge(object_desc_02_df, object_desc_04_df, how='outer')
totaldata = pd.merge(totaldata, object_desc_06_df, how = "outer")
#totaldata = pd.merge(totaldata, object_desc_08_df, how = "outer")
totaldata
ClassType Prob02 Prob04 Prob06
0 sheep 17 7 4
1 bottle 42 9 3
2 horse 101 76 64
3 bicycle 168 92 61
4 motorbike 50 25 14
5 cow 17 5 1
6 bus 21 13 11
7 dog 74 46 29
8 cat 4 1 1
9 person 4688 3367 2699
10 train 41 21 11
11 diningtable 70 41 26
12 aeroplane 32 18 13
13 sofa 54 39 24
14 pottedplant 146 73 45
15 tvmonitor 85 48 36
16 chair 357 186 112
17 bird 42 22 14
18 boat 102 65 45
19 car 192 119 89

The next step was dataframe transformation, that is needed for plotting barplots using ggplot library. More details are in the next post. The visualization of the dataframe is in the Part 2.2

totaldata_melted = pd.melt(totaldata, id_vars=['ClassType'], value_vars=['Prob02', 'Prob04', 'Prob06'],
                          var_name='Probs', value_name='Amount')
totaldata_melted

ClassType Probs Amount
0 sheep Prob02 17
1 bottle Prob02 42
2 horse Prob02 101
3 bicycle Prob02 168
4 motorbike Prob02 50
5 cow Prob02 17
6 bus Prob02 21
7 dog Prob02 74
8 cat Prob02 4
9 person Prob02 4688
10 train Prob02 41
11 diningtable Prob02 70
12 aeroplane Prob02 32
13 sofa Prob02 54
14 pottedplant Prob02 146
15 tvmonitor Prob02 85
16 chair Prob02 357
17 bird Prob02 42
18 boat Prob02 102
19 car Prob02 192
20 sheep Prob04 7
21 bottle Prob04 9
22 horse Prob04 76
23 bicycle Prob04 92
24 motorbike Prob04 25
25 cow Prob04 5
26 bus Prob04 13
27 dog Prob04 46
28 cat Prob04 1
29 person Prob04 3367
30 train Prob04 21
31 diningtable Prob04 41
32 aeroplane Prob04 18
33 sofa Prob04 39
34 pottedplant Prob04 73
35 tvmonitor Prob04 48
36 chair Prob04 186
37 bird Prob04 22
38 boat Prob04 65
39 car Prob04 119
40 sheep Prob06 4
41 bottle Prob06 3
42 horse Prob06 64
43 bicycle Prob06 61
44 motorbike Prob06 14
45 cow Prob06 1
46 bus Prob06 11
47 dog Prob06 29
48 cat Prob06 1
49 person Prob06 2699
50 train Prob06 11
51 diningtable Prob06 26
52 aeroplane Prob06 13
53 sofa Prob06 24
54 pottedplant Prob06 45
55 tvmonitor Prob06 36
56 chair Prob06 112
57 bird Prob06 14
58 boat Prob06 45
59 car Prob06 89
totaldata_melted.to_csv("/home/veronika/materials/cv/detection/total_classes.csv")
All details about SSD are here: https://github.com/weiliu89/caffe/tree/ssd/examples