任務描述：用Tensorflow object detection API檢測出來的結果是一整張圖片，想要把檢測出的bounding box部分單獨截取出來並儲存
執行環境：spyder
效果展示：
測試圖片：test_images --> 檢測圖片：testsave_images --> 裁剪bounding box：test_cropped

1. 我的修改過後的test檔案

# coding: utf-8

# # Object Detection Demo
# Welcome to the object detection inference walkthrough!  This notebook will walk you step by step through the process of using a pre-trained model to detect objects in an image. Make sure to follow the [installation instructions](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/installation.md) before you start.

# # Imports

# In[19]:
from skimage import data_dir
import skimage.io as io
import numpy as np
import os
import six.moves.urllib as urllib
import sys
import tarfile
import tensorflow as tf
import zipfile

from collections import defaultdict
from io import StringIO
from matplotlib import pyplot as plt
from PIL import Image

# This is needed since the notebook is stored in the object_detection folder.
sys.path.append("..")
from object_detection.utils import ops as utils_ops

if tf.__version__ < '1.4.0':
  raise ImportError('Please upgrade your tensorflow installation to v1.4.* or later!')


# ## Env setup

# In[20]:


# This is needed to display the images.
#get_ipython().run_line_magic('matplotlib', 'inline')  我的電腦上這句如果不註釋掉會報錯


# ## Object detection imports
# Here are the imports from the object detection module.

# In[21]:


from utils import label_map_util

from utils import visualization_utils as vis_util #主要就是用到了utils目錄下的visualization_utils檔案


# # Model preparation 

# ## Variables
# 
# Any model exported using the `export_inference_graph.py` tool can be loaded here simply by changing `PATH_TO_CKPT` to point to a new .pb file.  
# 
# By default we use an "SSD with Mobilenet" model here. See the [detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) for a list of other models that can be run out-of-the-box with varying speeds and accuracies.

# In[22]:


# What model to download.
MODEL_NAME = 'headshoulder0603'  #我的模型名字


# Path to frozen detection graph. This is the actual model that is used for the object detection.
PATH_TO_CKPT = MODEL_NAME + '/frozen_inference_graph.pb'

# List of the strings that is used to add correct label for each box.
PATH_TO_LABELS = os.path.join('data', 'headshoulder.pbtxt')  #我的模型相關檔案

NUM_CLASSES = 1 #我的任務是檢測頭肩，只有一類


# ## Download Model

# ## Load a (frozen) Tensorflow model into memory.

# In[ ]:


detection_graph = tf.Graph()
with detection_graph.as_default():
  od_graph_def = tf.GraphDef()
  with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
    serialized_graph = fid.read()
    od_graph_def.ParseFromString(serialized_graph)
    tf.import_graph_def(od_graph_def, name='')


# ## Loading label map
# Label maps map indices to category names, so that when our convolution network predicts `5`, we know that this corresponds to `airplane`.  Here we use internal utility functions, but anything that returns a dictionary mapping integers to appropriate string labels would be fine

# In[ ]:


label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
category_index = label_map_util.create_category_index(categories)


# ## Helper code

# In[23]:

#自己在實踐中發現，圖片經常需要在Image和numpy array兩種格式中切換，這個函式就是將Image格式轉換成numpy array
def load_image_into_numpy_array(image):
  (im_width, im_height) = image.size
  return np.array(image.getdata()).reshape(
      (im_height, im_width, 3)).astype(np.uint8)


# # Detection

# In[24]:


# For the sake of simplicity we will use only 2 images:
# image1.jpg
# image2.jpg
# If you want to test the code with Aayour images, just add path to the images to the TEST_IMAGE_PATHS.

PATH_TO_TEST_IMAGES_DIR = os.getcwd()+'\\test_images'   #存放測試圖片的目錄路徑
os.chdir(PATH_TO_TEST_IMAGES_DIR)  
TEST_IMAGE_PATHS = os.listdir(PATH_TO_TEST_IMAGES_DIR) 
# Size, in inches, of the output images.
IMAGE_SIZE = (12, 8)


# In[25]:


def run_inference_for_single_image(image, graph):
  with graph.as_default():
    with tf.Session() as sess:
      # Get handles to input and output tensors
      ops = tf.get_default_graph().get_operations()
      all_tensor_names = {output.name for op in ops for output in op.outputs}
      tensor_dict = {}
      for key in [
          'num_detections', 'detection_boxes', 'detection_scores',
          'detection_classes', 'detection_masks'
      ]:
        tensor_name = key + ':0'
        if tensor_name in all_tensor_names:
          tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(
              tensor_name)
      if 'detection_masks' in tensor_dict:
        # The following processing is only for single image
        detection_boxes = tf.squeeze(tensor_dict['detection_boxes'], [0])
        detection_masks = tf.squeeze(tensor_dict['detection_masks'], [0])
        # Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.
        real_num_detection = tf.cast(tensor_dict['num_detections'][0], tf.int32)
        detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1])
        detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1])
        detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(
            detection_masks, detection_boxes, image.shape[0], image.shape[1])
        detection_masks_reframed = tf.cast(
            tf.greater(detection_masks_reframed, 0.5), tf.uint8)
        # Follow the convention by adding back the batch dimension
        tensor_dict['detection_masks'] = tf.expand_dims(
            detection_masks_reframed, 0)
      image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0')

      # Run inference
      output_dict = sess.run(tensor_dict,
                             feed_dict={image_tensor: np.expand_dims(image, 0)})

      # all outputs are float32 numpy arrays, so convert types as appropriate
      output_dict['num_detections'] = int(output_dict['num_detections'][0])
      output_dict['detection_classes'] = output_dict[
          'detection_classes'][0].astype(np.uint8)
      output_dict['detection_boxes'] = output_dict['detection_boxes'][0]
      output_dict['detection_scores'] = output_dict['detection_scores'][0]
      if 'detection_masks' in output_dict:
        output_dict['detection_masks'] = output_dict['detection_masks'][0]
  return output_dict


# In[26]:


for image_path in TEST_IMAGE_PATHS:
    image= Image.open(image_path) #注意這裡的image_path是個路徑，也就是說是個字串str，下文會用到
  # the array based representation of the image will be used later in order to prepare the
  # result image with boxes and labels on it.
    image_np = load_image_into_numpy_array(image)
  # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
    image_np_expanded = np.expand_dims(image_np, axis=0)
  # Actual detection.
    output_dict = run_inference_for_single_image(image_np, detection_graph)
  # Visualization of the results of a detection.
    vis_util.visualize_boxes_and_labels_on_image_array(
      image_np,
      image_path, #原有的程式碼沒有這一行，但是我需要傳遞測試圖片image的檔名給visualize_utils檔案中，所以加上，對應的visualize_utils中的visualize_boxes_and_labels_on_image_array函式也要加上這個引數
      output_dict['detection_boxes'],
      output_dict['detection_classes'],
      output_dict['detection_scores'],
      category_index,
      instance_masks=output_dict.get('detection_masks'),
      use_normalized_coordinates=True,
      line_thickness=1, #預設的框到粗細是8，但是實在太粗了
      )#the width of bounding box,default is 8
    plt.figure(figsize=IMAGE_SIZE)
    plt.imshow(image_np)
    #save_dir = TEST_IMAGE_PATHS + '{}.jpg'.format(image_path)
    PATH_TO_TEST_IMAGES_1_DIR = r'C:\\models\\research\\object_detection\\testsave_images' #這裡可能要加上r，不然可能會有編碼ucf8錯誤
    save_dir = os.path.join(PATH_TO_TEST_IMAGES_1_DIR, image_path)
    vis_util.save_image_array_as_png(image_np,save_dir) #我將檢測出來的圖片儲存在testsave_images資料夾下
     
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)


 

2. 修改過後的visualization_utils.py檔案中 visualize_boxes_and_labels_on_image_array 函式

def visualize_boxes_and_labels_on_image_array(
    image,
    image_path, #我新增的，因為要用到測試圖片的檔名
    boxes,
    classes,
    scores,
    category_index,
    instance_masks=None,
    instance_boundaries=None,
    keypoints=None,
    use_normalized_coordinates=False,
    max_boxes_to_draw=20,
    min_score_thresh=.5,
    agnostic_mode=False,
    line_thickness=1,
    groundtruth_box_visualization_color='black',
    skip_scores=False,
    skip_labels=False,
    ):
  """Overlay labeled boxes on an image with formatted scores and label names.

  This function groups boxes that correspond to the same location
  and creates a display string for each detection and overlays these
  on the image. Note that this function modifies the image in place, and returns
  that same image.

  Args:
    image: uint8 numpy array with shape (img_height, img_width, 3)
    boxes: a numpy array of shape [N, 4]
    classes: a numpy array of shape [N]. Note that class indices are 1-based,
      and match the keys in the label map.
    scores: a numpy array of shape [N] or None.  If scores=None, then
      this function assumes that the boxes to be plotted are groundtruth
      boxes and plot all boxes as black with no classes or scores.
    category_index: a dict containing category dictionaries (each holding
      category index `id` and category name `name`) keyed by category indices.
    instance_masks: a numpy array of shape [N, image_height, image_width] with
      values ranging between 0 and 1, can be None.
    instance_boundaries: a numpy array of shape [N, image_height, image_width]
      with values ranging between 0 and 1, can be None.
    keypoints: a numpy array of shape [N, num_keypoints, 2], can
      be None
    use_normalized_coordinates: whether boxes is to be interpreted as
      normalized coordinates or not.
    max_boxes_to_draw: maximum number of boxes to visualize.  If None, draw
      all boxes.
    min_score_thresh: minimum score threshold for a box to be visualized
    agnostic_mode: boolean (default: False) controlling whether to evaluate in
      class-agnostic mode or not.  This mode will display scores but ignore
      classes.
    line_thickness: integer (default: 4) controlling line width of the boxes.
    groundtruth_box_visualization_color: box color for visualizing groundtruth
      boxes
    skip_scores: whether to skip score when drawing a single detection
    skip_labels: whether to skip label when drawing a single detection

  Returns:
    uint8 numpy array with shape (img_height, img_width, 3) with overlaid boxes.
  """
  # Create a display string (and color) for every box location, group any boxes
  # that correspond to the same location.

  box_to_display_str_map = collections.defaultdict(list)
  box_to_color_map = collections.defaultdict(str)
  box_to_instance_masks_map = {}
  box_to_instance_boundaries_map = {}
#  box_to_keypoints_map = collections.defaultdict(list)
  if not max_boxes_to_draw:
    max_boxes_to_draw = boxes.shape[0]
  for i in range(min(max_boxes_to_draw, boxes.shape[0])):
    if scores is None or scores[i] > min_score_thresh:
      box = tuple(boxes[i].tolist())
      if instance_masks is not None:
        box_to_instance_masks_map[box] = instance_masks[i]
      if instance_boundaries is not None:
        box_to_instance_boundaries_map[box] = instance_boundaries[i]
     
      if scores is None:
        box_to_color_map[box] = groundtruth_box_visualization_color
      else:
        display_str = ''
        if not skip_labels:
          if not agnostic_mode:
            if classes[i] in category_index.keys():
              class_name = category_index[classes[i]]['name']
            else:
              class_name = 'N/A'
            display_str = str(class_name)
        if not skip_scores:
          if not display_str:
            display_str = '{}%'.format(int(100*scores[i]))
          else:
            display_str = '{}: {}%'.format(display_str, int(100*scores[i]))
        box_to_display_str_map[box].append(display_str)
        if agnostic_mode:
          box_to_color_map[box] = 'DarkOrange'
        else:
          box_to_color_map[box] = STANDARD_COLORS[
              classes[i] % len(STANDARD_COLORS)]
#==================================================================================#
#===============crop bounding box images===========================================#
#==================================================================================# 	
  t = 0 #這行記得是放在下面的迴圈體外的
  for box, color in box_to_color_map.items():
    ymin, xmin, ymax, xmax = box  #前文已經得到bounding box的座標，但是，由於使用了use_normalized_coordinates，將座標歸一化了，所以要映射回來
    img = Image.fromarray(np.uint8(image))
    im_width, im_height = img.size
    print('im_width, im_height:', im_width, im_height)
    new_xmin = int(xmin * im_width)
    new_xmax = int(xmax * im_width)
    new_ymin = int(ymin * im_height)
    new_ymax = int(ymax * im_height)
#    print('xmin,xmax,ymin,ymax:',xmin,xmax,ymin,ymax)
    print('new_xmin,new_xmax,new_ymin,new_ymax:',new_xmin,new_xmax,new_ymin,new_ymax)

    image_ = image[new_ymin:new_ymax,new_xmin:new_xmax] #將測試圖片做裁剪，只剩bounding box部分，注意此處的image_命名，之前我寫的是image,會出錯。而且要注意的是，由於程式碼中用的是PIL圖片格式，原點在左上角，而此處用numpy的方式裁剪圖片，原點在左下角，所以x和y是反過來的。
    
    plt.imshow(image_)
    plt.show()
#       
    PATH_TO_crop_DIR = r'C:\\models\\research\\object_detection\\test_cropped' #儲存的裁剪圖片位置
    image_ = Image.fromarray(np.uint8(image_))
    t+=1
#    image_.save( 'C:\\models\\research\\object_detection\\test_cropped\\cropped_images.jpg')
#    image_.save(os.path.join(PATH_TO_crop_DIR, str(t)+'.jpg'))
    image_.save(os.path.join(PATH_TO_crop_DIR, (str(t)+'_')+os.path.basename(image_path))) #這裡用上了image_path,因為os.path.join需要字串的引數，由於一張測試圖片可能會檢測出多個bounding box，所以在儲存命名的時候加上了t這個引數（這一步應該會有更優雅的做法，但是我懶得去試了）
####################################################################################
  # Draw all boxes onto image.把框和檔案畫到測試圖片上
  for box, color in box_to_color_map.items():
    ymin, xmin, ymax, xmax = box
    draw_bounding_box_on_image_array(
        image,
        ymin,
        xmin,
        ymax,
        xmax,
        color=color,
        thickness=line_thickness,
        display_str_list=box_to_display_str_map[box],
        use_normalized_coordinates=use_normalized_coordinates)、
  return image