how to serve keras model for inference using flask?

Question

I have a inference object detection code of yolo v3 keras model

#! /usr/bin/env python

import os
import argparse
import json
import cv2
from utils.utils import get_yolo_boxes, makedirs
from utils.bbox import draw_boxes
from keras.models import load_model
from tqdm import tqdm
import numpy as np
import flask
import io
from PIL import Image
from keras.preprocessing.image import img_to_array

config_path  = "config.json"
input_path   = "test.jpg"
output_path  = "output"

with open(config_path) as config_buffer:    
        config = json.load(config_buffer)

makedirs(output_path)

net_h, net_w = 416, 416 
obj_thresh, nms_thresh = 0.5, 0.45

os.environ['CUDA_VISIBLE_DEVICES'] = config['train']['gpus']
infer_model = load_model(config['train']['saved_weights_name'])


image = cv2.imread(input_path)

# predict the bounding boxes
boxes = get_yolo_boxes(infer_model, [image], net_h, net_w, config['model']['anchors'], obj_thresh, nms_thresh)[0]

# draw bounding boxes on the image using labels
_,outputs =  draw_boxes(image, boxes, config['model']['labels'], obj_thresh) 

print(outputs)

# write the image with bounding boxes to file
cv2.imwrite(output_path + input_path.split('/')[-1], np.uint8(image))

This is working completely fine ,giving the expected output class & co-ordinate in terminal

{'classes': 'person 99.97%', 'X2': '389', 'X1': '174', 'Y1': '8', 'Y2': '8'}

But when I convert that for REST api based for serving by converting above code using official keras conversion to flask documentation ,like this :

#! /usr/bin/env python

import os
import argparse
import json
import cv2
from utils.utils import get_yolo_boxes, makedirs
from utils.bbox import draw_boxes
from keras.models import load_model
from tqdm import tqdm
import numpy as np
import flask
import io
from PIL import Image
from keras.preprocessing.image import img_to_array


config_path  = "config.json"
input_path   = "test.jpg"
output_path  = "output"

with open(config_path) as config_buffer:    
        config = json.load(config_buffer)

makedirs(output_path)

net_h, net_w = 416, 416 
obj_thresh, nms_thresh = 0.5, 0.45

app = flask.Flask(__name__)

os.environ['CUDA_VISIBLE_DEVICES'] = config['train']['gpus']
infer_model = load_model(config['train']['saved_weights_name'])

def prepare_image(image_path):
    image = cv2.imread(image_path)
    return image

@app.route("/predict", methods=["POST"])
def predict():
    # initialize the data dictionary that will be returned from the
    # view
    data = {"success": False}

    # ensure an image was properly uploaded to our endpoint
    if flask.request.method == "POST":
        if flask.request.files.get("image"):
            # read the image in PIL format
            image = flask.request.files["image"].read()
            image = Image.open(io.BytesIO(image))

            # preprocess the image and prepare it for classification            
            image = img_to_array(image)


            boxes = get_yolo_boxes(infer_model, [image], net_h, net_w, config['model']['anchors'], obj_thresh, nms_thresh)[0]


            _,outputs =  draw_boxes(image, boxes, config['model']['labels'], obj_thresh)


            data.append(outputs)

            print(data)


            # indicate that the request was a success
            data["success"] = True

    # return the data dictionary as a JSON response
    return flask.jsonify(data)



if __name__ == "__main__":
    print(("* Loading Keras model and Flask starting server..."
        "please wait until server has fully started"))
    app.run()

Running successfully on port 5000

But when I'm trying to predict via POST api using

curl -X POST -F [email protected] 'http://localhost:5000/predict'

is giving this error

raise ValueError("Tensor %s is not an element of this graph." % obj) ValueError: Tensor Tensor("conv2d_59/BiasAdd:0", shape=(?, ?, ?,
255), dtype=float32) is not an element of this graph. 127.0.0.1 - - [15/Aug/2019 15:11:23] "POST /predict HTTP/1.1" 500 -

I don't understand why same prediction function was working without flask but getting error in this.

Asmita Khaneja Asmita Khaneja · Accepted Answer · 2019-10-21T17:18:25

I was facing the same problem, this is a keras issue. Primarily seems to be triggered when when there is asynchronous event handler Adding model._make_predict_function() right after loading the trained model worked for me. For example,

from keras.models import load_model
   model=load_model('yolo.h5')
   model._make_predict_function()

Another approach that has worked for other people is to use graph and have inference within the context something like:

global graph
graph = tf.get_default_graph()
with graph.as_default():
res = model.predict()

For more insights , please refer below links:

https://github.com/keras-team/keras/issues/6462

https://github.com/keras-team/keras/issues/2397

how to serve keras model for inference using flask?

1 Answers