Keras custom matric: ROC curve

Question

I am using custom matric as given on keras website:

def compute_roc (y_true, y_pred):
    y_true =K.eval(y_true)#.eval(session=sess)
    y_pred = K.eval(y_pred)#.eval(session=sess)
    roc = metrics.roc_curve(y_true, y_pred, pos_label=2)
    roc = tf.to_float(roc, name='ToFloat')
    return roc

But I am getting error on "K.eval(y_true)"

    InvalidArgumentError                      Traceback (most recent call last)
<ipython-input-7-1de7569967ec> in <module>()
     20 #disc.load_weights(os.path.join(path, model_name+'.h5'))
     21 sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
---> 22 disc.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy',
compute_roc]) 23 #plot_model(model, to_file='model.png') 24 print('disc model loaded')
/usr/local/lib/python2.7/dist-packages/keras/engine/training.pyc in compile(self, optimizer, loss, metrics, loss_weights,
sample_weight_mode, weighted_metrics, target_tensors, **kwargs) 912 append_metric(i, metric_name, metric_result) 913 --> 914 handle_metrics(output_metrics) 915 handle_metrics(output_weighted_metrics, weights=weights) 916
/usr/local/lib/python2.7/dist-packages/keras/engine/training.pyc in handle_metrics(metrics, weights)
    909                             metric_result = weighted_metric_fn(y_true, y_pred,
    910                                                                weights=weights,
--> 911                                                                mask=masks[i])
    912                         append_metric(i, metric_name, metric_result)
    913 

/usr/local/lib/python2.7/dist-packages/keras/engine/training.pyc in weighted(y_true, y_pred, weights, mask)
    424         """
    425         # score_array has ndim >= 2
--> 426         score_array = fn(y_true, y_pred)
    427         if mask is not None:
    428             # Cast the mask to floatX to avoid float64 upcasting in theano

<ipython-input-7-1de7569967ec> in compute_roc(y_true, y_pred)
      4 
      5 def compute_roc (y_true, y_pred):
----> 6     y_true1 =K.eval(y_true)#.eval(session=sess)
      7    # y_pred1 = K.eval(y_pred)#.eval(session=sess)
      8     roc = 0#metrics.roc_curve(y_true, y_pred, pos_label=2)

/usr/local/lib/python2.7/dist-packages/keras/backend/tensorflow_backend.pyc
in eval(x) 658 ``` 659 """ --> 660 return to_dense(x).eval(session=get_session()) 661 662
/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc
in eval(self, feed_dict, session) 539 540 """ --> 541 return _eval_using_default_session(self, feed_dict, self.graph, session) 542 543
/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc
in _eval_using_default_session(tensors, feed_dict, graph, session) 4083 "the tensor's graph is different from the session's " 4084 "graph.") -> 4085 return session.run(tensors, feed_dict) 4086 4087
/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc
in run(self, fetches, feed_dict, options, run_metadata) 893 try: 894 result = self._run(None, fetches, feed_dict, options_ptr, --> 895 run_metadata_ptr) 896 if run_metadata: 897 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc
in _run(self, handle, fetches, feed_dict, options, run_metadata) 1122 if final_fetches or final_targets or (handle and feed_dict_tensor): 1123 results = self._do_run(handle, final_targets, final_fetches, -> 1124 feed_dict_tensor, options, run_metadata) 1125 else: 1126 results = []
/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc
in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata) 1319 if handle is None: 1320 return self._do_call(_run_fn, self._session, feeds, fetches, targets, -> 1321 options, run_metadata) 1322 else: 1323 return self._do_call(_prun_fn, self._session, handle, feeds, fetches)
/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc
in _do_call(self, fn, *args) 1338 except KeyError: 1339 pass -> 1340 raise type(e)(node_def, op, message) 1341 1342 def _extend_graph(self):
InvalidArgumentError: You must feed a value for placeholder tensor 'dense_6_target' with dtype float and shape [?,?]
   [[Node: dense_6_target = Placeholder[dtype=DT_FLOAT, shape=[?,?], _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
   [[Node: dense_6_target/_833 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0",
send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_4_dense_6_target", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Caused by op u'dense_6_target', defined at:
  File "/usr/lib/python2.7/runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py", line
16, in app.launch_new_instance() File "/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py", line 658, in launch_instance app.start() File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py", line 478, in start self.io_loop.start() File "/usr/local/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py", line 177, in start super(ZMQIOLoop, self).start() File "/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py", line 888, in start handler_func(fd_obj, events) File "/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py", line 277, in null_wrapper return fn(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events self._handle_recv() File "/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv self._run_callback(callback, msg) File "/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback callback(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py", line 277, in null_wrapper return fn(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 283, in dispatcher return self.dispatch_shell(stream, msg) File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell handler(stream, idents, msg) File "/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py", line 399, in execute_request user_expressions, allow_stdin) File "/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py", line 208, in do_execute res = shell.run_cell(code, store_history=store_history, silent=silent) File "/usr/local/lib/python2.7/dist-packages/ipykernel/zmqshell.py", line 537, in run_cell return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py", line 2718, in run_cell interactivity=interactivity, compiler=compiler, result=result) File "/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py", line 2822, in run_ast_nodes if self.run_code(code, result): File "/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py", line 2882, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "", line 22, in disc.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy', compute_roc]) File "/usr/local/lib/python2.7/dist-packages/keras/engine/training.py", line 722, in compile dtype=K.dtype(self.outputs[i])) File "/usr/local/lib/python2.7/dist-packages/keras/backend/tensorflow_backend.py", line 504, in placeholder x = tf.placeholder(dtype, shape=shape, name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py", line 1548, in placeholder return gen_array_ops._placeholder(dtype=dtype, shape=shape, name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 2094, in _placeholder name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op op_def=op_def) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2630, in create_op original_op=self._default_original_op, op_def=op_def) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1204, in init self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'dense_6_target' with dtype float and
shape [?,?] [[Node: dense_6_target = Placeholderdtype=DT_FLOAT, shape=[?,?], _device="/job:localhost/replica:0/task:0/gpu:0"]] [[Node: dense_6_target/_833 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_4_dense_6_target", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]]

The other way is to compute "y_pred" at the end of each epoch and compute roc.

Why don't you use sklearn roc_auc after getting predictions from your model? — enterML
The problem is with: y_true =K.eval(y_true) AND y_pred = K.eval(y_pred) — Mohbat Tharani

W. Sam W. Sam · Accepted Answer · 2018-07-02T19:34:38

You have two options:

Using ski learn function within custom Callback ( on-epoch_end).

In this case you need to get the batch_x, batch_y by passing your train generator to you callback, then calculate the y_pred ( on_batch_begin) through model.predict(batch_x) then accumulate it at (on_epoch_end) callback and then calculate the AUC using your function.
Building custom metric in Keras using tf.meteric auc

AUC, update_op = tf.metric.auc(y_true, y_pred)

In this case you don't need to calculate the y_pred for each batch. just build custom metric function and pass it to callback. You need to ensure to update the ( update_op) for each batch because Auc metric need to be calculated at the end of epoch not on batch level.

you can also look the comment of Tom here where you can find sample code for the above suggestion. However they refer to tf.contrib.metrics.streaming_auc which is deprecated and replaced by tf.metric.auc

Keras custom matric: ROC curve

1 Answers