I'm trying to build a scalable min max scaler with tensorflow dataset API and map function.
First I loop through my dataset to find the min and max values for all features (3) and then I would like to apply the min/max scaler to the dataset using map function.
Here is my simple code.
import numpy as np
import tensorflow as tf
b = np.array([[1, 2, 3], [4, 5, 6], [7,8,9],[10,11,12]])
b_ds = tf.data.Dataset.from_tensor_slices(b).batch(2)
my_iterator = b_ds.make_one_shot_iterator()
def compute_min_max(i, my_min, my_max):
new_batch = my_iterator.get_next()
my_min = tf.minimum(my_min,tf.reduce_min(new_batch, axis=0))
my_max = tf.maximum(my_max,tf.reduce_max(new_batch, axis=0))
return [i+1, my_min, my_max]
i = tf.constant(0)
feat_min = tf.Variable([10,10,10],dtype=tf.int64)
feat_max = tf.Variable([0,0,0],dtype=tf.int64)
c = lambda i, min, max: i < 2
b = lambda i, min, max: compute_min_max(i, min, max)
res_i, res_min, res_max = tf.while_loop(c, b, loop_vars=[i, feat_min, feat_max])
def min_max_ds(feat):
return tf.cast(feat-res_min,dtype=tf.float64)/tf.cast(res_max-res_min, dtype=tf.float64)
minmax_scaled_ds = b_ds.map(min_max_ds)
scaled_batch = minmax_scaled_ds.make_one_shot_iterator().get_next()
with tf.Session() as sess:
init=tf.global_variables_initializer()
sess.run(init)
print(sess.run((res_min, res_max, scaled_batch)))
When I execute this code I get a
RecursionError: maximum recursion depth exceeded
My guess is that the min_max_ds function calls back the tf.while_loop statement for every batch or so, but I can't figure how to kind of freeze res_min and res_max so they are used as constant in the min_max_ds function.