Tensorflow(python): Neural network produces differernt output when using tf.Variable instead of tf.constant as input?

Question

I am using a fixed neaural network model: vgg16. After I load the weights and biases of the network, I feed an image as the input of the network.

If I make the input tf.constant, I get the same output(response of certain layer) everytime I forward the net. However, if I set the input image as a tf.Variable, I get different output from the results I get by setting it as tf.constant and the output changes if I run it many times.

Here's some code for illustration: vgg.py

VGG_MEAN = [103.939, 116.779, 123.68]
class Vgg16:
    def __init__(self, vgg16_npy_path=None):
        self.data_dict = np.load(vgg16_npy_path, encoding='latin1').item()
        print("npy file loaded")

        self.layers = {}

    def build(self, rgb):
        """
        load variable from npy to build the VGG
        :param rgb: rgb image [batch, height, width, 3] values scaled [0, 1]
        """

        rgb_scaled = rgb * 255.0
        # Convert RGB to BGR
        red, green, blue = tf.split(rgb_scaled,3,3)
        assert red.get_shape().as_list()[1:] == [224, 224, 1]
        assert green.get_shape().as_list()[1:] == [224, 224, 1]
        assert blue.get_shape().as_list()[1:] == [224, 224, 1]
        bgr = tf.concat([
            blue - VGG_MEAN[0],
            green - VGG_MEAN[1],
            red - VGG_MEAN[2],
        ],3)
        assert bgr.get_shape().as_list()[1:] == [224, 224, 3]

        self.conv1_1 = self.conv_layer(bgr, "conv1_1")
        self.conv1_2 = self.conv_layer(self.conv1_1, "conv1_2")
        self.pool1 = self.max_pool(self.conv1_2, 'pool1')
        self.conv2_1 = self.conv_layer(self.pool1, "conv2_1")
        self.conv2_2 = self.conv_layer(self.conv2_1, "conv2_2")
        self.pool2 = self.max_pool(self.conv2_2, 'pool2')
        self.conv3_1 = self.conv_layer(self.pool2, "conv3_1")
        self.conv3_2 = self.conv_layer(self.conv3_1, "conv3_2")
        self.conv3_3 = self.conv_layer(self.conv3_2, "conv3_3")
        self.pool3 = self.max_pool(self.conv3_3, 'pool3')
        self.conv4_1 = self.conv_layer(self.pool3, "conv4_1")
        self.conv4_2 = self.conv_layer(self.conv4_1, "conv4_2")
        self.conv4_3 = self.conv_layer(self.conv4_2, "conv4_3")
        self.pool4 = self.max_pool(self.conv4_3, 'pool4')
        self.conv5_1 = self.conv_layer(self.pool4, "conv5_1")
        self.conv5_2 = self.conv_layer(self.conv5_1, "conv5_2")
        self.conv5_3 = self.conv_layer(self.conv5_2, "conv5_3")
        self.pool5 = self.max_pool(self.conv5_3, 'pool5')
        self.fc6 = self.fc_layer(self.pool5, "fc6")
        assert self.fc6.get_shape().as_list()[1:] == [4096]
        self.relu6 = tf.nn.relu(self.fc6)
        self.fc7 = self.fc_layer(self.relu6, "fc7")
        self.relu7 = tf.nn.relu(self.fc7)
        self.fc8 = self.fc_layer(self.relu7, "fc8")
        self.prob = tf.nn.softmax(self.fc8, name="prob")

    def avg_pool(self, bottom, name):
        return tf.nn.avg_pool(bottom, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name=name)

    def max_pool(self, bottom, name):
        return tf.nn.max_pool(bottom, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name=name)

    def conv_layer(self, bottom, name):
        filt = self.get_conv_filter(name)
        conv = tf.nn.conv2d(bottom, filt, [1, 1, 1, 1], padding='SAME')
        conv_biases = self.get_bias(name)
        bias = tf.nn.bias_add(conv, conv_biases)
        relu = tf.nn.relu(bias)
        return relu

    def fc_layer(self, bottom, name):
        shape = bottom.get_shape().as_list()
        dim = 1
        for d in shape[1:]:
            dim *= d
        x = tf.reshape(bottom, [-1, dim])

        weights = self.get_fc_weight(name)
        biases = self.get_bias(name)

        fc = tf.nn.bias_add(tf.matmul(x, weights), biases)

        return fc

    def get_conv_filter(self, name):
        return tf.constant(self.data_dict[name][0], name="filter")

    def get_bias(self, name):
        return tf.constant(self.data_dict[name][1], name="biases")

    def get_fc_weight(self, name):
        return tf.constant(self.data_dict[name][0], name="weights")

test.py

img = utils.load_image(img_path).reshape(1,224,224,3).astype('float32')
model = vgg.Vgg16(vgg16_npy_path)
sess = tf.Session()

input_tensor = tf.constant(img)
model.build(input_tensor)
res1 = sess.run(model.fc8)

input_var = tf.Variable(img)
model.build(input_var)
init_op = tf.global_variables_initializer()
sess.run(init_op)
res2 = sess.run(model.fc8)

input_tensor2 = tf.constant(img)
model.build(input_tensor2)
res3 = sess.run(model.fc8)

print(res1 == res2) # false
print(res1 == res3) # true

In test.py, res1 and res2 are different, which doesn't make sense since they are both generated from the same input image.

How can I solve this problem? Hope for your help!

Are res1 and res2 very different, not numerical problem? Can you print them out and compare? Is there only one tf.Variable when you test the model, just for excluding the problem of initialization from init_op? — LI Xuhong

Alexandre Passos Alexandre Passos · Accepted Answer · 2017-05-24T17:23:12

This is likely an interaction between constant folding and non-deterministic floating point operations. Does the problem still happen with constants if you disable constant folding by creating your session with

config = tf.ConfigProto(graph_options=tf.GraphOptions(optimizer_options=tf.OptimizerOptions(opt_level=tf.OptimizerOptions.L0))) sess = tf.Session(config=config) ?

Due to parallelism some floating point non-determinism is expected in tensorflow models.

Tensorflow(python): Neural network produces differernt output when using tf.Variable instead of tf.constant as input?

1 Answers