Tensorflow 1.9.0: A question about training my model with generators

Question

Situation

I am trying to build my own CNN with images I compiled. I have two folders with training and validation data. This is my code so far:

import os
import sys
import shutil
import numpy as np
import tensorflow as tf

SCRIPT_DIR = os.getcwd()

TRAIN_GRAPH = 'training_graph.pb'
CHKPT_FILE = 'float_model.ckpt'

CHKPT_DIR = os.path.join(SCRIPT_DIR, 'chkpts')
TB_LOG_DIR = os.path.join(SCRIPT_DIR, 'tb_logs')
CHKPT_PATH = os.path.join(CHKPT_DIR, CHKPT_FILE)
CNN_DIR = os.path.join(SCRIPT_DIR, 'cnn_dir')

# create a directory for the dataset if it doesn't already exist
if not (os.path.exists(CNN_DIR)):
    os.makedirs(CNN_DIR)
    print("Directory " , CNN_DIR ,  "created ") 

# create a directory for the TensorBoard data if it doesn't already exist
# delete it and recreate if it already exists
if (os.path.exists(TB_LOG_DIR)):
    shutil.rmtree(TB_LOG_DIR)
os.makedirs(TB_LOG_DIR)
print("Directory " , TB_LOG_DIR ,  "created ") 

# create a directory for the checkpoint if it doesn't already exist
# delete it and recreate if it already exists
if (os.path.exists(CHKPT_DIR)):
    shutil.rmtree(CHKPT_DIR)
os.makedirs(CHKPT_DIR)
print("Directory " , CHKPT_DIR ,  "created ")

LEARN_RATE = 0.0001
BATCHSIZE = 64
EPOCHS = 10
IMG_SIZE = 224

# ===== IMPORTANT PART STARTS HERE ================================

# SETTING UP DATA ---------------
datagen_training = tf.keras.preprocessing.image.ImageDataGenerator(
    rescale = 1./255,
    shear_range = 0.2,
    zoom_range = 0.2,
    horizontal_flip = True)

dataset_training = datagen_training.flow_from_directory(
        "./dataset/prepared/training",
        target_size = (IMG_SIZE, IMG_SIZE),
        batch_size = BATCHSIZE,
        class_mode = "binary")

datagen_test = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255)

dataset_test = datagen_training.flow_from_directory(
        "./dataset/prepared/validation",
        target_size=(IMG_SIZE, IMG_SIZE),
        batch_size=BATCHSIZE,
        class_mode="binary")
# SETTING UP CNN ---------------

cnn = tf.keras.models.Sequential()

# First convolutional layer
cnn.add(tf.keras.layers.Conv2D(
    filters = 16, 
    kernel_size = 3, 
    activation = tf.nn.relu, 
    input_shape = [IMG_SIZE, IMG_SIZE, 3]))

# Maxpooling layer
cnn.add(tf.keras.layers.MaxPool2D(
    pool_size = 2, 
    strides = 2))

# Second convolutional layer
cnn.add(tf.keras.layers.Conv2D(
    filters = 32, 
    kernel_size = 3, 
    activation = tf.nn.relu))
  
# Flattening the vector
cnn.add(tf.keras.layers.Flatten())
        
# Fully connected layer #1
# Hidden layer with 256 neurons
cnn.add(tf.keras.layers.Dense(
    units = 256,
    activation = tf.keras.activations.softmax))

# Fully connected layer #2
# Densing down to 8 variables, output layer
cnn.add(tf.keras.layers.Dense(
    units = 8,
    activation = None))

cnn.compile(optimizer = "adam", loss = "binary_crossentropy", metrics = ["accuracy"])

# PROBLEM LIES HERE ------------------------
cnn.fit(x = dataset_training, 
    validation_data = dataset_test,
    epochs = 10)

When I execute this, I get this error:

--------------------------------------------------------------------------- AttributeError Traceback (most recent call last) in 1 cnn.fit(x = dataset_training, 2 validation_data = dataset_test, ----> 3 epochs = 10)

~/anaconda3/envs/decent/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs) 1261 steps_name='steps_per_epoch', 1262 steps=steps_per_epoch, -> 1263 validation_split=validation_split) 1264 1265 # Prepare validation data.

~/anaconda3/envs/decent/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, batch_size, check_steps, steps_name, steps, validation_split) 866 feed_input_shapes, 867 check_batch_axis=False, # Don't enforce the batch size. --> 868 exception_prefix='input') 869 870 if y is not None:

~/anaconda3/envs/decent/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_utils.py in standardize_input_data(data, names, shapes, check_batch_axis, exception_prefix) 141 data = data.values if data.class.name == 'DataFrame' else data 142 data = [data] --> 143 data = [standardize_single_array(x) for x in data] 144 145 if len(data) != len(names):

~/anaconda3/envs/decent/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_utils.py in (.0) 141 data = data.values if data.class.name == 'DataFrame' else data 142 data = [data] --> 143 data = [standardize_single_array(x) for x in data] 144 145 if len(data) != len(names):

~/anaconda3/envs/decent/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_utils.py in standardize_single_array(x) 79 elif tensor_util.is_tensor(x): 80 return x ---> 81 elif x.ndim == 1: 82 x = np.expand_dims(x, 1) 83 return x

AttributeError: 'DirectoryIterator' object has no attribute 'ndim'

The tutorial I am following probably uses a more recent version of Tensorflow. However, due to restrictions in my system I need to use the version I am using. Now according to this SO post, I need to use fit_generator(...). Looking at this post the syntax of fit_generator(...) is:

fit_generator(object, generator, steps_per_epoch, epochs = 1,
  verbose = getOption("keras.fit_verbose", default = 1),
  callbacks = NULL, view_metrics = getOption("keras.view_metrics",
  default = "auto"), validation_data = NULL, validation_steps = NULL,
  class_weight = NULL, max_queue_size = 10, workers = 1,
  initial_epoch = 0)

Which means, I need to give the function a generator. However, the example given in the post looks like this:

# performing data argumentation by training image generator
dataAugmentaion = ImageDataGenerator(rotation_range = 30, zoom_range = 0.20, 
fill_mode = "nearest", shear_range = 0.20, horizontal_flip = True, 
width_shift_range = 0.1, height_shift_range = 0.1)

# training the model
model.fit_generator(dataAugmentaion.flow(trainX, trainY, batch_size = 32),
 validation_data = (testX, testY), steps_per_epoch = len(trainX) // 32,
 epochs = 10)

Which is weird for me. Where does the trainX and trainY come from? I don't have variables like that, because I (don't seem to) have some kind of vector of input and output data, at least it's not obvious for me where those arrays/vectors/tuples come from. I also noticed in another example from Avnet, that they also get such kind of data:

mnist_dataset = tf.keras.datasets.mnist.load_data('mnist_data')
(x_train, y_train), (x_test, y_test) = mnist_dataset

I don't get what is in those arrays, how that data is formatted, and what to use them for. I understand that I need some kind of input vector/image, which, in my code, should be provided by the ImageDataGenerator, right? There seems to be a big thing I am missing to understand how this works. The tutorial I am following doesn't elaborate further on this, since they can just plug in the generators into cnn.fit(...).

---

Questions

So I guess my questions are the following:

1. What are those arrays like trainX and trainY or (x_train, y_train), (x_test, y_test)?
2. What do I need them for?
3. How are they formatted?
4. How can I extract the data in that format from my ImageDataGenerator or rather from the DirectoryIterator which I use to "load" my data with flow_from_directory(...) (here: dataset_training and dataset_test)
5. Any other recommendations or suggestions for my code/methodology?

Tensorflow Version: 1.9.0

Keras Version: 2.1.6-tf

---

Update

According to this post, the function fit_generator(...) should look a bit like this:

# ...
model.fit(
        train_generator,
        steps_per_epoch=2000,
        epochs=50,
        validation_data=validation_generator,
        validation_steps=800)
# ...

I changed the fit into fit_generator. Which basically results in this:

cnn.fit_generator(
        generator=datagen_training,
        steps_per_epoch=50000/32,
        epochs=EPOCHS,
        validation_data=datagen_test,
        validation_steps=800
)

Unfortunately, that throws this error:

--------------------------------------------------------------------------- TypeError Traceback (most recent call last) in 9 epochs=EPOCHS, 10 validation_data=datagen_test, ---> 11 validation_steps=800 12 )

~/anaconda3/envs/decent/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch) 1759 use_multiprocessing=use_multiprocessing, 1760
shuffle=shuffle, -> 1761 initial_epoch=initial_epoch) 1762 1763 def evaluate_generator(self,

~/anaconda3/envs/decent/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch) 112 if do_validation and not val_gen: 113 # Prepare data for validation --> 114 if len(validation_data) == 2: 115 val_x, val_y = validation_data # pylint: disable=unpacking-non-sequence 116 val_sample_weight = None

TypeError: object of type 'ImageDataGenerator' has no len()

Answer 1

My knowledge is limited, and I'm sure someone else will come along to solve. However, the ImageDataGenerator class would seem to be a good fit - creating a ready-made and customizable generator object for fit_generator() method - basically yielding subsets of the input tensors to optimize processing in batches.

Look to use ImageDataGenerator's flow() method since you are already pre-processing your image data in code (assuming that's all done effectively). flow_from_directory() is a good off-the-shelf solution if you haven't set up loading and preprocessing and have images in a folder.

Something like this might get you started (default parameters used, and some arbitrary batch/step values):

train_datagen = ImageDataGenerator() #Using rescale=1./255 (for 8 bit images) will optimize your model's performance
val_datagen = ImageDataGenerator() #Using rescale=1./255 (for 8 bit images) will optimize your model's performance

train_generator = train_datagen.flow( dataset_training, batch_size=20, class_mode='binary')
val_generator = val_datagen.flow( dataset_test, batch_size=20, class_mode='binary')

history = cnn.fit_generator(train_generator, 
                            steps_per_epoch=100,
                            epochs=10,
                            validation_data=val_generator,
                            validation_steps=50)

Whether that takes you to next step is uncertain, but you'll want to save the history object - it retains data for 'acc', 'val_acc', 'loss', and 'val_loss'.