Bayesian Optimization for LSTM

1
votes

I am trying to optimize the hyperparameters of a LSTM with Bayesian Optimization. But I received the error message TypeError: only integer scalar arrays can be converted to a scalar index when I run the code. A solution I found is to convert the training data and validation data into arrays, but in my code they are already arrays not lists. Or convert them into tuples but I cannot see how I would do this

X_train shape: (946, 60, 1)

y_train shape: (946,)

X_val shape: (192, 60, 1)

y_val shape: (192,)

def build(hp):


    activation = hp.Choice('activation', 
                        [
                          'relu',
                          'tanh',
                          'linear',
                          'selu',
                          'elu'
                        ])

    num_rnn_layers = hp.Int(
                        'num_rnn_layers', 
                        min_value=0,
                        max_value=12,
                        default=3)

    recurrent_dropout = hp.Float(
                        'recurrent_dropout', 
                        min_value=0.0,
                        max_value=0.99,
                        default=0.2)
    num_units = hp.Int(
                        'num_units', 
                        min_value=0,
                        max_value=64,
                        default=32)
    
    model = Sequential()
    model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train[1], 1)))

    model.add(Dense(1))
    model.compile(loss='mse', metrics=['mse'], optimizer=keras.optimizers.Adam(
      hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])))



    model.compile(
      optimizer=keras.optimizers.Adam(
      hp.Float(
        'learning_rate',
        min_value=1e-10,
        max_value=1e-2,
        sampling='LOG',
        default=1e-6
            ),

        ),
        loss=tf.losses.MeanSquaredError(),
        metrics=[tf.metrics.MeanAbsoluteError()]
    )
    return model

bayesian_opt_tuner = BayesianOptimization(
    build,
    objective='mse',
    max_trials=3,
    executions_per_trial=1,
    directory=os.path.normpath('C:/keras_tuning'),
    project_name='kerastuner_bayesian_poc',
    overwrite=True)
n_epochs=100

bayesian_opt_tuner.search(X_train, y_train,epochs=n_epochs,
     validation_data=(X_val, y_val),
     validation_split=0.2,verbose=1)


bayes_opt_model_best_model = bayesian_opt_tuner.get_best_models(num_models=1)
model = bayes_opt_model_best_model[0]

Error log:

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "<ipython-input-80-00452994e0d6>", line 33, in build
    model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train[1], 1)))
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py", line 517, in _method_wrapper
    result = method(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py", line 204, in add
    batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 309, in Input
    input_layer = InputLayer(**input_layer_config)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 160, in __init__
    ragged=ragged)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/backend.py", line 1247, in placeholder
    shape=shape, dtype=dtype, name=name)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_spec.py", line 51, in __init__
    self._shape = tensor_shape.TensorShape(shape)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in __init__
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in <listcomp>
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 201, in __init__
    self._value = int(value.__index__())
TypeError: only integer scalar arrays can be converted to a scalar index
[Warning] Invalid model 0/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "<ipython-input-80-00452994e0d6>", line 33, in build
    model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train[1], 1)))
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py", line 517, in _method_wrapper
    result = method(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py", line 204, in add
    batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 309, in Input
    input_layer = InputLayer(**input_layer_config)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 160, in __init__
    ragged=ragged)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/backend.py", line 1247, in placeholder
    shape=shape, dtype=dtype, name=name)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_spec.py", line 51, in __init__
    self._shape = tensor_shape.TensorShape(shape)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in __init__
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in <listcomp>
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 201, in __init__
    self._value = int(value.__index__())
TypeError: only integer scalar arrays can be converted to a scalar index
[Warning] Invalid model 1/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "<ipython-input-80-00452994e0d6>", line 33, in build
    model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train[1], 1)))
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py", line 517, in _method_wrapper
    result = method(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py", line 204, in add
    batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 309, in Input
    input_layer = InputLayer(**input_layer_config)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 160, in __init__
    ragged=ragged)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/backend.py", line 1247, in placeholder
    shape=shape, dtype=dtype, name=name)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_spec.py", line 51, in __init__
    self._shape = tensor_shape.TensorShape(shape)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in __init__
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in <listcomp>
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 201, in __init__
    self._value = int(value.__index__())
TypeError: only integer scalar arrays can be converted to a scalar index
[Warning] Invalid model 2/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "<ipython-input-80-00452994e0d6>", line 33, in build
    model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train[1], 1)))
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py", line 517, in _method_wrapper
    result = method(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py", line 204, in add
    batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 309, in Input
    input_layer = InputLayer(**input_layer_config)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 160, in __init__
    ragged=ragged)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/backend.py", line 1247, in placeholder
    shape=shape, dtype=dtype, name=name)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_spec.py", line 51, in __init__
    self._shape = tensor_shape.TensorShape(shape)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in __init__
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in <listcomp>
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 201, in __init__
    self._value = int(value.__index__())
TypeError: only integer scalar arrays can be converted to a scalar index
[Warning] Invalid model 3/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "<ipython-input-80-00452994e0d6>", line 33, in build
    model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train[1], 1)))
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py", line 517, in _method_wrapper
    result = method(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py", line 204, in add
    batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 309, in Input
    input_layer = InputLayer(**input_layer_config)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 160, in __init__
    ragged=ragged)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/backend.py", line 1247, in placeholder
    shape=shape, dtype=dtype, name=name)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_spec.py", line 51, in __init__
    self._shape = tensor_shape.TensorShape(shape)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in __init__
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in <listcomp>
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 201, in __init__
    self._value = int(value.__index__())
TypeError: only integer scalar arrays can be converted to a scalar index
[Warning] Invalid model 4/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "<ipython-input-80-00452994e0d6>", line 33, in build
    model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train[1], 1)))
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py", line 517, in _method_wrapper
    result = method(self, *args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/sequential.py", line 204, in add
    batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 309, in Input
    input_layer = InputLayer(**input_layer_config)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_layer.py", line 160, in __init__
    ragged=ragged)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/backend.py", line 1247, in placeholder
    shape=shape, dtype=dtype, name=name)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_spec.py", line 51, in __init__
    self._shape = tensor_shape.TensorShape(shape)
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in __init__
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 758, in <listcomp>
    self._dims = [Dimension(d) for d in dims]
  File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 201, in __init__
    self._value = int(value.__index__())
TypeError: only integer scalar arrays can be converted to a scalar index
[Warning] Invalid model 5/5
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/kerastuner/engine/hypermodel.py in build(self, hp)
    104                 with maybe_distribute(self.distribution_strategy):
--> 105                     model = self.hypermodel.build(hp)
    106             except:

17 frames
TypeError: only integer scalar arrays can be converted to a scalar index

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/kerastuner/engine/hypermodel.py in build(self, hp)
    113                 if i == self._max_fail_streak:
    114                     raise RuntimeError(
--> 115                         'Too many failed attempts to build model.')
    116                 continue
    117 

RuntimeError: Too many failed attempts to build model.
1
pythonmachine-learningkerasdeep-learninglstm
At what line you are getting this error ? - KnowledgeGainer
The overwrite=True) line - jeep740
Can you include the complete error log into the question after running this script. - KnowledgeGainer
I have added the error log - jeep740
I think this line should be: LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train.shape[1], 1)) - KnowledgeGainer

1 Answers

2
votes

Your code should look like:

def build(hp):


    activation = hp.Choice('activation', 
                        [
                          'relu',
                          'tanh',
                          'linear',
                          'selu',
                          'elu'
                        ])

    num_rnn_layers = hp.Int(
                        'num_rnn_layers', 
                        min_value=0,
                        max_value=12,
                        default=3)

    recurrent_dropout = hp.Float(
                        'recurrent_dropout', 
                        min_value=0.0,
                        max_value=0.99,
                        default=0.2)
    num_units = hp.Int(
                        'num_units', 
                        min_value=0,
                        max_value=64,
                        default=32)
    
    model = Sequential()
    model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train.shape[1], 1)))

    model.add(Dense(1))
    model.compile(loss='mse', metrics=['mse'], optimizer=keras.optimizers.Adam(
      hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])))



    model.compile(
      optimizer=keras.optimizers.Adam(
      hp.Float(
        'learning_rate',
        min_value=1e-10,
        max_value=1e-2,
        sampling='LOG',
        default=1e-6
            ),

        ),
        loss=tf.losses.MeanSquaredError(),
        metrics=[tf.metrics.MeanAbsoluteError()]
    )
    return model

bayesian_opt_tuner = BayesianOptimization(
    build,
    objective='mse',
    max_trials=3,
    executions_per_trial=1,
    directory=os.path.normpath('C:/keras_tuning'),
    project_name='kerastuner_bayesian_poc',
    overwrite=True)
n_epochs=100

bayesian_opt_tuner.search(X_train, y_train,epochs=n_epochs,
     validation_data=(X_val, y_val),
     validation_split=0.2,verbose=1)


bayes_opt_model_best_model = bayesian_opt_tuner.get_best_models(num_models=1)
model = bayes_opt_model_best_model[0]

This line is causing the problem I think:

model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train[1], 1)))

Change it to this:

model.add(LSTM(units=num_units, activation=activation, recurrent_dropout = recurrent_dropout,input_shape=(X_train.shape[1], 1)))