self.embed = Sequential([Embedding(9488, output_dim=512,input_length=14),
Activation('relu'),
Dropout(0.5)], name='embed.0')
self.fc_embed = Sequential([Dense(512, input_shape=(10,2048)),
Activation('relu'),
Dropout(0.5)], name='fc_embed.0')
inputs_bedding = Input(shape=(10,))
xt = self.embed(inputs_bedding)
input_feats = Input(shape=(10,2048))
fc_feats = self.fc_embed(input_feats)
fc_feats_new = K.reshape(fc_feats, [fc_feats.shape[1], fc_feats.shape[2]])
xt_new = K.reshape(xt, [xt.shape[1], xt.shape[2]])
prev_h = state[0][-1] (shape is (10,512))
att_lstm_input = Concatenate([prev_h, fc_feats_new, xt_new], axis=1)
lstm, h_att, c_att = LSTM(units=512, name='core.att_lstm', return_state=True)(att_lstm_input)
model = Model([input_feats, inputs_att, inputs_bedding], lstm)
model.summary()
This is the error I get:
File "copy_eval.py", line 165, in <module>
model1 = TopDownModel.forward(fc_feats, att_feats, seq, att_masks)
File "/home/ubuntu/misc/customize_keras.py", line 127, in forward
lstm, h_att, c_att = LSTM(units=512, name='core.att_lstm', return_state=True)(att_lstm_input)
File "/usr/local/lib/python2.7/dist-packages/keras/layers/recurrent.py", line 500, in call
return super(RNN, self).call(inputs, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/keras/engine/topology.py", line 575, in call
self.assert_input_compatibility(inputs)
File "/usr/local/lib/python2.7/dist-packages/keras/engine/topology.py", line 448, in assert_input_compatibility
str(inputs) + '. All inputs to the layer '
ValueError: Layer core.att_lstm was called with an input that isn't a symbolic tensor. Received type: . Full input: []. All inputs to the layer should be tensors.
For more input, how to merge them into one output?
Concatenate should be used as a layer, like this:
att_lstm_input = Concatenate(axis=1)([prev_h, fc_feats_new, xt_new])
Related
The environment is Python 3.7.6, below is my import:
import os, sys
import tensorflow as tf # v2.2.0
tf.compat.v1.enable_eager_execution()
import numpy as np
import matplotlib.pyplot as plt
from sys import platform
import time
import random
import pickle
from tensorflow.keras.layers import ReLU
I tried to clone a tf.keras.Model without success, because ReLU is an unknown activation. Nevertheless, the initiation was successful, so the system should know what ReLU is. I wonder how to fix this.
def init_model(D=8, W=256):
# This is a simple MLP neural network
# D: The number of layers
# H: The neurons in each layer
relu = ReLU()
dense = lambda W=W, act=relu: tf.keras.layers.Dense(W, activation=act, dtype='float32')
inputs = tf.keras.Input(shape=(3 + 3 * 2 * L_embed))
outputs = inputs
for i in range(D):
outputs = dense()(outputs)
if i % 4 == 0 and i > 0:
outputs = tf.concat([outputs, inputs], -1)
outputs = dense(4, act=None)(outputs)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
return model
Then I called:
model_inner_copy = tf.keras.models.clone_model(model)
The error message is:
File "D:/Code Repository/Meta-NeRF/Code/NeRF/Tiny_MAML_NeRF.py", line 211, in train_maml_nerf
model_inner_copy = tf.keras.models.clone_model(model)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\models.py", line 427, in clone_model
model, input_tensors=input_tensors, layer_fn=clone_function)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\models.py", line 196, in _clone_functional_model
model, new_input_layers, layer_fn)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\models.py", line 246, in _clone_layers_and_model_config
config = network.get_network_config(model, serialize_layer_fn=_copy_layer)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\engine\network.py", line 2119, in get_network_config
layer_config = serialize_layer_fn(layer)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\models.py", line 243, in _copy_layer
created_layers[layer.name] = layer_fn(layer)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\models.py", line 61, in _clone_layer
return layer.__class__.from_config(layer.get_config())
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\engine\base_layer.py", line 655, in from_config
return cls(**config)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\layers\core.py", line 1135, in __init__
self.activation = activations.get(activation)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\activations.py", line 465, in get
identifier, printable_module_name='activation')
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\utils\generic_utils.py", line 362, in deserialize_keras_object
config, module_objects, custom_objects, printable_module_name)
File "C:\Users\Jack\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\keras\utils\generic_utils.py", line 321, in class_and_config_for_serialized_keras_object
raise ValueError('Unknown ' + printable_module_name + ': ' + class_name)
ValueError: Unknown activation: ReLU
So the problem is that tf.keras.layers.ReLU is a layer that implements the ReLU activation, but it is not an activation function by itself. It is meant to be used as a layer inside your model, not as a parameter of your Dense layer.
To have a function that works as an activation to give as a parameter to Dense, you should use tf.keras.activations.relu.
I've tried running the following code, but got this error:
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training.py",
line 819, in fit
use_multiprocessing=use_multiprocessing)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py",
line 235, in fit
use_multiprocessing=use_multiprocessing)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py",
line 593, in _process_training_inputs
use_multiprocessing=use_multiprocessing)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py",
line 706, in _process_inputs
use_multiprocessing=use_multiprocessing)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\data_adapter.py",
line 702, in init
x = standardize_function(x)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py",
line 660, in standardize_function
standardize(dataset, extract_tensors_from_dataset=False)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training.py",
line 2346, in _standardize_user_data
all_inputs, y_input, dict_inputs = self._build_model_with_inputs(x, y)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training.py",
line 2572, in _build_model_with_inputs
self._set_inputs(cast_inputs)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training.py",
line 2647, in _set_inputs
inputs = self._set_input_attrs(inputs)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\training\tracking\base.py",
line 457, in _method_wrapper
result = method(self, *args, **kwargs)
File
"C:\Users\TomerK\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\training.py",
line 2681, in _set_input_attrs
raise ValueError('Passing a dictionary input to a Sequential Model '
ValueError: Passing a dictionary input to a Sequential Model which
doesn't have FeatureLayer as the first layer is an error.
Code:
# -*- coding: utf-8 -*-
import os
#os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
import tensorflow_datasets as tfds
try:
model = keras.models.load_model("passrockmodel.h5")
except:
print('\nDownloading Train Dataset...\n')
train_dataset = tfds.load(name="rock_you", split="train[:75%]")
assert isinstance(train_dataset, tf.data.Dataset)
print('\nDownloading Test Dataset...\n')
test_dataset = tfds.load("rock_you", split='train[-25%:]')
assert isinstance(test_dataset, tf.data.Dataset)
model = tf.keras.Sequential([
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid'),
])
model.compile(
loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(train_dataset, epochs=20)
model.save("passrockmodel.h5")
test_loss, test_accuracy = model.evaluate(test_dataset)
print('\nPredicting...\n')
predictions = model.predict(test_dataset)
print(predictions[0])
I've had your problem yesterday. Here's what solved it for me:
your first layer should be of type tf.keras.layers.DenseFeatures
This first layer must be instantiated with an array of tf.feature_column objects. It happens that all my columns were numeric so my array was:
featureColumns = [tf.feature_column.numeric_column(columnNames[i], normalizer_fn= lambda x: (x - mean[i])/std[i]) for i in range(len(columnNames[:-1]))]
Note: the normalizer_fn arg is very useful as you can see, as well. It can eliminate the need of any additional normalization preprocessing layer should you need it.
And so my layer became:
layers.DenseFeatures(feature_columns=featureColumns, trainable=True)
I believe this should solve the error mentioned above in your question. Which is quoted as
ValueError: Passing a dictionary input to a Sequential Model which
doesn't have FeatureLayer as the first layer is an error.
I'm attempting to train a regression model to predict attributes of music such as BPM. The model takes in spectrograms of audio snippets that are 256x128px png files and outputs a couple continuous values. I have the following code so far that I have developed based upon this guide on the tensorflow website:
import tensorflow as tf
import os
import random
import pathlib
AUTOTUNE = tf.data.experimental.AUTOTUNE
TRAINING_DATA_DIR = r'specgrams'
def gen_model():
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(256, 128, 3)),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dense(2)
])
model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),
loss='mse',
metrics=['mse', 'mae'])
return model
def fetch_batch(batch_size=1000):
all_image_paths = []
all_image_labels = []
data_root = pathlib.Path(TRAINING_DATA_DIR)
files = data_root.iterdir()
for file in files:
file = str(file)
all_image_paths.append(os.path.abspath(file))
label = file[:-4].split('-')[2:]
label = float(label[0]) / 200, int(label[1]) / 1000.0
all_image_labels.append(label)
def preprocess_image(path):
img_raw = tf.io.read_file(path)
image = tf.image.decode_png(img_raw, channels=3)
image = tf.image.resize(image, [256, 128])
image /= 255.0
return image
def preprocess(path, label):
return preprocess_image(path), label
path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)
image_ds = path_ds.map(preprocess_image, num_parallel_calls=AUTOTUNE)
label_ds = tf.data.Dataset.from_tensor_slices(all_image_labels)
ds = tf.data.Dataset.zip((image_ds, label_ds))
ds = ds.shuffle(buffer_size=len(os.listdir(TRAINING_DATA_DIR)))
ds = ds.repeat()
ds = ds.batch(batch_size)
ds = ds.prefetch(buffer_size=AUTOTUNE)
return ds
ds = fetch_batch()
model = gen_model()
model.fit(ds, epochs=1, steps_per_epoch=10)
However I believe I have made a mistake with the structure of my model or how I am preprocessing the training data because I get an error about incorrect dimensions but I'm struggling to narrow down exactly where the issue is. I understand that the guide I followed was for classification problem as opposed to regression and my "labels" are an array of 2 value which is what is causing the problem but I'm not sure how to resolve this.
For context the filenames are in the format xxx-xxx-A-B.png where A and B are the two desired output values of the model. A is a floating-point value somewhere between 70 and 180 and B is an integer value between 0-1000. As such the label variable for each image looks something like this: (0.64, 0.319).
This is the error I am seeing when I attempt to execute the above script:
Traceback (most recent call last):
File "C:\Users\cainy\Desktop\BeatNet\training.py", line 60, in <module>
model.fit(ds, epochs=1, steps_per_epoch=3)
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\training.py", line 791, in fit
initial_epoch=initial_epoch)
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1515, in fit_generator
steps_name='steps_per_epoch')
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\training_generator.py", line 257, in model_iteration
batch_outs = batch_function(*batch_data)
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1259, in train_on_batch
outputs = self._fit_function(ins) # pylint: disable=not-callable
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\backend.py", line 3217, in __call__
outputs = self._graph_fn(*converted_inputs)
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\function.py", line 558, in __call__
return self._call_flat(args)
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\function.py", line 627, in _call_flat
outputs = self._inference_function.call(ctx, args)
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\function.py", line 415, in call
ctx=ctx)
File "C:\Users\cainy\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\execute.py", line 66, in quick_execute
six.raise_from(core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: Can not squeeze dim[1], expected a dimension of 1, got 2
[[{{node metrics/accuracy/Squeeze}}]] [Op:__inference_keras_scratch_graph_734]
Edit: I have uploaded the source code to GitHub here.
You currently only have 1 output - a tensor with length 2 (per batch element). If you want to use/monitor separate losses you'll need to unstack it in both the model output and the labels.
I'm not sure if models.Sequential will be suitable, but you can definitely use the functional API:
def gen_model():
inputs = tf.keras.layers.Input(shape=(256, 128, 3), dtype=tf.float32)
x = inputs
x = tf.keras.layers.Dense(256, activation='relu')
x = tf.keras.layers.Dense(2)
a, b = tf.keras.layers.Lambda(tf.unstack, arguments=dict(axis=-1))(x)
model = tf.keras.models.Model(inputs=inputs, outputs=[a, b])
model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),
loss=['mse', 'mae'],
metrics=[['mse'], ['mae']])
return model
And in your preprocessing:
def preprocess(path, label):
return preprocess_image(path), tf.unstack(label, axis=-1)
I'm trying to create a recurrent neural network with the Keras functional API in TensorFlow. The RNN takes in tweets and classifies them as positive or negative.
attention_input = keras.Input(shape=(512,), name='attention')
a = keras.layers.Dense(1, activation='sigmoid')(attention_input)
attention_output = keras.layers.Multiply()([attention_input, a])
attention = keras.Model(inputs=attention_input, outputs=attention_output, name='attention_model')
inputs1 = keras.Input(shape=(100,), name='lstm')
x = keras.layers.Embedding(len(tokenizer.word_counts)+1,
100,
weights=[embedding_matrix],
input_length=100,
trainable=True)(inputs1)
x = keras.layers.Bidirectional(tf.keras.layers.LSTM(256, return_sequences=True))(x)
x = keras.layers.TimeDistributed(attention)(x)
x = tf.unstack(x, num=256)
t_sum = x[0]
for i in range(256 - 1):
t_sum = keras.layers.Add()([t_sum, x[i+1]])
lstm = keras.Model(inputs=inputs1, outputs=t_sum, name='lstm_model')
inputs2 = keras.Input(shape=(100,), name='dense')
x = keras.layers.Dense(256, activation='relu')(inputs2)
x = keras.layers.Dropout(0.2)(x)
x = keras.layers.Dense(128, activation='relu')(x)
x = keras.layers.Dropout(0.2)(x)
outputs2 = keras.layers.Dense(1, activation='sigmoid')(x)
dense = keras.Model(inputs=inputs2, outputs=outputs2, name='txt_model')
inputs = keras.Input(shape=(100,), name='text')
x = lstm(inputs)
outputs = dense(x)
model = keras.Model(inputs=inputs, outputs=outputs, name='text_model')
model.compile(
loss = 'binary_crossentropy',
optimizer = 'adam',
metrics = ['acc',
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall()])
I get the following runtime error
2019-04-13 10:29:34.855192: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
Traceback (most recent call last):
File ".\main.py", line 25, in <module>
' -> '.join(permutation).lower() : { ** results.get(' -> '.join(permutation).lower(), {}), ** framework.runtime.evaluate(path, permutation, classifiers, cached) }
File "C:\Users\steff\Desktop\Skole\MsT\framework\framework\runtime.py", line 30, in evaluate
classifier.lower() : framework.classifiers.list[classifier.lower()](data)
File "C:\Users\steff\Desktop\Skole\MsT\framework\framework\classifiers\rnn.py", line 93, in evaluate
x = lstm(inputs)
File "C:\Users\steff\AppData\Roaming\Python\Python36\site-packages\tensorflow\python\keras\engine\base_layer.py", line 612, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "C:\Users\steff\AppData\Roaming\Python\Python36\site-packages\tensorflow\python\keras\engine\network.py", line 870, in call
return self._run_internal_graph(inputs, training=training, mask=mask)
File "C:\Users\steff\AppData\Roaming\Python\Python36\site-packages\tensorflow\python\keras\engine\network.py", line 1011, in _run_internal_graph
output_tensors = layer(computed_tensors, **kwargs)
File "C:\Users\steff\AppData\Roaming\Python\Python36\site-packages\tensorflow\python\keras\engine\base_layer.py", line 669, in __call__
self.set_weights(self._initial_weights)
File "C:\Users\steff\AppData\Roaming\Python\Python36\site-packages\tensorflow\python\keras\engine\base_layer.py", line 938, in set_weights
param_values = backend.batch_get_value(params)
File "C:\Users\steff\AppData\Roaming\Python\Python36\site-packages\tensorflow\python\keras\backend.py", line 2837, in batch_get_value
raise RuntimeError('Cannot get value inside Tensorflow graph function.')
RuntimeError: Cannot get value inside Tensorflow graph function.
I can see from the errors that it has something to do with my LSTM model, but I can't see what is the cause of the problem.
I think that you are using Tensorflow 2.0. If this is the case then using the parameter embeddings_initializer= instead of weights= worked.
x = tf.keras.layers.Embedding(vocabulary_size, embedding_dim, embeddings_initializer=tf.keras.initializers.Constant(embedding_matrix), trainable=False)
I'm trying to create a LSTM model. Before passing the data to the first LSTM layer, I want to add a Masking layer. I am able to do this using Sequential approach in Keras. See example. However when I try to code it differently I get a value error (see below). Any Idea on how to fix this?
import keras
def network_structure(window_len, n_features, lstm_neurons):
masking = keras.layers.Masking(
mask_value=0.0, input_shape=(window_len, n_features)
)
lstm_h1 = keras.layers.LSTM(lstm_neurons)(masking)
lstm_h2 = keras.layers.LSTM(lstm_neurons)(lstm_h1)
cte = keras.layers.Dense(
1,
activation='linear',
name='CTE',
)(lstm_h2)
ate = keras.layers.Dense(
1,
activation='linear',
name='ATE',
)(lstm_h2)
pae = keras.layers.Dense(
1,
activation='linear',
name='PAE',
)(lstm_h2)
model = keras.models.Model(
inputs=masking,
outputs=[cte, ate, pae]
)
model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mae'])
model.summary()
return model
model = network_structure(32, 44, 125)
Error Message:
Using TensorFlow backend.
Traceback (most recent call last):
File "C:\Python35\lib\site-packages\keras\engine\topology.py", line 442, in assert_input_compatibility
K.is_keras_tensor(x)
File "C:\Python35\lib\site-packages\keras\backend\tensorflow_backend.py", line 468, in is_keras_tensor
raise ValueError('Unexpectedly found an instance of type `' + str(type(x)) + '`. '
ValueError: Unexpectedly found an instance of type `<class 'keras.layers.core.Masking'>`. Expected a symbolic tensor instance.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/Master Tk/PycharmProjects/FPL/testcompile.py", line 46, in <module>
model = network_structure(32, 44, 125)
File "C:/Users/Master Tk/PycharmProjects/FPL/testcompile.py", line 12, in network_structure
lstm_h1 = keras.layers.LSTM(lstm_neurons)(masking)
File "C:\Python35\lib\site-packages\keras\layers\recurrent.py", line 499, in __call__
return super(RNN, self).__call__(inputs, **kwargs)
File "C:\Python35\lib\site-packages\keras\engine\topology.py", line 575, in __call__
self.assert_input_compatibility(inputs)
File "C:\Python35\lib\site-packages\keras\engine\topology.py", line 448, in assert_input_compatibility
str(inputs) + '. All inputs to the layer '
ValueError: Layer lstm_1 was called with an input that isn't a symbolic tensor. Received type: <class 'keras.layers.core.Masking'>. Full input: [<keras.layers.core.Masking object at 0x000002224683A780>]. All inputs to the layer should be tensors.
You have forgotten to create an input layer. First define the input layer and then pass the placeholder tensor to the Masking layer:
inp = Input(shape=(window_len, n_features))
masking = keras.layers.Masking(mask_value=0.0)(inp)
lstm_h1 = keras.layers.LSTM(lstm_neurons)(masking)
And don't forget to change the model definition accordingly by passing the input tensor as the inputs argument:
model = keras.models.Model(inputs=inp, outputs=[cte, ate, pae])