I am trying to use make_template() to avoid passing reuse flag throughout my model. But it seems that make_template() doesn't work correctly when it is used inside of a python class. I pasted ]my model code and the error I am getting below. It is a simple MLP to train on the MNIST dataset.
Since the code is kinda long, the main part here is the _weights() function. I try to wrap it using make_template() and then use get_variables() inside it to create and reuse weights throughout my model. _weights() is used by _create_dense_layer() and that in turn is used by _create_model() to create the graph. The train() function accepts tensors that I get from a data reader.
Model
class MLP(object):
def __init__(self, hidden=[], biases=False, activation=tf.nn.relu):
self.graph = tf.get_default_graph()
self.hidden = hidden
self.activation = activation
self.biases = biases
self.n_features = 784
self.n_classes = 10
self.bsize = 100
self.l2 = 0.1
def _real_weights(self, shape):
initializer=tf.truncated_normal_initializer(stddev=0.1)
weights = tf.get_variable('weights', shape, initializer=initializer)
return weights
# use make_template to make variable reuse transparent
_weights = tf.make_template('_weights', _real_weights)
def _real_biases(self, shape):
initializer=tf.constant_initializer(0.0)
return tf.get_variable('biases', shape, initializer=initializer)
# use make_template to make variable reuse transparent
_biases = tf.make_template('_biases', _real_biases)
def _create_dense_layer(self, name, inputs, n_in, n_out, activation=True):
with tf.variable_scope(name):
weights = self._weights([n_in, n_out])
layer = tf.matmul(inputs, weights)
if self.biases:
biases = self._biases([n_out])
layer = layer + biases
if activation:
layer = self.activation(layer)
return layer
def _create_model(self, inputs):
n_in = self.n_features
for i in range(len(self.hidden)):
n_out = self.hidden[i]
name = 'hidden%d' % (i)
inputs = self._create_dense_layer(name, inputs, n_in, n_out)
n_in = n_out
output = self._create_dense_layer('output', inputs, n_in, self.n_classes, activation=False)
return output
def _create_loss_op(self, logits, labels):
cent = tf.nn.softmax_cross_entropy_with_logits(logits, labels)
weights = self.graph.get_collection('weights')
l2 = (self.l2 / self.bsize) * tf.reduce_sum([tf.reduce_sum(tf.square(w)) for w in weights])
return tf.reduce_mean(cent, name='loss') + l2
def _create_train_op(self, loss):
optimizer = tf.train.AdamOptimizer()
return optimizer.minimize(loss)
def _create_accuracy_op(self, logits, labels):
predictions = tf.nn.softmax(logits)
errors = tf.equal(tf.argmax(predictions, 1), tf.argmax(labels, 1))
return tf.reduce_mean(tf.cast(errors, tf.float32))
def train(self, images, labels):
logits = model._create_model(images)
loss = model._create_loss_op(logits, labels)
return model._create_train_op(loss)
def accuracy(self, images, labels):
logits = model._create_model(images)
return model._create_accuracy_op(logits, labels)
def predict(self, images):
return model._create_model(images)
The error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
in ()
25 model = MLP(hidden=[128])
26 # define ops
---> 27 train = model.train(images, labels)
28 accuracy = model.accuracy(eval_images, eval_labels)
29 # load test data and create a prediction op
in train(self, images, labels)
60
61 def train(self, images, labels):
---> 62 logits = model._create_model(images)
63 loss = model._create_loss_op(logits, labels)
64 return model._create_train_op(loss)
in _create_model(self, inputs)
39 n_out = self.hidden[i]
40 name = 'hidden%d' % (i)
---> 41 inputs = self._create_dense_layer(name, inputs, n_in, n_out)
42 n_in = n_out
43 output = self._create_dense_layer('output', inputs, n_in, self.n_classes, activation=False)
in _create_dense_layer(self, name, inputs, n_in, n_out, activation)
25 def _create_dense_layer(self, name, inputs, n_in, n_out, activation=True):
26 with tf.variable_scope(name):
---> 27 weights = self._weights([n_in, n_out])
28 layer = tf.matmul(inputs, weights)
29 if self.biases:
/usr/local/lib/python3.5/site-packages/tensorflow/python/ops/template.py in __call__(self, *args, **kwargs)
265 self._unique_name, self._name) as vs:
266 self._var_scope = vs
--> 267 return self._call_func(args, kwargs, check_for_new_variables=False)
268
269 #property
/usr/local/lib/python3.5/site-packages/tensorflow/python/ops/template.py in _call_func(self, args, kwargs, check_for_new_variables)
206 ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
207
--> 208 result = self._func(*args, **kwargs)
209 if check_for_new_variables:
210 trainable_variables = ops.get_collection(
TypeError: _real_weights() missing 1 required positional argument: 'shape'
originally defined at:
File "", line 1, in
class MLP(object):
File "", line 17, in MLP
_weights = tf.make_template('_weights', _real_weights)
There are multiple problems with this code as it is here, e.g. the model references in the train, accuracy and predict methods. I assume this is due to cutting the code from its natural habitat.
The reason for the TypeError you mention,
TypeError: _real_weights() missing 1 required positional argument: 'shape'
most likely comes from the fact that _real_weights itself is an instance method of the MLP class, not a regular function or static method. As such, the first parameter to the function is always the self reference pointing to the instance of the class at the time of the call (an explicit version of the this pointer in C-like languages), as can be seen in the function declaration:
def _real_weights(self, shape):
initializer=tf.truncated_normal_initializer(stddev=0.1)
weights = tf.get_variable('weights', shape, initializer=initializer)
return weights
Note that even though you don't use the argument, it's still required in this case. Thus when creating a template of the function using
tf.make_template('_weights', self._real_weights)
you basically state that the _weights template you create should take two positional arguments: self and weights (as does the _real_weights method). Consequently, when you call the function created from the template as
weights = self._weights([n_in, n_out])
you pass the array to the self argument, leaving the (required) shape argument unspecified.
From what it looks like you'd have two options here: You could either make _real_weights a regular function outside of the MLP class, so that
def _real_weights(shape):
initializer=tf.truncated_normal_initializer(stddev=0.1)
weights = tf.get_variable('weights', shape, initializer=initializer)
return weights
class MLP():
# etc.
which is probably not what you want, given that you already created a class for the model - or you could explicitly make it a static method of the MLP class, so that
class MLP():
#staticmethod
def _real_weights(shape):
initializer=tf.truncated_normal_initializer(stddev=0.1)
weights = tf.get_variable('weights', shape, initializer=initializer)
return weights
Since static methods by definition do not operate on a class instance, you can (and have to) omit the self reference.
You would then create the templates as
tf.make_template('_weights', _real_weights)
in the first case and
tf.make_template('_weights', MLP._real_weights)
in the second case, explicitly specifying the class MLP as the name scope of the static method. Either way, the _real_weights function/method and the _weights template both now have only one argument, the shape of the variable to create.
Related
I just need a little help regarding my project. I've already written a code but I am facing an error in that. I am using a few-shot learning technique triplet neural network. The triplet neural network(TNN) is a horizontal concatenation triplet consisting of three identical Convolutional Neural Networks (with common parameters) that are trained with triplets of inputs. An anchor instance, a positive instance (of the same class as the anchor), and a negative instance make up the input triplet (different class from the anchor). After that, the network is trained to learn a triplet loss embedding function. To compute triplet loss, three training examples are required. Each triplet is formed by intentionally selecting training examples such that each triplet has:
• a reference image called anchor image
• an image having the same label as anchor is called a positive image
• an image has a different label than the anchor called a negative image.
The TNN learns to create k-dimensional feature vector representation of images in such a manner that similar images lie closer in an embedding space of k-dimensions.
embedding_dimension = 128
from tensorflow.keras.applications.vgg16 import VGG16
pre_trained_vgg16 = tf.keras.applications.VGG16(
input_shape=(size, size,3),
include_top=False,
weights="imagenet",
input_tensor=None,
pooling=None,
classes=1000,
classifier_activation=None
)
pre_trained_vgg16.save('vgg16.h5')
pre_trained_vgg16 = tf.keras.models.load_model('vgg16.h5')
def build_embedding_network(embedding_dimension):
embedding_network = pre_trained_vgg16
embedding_network.trainable = False
x = embedding_network.output
x = tf.keras.layers.GlobalAveragePooling2D()(x)
x = tf.keras.layers.Flatten()(x)
x = tf.keras.layers.Dropout(0.2)(x)
x = tf.keras.layers.Dense(2*embedding_dimension, activation='sigmoid')(x)
x = tf.keras.layers.Dense(embedding_dimension, activation='sigmoid')(x)
embedding_network = tf.keras.Model(embedding_network.input, x, name="embedding_network")
return embedding_network
def build_metric_network(single_embedding_dim):
input1 = tf.keras.layers.Input((single_embedding_dim), name="input1")
input2 = tf.keras.layers.Input((single_embedding_dim), name="input2")
embedded_distance = tf.keras.layers.Subtract(name='subtract_embeddings')([input1, input2])
embedded_distance = tf.keras.layers.Lambda(lambda x: K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True)),
name='euclidean_distance')(embedded_distance)
metric_network = tf.keras.Model(inputs=[input1, input2],
outputs=[embedded_distance],
name="metric_network")
return metric_network
class TripletLossLayer(tf.keras.layers.Layer):
def __init__(self, margin, **kwargs):
self.margin = margin
super(TripletLossLayer, self).__init__(**kwargs)
def triplet_loss(self, inputs):
ap_dist, an_dist = inputs
#square
ap_dist2 = K.square(ap_dist)
an_dist2 = K.square(an_dist)
return K.sum(K.maximum(ap_dist2 - an_dist2 + self.margin, 0))
def call(self, inputs):
loss = self.triplet_loss(inputs)
self.add_loss(loss)
return loss
def get_config(self):
config = super().get_config().copy()
config.update({
'margin': self.margin
})
return config
def build_triplet_snn(input_shape, embedding_network, metric_network, margin=0.1):
# Define the tensors for the three input images
anchor_input = tf.keras.layers.Input(input_shape, name="anchor_input")
positive_input = tf.keras.layers.Input(input_shape, name="positive_input")
negative_input = tf.keras.layers.Input(input_shape, name="negative_input")
# Generate the embeddings (feature vectors) for the three images
embedding_a = embedding_network(anchor_input)
embedding_p = embedding_network(positive_input)
embedding_n = embedding_network(negative_input)
ap_dist = metric_network([embedding_a,embedding_p])
an_dist = metric_network([embedding_a,embedding_n])
# Triplet loss layer
loss_layer = TripletLossLayer(margin=margin, name='TripletLossLayer')([ap_dist, an_dist])
# Compute the concatenated pairs
all_concatenated = tf.keras.layers.Concatenate(axis=-1,name="All-Embeddings")([embedding_a,embedding_p,embedding_n])
# Connect the inputs with the outputs
triplet_snn = tf.keras.Model(inputs=[anchor_input, positive_input, negative_input],
outputs=[loss_layer, all_concatenated],
name="triplet_snn")
# Return the model
return triplet_snn
embedding_network = build_embedding_network(embedding_dimension)
metric_network = build_metric_network(embedding_dimension)
triplet_snn = build_triplet_snn(
input_shape=(size, size,3),
embedding_network=embedding_network,
metric_network=metric_network,
margin=0.1
)
learning_rate = 0.0001
epochs = 5
class TripletDataGenerator(tf.keras.utils.Sequence):
def __init__(self, triplet_dataset, shuffle=False):
self.triplet_dataset = triplet_dataset
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
return len(self.triplet_dataset)
def __getitem__(self, index):
return triplet_dataset[index][0]
#return (np.array(triplet_dataset[index][0]).reshape(1,224,224,3))
def on_epoch_end(self):
if self.shuffle == True:
random.shuffle(self.triplet_dataset)
data_gen = TripletDataGenerator(triplet_dataset)
filepath = 'C:\\Users\\Y540\\Desktop\\Retinal Disease\\TrainedSNN\\temp\\weights.{epoch}'
save_model_weights_at_every_epoch = tf.keras.callbacks.ModelCheckpoint(
filepath,
monitor="loss",
verbose=1,
save_best_only=False,
save_weights_only=True,
mode="auto",
save_freq="epoch"
)
optimizer = tf.keras.optimizers.Adam(lr=learning_rate)
triplet_snn.compile(loss=None, optimizer=optimizer, run_eagerly=True)
%%time
history = triplet_snn.fit(data_gen, epochs=epochs, verbose=1, callbacks=[save_model_weights_at_every_epoch])
The error I am getting is:
this is the error that I am getting
Epoch 1/5
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<timed exec> in <module>
~\anaconda3\lib\site-packages\keras\utils\traceback_utils.py in error_handler(*args, **kwargs)
65 except Exception as e: # pylint: disable=broad-except
66 filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67 raise e.with_traceback(filtered_tb) from None
68 finally:
69 del filtered_tb
~\anaconda3\lib\site-packages\keras\engine\input_spec.py in assert_input_compatibility(input_spec, inputs, layer_name)
198
199 if len(inputs) != len(input_spec):
--> 200 raise ValueError(f'Layer "{layer_name}" expects {len(input_spec)} input(s),'
201 f' but it received {len(inputs)} input tensors. '
202 f'Inputs received: {inputs}')
ValueError: Layer "triplet_snn" expects 3 input(s), but it received 1 input tensors. Inputs received: [<tf.Tensor: shape=(1, 224, 224, 3), dtype=float32, numpy=
I got a custom keras Model which I want to optimize for hyperparameters while having a good tracking of whats going on and visualization. Therefor I want to pass hparams to the custom model like this:
class Model_hparams(tf.keras.Model):
def __init__(self, hparams):
super(Model_hparams, self).__init__()
self.hps = hparams
def build(self, inputs_shape):
self.conv1 = tf.keras.layers.Conv1D(filters=self.hps[HP_NUM_UNITS_1],
kernel_size=self.hps[HP_LEN_CONV_1],
activation='relu',
input_shape=inputs_shape[1:])
self.pool1 = tf.keras.layers.MaxPool1D(pool_size=2)
self.bn1 = tf.keras.layers.BatchNormalization()
self.dense1 = tf.keras.layers.Dense(1)
# actually, here are even more layers
def call(self, x, training=True):
x = self.conv1(x)
x = self.pool1(x)
x = self.bn1(x, training=training)
x = self.dense1(x)
return x
I followed the guide from TF:
from tensorboard.plugins.hparams import api as hp
HP_NUM_UNITS_1 = hp.HParam('num_units_1', hp.Discrete([16, 32]))
HP_LEN_CONV_1 = hp.HParam('len_conv_1', hp.Discrete([3]))
METRIC = 'mae'
with tf.summary.create_file_writer("../../model_output/hparams").as_default():
hp.hparams_config(
hparams=[HP_NUM_UNITS_1,
HP_LEN_CONV_1,],
metrics=[hp.Metric(METRIC, display_name='Test_MAE')],
)
def run(run_dir, hparams):
with tf.summary.create_file_writer(run_dir).as_default():
hp.hparams(hparams) # record the values used in this trial
test_mae = train_model(hparams)
tf.summary.scalar('Mean_Average_Error', test_mae, step=1)
Now my training fuction calls the model with my training procedure which looks like this (simplified):
def train_model(hparams):
model=Model_hparams(hparams)
for batch in dataset:
#...
with tf.GradientTape() as tape:
predictions = model(batch, training=True)
#...
The actual optimization starts here:
n=0
for num_units_1 in HP_NUM_UNITS_1.domain.values:
for len_conv_1 in HP_LEN_CONV_1.domain.values:
hparams = {HP_NUM_UNITS_1: num_units_1,
HP_LEN_CONV_1: len_conv_1}
run_name = "run-%d" % n
run("../../model_output/hparams/" + run_name, hparams)
n += 1
However, if I run this, an error occures when I want to instantiate my model:
<ipython-input-99-17dd66300f5b> in __init__(self, hparams)
72 def __init__(self, hparams):
73 super(Model_hparams, self).__init__()
---> 74 self.hps = hparams
75
76 def build(self, inputs_shape):
c:\users\123\anaconda3\envs\python_3_8_env1\lib\site-packages\tensorflow\python\keras\engine\training.py in __setattr__(self, name, value)
312 isinstance(v, (base_layer.Layer,
313 data_structures.TrackableDataStructure)) or
--> 314 base_layer_utils.has_weights(v) for v in nest.flatten(value)):
315 try:
316 self._base_model_initialized
c:\users\123\anaconda3\envs\python_3_8_env1\lib\site-packages\tensorflow\python\util\nest.py in flatten(structure, expand_composites)
339 return [None]
340 expand_composites = bool(expand_composites)
--> 341 return _pywrap_utils.Flatten(structure, expand_composites)
342
343
TypeError: '<' not supported between instances of 'HParam' and 'HParam'
I´m not sure why this happens and I cannot get it to work. I cannot find anything in the docs.
Is there anything I´m missing??
Thanks for the support.
tf.keras.Model class overrides __setattr__ function, so you can not set mismatched variables. However, you can bypass this function below trick.
object.__setattr__(self, 'hps', hparams)
.. instead of
self.hps = hparams
class Model_hparams(tf.keras.Model):
def __init__(self, hparams):
super(Model_hparams, self).__init__()
object.__setattr__(self, 'hps', hparams)
When I want to do a forward pass with an initialized model = nn.Sequential object, I simply use:
out = model(X)
# OR
out = model.forward(X)
However, I have tried extending the Sequential class, and now both of these methods suddenly require a second argument. For example, note in the following method my call to self(x):
def train(self, trainloader, epochs):
for e in range(epochs):
for x, y in trainloader:
x = x.view(x.shape[0], -1)
self.optimizer.zero_grad()
loss = self.criterion(self(x), y) # CALL OCCURS HERE
loss.backward()
self.optimizer.step()
This code now gives me TypeError: forward() missing 1 required positional argument: 'target'.
My Question: Since I have done nothing but extend the class, why is this?
Code for full class below:
class Network(nn.Sequential):
def __init__(self, layers):
super().__init__(self.init_modules(layers))
self.criterion = nn.NLLLoss()
self.optimizer = optim.Adam(self.parameters(), lr=0.003)
def init_modules(self, layers):
n_layers = len(layers)
modules = OrderedDict()
# Layer definitions for input and inner layers:
for i in range(n_layers - 2):
modules[f'fc{i}'] = nn.Linear(layers[i], layers[i+1])
modules[f'relu{i}'] = nn.ReLU()
# Definition for output layer:
modules['fc_out'] = nn.Linear(layers[-2], layers[-1])
modules['smax_out'] = nn.LogSoftmax(dim=1)
return modules
def train(self, trainloader, epochs):
for e in range(epochs):
for x, y in trainloader:
x = x.view(x.shape[0], -1)
self.optimizer.zero_grad()
loss = self.criterion(self(x), y)
loss.backward()
self.optimizer.step()
Full stack trace:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-63-490e0b9eef22> in <module>
----> 1 model2.train(trainloader, 5, plot_loss=True)
<ipython-input-61-e173e5672f18> in train(self, trainloader, epochs, plot_loss)
32 x = x.view(x.shape[0], -1)
33 self.optimizer.zero_grad()
---> 34 loss = self.criterion(self(x), y)
35 loss.backward()
36 self.optimizer.step()
c:\program files\python38\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
548 result = self._slow_forward(*input, **kwargs)
549 else:
--> 550 result = self.forward(*input, **kwargs)
551 for hook in self._forward_hooks.values():
552 hook_result = hook(self, input, result)
c:\program files\python38\lib\site-packages\torch\nn\modules\container.py in forward(self, input)
98 def forward(self, input):
99 for module in self:
--> 100 input = module(input)
101 return input
102
c:\program files\python38\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
548 result = self._slow_forward(*input, **kwargs)
549 else:
--> 550 result = self.forward(*input, **kwargs)
551 for hook in self._forward_hooks.values():
552 hook_result = hook(self, input, result)
TypeError: forward() missing 1 required positional argument: 'target'
Q: Since I have done nothing but extend the class, why is this?
Actually, you have. You chose the "wrong" base class. The forward of the nn.Sequential simply goes through all modules, and when you defined:
self.criterion = nn.NLLLoss()
you registered the loss as a module. Therefore, when you call self(x), you're actually calling self.criterion(x) at some point, hence the TypeError.
Note that self.criterion is an instance of the class nn.NLLLoss. You need to pass a torch tensor as a second argument, make sure y satisfies that type requirement. Also, I am not sure about this line loss = self.criterion(self(x), y) why call self(x)? The first argument is supposed to be the input data passed through an activation function.
I'm trying to make a neural network in pytorch that has a variable number of layers. My problem is that apparently I am passing some sort of iterable with more than one item to a linear layer which can only take one argument. I just can't see why.
So here is some code. First I created my own module and import it to my notebook later
import torch
class NNet(torch.nn.Module):
def __init__(self, layer_shapes, activation_functions):
super(NNet, self).__init__()
assert len(layer_shapes) == len(activation_functions) + 1
self.layer_shapes = layer_shapes
self.activation_functions = activation_functions
linear_functions = list()
for i in range(len(self.layer_shapes)-1):
linear_functions.append(torch.nn.Linear(
self.layer_shapes[i], self.layer_shapes[i+1]))
self.linear_functions = linear_functions
def parameters(self):
parameters = list()
for function in self.linear_functions:
parameters = parameters+list(function.parameters())
return parameters
def forward(self, x):
assert x.shape[1] == self.layer_shapes[0]
y = x
for i in range(len(self.layer_shapes)-1):
lin = self.linear_functions[i](y)
y = self.activation_functions[i](lin)
return y
In the notebook, the error is in the forward function at y = self.activation_functions[i](self.linear_functions[i](y))
Now I try to use the MNIST dataset supplied by torchvision and use my own module.
batch_size = 100
epochs = 500
learning_rate = 0.001
train_set = torchvision.datasets.MNIST(root =
'../../data',
train=True,
transform=torchvision.transforms.ToTensor(),
download=True)
test_set = torchvision.datasets.MNIST(root =
'../../data',
train=False,
transform=torchvision.transforms.ToTensor(),
download=True)
train_loader = torch.utils.data.DataLoader(dataset=train_set,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_set,
batch_size=batch_size,
shuffle=False)
model = nnet.NNet([784, 16, 10], [torch.nn.Tanh,
torch.nn.Softmax(dim=1)])
loss_function = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
loss_items = list()
for t in range(epochs):
for i, (images, labels) in enumerate(train_loader):
images = images.reshape(-1,28*28)
outputs = model(images)
loss = loss_function(outputs, labels)
loss_items.append(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
This last for loop yields the error:
TypeError Traceback (most recent call last)
<ipython-input-6-4ccb4b105a41> in <module>()
5 images = images.reshape(-1,28*28)
6
----> 7 outputs = model(images)
8 loss = loss_function(outputs, labels)
9 loss_items.append(loss.item())
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in
__call__(self, *input, **kwargs)
475 result = self._slow_forward(*input, **kwargs)
476 else:
--> 477 result = self.forward(*input, **kwargs)
478 for hook in self._forward_hooks.values():
479 hook_result = hook(self, input, result)
~/Desktop/Archive/Computing/Projects/Python/ai/neural_network.py in
forward(self, x)
28 for i in range(len(self.layer_shapes)-1):
29 lin = self.linear_functions[i](y)
---> 30 y = self.activation_functions[i](lin)
31 return y
32
TypeError: __init__() takes 1 positional argument but 2 were given
I'm sure someone could tell me why this is happening, but could someone please give me a helpful strategy for debugging here? I'm new to pytorch and I doubt this will be the last trouble that I have. So a strategy for investigating these things would be helpful.
I would appreciate any input.
In the definition of model I forgot the parentheses on the torch.nn.Tanh class. It should be torch.nn.Tanh()
I keep thinking that these are functions and not classes. I still have some things to fix, but I'm glad I saw that. So frustrating. I found it by basically putting assert and print statements all over my code.
You might want to use the Sequential class
import torch.nn as nn
class NNet(nn.Module):
def __init__(self, idim, hdim, odim, depth):
super().__init__()
layers = [nn.Linear(idim, hdim)]
layers += [nn.Linear(hdim, hdim)
for i in range(depth)]
layers += [nn.Linear(hdim, odim)]
self.net = nn.Sequential(*layers)
def forward(self, x):
return self.net(x)
This takes care of the parameters etc too.
I would want to create a neural network with Python and I have some problems with the estimator.
First, I read some documentation about estimators specification, and I think I created my estimators type correctly:
estimate_train = tf.estimator.EstimatorSpec(mode=tf.estimator.ModeKeys.TRAIN, loss=loss, train_op=train_op)
estimate_test = tf.estimator.EstimatorSpec(mode=tf.estimator.ModeKeys.EVAL, loss=loss)
But when I want to create the estimator that will be used to train my network:
estimator_ = tf.estimator.Estimator(model_fn= estimate_train, model_dir="Path")
There is the following error:
TypeError Traceback (most recent call last)
/usr/lib/python3.5/inspect.py in getfullargspec(func)
1088 skip_bound_arg=False,
-> 1089 sigcls=Signature)
1090 except Exception as ex:
/usr/lib/python3.5/inspect.py in _signature_from_callable(obj, follow_wrapper_chains, skip_bound_arg, sigcls)
2155 if not callable(obj):
-> 2156 raise TypeError('{!r} is not a callable object'.format(obj))
2157
TypeError: EstimatorSpec(mode='eval', predictions={}, loss=<tf.Tensor 'mean_squared_error/value:0' shape=() dtype=float32>, train_op=None, eval_metric_ops={}, export_outputs=None, training_chief_hooks=(), training_hooks=(), scaffold=<tensorflow.python.training.monitored_session.Scaffold object at 0x7f9ddc083748>, evaluation_hooks=(), prediction_hooks=()) is not a callable object
The above exception was the direct cause of the following exception:
TypeError Traceback (most recent call last)
<ipython-input-2-c86a69b4da46> in <module>()
39
40
---> 41 estimatorn = tf.estimator.Estimator(model_fn= estimate_test, model_dir="/home/jabou/Bureau")
/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py in __init__(self, model_fn, model_dir, config, params, warm_start_from)
221 if model_fn is None:
222 raise ValueError('model_fn must be provided to Estimator.')
--> 223 _verify_model_fn_args(model_fn, params)
224 self._model_fn = model_fn
225 self._params = copy.deepcopy(params or {})
/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py in _verify_model_fn_args(model_fn, params)
1212 def _verify_model_fn_args(model_fn, params):
1213 """Verifies model fn arguments."""
-> 1214 args = set(util.fn_args(model_fn))
1215 if 'features' not in args:
1216 raise ValueError('model_fn (%s) must include features argument.' % model_fn)
/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/util.py in fn_args(fn)
58 if _is_callable_object(fn):
59 fn = fn.__call__
---> 60 args = tf_inspect.getfullargspec(fn).args
61 if _is_bounded_method(fn):
62 args.remove('self')
/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/tf_inspect.py in getfullargspec(obj)
88 decorators, target = tf_decorator.unwrap(obj)
89 return next((d.decorator_argspec for d in decorators
---> 90 if d.decorator_argspec is not None), spec_fn(target))
91
92
/usr/lib/python3.5/inspect.py in getfullargspec(func)
1093 # else. So to be fully backwards compatible, we catch all
1094 # possible exceptions here, and reraise a TypeError.
-> 1095 raise TypeError('unsupported callable') from ex
1096
1097 args = []
TypeError: unsupported callable
Here is my complete code:
import tensorflow as tf
sess = tf.Session()
tf.reset_default_graph()
batch_size = 20
# Values needed to create the network
input_ = tf.placeholder(tf.float32, shape=(batch_size, 1 , 1 ,1))
filter_ = tf.placeholder(tf.float32, shape=(batch_size, 1 , 2700 ,1))
output_network = tf.placeholder(tf.int32, shape=(4,))
output_real = tf.placeholder(tf.float32)
x_var = tf.get_variable(name = 'x_var', dtype = tf.float32, initializer = tf.random_normal((batch_size,1,1,1), 0, 0.001)) # Initialised values
bias = tf.Variable(tf.zeros([2700]))
# Network
logits = tf.nn.conv2d_transpose(x_var, filter_ ,output_network,[1,1,3,1],'SAME') + bias
loss = tf.losses.mean_squared_error(output_real,logits) # loss function
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(loss=loss,global_step=tf.train.get_global_step())
# Estimators specification
estimate_train = tf.estimator.EstimatorSpec(mode=tf.estimator.ModeKeys.TRAIN, loss=loss, train_op=train_op)
estimate_test = tf.estimator.EstimatorSpec(mode=tf.estimator.ModeKeys.EVAL, loss=loss)
# Estimator
estimator_ = tf.estimator.Estimator(model_fn= estimate_test, model_dir="Path")
Can you help me ?
EDIT
Following the answer of #f4, I corrected my code but I have still the same error:
import tensorflow as tf
sess = tf.Session()
tf.reset_default_graph()
batch_size = 20
def model(param):
# Values needed to create the network
input_ = tf.placeholder(tf.float32, shape=(batch_size, 1 , 1 ,1))
filter_ = tf.placeholder(tf.float32, shape=(batch_size, 1 , 2700 ,1))
output_network = tf.placeholder(tf.int32, shape=(4,))
output_real = tf.placeholder(tf.float32)
x_var = tf.get_variable(name = 'x_var', dtype = tf.float32, initializer = tf.random_normal((batch_size,1,1,1), 0, 0.001)) # Initialised values
bias = tf.Variable(tf.zeros([2700]))
# Network
logits = tf.nn.conv2d_transpose(x_var, filter_ ,output_network,[1,1,3,1],'SAME') + bias
loss = tf.losses.mean_squared_error(output_real,logits) # loss function
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(loss=loss,global_step=tf.train.get_global_step())
# Estimators specification
if param == "train":
return tf.estimator.EstimatorSpec(mode=tf.estimator.ModeKeys.TRAIN, loss=loss, train_op=train_op)
if param == "test":
return tf.estimator.EstimatorSpec(mode=tf.estimator.ModeKeys.EVAL, loss=loss)
# Estimator
estimator_ = tf.estimator.Estimator(model_fn= model("train"), model_dir="Path")
What's wrong again ?
You have given the Estimator an EstimatorSpec directly and it isn't correct.
model_fn should be a function which returns an instance of EstimatorSpec.
This function will be called later on. As a result it is complaining that what you have give is not callable.
EDIT
No again you're giving the return value of the function, what you need to pass is the function itself, something that's callable:
estimator_ = tf.estimator.Estimator(model_fn = model, model_dir="Path")
Also your model_fn is not good. I suggest you read the documentation https://www.tensorflow.org/get_started/custom_estimators
it should have this signature:
def my_model_fn(
features, # This is batch_features from input_fn
labels, # This is batch_labels from input_fn
mode, # An instance of tf.estimator.ModeKeys
params): # Additional configuration
The first two arguments are the batches of features and labels
returned from the input function; that is, features and labels are the
handles to the data your model will use. The mode argument indicates
whether the caller is requesting training, predicting, or evaluation.
You should make use of features and labels instead of creating palceholders.
Additionally, in your model_fn, you should not be creating the training ops when you are not in training mode.