class BinaryTruePositives(tf.keras.metrics.Metric):
def __init__(self, name='binary_true_positives', **kwargs):
super(BinaryTruePositives, self).__init__(name=name, **kwargs)
self.true_positives = self.add_weight(name='tp', initializer='zeros')
def update_state(self, y_true, y_pred, sample_weight=None):
y_true = tf.squeeze(y_true)
y_pred = tf.sign(y_pred)
y_pred=tf.reshape(y_pred,[-1])
self.true_positives.assign_add(tf.keras.backend.mean(tf.keras.backend.equal(y_true,
y_pred)))
def result(self):
return self.true_positives
def reset_states(self):
self.true_positives.assign(0)
def model_fn():
keras_model = create_keras_model()
return tff.learning.from_keras_model(keras_model,
input_spec=preprocessed_example_dataset.element_spec,
loss=tf.keras.losses.MSE,
metrics=[BinaryTruePositives()])
TypeError: Expected tensorflow.python.keras.losses.Loss or collections.abc.Sequence, found function.
Some more of the stacktrace might be useful here, but I believe the issue in the code above is the fact that tf.keras.losses.MSE is a function defining the loss logic, rather than an instance of tf.keras.losses.Loss itself.
Looking at an old version of TFF, it seems you are hitting this line, though note that you'd get a different error with a newer version of TFF (I believe you'd hit this line instead).
You can fix this by passing
loss=tf.keras.losses.MeanSquaredError()
instead of the existing loss argument in your model_fn above.
Related
I am new to PyTorch and have some custom nn.Modules that I would like to run on a GPU. Let's call them M_outer, M_inner, and M_sub. In general, the structure looks like:
class M_outer(nn.Module):
def __init__(self, **kwargs):
self.inner_1 = M_inner(**kwargs)
self.inner_2 = M_inner(**kwargs)
# ...
def forward(self, input):
result = self.inner_1(input)
result = self.inner_2(result)
# ...
return result
class M_inner(nn.Module):
def __init__(self, **kwargs):
self.sub_1 = M_sub(**kwargs)
self.sub_2 = M_sub(**kwargs)
# ...
def forward(self, input):
result = self.sub_1(input)
result = self.sub_2(result)
# ...
return result
class M_sub(nn.Module):
def __init__(self, **kwargs):
self.emb_1 = nn.Embedding(x, y)
self.emb_2 = nn.Embedding(x, y)
# ...
self.norm = nn.LayerNorm()
def forward(self, input):
emb = (self.emb_1(input) + self.emb_2(input))
# ...
return self.norm(emb)
and I try to get my module on a gpu via:
model = M_outer(params).to(device)
Yet I am still getting errors from the embedding layers saying that some operations are on the cpu.
I have read the documentation. I have read useful Discuss posts like this and related StackOverflow posts like this.
I can not register an nn.EmbeddingLayer via nn.Parameter. What am I missing?
In Tensorflow Federated (TFF), you can pass to the tff.learning.build_federated_averaging_process a broadcast_process and an aggregation_process, which can embed customized encoders e.g. to apply custom compressions.
Getting to the point of my question, I am trying to implement an encoder to sparsify model updates/model weights.
I am trying to build such an encoder by implementing the EncodingStageInterface, from tensorflow_model_optimization.python.core.internal.
However, I am struggling to implement a (local) state to accumulate the zeroed-out coordinates of model updates/model weights round by round. Note that this state should not be communicated, and just need to be maintained locally (so the AdaptiveEncodingStageInterface should not be helpful). In general, the question is how to maintain a local state inside an Encoder to be then passed to the fedavg process.
I attach the code of my encoder implementation (that, besides the state I would like to add, works fine as stateless as expected).
I then attach the excerpt of my code where I use the encoder implementation.
If I decomment the commented parts in stateful_encoding_stage_topk.py the code does not work: I can't figure out how manage the state (that is a Tensor) in TF non eager mode.
stateful_encoding_stage_topk.py
import tensorflow as tf
import numpy as np
from tensorflow_model_optimization.python.core.internal import tensor_encoding as te
#te.core.tf_style_encoding_stage
class StatefulTopKEncodingStage(te.core.EncodingStageInterface):
ENCODED_VALUES_KEY = 'stateful_topk_values'
INDICES_KEY = 'indices'
def __init__(self):
super().__init__()
# Here I would like to init my state
#self.A = tf.zeros([800], dtype=tf.float32)
#property
def name(self):
"""See base class."""
return 'stateful_topk'
#property
def compressible_tensors_keys(self):
"""See base class."""
return [self.ENCODED_VALUES_KEY]
#property
def commutes_with_sum(self):
"""See base class."""
return True
#property
def decode_needs_input_shape(self):
"""See base class."""
return True
def get_params(self):
"""See base class."""
return {}, {}
def encode(self, x, encode_params):
"""See base class."""
del encode_params # Unused.
dW = tf.reshape(x, [-1])
# Here I would like to retrieve the state
A = tf.zeros([800], dtype=tf.float32)
#A = self.residual
dW_and_A = tf.math.add(A, dW)
percentage = tf.constant(0.4, dtype=tf.float32)
k_float = tf.multiply(percentage, tf.cast(tf.size(dW), tf.float32))
k_int = tf.cast(tf.math.round(k_float), dtype=tf.int32)
values, indices = tf.math.top_k(tf.math.abs(dW_and_A), k = k_int, sorted = False)
indices = tf.expand_dims(indices, 1)
sparse_dW = tf.scatter_nd(indices, values, tf.shape(dW_and_A))
# Here I would like to update the state
A_updated = tf.math.subtract(dW_and_A, sparse_dW)
#self.A = A_updated
encoded_x = {self.ENCODED_VALUES_KEY: values,
self.INDICES_KEY: indices}
return encoded_x
def decode(self,
encoded_tensors,
decode_params,
num_summands=None,
shape=None):
"""See base class."""
del decode_params, num_summands # Unused.
indices = encoded_tensors[self.INDICES_KEY]
values = encoded_tensors[self.ENCODED_VALUES_KEY]
tensor = tf.fill([800], 0.0)
decoded_values = tf.tensor_scatter_nd_update(tensor, indices, values)
return tf.reshape(decoded_values, shape)
def sparse_quantizing_encoder():
encoder = te.core.EncoderComposer(
StatefulTopKEncodingStage() )
return encoder.make()
fedavg_with_sparsification.py
[...]
def sparsification_broadcast_encoder_fn(value):
spec = tf.TensorSpec(value.shape, value.dtype)
return te.encoders.as_simple_encoder(te.encoders.identity(), spec)
def sparsification_mean_encoder_fn(value):
spec = tf.TensorSpec(value.shape, value.dtype)
if value.shape.num_elements() == 800:
return te.encoders.as_gather_encoder(
stateful_encoding_stage_topk.sparse_quantizing_encoder(), spec)
else:
return te.encoders.as_gather_encoder(te.encoders.identity(), spec)
encoded_broadcast_process = (
tff.learning.framework.build_encoded_broadcast_process_from_model(
model_fn, sparsification_broadcast_encoder_fn))
encoded_mean_process = (
tff.learning.framework.build_encoded_mean_process_from_model(
model_fn, sparsification_mean_encoder_fn))
iterative_process = tff.learning.build_federated_averaging_process(
model_fn,
client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.004),
client_weight_fn=lambda _: tf.constant(1.0),
broadcast_process=encoded_broadcast_process,
aggregation_process=encoded_mean_process)
[...]
I am using:
tensorflow 2.4.0
tensorflow-federated 0.17.0
I'll try to answer in two parts; (1) top_k encoder without state and (2) realizing the stateful idea you seem to want in TFF.
(1)
To get the TopKEncodingStage working without state, I see a few details to change.
The commutes_with_sum property should be set to False. In pseudo-code, its meaning is whether sum_x(decode(encode(x))) == decode(sum_x(encode(x))) . This is not true for the representation your encode method returns -- summing the indices would not work well. I think implementation of the decode method can be simplified to
return tf.scatter_nd(
indices=encoded_tensors[self.INDICES_KEY],
updates=encoded_tensors[self.ENCODED_VALUES_KEY],
shape=shape)
(2)
What you refer to cannot be achieved in this manner using tff.learning.build_federated_averaging_process. The process returned by this method does not have any mechanism for maintaining client/local state. Whatever is the state expressed in your StatefulTopKEncodingStage would end up being the server state, not local state.
To work with the client/local state, you may need to write more custom code. For a starter, see examples/stateful_clients which you can adapt to store the state you refer to.
Keep in mind that in TFF, this will need to be represented as functional transformations. Storing values in attributes of a class and use them elsewhere can lead to surprising errors.
I've created a simple class which I'd like to have return a model.
One of the methods is a simple normalizer which I'd like to be able to call from within another block of code. However it's not finding it. Here's what I have:
class MyClass():
def __init__(self):
return
def norm(self, train, x):
return(x - train['mean'] / train['std'])
def modelCreator(self, norm, df):
train = df.sample(frac=0.8, random_state=0)
test = df.drop(train.index)
train_stats = train.describe()
train_stats = train_stats.transpose()
normed_train = norm(train)
normed_test = norm(test)
However, when I try to run this, I get the following error:
modelCreator() is missing 1 required positional argument: 'df'
(I was trying to call the modelCreator method directly and passing it a df)
I also tried to remove "norm" from the definition of modelCreator:
def modelCreator(self, df):
But in this case I get this error:
name 'norm' is not defined
Thank you in advance!
You only need df as a parameter to modelCreator(). You then access your norm method via self.norm. Below is your code updated.
class MyClass():
def __init__(self):
return
def norm(self, train, x):
return(x - train['mean'] / train['std'])
def modelCreator(self, df):
train = df.sample(frac=0.8, random_state=0)
test = df.drop(train.index)
train_stats = train.describe()
train_stats = train_stats.transpose()
# These lines below are changed from original
normed = self.norm(train, test)
In PyMC3 you can do this
basic_model = pm.Model()
with basic_model:
# Priors for unknown model parameters
alpha = pm.Normal('alpha', mu=0, sd=10)
beta = pm.Normal('beta', mu=0, sd=10, shape=2)
sigma = pm.HalfNormal('sigma', sd=1)
# Expected value of outcome
mu = alpha + beta[0]*X1 + beta[1]*X2
# Likelihood (sampling distribution) of observations
Y_obs = pm.Normal('Y_obs', mu=mu, sd=sigma, observed=Y)
and all the variables (pm.Normal, ...) will be "assigned" to the basic_model instance.
From the docs
The first line,
basic_model = Model()
creates a new Model object which is a container for the model random
variables.
Following instantiation of the model, the subsequent specification of
the model components is performed inside a with statement:
with basic_model:
This creates a context manager, with our basic_model as the context,
that includes all statements until the indented block ends. This means
all PyMC3 objects introduced in the indented code block below the with
statement are added to the model behind the scenes. Absent this
context manager idiom, we would be forced to manually associate each
of the variables with basic_model right after we create them. If you
try to create a new random variable without a with model: statement,
it will raise an error since there is no obvious model for the
variable to be added to.
I think it is very elegant for the purpose of the library. How is that actually implemented?
The only way I can think of is something in the spirit of this:
class Model:
active_model = None
def __enter__(self):
Model.active_model = self
def __exit__(self, *args, **kwargs):
Model.active_model = None
class Normal:
def __init__(self):
if Model.active_model is None:
raise ValueError("cant instantiate variable outside of Model")
else:
self.model = Model.active_model
It works with my simple REPL tests but I am not sure if this have some pitfalls and is actually that simple.
You are very close, and it was even quite similar to your implementation for a while. Note that threading.local is used to store objects, and it is maintained as a list to allow nesting multiple models, and allows multiprocessing. There is a little extra in the actual implementation to allow setting theano configuration when entering a model context that I deleted:
class Context(object):
contexts = threading.local()
def __enter__(self):
type(self).get_contexts().append(self)
return self
def __exit__(self, typ, value, traceback):
type(self).get_contexts().pop()
#classmethod
def get_contexts(cls):
if not hasattr(cls.contexts, 'stack'):
cls.contexts.stack = []
return cls.contexts.stack
#classmethod
def get_context(cls):
"""Return the deepest context on the stack."""
try:
return cls.get_contexts()[-1]
except IndexError:
raise TypeError("No context on context stack")
The Model class subclasses Context, so when writing algorithms we can call Model.get_context() from inside a context manager and have access to the object. This is equivalent to your Model.active_model.
I have subclassed RNNCell as the building block of my RNN. I put an instance of this object into tf.dynamic_rnn and then I define a prediction function in my Agent class:
class Agent():
def __init__(self):
...
def predictions(self):
cell = RNNCell()
output, last_state = tf.dynamic_rnn(cell, inputs = ...)
return output
Everything works fine, but how do I add a histogram for the layers now? I've tried to do it in the RNNCell but it doesn't work:
class RNNCell(tf.nn.rnn_cell.RNNCell):
def __init__(self):
super(RNNCell, self).__init__()
self._output_size = 15
self._state_size = 15
self._histogram1 = None
def __call__(self, X, state):
network = tflearn.layers.conv_2d(X, 5, [1, 3], activation='relu', weights_init=tflearn.initializations.variance_scaling(), padding="valid")
self._histogram1 = tf.summary.histogram("layer1_hist_summary", network)
...
#property
def histogram1(self):
return self._histogram1
and then
class Agent():
def __init__(self):
...
def predictions(self):
cell = RNNCell()
self.histogram1 = cell.histogram1
output, last_state = tf.dynamic_rnn(cell, inputs = ...)
return output
Later when I run sess.run(agent.histogram1, feed_dict=...) I get the error TypeError: Fetch argument None has invalid type <class 'NoneType'>
I think the problem is that the value of Agent's self.histogram1 never got updated to reflect that summary assigned in RNNCell.
Your code for the Agent predictions() method initializes Agent's histogram1 value to None here:
cell = RNNCell() #invoks __init__() so RNNCELL's histogram1 is now None
self.histogram1 = cell.histogram1
When RNNCell's __call__() method is invoked, it updates the RNNCell's value of histogram1
self._histogram1 = tf.summary.histogram("layer1_hist_summary", network)
But the Agent's copy of histogram1 was apparently not updated, so when the call is made:
sess.run(agent.histogram1, feed_dict=...)
agent.histogram1 is still None.
I don't see in the posted code where the summaries were merged before training, so the missing step is likely in unposted code somewhere.