Develop Reference

I got this error while attempting to save the model with Tensoflow

This is a project I found on GitHub and I tried reproducing it. This code was originally run on English dataset, I reproduced it on Chinese dataset. It is supposed to save a checkpoint model every 100 iterations, during saving the checkpoint model for the first 100 iterations, this error occurred, and then the process stopped.
[libprotobuf ERROR google/protobuf/wire_format_lite.cc:577] String field 'tensorflow.TensorShapeProto.Dim.name' contains invalid UTF-8 data when parsing a protocol buffer. Use the 'bytes' type if you intend to send raw bytes.
Traceback (most recent call last):
File "train.py", line 77, in <module>
train(model)
File "train.py", line 24, in train
model.optimize()
File "/root/autodl-tmp/HyperBox-main/script/model/box_model.py", line 328, in optimize
self.save_model(itr)
File "/root/autodl-tmp/HyperBox-main/script/model/box_model.py", line 140, in save_model
self.saver.save(self.sess, filename)
File "/root/miniconda3/lib/python3.8/site-packages/tensorflow/python/training/saver.py", line 1310, in save
self.export_meta_graph(
File "/root/miniconda3/lib/python3.8/site-packages/tensorflow/python/training/saver.py", line 1356, in export_meta_graph
graph_def=ops.get_default_graph().as_graph_def(add_shapes=True),
File "/root/miniconda3/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 3592, in as_graph_def
result, _ = self._as_graph_def(from_version, add_shapes)
File "/root/miniconda3/lib/python3.8/site-packages/tensorflow/python/framework/ops.py", line 3506, in _as_graph_def
graph.ParseFromString(compat.as_bytes(data))
google.protobuf.message.DecodeError: Error parsing message
And here's the code
import math
import os
import pickle
import random
from random import shuffle
import numpy as np
import tensorflow as tf
from reader import Reader
def train(model):
model.setup_reader()
print("Number of batches: ", model.num_batch)
print("Number of words: ", model.num_ent)
model.setup_weights()
model.setup_saver()
model.create_train_placeholders()
model.gather_train_embeddings()
model.create_train_model()
model.create_optimizer()
model.create_session()
model.optimize()
model.close_session()
class BoxE:
def __init__(self, params, corpus_type, work_dir):
self.params = params
self.alpha = params.alpha
self.num_rel = 1
self.bounded_norm = params.bounded_norm
self.normed_bumps = params.normed_bumps
self.fixed_width = params.fixed_width
self.hard_size = params.hard_size
self.total_size = params.total_size
self.learnable_shape = params.learnable_shape
self.corpus_type = corpus_type
self.cwd = work_dir
self.word_vectors = np.load(
f"{self.cwd}/../../word_vectors_processed/{self.corpus_type}_word_vectors_processed.npy"
)
def setup_reader(self):
self.reader = Reader(self.corpus_type)
self.reader.read_triples()
self.reader.set_batch_size(self.params.batch_size)
self.reader.set_neg_samples(self.params.no_neg_samples)
self.num_batch = self.reader.num_batch()
self.num_ent = self.reader.num_ent()
def setup_loader(self):
self.loader = tf.compat.v1.train.Saver(self.var_list)
def setup_saver(self):
self.saver = tf.compat.v1.train.Saver(max_to_keep=0)
def create_session(self):
self.sess = tf.compat.v1.Session()
self.sess.run(tf.compat.v1.global_variables_initializer())
def load_session(self, itr):
self.loader.restore(
self.sess,
f"{self.cwd}/BoxModel_"
+ self.corpus_type
+ "_weights/"
+ "/"
+ itr
+ ".ckpt",
)
def close_session(self):
self.sess.close()
def product_normalise(self, input_tensor, bounded_norm=True):
step1_tensor = tf.abs(input_tensor)
step2_tensor = step1_tensor + (10 ** -8)
log_norm_tensor = tf.math.log(step2_tensor)
step3_tensor = tf.reduce_mean(log_norm_tensor, axis=2, keepdims=True)
norm_volume = tf.math.exp(step3_tensor)
pre_norm_out = input_tensor / norm_volume
if not bounded_norm:
return pre_norm_out
else:
minsize_tensor = tf.minimum(
tf.reduce_min(log_norm_tensor, axis=2, keepdims=True), -1
)
maxsize_tensor = tf.maximum(
tf.reduce_max(log_norm_tensor, axis=2, keepdims=True), 1
)
minsize_ratio = -1 / minsize_tensor
maxsize_ratio = 1 / maxsize_tensor
size_norm_ratio = tf.minimum(minsize_ratio, maxsize_ratio)
normed_tensor = log_norm_tensor * size_norm_ratio
return tf.exp(normed_tensor)
def create_train_placeholders(self):
self.ph = tf.compat.v1.placeholder(tf.int32, [None])
self.pt = tf.compat.v1.placeholder(tf.int32, [None])
self.nh = tf.compat.v1.placeholder(tf.int32, [None])
self.nt = tf.compat.v1.placeholder(tf.int32, [None])
self.r = tf.compat.v1.placeholder(tf.int32, [None])
def create_test_placeholders(self):
self.head = tf.compat.v1.placeholder(tf.int32, [None])
self.rel = tf.compat.v1.placeholder(tf.int32, [None])
self.tail = tf.compat.v1.placeholder(tf.int32, [None])
def distance_function(self, points):
self.rel_bx_low, self.rel_bx_high = self.compute_box(
self.rel_bases_emb, self.rel_deltas_emb
)
lower_corner = self.rel_bx_low
upper_corner = self.rel_bx_high
centres = 1 / 2 * (lower_corner + upper_corner)
widths = upper_corner - lower_corner
widths_p1 = widths + tf.constant(1.0)
width_cond = tf.where(
tf.logical_and(lower_corner <= points, points <= upper_corner),
tf.abs(points - centres) / widths_p1,
widths_p1 * tf.abs(points - centres)
- (widths / 2) * (widths_p1 - 1 / widths_p1),
)
distance = tf.norm(
width_cond, axis=2, ord=self.params.p_norm
) ###batch*2*1 after norm
distance = tf.reduce_sum(distance, axis=1)
return distance
def create_optimizer(self):
self.loss = -1 * tf.math.reduce_mean(
tf.math.log_sigmoid(self.params.gamma - self.pos_dissims)
) - tf.math.reduce_mean(
tf.math.log_sigmoid(self.neg_dissims - self.params.gamma)
)
tf.compat.v1.disable_eager_execution()
var_list = (tf.trainable_variables()
+ tf.get_collection(tf.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))
self.optimizer = tf.compat.v1.train.AdamOptimizer(
self.params.learning_rate
).minimize(self.loss)
def save_model(self, itr):
filename = (
f"{self.cwd}/BoxModel_"
+ self.corpus_type
+ "_weights/"
+ str(itr)
+ ".ckpt"
)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
self.saver.save(self.sess, filename)
def setup_weights(self):
sqrt_size = 6.0 / math.sqrt(self.params.emb_size)
self.ent_emb = tf.Variable(self.word_vectors, dtype=tf.float32, name="ent_emb")
self.ent_emb_bmp = tf.Variable(
self.word_vectors, dtype=tf.float32, name="ent_emb_bmp"
)
self.base_weight_ent_emb = tf.Variable(
name="base_weight_ent_emb",
initial_value=tf.random.uniform(
# shape=[300, self.params.emb_size], minval=-sqrt_size, maxval=sqrt_size
shape=[400, self.params.emb_size], minval=-sqrt_size, maxval=sqrt_size
),
)
self.bump_weight_ent_emb = tf.Variable(
name="bump_weight_ent_emb",
initial_value=tf.random.uniform(
# shape=[300, self.params.emb_size], minval=-sqrt_size, maxval=sqrt_size
shape=[400, self.params.emb_size], minval=-sqrt_size, maxval=sqrt_size
),
)
if self.learnable_shape: # If shape is learnable, define variables accordingly
self.rel_shapes = tf.Variable(
name="rel_shapes",
initial_value=tf.random.uniform(
shape=[self.num_rel, 2, self.params.emb_size],
minval=-sqrt_size,
maxval=sqrt_size,
),
)
self.norm_rel_shapes = self.product_normalise(
self.rel_shapes, self.bounded_norm
)
else:
self.norm_rel_shapes = tf.ones(
[self.num_rel, 2, self.params.emb_size], name="norm_rel_shapes"
)
self.rel_bases = tf.Variable(
name="rel_bases",
initial_value=tf.random.uniform(
shape=[self.num_rel, 2, self.params.emb_size],
minval=-sqrt_size,
maxval=sqrt_size,
),
)
if self.fixed_width:
self.rel_multiples1 = tf.zeros([self.num_rel, 2, 1])
else:
self.rel_multiples1 = tf.Variable(
name="rel_multiples",
initial_value=tf.random.uniform(
shape=[self.num_rel, 2, 1], minval=-sqrt_size, maxval=sqrt_size
),
)
if self.hard_size:
self.rel_multiples = self.total_size * tf.nn.softmax(
self.rel_multiples1, axis=0
)
else:
self.rel_multiples = tf.nn.elu(self.rel_multiples1) + tf.constant(1.0)
self.rel_deltas = tf.multiply(
self.rel_multiples, self.norm_rel_shapes, name="rel_deltas"
)
self.var_list = [
self.rel_bases,
self.rel_shapes,
self.rel_multiples1,
self.base_weight_ent_emb,
self.bump_weight_ent_emb,
self.ent_emb,
self.ent_emb_bmp,
]
def gather_train_embeddings(self):
temp = tf.matmul(self.ent_emb, self.base_weight_ent_emb)
self.ph_base_emb = tf.gather(temp, self.ph)
self.pt_base_emb = tf.gather(temp, self.pt)
self.nh_base_emb = tf.gather(temp, self.nh)
self.nt_base_emb = tf.gather(temp, self.nt)
temp1 = tf.matmul(self.ent_emb_bmp, self.bump_weight_ent_emb)
if self.normed_bumps: # Normalization of bumps option
temp1 = tf.math.l2_normalize(temp1, axis=1)
self.ph_bump_emb = tf.gather(temp1, self.ph)
self.pt_bump_emb = tf.gather(temp1, self.pt)
self.nh_bump_emb = tf.gather(temp1, self.nh)
self.nt_bump_emb = tf.gather(temp1, self.nt)
self.rel_bases_emb = tf.math.tanh(tf.gather(self.rel_bases, self.r))
self.rel_deltas_emb = tf.math.tanh(tf.gather(self.rel_deltas, self.r))
def gather_test_embeddings(self):
temp = tf.matmul(self.ent_emb, self.base_weight_ent_emb)
self.h_base_emb = tf.gather(temp, self.head)
self.t_base_emb = tf.gather(temp, self.tail)
temp1 = tf.matmul(self.ent_emb_bmp, self.bump_weight_ent_emb)
if self.normed_bumps: # Normalization of bumps option
temp1 = tf.math.l2_normalize(temp1, axis=1)
self.h_bump_emb = tf.gather(temp1, self.head)
self.t_bump_emb = tf.gather(temp1, self.tail)
self.rel_bases_emb = tf.math.tanh(tf.gather(self.rel_bases, self.rel))
self.rel_deltas_emb = tf.math.tanh(tf.gather(self.rel_deltas, self.rel))
def compute_box(self, box_base, box_delta):
box_second = box_base + tf.constant(0.5) * box_delta
box_first = box_base - tf.constant(0.5) * box_delta
box_low = tf.minimum(box_first, box_second, "box_low")
box_high = tf.maximum(box_first, box_second, "box_high")
return box_low, box_high
def create_train_model(self):
self.pos_h_points = tf.expand_dims(self.ph_base_emb + self.pt_bump_emb, 1)
self.pos_t_points = tf.expand_dims(self.pt_base_emb + self.ph_bump_emb, 1)
self.neg_h_points = tf.expand_dims(self.nh_base_emb + self.nt_bump_emb, 1)
self.neg_t_points = tf.expand_dims(self.nt_base_emb + self.nh_bump_emb, 1)
self.pos_points = tf.math.tanh(
tf.concat([self.pos_h_points, self.pos_t_points], 1)
)
self.neg_points = tf.math.tanh(
tf.concat([self.neg_h_points, self.neg_t_points], 1)
)
#### concat dimension is batch*2*100 ####
self.pos_dissims = self.distance_function(self.pos_points)
self.neg_dissims = self.distance_function(self.neg_points)
def create_test_model(self):
self.h_points = tf.math.tanh(
tf.expand_dims(self.h_base_emb + self.t_bump_emb, 1)
)
self.t_points = tf.math.tanh(
tf.expand_dims(self.t_base_emb + self.h_bump_emb, 1)
)
self.test_pt = tf.concat([self.h_points, self.t_points], 1)
self.dissims = self.distance_function(self.test_pt)
def optimize(self):
for itr in range(0, self.params.max_iterate + 1):
total_loss = 0.0
for b in range(self.num_batch):
ph, pt, nh, nt, r = self.reader.next_batch()
_, err = self.sess.run(
[self.optimizer, self.loss],
feed_dict={
self.ph: ph,
self.pt: pt,
self.nh: nh,
self.nt: nt,
self.r: r,
},
)
total_loss += err
if math.isnan(total_loss):
break
print("Loss in iteration", itr, "=", total_loss)
if itr % self.params.save_each == 0 and itr >= self.params.save_after:
self.save_model(itr)

'tensorflow_federated' has no attribute 'NamedTupleType

I am following this code https://github.com/BUAA-BDA/FedShapley/tree/master/TensorflowFL and trying to run the file same_OR.py
I also place input file "initial_model_parameters.txt" and data folder "MNIST_data" in same folder
from __future__ import absolute_import, division, print_function
import tensorflow_federated as tff
import tensorflow.compat.v1 as tf
import numpy as np
import time
from scipy.special import comb, perm
import os
# tf.compat.v1.enable_v2_behavior()
# tf.compat.v1.enable_eager_execution()
# NUM_EXAMPLES_PER_USER = 1000
BATCH_SIZE = 100
NUM_AGENT = 5
def get_data_for_digit(source, digit):
output_sequence = []
all_samples = [i for i, d in enumerate(source[1]) if d == digit]
for i in range(0, len(all_samples), BATCH_SIZE):
batch_samples = all_samples[i:i + BATCH_SIZE]
output_sequence.append({
'x': np.array([source[0][i].flatten() / 255.0 for i in batch_samples],
dtype=np.float32),
'y': np.array([source[1][i] for i in batch_samples], dtype=np.int32)})
return output_sequence
def get_data_for_digit_test(source, digit):
output_sequence = []
all_samples = [i for i, d in enumerate(source[1]) if d == digit]
for i in range(0, len(all_samples)):
output_sequence.append({
'x': np.array(source[0][all_samples[i]].flatten() / 255.0,
dtype=np.float32),
'y': np.array(source[1][all_samples[i]], dtype=np.int32)})
return output_sequence
def get_data_for_federated_agents(source, num):
output_sequence = []
Samples = []
for digit in range(0, 10):
samples = [i for i, d in enumerate(source[1]) if d == digit]
samples = samples[0:5421]
Samples.append(samples)
all_samples = []
for sample in Samples:
for sample_index in range(int(num * (len(sample) / NUM_AGENT)), int((num + 1) * (len(sample) / NUM_AGENT))):
all_samples.append(sample[sample_index])
# all_samples = [i for i in range(int(num*(len(source[1])/NUM_AGENT)), int((num+1)*(len(source[1])/NUM_AGENT)))]
for i in range(0, len(all_samples), BATCH_SIZE):
batch_samples = all_samples[i:i + BATCH_SIZE]
output_sequence.append({
'x': np.array([source[0][i].flatten() / 255.0 for i in batch_samples],
dtype=np.float32),
'y': np.array([source[1][i] for i in batch_samples], dtype=np.int32)})
return output_sequence
BATCH_TYPE = tff.NamedTupleType([
('x', tff.TensorType(tf.float32, [None, 784])),
('y', tff.TensorType(tf.int32, [None]))])
MODEL_TYPE = tff.NamedTupleType([
('weights', tff.TensorType(tf.float32, [784, 10])),
('bias', tff.TensorType(tf.float32, [10]))])
#tff.tf_computation(MODEL_TYPE, BATCH_TYPE)
def batch_loss(model, batch):
predicted_y = tf.nn.softmax(tf.matmul(batch.x, model.weights) + model.bias)
return -tf.reduce_mean(tf.reduce_sum(
tf.one_hot(batch.y, 10) * tf.log(predicted_y), axis=[1]))
#tff.tf_computation(MODEL_TYPE, BATCH_TYPE, tf.float32)
def batch_train(initial_model, batch, learning_rate):
# Define a group of model variables and set them to `initial_model`.
model_vars = tff.utils.create_variables('v', MODEL_TYPE)
init_model = tff.utils.assign(model_vars, initial_model)
# Perform one step of gradient descent using loss from `batch_loss`.
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
with tf.control_dependencies([init_model]):
train_model = optimizer.minimize(batch_loss(model_vars, batch))
# Return the model vars after performing this gradient descent step.
with tf.control_dependencies([train_model]):
return tff.utils.identity(model_vars)
LOCAL_DATA_TYPE = tff.SequenceType(BATCH_TYPE)
#tff.federated_computation(MODEL_TYPE, tf.float32, LOCAL_DATA_TYPE)
def local_train(initial_model, learning_rate, all_batches):
# Mapping function to apply to each batch.
#tff.federated_computation(MODEL_TYPE, BATCH_TYPE)
def batch_fn(model, batch):
return batch_train(model, batch, learning_rate)
l = tff.sequence_reduce(all_batches, initial_model, batch_fn)
return l
#tff.federated_computation(MODEL_TYPE, LOCAL_DATA_TYPE)
def local_eval(model, all_batches):
#
return tff.sequence_sum(
tff.sequence_map(
tff.federated_computation(lambda b: batch_loss(model, b), BATCH_TYPE),
all_batches))
SERVER_MODEL_TYPE = tff.FederatedType(MODEL_TYPE, tff.SERVER, all_equal=True)
CLIENT_DATA_TYPE = tff.FederatedType(LOCAL_DATA_TYPE, tff.CLIENTS)
#tff.federated_computation(SERVER_MODEL_TYPE, CLIENT_DATA_TYPE)
def federated_eval(model, data):
return tff.federated_mean(
tff.federated_map(local_eval, [tff.federated_broadcast(model), data]))
SERVER_FLOAT_TYPE = tff.FederatedType(tf.float32, tff.SERVER, all_equal=True)
#tff.federated_computation(
SERVER_MODEL_TYPE, SERVER_FLOAT_TYPE, CLIENT_DATA_TYPE)
def federated_train(model, learning_rate, data):
l = tff.federated_map(
local_train,
[tff.federated_broadcast(model),
tff.federated_broadcast(learning_rate),
data])
return l
# return tff.federated_mean()
def readTestImagesFromFile(distr_same):
ret = []
if distr_same:
f = open(os.path.join(os.path.dirname(__file__), "test_images1_.txt"), encoding="utf-8")
else:
f = open(os.path.join(os.path.dirname(__file__), "test_images1_.txt"), encoding="utf-8")
lines = f.readlines()
for line in lines:
tem_ret = []
p = line.replace("[", "").replace("]", "").replace("\n", "").split("\t")
for i in p:
if i != "":
tem_ret.append(float(i))
ret.append(tem_ret)
return np.asarray(ret)
def readTestLabelsFromFile(distr_same):
ret = []
if distr_same:
f = open(os.path.join(os.path.dirname(__file__), "test_labels_.txt"), encoding="utf-8")
else:
f = open(os.path.join(os.path.dirname(__file__), "test_labels_.txt"), encoding="utf-8")
lines = f.readlines()
for line in lines:
tem_ret = []
p = line.replace("[", "").replace("]", "").replace("\n", "").split(" ")
for i in p:
if i!="":
tem_ret.append(float(i))
ret.append(tem_ret)
return np.asarray(ret)
def getParmsAndLearningRate(agent_no):
f = open(os.path.join(os.path.dirname(__file__), "weights_" + str(agent_no) + ".txt"))
content = f.read()
g_ = content.split("***\n--------------------------------------------------")
parm_local = []
learning_rate_list = []
for j in range(len(g_) - 1):
line = g_[j].split("\n")
if j == 0:
weights_line = line[0:784]
learning_rate_list.append(float(line[784].replace("*", "").replace("\n", "")))
else:
weights_line = line[1:785]
learning_rate_list.append(float(line[785].replace("*", "").replace("\n", "")))
valid_weights_line = []
for l in weights_line:
w_list = l.split("\t")
w_list = w_list[0:len(w_list) - 1]
w_list = [float(i) for i in w_list]
valid_weights_line.append(w_list)
parm_local.append(valid_weights_line)
f.close()
f = open(os.path.join(os.path.dirname(__file__), "bias_" + str(agent_no) + ".txt"))
content = f.read()
g_ = content.split("***\n--------------------------------------------------")
bias_local = []
for j in range(len(g_) - 1):
line = g_[j].split("\n")
if j == 0:
weights_line = line[0]
else:
weights_line = line[1]
b_list = weights_line.split("\t")
b_list = b_list[0:len(b_list) - 1]
b_list = [float(i) for i in b_list]
bias_local.append(b_list)
f.close()
ret = {
'weights': np.asarray(parm_local),
'bias': np.asarray(bias_local),
'learning_rate': np.asarray(learning_rate_list)
}
return ret
def train_with_gradient_and_valuation(agent_list, grad, bi, lr, distr_type):
f_ini_p = open(os.path.join(os.path.dirname(__file__), "initial_model_parameters.txt"), "r")
para_lines = f_ini_p.readlines()
w_paras = para_lines[0].split("\t")
w_paras = [float(i) for i in w_paras]
b_paras = para_lines[1].split("\t")
b_paras = [float(i) for i in b_paras]
w_initial_g = np.asarray(w_paras, dtype=np.float32).reshape([784, 10])
b_initial_g = np.asarray(b_paras, dtype=np.float32).reshape([10])
f_ini_p.close()
model_g = {
'weights': w_initial_g,
'bias': b_initial_g
}
for i in range(len(grad[0])):
# i->迭代轮数
gradient_w = np.zeros([784, 10], dtype=np.float32)
gradient_b = np.zeros([10], dtype=np.float32)
for j in agent_list:
gradient_w = np.add(np.multiply(grad[j][i], 1/len(agent_list)), gradient_w)
gradient_b = np.add(np.multiply(bi[j][i], 1/len(agent_list)), gradient_b)
model_g['weights'] = np.subtract(model_g['weights'], np.multiply(lr[0][i], gradient_w))
model_g['bias'] = np.subtract(model_g['bias'], np.multiply(lr[0][i], gradient_b))
test_images = readTestImagesFromFile(False)
test_labels_onehot = readTestLabelsFromFile(False)
m = np.dot(test_images, np.asarray(model_g['weights']))
test_result = m + np.asarray(model_g['bias'])
y = tf.nn.softmax(test_result)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.arg_max(test_labels_onehot, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
return accuracy.numpy()
def remove_list_indexed(removed_ele, original_l, ll):
new_original_l = []
for i in original_l:
new_original_l.append(i)
for i in new_original_l:
if i == removed_ele:
new_original_l.remove(i)
for i in range(len(ll)):
if set(ll[i]) == set(new_original_l):
return i
return -1
def shapley_list_indexed(original_l, ll):
for i in range(len(ll)):
if set(ll[i]) == set(original_l):
return i
return -1
def PowerSetsBinary(items):
N = len(items)
set_all = []
for i in range(2 ** N):
combo = []
for j in range(N):
if (i >> j) % 2 == 1:
combo.append(items[j])
set_all.append(combo)
return set_all
if __name__ == "__main__":
start_time = time.time()
#data_num = np.asarray([5923,6742,5958,6131,5842])
#agents_weights = np.divide(data_num, data_num.sum())
for index in range(NUM_AGENT):
f = open(os.path.join(os.path.dirname(__file__), "weights_"+str(index)+".txt"), "w")
f.close()
f = open(os.path.join(os.path.dirname(__file__), "bias_" + str(index) + ".txt"), "w")
f.close()
mnist_train, mnist_test = tf.keras.datasets.mnist.load_data()
DISTRIBUTION_TYPE = "SAME"
federated_train_data_divide = None
federated_train_data = None
if DISTRIBUTION_TYPE == "SAME":
federated_train_data_divide = [get_data_for_federated_agents(mnist_train, d) for d in range(NUM_AGENT)]
federated_train_data = federated_train_data_divide
f_ini_p = open(os.path.join(os.path.dirname(__file__), "initial_model_parameters.txt"), "r")
para_lines = f_ini_p.readlines()
w_paras = para_lines[0].split("\t")
w_paras = [float(i) for i in w_paras]
b_paras = para_lines[1].split("\t")
b_paras = [float(i) for i in b_paras]
w_initial = np.asarray(w_paras, dtype=np.float32).reshape([784, 10])
b_initial = np.asarray(b_paras, dtype=np.float32).reshape([10])
f_ini_p.close()
initial_model = {
'weights': w_initial,
'bias': b_initial
}
model = initial_model
learning_rate = 0.1
for round_num in range(50):
local_models = federated_train(model, learning_rate, federated_train_data)
print("learning rate: ", learning_rate)
#print(local_models[0][0])#第0个agent的weights矩阵
#print(local_models[0][1])#第0个agent的bias矩阵
#print(len(local_models))
for local_index in range(len(local_models)):
f = open(os.path.join(os.path.dirname(__file__), "weights_"+str(local_index)+".txt"),"a",encoding="utf-8")
for i in local_models[local_index][0]:
line = ""
arr = list(i)
for j in arr:
line += (str(j)+"\t")
print(line, file=f)
print("***"+str(learning_rate)+"***",file=f)
print("-"*50,file=f)
f.close()
f = open(os.path.join(os.path.dirname(__file__), "bias_" + str(local_index) + ".txt"), "a", encoding="utf-8")
line = ""
for i in local_models[local_index][1]:
line += (str(i) + "\t")
print(line, file=f)
print("***" + str(learning_rate) + "***",file=f)
print("-"*50,file=f)
f.close()
m_w = np.zeros([784, 10], dtype=np.float32)
m_b = np.zeros([10], dtype=np.float32)
for local_model_index in range(len(local_models)):
m_w = np.add(np.multiply(local_models[local_model_index][0], 1/NUM_AGENT), m_w)
m_b = np.add(np.multiply(local_models[local_model_index][1], 1/NUM_AGENT), m_b)
model = {
'weights': m_w,
'bias': m_b
}
learning_rate = learning_rate * 0.9
loss = federated_eval(model, federated_train_data)
print('round {}, loss={}'.format(round_num, loss))
print(time.time()-start_time)
gradient_weights = []
gradient_biases = []
gradient_lrs = []
for ij in range(NUM_AGENT):
model_ = getParmsAndLearningRate(ij)
gradient_weights_local = []
gradient_biases_local = []
learning_rate_local = []
for i in range(len(model_['learning_rate'])):
if i == 0:
gradient_weight = np.divide(np.subtract(initial_model['weights'], model_['weights'][i]),
model_['learning_rate'][i])
gradient_bias = np.divide(np.subtract(initial_model['bias'], model_['bias'][i]),
model_['learning_rate'][i])
else:
gradient_weight = np.divide(np.subtract(model_['weights'][i - 1], model_['weights'][i]),
model_['learning_rate'][i])
gradient_bias = np.divide(np.subtract(model_['bias'][i - 1], model_['bias'][i]),
model_['learning_rate'][i])
gradient_weights_local.append(gradient_weight)
gradient_biases_local.append(gradient_bias)
learning_rate_local.append(model_['learning_rate'][i])
gradient_weights.append(gradient_weights_local)
gradient_biases.append(gradient_biases_local)
gradient_lrs.append(learning_rate_local)
all_sets = PowerSetsBinary([i for i in range(NUM_AGENT)])
group_shapley_value = []
for s in all_sets:
group_shapley_value.append(
train_with_gradient_and_valuation(s, gradient_weights, gradient_biases, gradient_lrs, DISTRIBUTION_TYPE))
print(str(s)+"\t"+str(group_shapley_value[len(group_shapley_value)-1]))
agent_shapley = []
for index in range(NUM_AGENT):
shapley = 0.0
for j in all_sets:
if index in j:
remove_list_index = remove_list_indexed(index, j, all_sets)
if remove_list_index != -1:
shapley += (group_shapley_value[shapley_list_indexed(j, all_sets)] - group_shapley_value[
remove_list_index]) / (comb(NUM_AGENT - 1, len(all_sets[remove_list_index])))
agent_shapley.append(shapley)
for ag_s in agent_shapley:
print(ag_s)
print("end_time", time.time()-start_time)
I installed tensor flow federated with this command
pip install --upgrade tensorflow_federated
and this line is also underlied with red color
import tensorflow.compat.v1 as tf
when i tried to execute go this error
File "same_OR.py", line 94, in
BATCH_TYPE = tff.NamedTupleType([ AttributeError: module 'tensorflow_federated' has no attribute 'NamedTupleType'
where is the problem? anyone can help?

tff.NamedTupleType was renamed to tff.StructType in TFF version 0.16.0 (release notes).
Two options:
Install a pre-0.16.0 version of TFF: this should be doable with pip install tensorflow_federated=0.15.0.
Update the code: the error should go away after replacing the tff.NamedTupleType with tff.StructType in the snippet:
BATCH_TYPE = tff.NamedTupleType([
('x', tff.TensorType(tf.float32, [None, 784])),
('y', tff.TensorType(tf.int32, [None]))])
MODEL_TYPE = tff.NamedTupleType([
('weights', tff.TensorType(tf.float32, [784, 10])),
('bias', tff.TensorType(tf.float32, [10]))])

Convolution preprocess consuming more RAM Python

I am trying to implement this CNN model for stock prize prediction: https://github.com/ZezhouLi/Convolutional-Networks-for-Stock-Predicting
But I am facing the issue while preprocessing the data for the implementation. The preprocess step is consuming a lot of RAM. (My system has 32 GB RAM and 256 GB SSD Hard disk)
Here is the file that is consuming the RAM and at last gives a Memory Error:
import numpy as np
import matplotlib.pyplot as plt
import glob
import math
from PIL import Image
import statsmodels.api as sm
def r_squared(y_true, y_hat):
ssr = 0
sst = 0
e = np.subtract(y_true, y_hat)
y_mean = np.mean(y_true)
for item in e:
ssr += item**2
for item in y_true:
sst += (item - y_mean)**2
r2 = 1 - ssr / sst
return r2
def data_process(data):
processed_data = []
for item in data:
m = np.mean(item)
s = np.std(item)
normal_item = [(float(i)-m)/s for i in item]
normal_item.insert(0, 1)
processed_data.append(normal_item)
return processed_data
def get_pixel_values():
file_name = r'\figures'
pixels = []
for filename in glob.glob(file_name + '\*.png'):
im = Image.open(filename)
temp_pixels = list(im.getdata())
pixels.append(temp_pixels)
return pixels
def find_returns(data):
returns = []
for group in data:
count = 30
while count <= (len(group)-5):
current_data = group[count-1]
future_data = group[count+4]
p1 = np.mean(current_data)
p2 = np.mean(future_data)
returns.append(math.log(p2/p1))
count += 1
return returns
def convert_image():
size = 54, 32
file_name = r'\figures'
for filename in glob.glob(file_name + '\*.png'):
img = Image.open(filename)
img.thumbnail(size)
img = img.convert('L')
img.save(filename)
def plot_data(data):
t = np.arange(0, 29, 1)
file_name_number = 0
fig = plt.figure(frameon=False)
for group in data:
count = 30
while count <= (len(group)-5):
high = []
low = []
for item in group[count-30:count]:
high.append(item[0])
low.append(item[1])
file_name = r'\fig_' + str(file_name_number)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
ax.plot(t, high[0:-1], 'b', t, low[0:-1], 'g')
fig.savefig(r'\figures' + file_name)
fig.clf()
file_name_number += 1
count += 1
print('Created %d files!' % file_name_number)
def extract_useful_data(data):
groups = []
for group in data:
temp_buffer = []
for item in group:
temp = [item[2], item[3]]
temp = [float(i) for i in temp]
temp_buffer.append(temp)
groups.append(temp_buffer)
return groups
def split_data(data):
groups = []
for item in data:
temp_buffer = []
for string in item:
number = string.split(',')
temp_buffer.append(number)
groups.append(temp_buffer)
return groups
def extract_data():
file_name = r'\data.txt'
infile = open(file_name, 'r')
temp_buffer = []
for line in infile:
temp_buffer.append(line.strip('\n'))
temp_buffer = temp_buffer[8:]
i = 0
groups = []
temp = []
for item in temp_buffer:
if i != 390:
temp.append(item)
i += 1
else:
groups.append(temp)
temp = []
i = 0
groups.append(temp)
infile.close()
return groups
def main():
original_data = extract_data()
splitted_data = split_data(original_data)
useful_data = extract_useful_data(splitted_data)
plot_data(useful_data)
convert_image()
returns = np.asarray(find_returns(useful_data))
training_data = np.asarray(get_pixel_values())
training_data = sm.add_constant(training_data, has_constant='add')
results = sm.OLS(returns[0:4340], training_data[0:4340]).fit()
y_in_sample = results.predict(training_data[0:4340])
r2 = r_squared(returns[0:4340], y_in_sample)
print r2
if __name__ == "__main__":
main()
I have got Memory Error, which occurs when the program consumes all of the RAM memory of the system. Please improve on the program.

Handwriting neural network weights don't change

from struct import unpack
import gzip
import numpy
from numpy import *
import matplotlib.pyplot as plt
learningRate = 0.1
def get_labeled_data(imagefile, labelfile):
"""Read input-vector (image) and target class (label, 0-9) and return
it as list of tuples.
"""
# Open the images with gzip in read binary mode
images = gzip.open(imagefile, 'rb')
labels = gzip.open(labelfile, 'rb')
# Read the binary data
# We have to get big endian unsigned int. So we need '>I'
# Get metadata for images
images.read(4) # skip the magic_number
number_of_images = images.read(4)
number_of_images = unpack('>I', number_of_images)[0]
rows = images.read(4)
rows = unpack('>I', rows)[0]
cols = images.read(4)
cols = unpack('>I', cols)[0]
# Get metadata for labels
labels.read(4) # skip the magic_number
N = labels.read(4)
N = unpack('>I', N)[0]
if number_of_images != N:
raise Exception('number of labels did not match the number of images')
# Get the data
x = zeros((N, rows, cols), dtype="float32") # Initialize numpy array
y = zeros((N, 1), dtype="uint8") # Initialize numpy array
for i in range(N):
if i % 1000 == 0:
print("i: %i" % i)
for row in range(rows):
for col in range(cols):
tmp_pixel = images.read(1) # Just a single byte
tmp_pixel = unpack('>B', tmp_pixel)[0]
x[i][row][col] = tmp_pixel
tmp_label = labels.read(1)
y[i] = unpack('>B', tmp_label)[0]
return (x, y)
ld = get_labeled_data("C:/Users/XBGFD/Desktop/Programming/NeuralNetworks/HRR/train-images-idx3-ubyte.gz", "C:/Users/XBGFD/Desktop/Programming/NeuralNetworks/HRR/train-labels-idx1-ubyte.gz")
def sigmoid(x):
return 1/(1+numpy.exp(-x))
def sigmoid_P(x):
return sigmoid(x) * (1 - sigmoid(x))
def cost(i, t):
return (i - t) ** 2
def cost_P(i, t):
return 2 * (i - t)
# 10x28x28 - number x row x column
weights = numpy.random.random((10, 28, 28))
biases = numpy.random.random((10, 28, 28))
dr = 0
da = 0
for loopi in range(10000):
r = numpy.random.randint(0, len(ld[0][0]))
targets = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
image = ld[0][r]
label = ld[1][r][0]
# weighted 3D Matrix of [number][row][column]
predictions = []
wPredictions = []
# average of predictions for each number
avgPred = []
avgPred2 = []
img = list(image)
for i in range(10):
x = []
y = []
for b, w in zip(biases[i], weights[i]):
x.append(sigmoid(numpy.dot(w, image) + b))
y.append(numpy.dot(w, image) + b)
predictions.append(x)
avgPred.append(numpy.average(list(x)))
avgPred2.append(numpy.average(list(y)))
for i in range(10):
sqError = cost(avgPred[i], targets[i])
# derivative of the cost with respect to each of the weights and biases
dc_dp = cost_P(avgPred[i], targets[i])
dp_dz = sigmoid_P(avgPred2[i])
#for b, w in zip(biases[i], weights[i]):
for imgRow in range(28):
for imgCol in range(28):
dz_dw = image[imgRow][imgCol]
dz_db = 1
print("dc_dp: " + str(dc_dp) + "\ndp_dz: "+ str(dp_dz) + "\ndz_dw: " + str(dz_dw))
dc_dw = dc_dp * dp_dz * dz_dw
dc_db = dc_dp * dp_dz * dz_db
dr = dc_dw
weights[i][imgRow][imgCol] -= learningRate * dc_dw
da = weights[i][imgRow][imgCol]
biases[i][imgRow][imgCol] -= learningRate * dc_db
while True:
big = 0
intid = int(input())
imag = ld[0][intid]
for l in range(10):
papa = []
for b, w in zip(biases[i], weights[i]):
papa.append(sigmoid(numpy.dot(w, imag) + b))
lol = numpy.average(papa)
if(lol > big):
big = l
print(str(dr) + " " + str(da))
print(big)
The weights aren't changing because dp_dz is always 0, I'm not sure what's causing that. I don't mean that they're changing but only a very small change, they're literally NOT changing at all. I believe it has to do with my approach in general, but I'm not sure how else I could approach this problem, I'm very new to neural networks. Any help would be greatly appreciated!

TypeError when using a method from a class - python

I have an improved kmeans algorithm (KPlusPlus) that builds on the class kmeans. Detk is another class inherited from KPlusPlus.
The objective of the KPlusPlus class is to find out the optimal seeding for finding the kmeans centroids (Source)
Detk calculates the gap statistic to find the optimal number of clusters. I have found this code from here
# kmeans class
class KMeans():
def __init__(self, K, X=None, N=0):
self.K = K
if X == None:
if N == 0:
raise Exception("If no data is provided, \
a parameter N (number of points) is needed")
else:
self.N = N
self.X = self._init_board_gauss(N, K)
else:
self.X = X
self.N = len(X)
self.mu = None
self.clusters = None
self.method = None
def _init_board_gauss(self, N, k):
n = float(N)/k
X = []
for i in range(k):
c = (random.uniform(-1,1), random.uniform(-1,1))
s = random.uniform(0.05,0.15)
x = []
while len(x) < n:
a,b = np.array([np.random.normal(c[0],s),np.random.normal(c[1],s)])
# Continue drawing points from the distribution in the range [-1,1]
if abs(a) and abs(b)<1:
x.append([a,b])
X.extend(x)
X = np.array(X)[:N]
return X
def plot_board(self):
X = self.X
fig = plt.figure(figsize=(5,5))
plt.xlim(-1,1)
plt.ylim(-1,1)
if self.mu and self.clusters:
mu = self.mu
clus = self.clusters
K = self.K
for m, clu in clus.items():
cs = cm.spectral(1.*m/self.K)
plt.plot(mu[m][0], mu[m][1], 'o', marker='*', \
markersize=12, color=cs)
plt.plot(zip(*clus[m])[0], zip(*clus[m])[1], '.', \
markersize=8, color=cs, alpha=0.5)
else:
plt.plot(zip(*X)[0], zip(*X)[1], '.', alpha=0.5)
if self.method == '++':
tit = 'K-means++'
else:
tit = 'K-means with random initialization'
pars = 'N=%s, K=%s' % (str(self.N), str(self.K))
plt.title('\n'.join([pars, tit]), fontsize=16)
plt.savefig('kpp_N%s_K%s.png' % (str(self.N), str(self.K)), \
bbox_inches='tight', dpi=200)
def _cluster_points(self):
mu = self.mu
clusters = {}
for x in self.X:
bestmukey = min([(i[0], np.linalg.norm(x-mu[i[0]])) \
for i in enumerate(mu)], key=lambda t:t[1])[0]
try:
clusters[bestmukey].append(x)
except KeyError:
clusters[bestmukey] = [x]
self.clusters = clusters
def _reevaluate_centers(self):
clusters = self.clusters
newmu = []
keys = sorted(self.clusters.keys())
for k in keys:
newmu.append(np.mean(clusters[k], axis = 0))
self.mu = newmu
def _has_converged(self):
K = len(self.oldmu)
return(set([tuple(a) for a in self.mu]) == \
set([tuple(a) for a in self.oldmu])\
and len(set([tuple(a) for a in self.mu])) == K)
def find_centers(self,K, method='random'):
self.method = method
X = self.X
K = self.K
self.oldmu = random.sample(X, K)
if method != '++':
# Initialize to K random centers
self.mu = random.sample(X, K)
while not self._has_converged():
self.oldmu = self.mu
# Assign all points in X to clusters
self._cluster_points()
# Reevaluate centers
self._reevaluate_centers()
The KPlusPlus class inherits from kmeans to find the optimal seeding
class KPlusPlus(KMeans):
def _dist_from_centers(self):
cent = self.mu
X = self.X
D2 = np.array([min([np.linalg.norm(x-c)**2 for c in cent]) for x in X])
self.D2 = D2
def _choose_next_center(self):
self.probs = self.D2/self.D2.sum()
self.cumprobs = self.probs.cumsum()
r = random.random()
ind = np.where(self.cumprobs >= r)[0][0]
return(self.X[ind])
def init_centers(self,K):
self.K = K
self.mu = random.sample(self.X, 1)
while len(self.mu) < self.K:
self._dist_from_centers()
self.mu.append(self._choose_next_center())
def plot_init_centers(self):
X = self.X
fig = plt.figure(figsize=(5,5))
plt.xlim(-1,1)
plt.ylim(-1,1)
plt.plot(zip(*X)[0], zip(*X)[1], '.', alpha=0.5)
plt.plot(zip(*self.mu)[0], zip(*self.mu)[1], 'ro')
plt.savefig('kpp_init_N%s_K%s.png' % (str(self.N),str(self.K)), \
bbox_inches='tight', dpi=200)
The class Detk inherits from KPlusPlus to find the optmal number of clusters based on gap statistic
class DetK(KPlusPlus):
def fK(self, thisk, Skm1=0):
X = self.X
Nd = len(X[0])
a = lambda k, Nd: 1 - 3/(4*Nd) if k == 2 else a(k-1, Nd) + (1-a(k-1, Nd))/6
self.find_centers(thisk, method='++')
mu, clusters = self.mu, self.clusters
Sk = sum([np.linalg.norm(mu[i]-c)**2 \
for i in range(thisk) for c in clusters[i]])
if thisk == 1:
fs = 1
elif Skm1 == 0:
fs = 1
else:
fs = Sk/(a(thisk,Nd)*Skm1)
return fs, Sk
def _bounding_box(self):
X = self.X
xmin, xmax = min(X,key=lambda a:a[0])[0], max(X,key=lambda a:a[0])[0]
ymin, ymax = min(X,key=lambda a:a[1])[1], max(X,key=lambda a:a[1])[1]
return (xmin,xmax), (ymin,ymax)
def gap(self, thisk):
X = self.X
(xmin,xmax), (ymin,ymax) = self._bounding_box()
self.init_centers(thisk)
self.find_centers(thisk, method='++')
mu, clusters = self.mu, self.clusters
Wk = np.log(sum([np.linalg.norm(mu[i]-c)**2/(2*len(c)) \
for i in range(thisk) for c in clusters[i]]))
# Create B reference datasets
B = 10
BWkbs = zeros(B)
for i in range(B):
Xb = []
for n in range(len(X)):
Xb.append([random.uniform(xmin,xmax), \
random.uniform(ymin,ymax)])
Xb = np.array(Xb)
kb = DetK(thisk, X=Xb)
kb.init_centers(thisk)
kb.find_centers(thisk, method='++')
ms, cs = kb.mu, kb.clusters
BWkbs[i] = np.log(sum([np.linalg.norm(ms[j]-c)**2/(2*len(c)) \
for j in range(thisk) for c in cs[j]]))
Wkb = sum(BWkbs)/B
sk = np.sqrt(sum((BWkbs-Wkb)**2)/float(B))*np.sqrt(1+1/B)
return Wk, Wkb, sk
def run(self, maxk, which='both'):
ks = range(1,maxk)
fs = zeros(len(ks))
Wks,Wkbs,sks = zeros(len(ks)+1),zeros(len(ks)+1),zeros(len(ks)+1)
# Special case K=1
self.init_centers(1)
if which == 'f':
fs[0], Sk = self.fK(1)
elif which == 'gap':
Wks[0], Wkbs[0], sks[0] = self.gap(1)
else:
fs[0], Sk = self.fK(1)
Wks[0], Wkbs[0], sks[0] = self.gap(1)
# Rest of Ks
for k in ks[1:]:
self.init_centers(k)
if which == 'f':
fs[k-1], Sk = self.fK(k, Skm1=Sk)
elif which == 'gap':
Wks[k-1], Wkbs[k-1], sks[k-1] = self.gap(k)
else:
fs[k-1], Sk = self.fK(k, Skm1=Sk)
Wks[k-1], Wkbs[k-1], sks[k-1] = self.gap(k)
if which == 'f':
self.fs = fs
elif which == 'gap':
G = []
for i in range(len(ks)):
G.append((Wkbs-Wks)[i] - ((Wkbs-Wks)[i+1]-sks[i+1]))
self.G = np.array(G)
else:
self.fs = fs
G = []
for i in range(len(ks)):
G.append((Wkbs-Wks)[i] - ((Wkbs-Wks)[i+1]-sks[i+1]))
self.G = np.array(G)
When I try to run the following program on a given number of points (locArray)
locArray = np.array(locArrayMaster[counter])
kmeanscluster = DetK(2, X = locArray)
kmeanscluster.run(5)
noClusters[counter] = np.where(kmeanscluster.fs == min(kmeanscluster.fs))[0][0]+ 1
it returns me the following error
File "C:\Users\Anaconda2\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 714, in runfile
execfile(filename, namespace)
File "C:\Users\Anaconda2\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 74, in execfile
exec(compile(scripttext, filename, 'exec'), glob, loc)
File "C:/Users/Documents/SUMOTraffic/kplusplus.py", line 355, in <module>
kmeanscluster.run(5)
File "C:/Users/Documents/SUMOTraffic/kplusplus.py", line 217, in run
Wks[0], Wkbs[0], sks[0] = self.gap(1)
File "C:/Users/Documents/SUMOTraffic/kplusplus.py", line 200, in gap
for j in range(thisk) for c in cs[j]]))
TypeError: 'NoneType' object has no attribute '__getitem__'
Thanks for any help.

The error is due to the failure of the kmeans algorithm to find the cluster centres when the number of clusters is just 1. Hence the cluster dictionary is not created for this case. So, added an extra line of code in the class DetK which checks if the type of cluster dictionary is 'NoneType' and if it returns TRUE, recalculates the cluster centres again.
class DetK(KPlusPlus):
def fK(self, thisk, Skm1=0):
X = self.X
Nd = len(X[0])
a = lambda k, Nd: 1 - 3/(4*Nd) if k == 2 else a(k-1, Nd) + (1-a(k-1, Nd))/6
self.find_centers(thisk, method='++')
while type(self.clusters) is not dict:
self.find_centers(thisk, method = '++')
mu, clusters = self.mu, self.clusters
Sk = sum([np.linalg.norm(mu[i]-c)**2 \
for i in range(thisk) for c in clusters[i]])
if thisk == 1:
fs = 1
elif Skm1 == 0:
fs = 1
else:
fs = Sk/(a(thisk,Nd)*Skm1)
return fs, Sk