I want to get a confusion matrix according to y_test and pred_test, but raise a question "At least one label specified must be in y_true",i don't know why
metrics.confusion_matrix(np.argmax(y_test,axis=1),pred_test)
y_test = [[0. 1. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 1.]
[0. 0. 0. 0. 1. 0.]
...
[0. 0. 0. 1. 0. 0.]
[0. 0. 1. 0. 0. 0.]
[0. 0. 1. 0. 0. 0.]]
pred_test = [1 4 5 ... 3 2 2]
np.argmax(y_test,axis=1) = [1 5 4 ... 3 2 2]
File "D:\Anaconda\lib\site-packages\sklearn\metrics\classification.py", line 259, in confusion_matrix
raise ValueError("At least one label specified must be in y_true")
ValueError: At least one label specified must be in y_true
I create a convolutional neural network. model and use cross validation for estimate, finally generate a confusion matrix. Now there are problems in generating confusion matrix.
The dataset is enter link description here.The complete code is as follows：
import matplotlib
#matplotlib.use('Agg')
import timing
from keras.layers import Input,Dense,Conv2D,MaxPooling2D,UpSampling2D,Flatten
from keras.models import Model
from keras import backend as K
from keras.utils.np_utils import to_categorical
import numpy as np
import pandas as pd
import seaborn as sns
from keras.models import Sequential# 导入Sequential
from keras.utils import np_utils, generic_utils
from keras.callbacks import LearningRateScheduler
import os
from keras.layers import Dropout
from keras.backend.tensorflow_backend import set_session
import tensorflow as tf
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.cross_validation import KFold, StratifiedKFold
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn import metrics
import time
from scipy import stats
from keras import optimizers
import matplotlib.pyplot as plt
from keras import regularizers
import keras
from keras.callbacks import TensorBoard
config = tf.ConfigProto(allow_soft_placement=True)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
time1 = time.time()
class LossHistory(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.losses = {'batch':[], 'epoch':[]}
self.accuracy = {'batch':[], 'epoch':[]}
self.val_loss = {'batch':[], 'epoch':[]}
self.val_acc = {'batch':[], 'epoch':[]}
def on_batch_end(self, batch, logs={}):
self.losses['batch'].append(logs.get('loss'))
self.accuracy['batch'].append(logs.get('acc'))
self.val_loss['batch'].append(logs.get('val_loss'))
self.val_acc['batch'].append(logs.get('val_acc'))
def on_epoch_end(self, batch, logs={}):
self.losses['epoch'].append(logs.get('loss'))
self.accuracy['epoch'].append(logs.get('acc'))
self.val_loss['epoch'].append(logs.get('val_loss'))
self.val_acc['epoch'].append(logs.get('val_acc'))
def loss_plot(self, loss_type):
iters = range(len(self.losses[loss_type]))
plt.figure()
# acc
plt.plot(iters, self.accuracy[loss_type], 'r', label='train acc')
# loss
plt.plot(iters, self.losses[loss_type], 'g', label='train loss')
if loss_type == 'epoch':
# val_acc
plt.plot(iters, self.val_acc[loss_type], 'b', label='val acc')
# val_loss
plt.plot(iters, self.val_loss[loss_type], 'k', label='val loss')
plt.grid(True)
plt.xlabel(loss_type)
plt.ylabel('acc-loss')
plt.legend(loc="center")
plt.show()
#plt.savefig('common.png')
#dataset
RANDOM_SEED = 42
def read_data(file_path):
column_names = ['user-id', 'activity', 'timestamp', 'x-axis', 'y-axis', 'z-axis']
m = pd.read_csv(file_path,names=column_names, header=None,sep=',')
return m
def feature_normalize(dataset):
mu = np.mean(dataset,axis=0)
sigma = np.std(dataset,axis=0)
return (dataset-mu)/sigma
dataset1 = read_data('ab.txt')
dataset = pd.DataFrame(dataset1)
dataset['x-axis'] = feature_normalize(dataset['x-axis'])
dataset['y-axis'] = feature_normalize(dataset['y-axis'])
dataset['z-axis'] = feature_normalize(dataset['z-axis'])
N_TIME_STEPS = 200
N_FEATURES = 3
step = 200
segments = []
labels = []
for i in range(0, len(dataset) - N_TIME_STEPS, step):
xs = dataset['x-axis'].values[i: i + N_TIME_STEPS]
ys = dataset['y-axis'].values[i: i + N_TIME_STEPS]
zs = dataset['z-axis'].values[i: i + N_TIME_STEPS]
label = stats.mode(dataset['activity'][i: i + N_TIME_STEPS])[0][0]
segments.append([xs, ys, zs])
labels.append(label)
print("reduced size of data", np.array(segments).shape)
reshaped_segments = np.asarray(segments,dtype=np.float32).reshape(-1,1, N_TIME_STEPS, 3)
print("Reshape the segments", np.array(reshaped_segments).shape)
#x_train1, x_val_test, y_train1, y_val_test = train_test_split(reshaped_segments, labels, test_size=0.25, random_state=RANDOM_SEED)
batch_size = 128
num_classes =6
def create_model():
input_shape = Input(shape=(1,200,3))
x = Conv2D(5, kernel_size=(1, 1), padding='valid')(input_shape)
x1 = keras.layers.concatenate([input_shape, x], axis=-1)
x = Conv2D(50, kernel_size=(1, 7),padding='valid',
kernel_initializer='glorot_uniform',
kernel_regularizer = keras.regularizers.l2(0.0015))(x1)
x = keras.layers.core.Activation('relu')(x)
x = MaxPooling2D(pool_size=(1, 2))(x)
x = Conv2D(50, kernel_size=(1, 7),padding='valid',kernel_initializer='glorot_uniform',
kernel_regularizer=keras.regularizers.l2(0.0015))(x)
x = keras.layers.core.Activation('relu')(x)
x = MaxPooling2D(pool_size=(1, 2))(x)
x = Flatten()(x)
x = Dropout(0.9)(x)
output = Dense(num_classes, activation='softmax',kernel_initializer='glorot_uniform',)(x)
model = Model(inputs=input_shape,outputs=output)
model.summary()
sgd = optimizers.SGD(lr=0.005,decay=1e-6,momentum=0.9,nesterov=True)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=sgd,
metrics=['accuracy'])
return model
history = LossHistory()
epochs = 4000
#setting learning rate
def scheduler(epoch):
if epoch > 0.75 * epochs:
lr = 0.0005
elif epoch > 0.25 * epochs:
lr = 0.001
else:
lr = 0.005
return lr
scheduler = LearningRateScheduler(scheduler)
estimator = KerasClassifier(build_fn=create_model)
#divide dataset
scores = []
confusions = []
sign = ['DOWNSTAIRS','JOGGING','SITTING','STANDING','UPSTAIRS','WALKING']
encoder = LabelEncoder()
encoder_y = encoder.fit_transform(labels)
train_labels = to_categorical(encoder_y,num_classes=None)
#kfold = StratifiedKFold(reshaped_segments.shape[0],n_folds=10,shuffle=True,random_state=42)
kfold = StratifiedKFold(labels,n_folds=3,shuffle=True,random_state=42)
for train_index,test_index in kfold:
print(test_index)
x_train, x_test = reshaped_segments[train_index], reshaped_segments[test_index]
y_train, y_test = train_labels[train_index], train_labels[test_index]
estimator.fit(x_train,y_train,callbacks=[scheduler,history],epochs=10,batch_size=128,verbose=0)
scores.append(estimator.score(x_test,y_test))
print(y_test)
print(type(y_test))
pred_test = estimator.predict(x_test)
print(pred_test)
print(np.argmax(y_test,axis=1))
confusions.append(metrics.confusion_matrix(np.argmax(y_test,axis=1),pred_test,sign))
matrix = [[0,0,0,0,0,0],[0,0,0,0,0,0],[0,0,0,0,0,0],[0,0,0,0,0,0],[0,0,0,0,0,0],[0,0,0,0,0,0]]
for i in np.arange(n_folds-1):
for j in len(confusions[0]):
for k in len(confusions[0][0]):
matrix[j][k] = matrix[j][k] + confusions[i][j][k] + confusions[i+1][j][k]
model.save('model.h5')
model.save_weights('my_model_weights.h5')
print('score:',scores)
scores = np.mean(scores)
print('mean:',scores)
plt.figure(figsize=(16,14))
sns.heatmap(matrix, xticklabels=sign, yticklabels=sign, annot=True, fmt="d");
plt.title("CONFUSION MATRIX : ")
plt.ylabel('True Label')
plt.xlabel('Predicted label')
plt.savefig('cmatrix.png')
plt.show();
The error isn't in your main code but rather in the definition of sign. When you define sign as
sign = ['DOWNSTAIRS','JOGGING','SITTING','STANDING','UPSTAIRS','WALKING']
the system cannot read your labels as it is looking for the labels 0,1,2,3,4,5 as what the error was trying to say i.e. it could not find any labels in sign in y_pred.
changing sign to
sign = [1,2,3,4,5]
should fix the error. As for what you do now , its rather simple just map your result as this array and then during the actual predictions(Deployment) just swap out the numeric values for the labels.
Related
I am trying to deploy classification model for fall and no fall using accelerometer data in raspberry pi 4. However something seems wrong with either tensorflow or numpy when reshaping them.
Exact error that I am getting is
ValueError: The target structure is of type <class 'NoneType'>
None
However the input structure is a sequence (<class 'list'>) of length 0.
[]
nest cannot guarantee that it is safe to map one to the other.
Deployment code can be found here.
import numpy as np
import pandas as pd
from tensorflow.keras.models import load_model
import board
import scipy.stats as stats
import adafruit_mpu6050
from math import atan2,degrees,pow
import warnings
warnings.filterwarnings("ignore")
import pandas as pd
import numpy as np
print("Inititating")
model_trained = load_model('model_cnn.h5')
print('\n',model_trained.summary())
i2c = board.I2C()
mpu = adafruit_mpu6050.MPU6050(i2c)
mpu_accelerometer_range = adafruit_mpu6050.Range.RANGE_4_G
data1 = []
i = 0
while(i<2000):
i+=1
ax,ay,az = mpu.acceleration
data1.append([ax,ay,az])
df = pd.DataFrame(data1)
#print('length of df',len(df))
columns = ['x','y','z']
df.columns = columns
#print(df)
Fs = 50
frame_size = Fs*4
hop_size = Fs*2
frames = []
N_FEATURES = 3
for i in range(0,len(df)-frame_size,hop_size):
x = df['x'].values[i:i+frame_size]
y = df['y'].values[i:i+frame_size]
z = df['z'].values[i:i+frame_size]
frames.append([x,y,z])
#print('leeeee',len(frames))
# converting frames to numpy array
frames = np.asarray(frames).reshape(-1,frame_size,N_FEATURES)
#print('frames',len(frames))
k = int((len(df)-frame_size)/hop_size)+1
print('type is',type(k))
X_test = frames.reshape(k,200,3,1)
y_pred = model_trained.predict_classes(X_test)
print(y_pred)
I am using hopping window approach on cnn data having 3 columns x-axis acceleration, y-axis acceleration, z-axis acceleration.
When I run below code outside of while loop, model gives output but that can not be used for real time.
X_test = frames.reshape(k,200,3,1)
y_pred = model_trained.predict_classes(X_test)
print(y_pred)
Training code
# Importing
import numpy as np
import pandas as pd
#import seaborn as sns
import tensorflow as tf
#import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Flatten, Dense, Dropout, BatchNormalization,MaxPooling2D
from tensorflow.keras.layers import Conv2D,MaxPool2D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import load_model
physical_devices = tf.config.experimental.list_physical_devices('GPU')
if len(physical_devices) > 0:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
import os
import tensorflow as tf
os.environ['TF_CPP_MIN_LOG_LEVEL'] = "2"
df = pd.read_csv('fall_nofall_10_with_labels.csv')
df.columns = ['a','activity','time','realtime','x','y','z','gyro-x','gyro-y','gyro-z']
data = df.drop(['a','realtime','gyro-x','gyro-y','gyro-z'],axis=1)
print(data['activity'].value_counts())
# # sampling rate
Fs = 50
activities = data['activity'].value_counts().index
balanced_data = data.drop(['time'], axis = 1).copy()
balanced_data['activity'].value_counts()
from sklearn.preprocessing import LabelEncoder
label = LabelEncoder()
balanced_data['label'] = label.fit_transform(df['activity'])
### Frame Prepration
import scipy.stats as stats
Fs = 50
frame_size = Fs*4 #(4 seconds)
# 200x200x3 will be feeded in
hop_size = Fs*2 #(How much overlap) make advancement with 100 data samples
def get_frames(df,frame_size,hop_size):
N_FEATURES = 3 # input feature is x,y and z
frames = []
labels = []
for i in range(0,len(df) - frame_size,hop_size):
x = df['x'].values[i:i+frame_size] # 0 to 4 second then 1 to 5 seconds
y = df['y'].values[i:i+frame_size]
z = df['z'].values[i:i+frame_size]
# activity which comes most number of time we'll be considering that
label = stats.mode(df['label'][i:i+frame_size])[0][0]
# labels = label[0][0]
frames.append([x,y,z])
labels.append(label)
# convert into numpy array
frames = np.asarray(frames).reshape(-1,frame_size,N_FEATURES)
labels = np.asarray(labels)
return frames,labels
X,y = get_frames(balanced_data,frame_size,hop_size)
print(X.shape,y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0, stratify = y)
X_train.shape, X_test.shape
X_train[0].shape, X_test[0].shape
X_train = X_train.reshape(39,200,3,1)
X_test = X_test.reshape(10,200,3,1)
X_train[0].shape,X_test[0].shape
# ## Creating CNN model
model = Sequential()
model.add(Conv2D(16,(2,2),activation='relu',input_shape=X_train[0].shape))
# model.add(MaxPooling2D(pool_size=(1,1)))
model.add(Dropout(0.1))
model.add(Conv2D(64,(2,2),activation='relu'))
# model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(64,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2,activation='sigmoid'))
model.summary()
model.compile(optimizer=Adam(learning_rate = 0.001), loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])
history = model.fit(X_train, y_train, epochs = 10, validation_data= (X_test, y_test), verbose=1)
model.save("model_cnn.h5")
# load the model
model_trained = load_model('model_cnn.h5')
# summary of the model
print('\n',model_trained.summary())
from sklearn.metrics import confusion_matrix,classification_report
y_pred = model_trained.predict_classes(X_test)
print(y_pred)
print(classification_report(y_pred,y_test))
print(confusion_matrix(y_pred,y_test))
complete traceback:
Traceback (most recent call last):
File "cnn_nofall_live.py", line 68, in <module>
y_pred = model_trained.predict_classes(X_test)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/sequential.py", line 338, in predict_classes
proba = self.predict(x, batch_size=batch_size, verbose=verbose)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training.py", line 1013, in predict
use_multiprocessing=use_multiprocessing)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 498, in predict
workers=workers, use_multiprocessing=use_multiprocessing, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 475, in _model_iteration
total_epochs=1)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 187, in run_one_epoch
aggregator.finalize()
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_utils.py", line 353, in finalize
self.results = nest.pack_sequence_as(self._structure, self.results)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/nest.py", line 504, in pack_sequence_as
return _pack_sequence_as(structure, flat_sequence, expand_composites)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/nest.py", line 453, in _pack_sequence_as
len(flat_sequence), truncate(flat_sequence, 100)))
ValueError: The target structure is of type `<class 'NoneType'>`
None
However the input structure is a sequence (<class 'list'>) of length 0.
[]
nest cannot guarantee that it is safe to map one to the other.
I think you need to post the full traceback. It is difficult to debug/help without seeing that
But OOTH one possible issue is that you're using "i" for the while loop and the for loop. Maybe change the for loop index to "j" or something.
Try running this code snippet to see what I mean
i=0
while(i<10):
i+=1
for i in range(100):
print(f"for loop i is {i}")
print(f"while loop i is {i}")
btw, you trying for this?
data1 = []
i = 0
while(i<2000):
i+=1
ax,ay,az = mpu.acceleration
data1.append([ax,ay,az])
df = pd.DataFrame(data1, columns=['x','y','z'])
frame_size = 200
hop_size = 100
frames = []
N_FEATURES = 3
for i in range(0,len(df)-frame_size,hop_size):
x = df['x'].values[i:i+frame_size]
y = df['y'].values[i:i+frame_size]
z = df['z'].values[i:i+frame_size]
frames.append([x,y,z])
frames = np.array(frames)
k = int((len(df)-frame_size)/hop_size)
X_test = frames.reshape(k,200,3,1)
I am trying to generate a LSTM model using Keras. I create a simple sine wave example which contain more thang 1000 point to predict the next point. But the result is not good as i expected. When i fit the model the result is moves between 0~1 not like the sine wave. I have tried to change parameter like epoch, batchsize, learning rate, but it is not better.
model predict image
What am I doing wrong?
import joblib
import numpy as np
import matplotlib.pyplot as plt
import copy
import gc
import os
import sys
from sklearn.preprocessing import MinMaxScaler
from tensorflow import keras
import tensorflow as tf
from tensorflow.keras.models import *
from tensorflow.keras.layers import *
from keras.callbacks import Callback
learning_rate = 0.001
len_train = 30
total_predict = 300
len_test = 400
epoch = 100
batch_size = 32
workers = -1
class Callback_Class(Callback):
def load_data(self, x_test, y_test):
self.x_test = x_test
self.y_test = np.array(y_test)
def model_predict(self, data_close):
output_predict = []
for i in range(total_predict):
if (i==0):
data_close_ = data_close.reshape(-1, len_train, 1)
else:
data_close_ = np.delete(data_close_, 0)
data_close_ = np.append(data_close_, pred_close)
data_close_ = data_close_.reshape(-1, len_train, 1)
pred_close = model.predict(data_close_)
pred_close = pred_close.ravel()
pred_close = np.array(pred_close).reshape(len(pred_close), 1)
pred_cl = sc.inverse_transform(pred_close)
output_predict.append(pred_cl)
output_predict = np.array(output_predict)
return output_predict
def on_epoch_end(self, epoch, logs=None):
if (epoch % 20 == 0):
output_predict = self.model_predict(self.x_test)
fig, ax = plt.subplots(figsize=(12,6))
ax.grid(True)
plt.title(f"Model predict")
plt.plot(output_predict.ravel(), color="red", label='Predict')
plt.plot(self.y_test.ravel(), color="blue", label='REAL')
fig.tight_layout()
plt.legend(loc='lower left')
plt.savefig(f'Demo_lstm_epoch_{epoch}.png')
plt.clf()
plt.close()
def lstm_reg(input_shape=(60, 1), unit=40, clustering_params=None):
inputs = Input(input_shape)
lstm1f = Bidirectional(LSTM(units=32, return_sequences=True))(inputs)
lstm1f = Bidirectional(LSTM(units=32, return_sequences=False))(lstm1f)
outputs = Dense(units=1, activation='linear')(lstm1f)
model = Model(inputs=inputs, outputs=outputs)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate), loss='mean_squared_error', metrics=["accuracy"])
return model
def create_data_train(data_time_series):
data_time_series = np.array(data_time_series).ravel()
X_train = []
y_train = []
for i in range(len_train, len(data_time_series)):
X_train.append(data_time_series[i-len_train:i])
y_train.append(data_time_series[i])
X_train, y_train = np.array(X_train), np.array(y_train)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
return X_train, y_train
x = np.linspace(-20*np.pi, 20*np.pi, 2001)
sin_alpha = np.sin(x).ravel()
sin_alpha_train = np.array(copy.deepcopy(sin_alpha))[:len(sin_alpha)-len_test]
sin_alpha_train = np.array(sin_alpha_train).reshape(len(sin_alpha_train), 1)
sc = MinMaxScaler(feature_range=(0, 1))
sin_alpha_train = sc.fit_transform(sin_alpha_train)
X_train, y_train = create_data_train(sin_alpha_train)
joblib.dump(sc, f'Demo_MinMaxScaler.gz')
sc = joblib.load(f"Demo_MinMaxScaler.gz")
X_test = np.array(copy.deepcopy(sin_alpha))[len(sin_alpha)-len_test:len(sin_alpha)-len_test+len_train]
X_test = np.array(X_test).reshape(len(X_test), 1)
X_test = sc.fit_transform(X_test)
y_test = np.array(copy.deepcopy(sin_alpha))[len(sin_alpha)-len_test+len_train:len(sin_alpha)-len_test+len_train+total_predict]
model = lstm_reg(input_shape=(len_train, 1), unit=int(2*(len_train+len(y_train))/3))
model.summary()
callback_class = Callback_Class()
callback_class.load_data(X_test, y_test)
model.fit(X_train, y_train, epochs=epoch, use_multiprocessing=True, verbose=1, callbacks=[callback_class], workers=workers, batch_size=batch_size)
It seems like you are normalizing your features and your labels in these lines
sc = MinMaxScaler(feature_range=(0, 1))
sin_alpha_train = sc.fit_transform(sin_alpha_train)
X_train, y_train = create_data_train(sin_alpha_train)
Try it without scaling your label set. Due to your output layer using the linear activation function, which is correct as you're working on a regression problem, the model should be able to handle non scaled labels. The model only learns your data in a range of 0 to 1 while your sine wave goes from -1 to 1.
I'm trying to implement a Bayesian neural network for genomic predictions. My X is a matrix that is scaled and gets normalized so that the values are between 0 and 1. The y is a vector of values that are again normalized so that the values are between 0 and 1.
The network seems to learn as seen here:
But, when I try to make predictions these look strange and seem to behave randomly. While the true values of y are distributed between 0 and 1. The predicted values are between ~ 0.4 - 0.6 and my R2 is negative. The MSE is around 0.02, what seems not to bad, but might be caused by the fact that the range of the predictions is quite narrow.
I'm a bit running out of ideas what could be wrong. Any suggestions are appreciated :).
I've also tried to predict the training data. That is also not working and I'm getting a negative R2.
X has the dimensions (5000,500) and y (5000,)
Increasing the number of hidden layers (up to 3) and units (upt to 128) doesn't change anything.
# Import necessary packages:
import sys
from os.path import join
import warnings
warnings.filterwarnings('ignore')
from IPython import display
import tensorflow as tf
from tensorflow import keras
import tensorflow_probability as tfp
import kerastuner as kt
from keras import backend as K
from keras import activations, initializers
from keras.layers import Layer
import tensorflow_docs as tfdocs
import tensorflow_docs.modeling
import tensorflow_docs.plots
import numpy as np
import numpy.ma as ma
import pandas as pd
import seaborn as sns
import matplotlib.pylab as plt
import time
import tempfile
import math
import statsmodels.api as sm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import r2_score,mean_absolute_error
from sklearn.linear_model import LinearRegression, BayesianRidge
from sklearn.utils import shuffle
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, scale
from pandas_plink import read_plink
from pandas_plink import read_plink1_bin
from pandas_plink import get_data_folder
tfd = tfp.distributions
# Set random seed and start timer
np.random.seed(12345)
start = time.time()
### functions
def get_optimizer():
return tf.keras.optimizers.SGD()
def get_callbacks():
return [
#tfdocs.modeling.EpochDots(),
tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1, patience=500, restore_best_weights=True),
]
def normalize_data(df):
return (df - df.min())/(df.max() - df.min())
def compile_model(model, optimizer=None):
if optimizer is None:
optimizer = get_optimizer()
model.compile(optimizer=optimizer,
loss=keras.losses.MeanSquaredError())
return model
def MSE(test,pred):
sqr_err = np.subtract(test,pred)**2
return sqr_err.mean()
# ## load & preprocess data
# ### load genotype data
G = np.genfromtxt("genotype.txt")
G[np.isnan(G)] = 0.
G = normalize_data(G)
print(G.mean())
print(G.var())
# ### load phenotype data
traits = np.genfromtxt("phenotype.txt")
traits = normalize_data(traits)
print(traits.mean())
print(traits.var())
# ### split training and validation set
train_X, test_X, train_y, test_y = train_test_split(G, traits, test_size = 0.2, random_state = 42)
X = np.concatenate((train_X, test_X), axis=0)
y = np.concatenate((train_y, test_y), axis=0)
# ### parameter definition
N = G.shape[0]
p = G.shape[1]
NUM_FOLDS = 5
kfold = KFold(n_splits=NUM_FOLDS, shuffle=True)
INPUT_SHAPE = X.shape[1]
OUTPUT_SHAPE = y.shape[0]
BATCH_SIZE = 32
STEPS_PER_EPOCH = math.ceil((X.shape[0]*(1-1/NUM_FOLDS)*0.8)/BATCH_SIZE)
MAX_EPOCHS = 5000
df = pd.DataFrame(columns = ['method','MSE','R2'])
histories = {}
# Specify the surrogate posterior over `keras.layers.Dense` `kernel` and `bias`.
def posterior_mean_field(kernel_size, bias_size=0, dtype=None):
n = kernel_size + bias_size
c = np.log(np.expm1(1.))
return tf.keras.Sequential([
tfp.layers.VariableLayer(2 * n, dtype=dtype),
tfp.layers.DistributionLambda(lambda t: tfd.Independent(
tfd.Normal(loc=t[..., :n],
scale=1e-5 + tf.nn.softplus(c + t[..., n:])),
reinterpreted_batch_ndims=1)),
])
# Specify the prior over `keras.layers.Dense` `kernel` and `bias`.
def prior_trainable(kernel_size, bias_size=0, dtype=None):
n = kernel_size + bias_size
return tf.keras.Sequential([
tfp.layers.VariableLayer(n, dtype=dtype),
tfp.layers.DistributionLambda(lambda t: tfd.Independent(
tfd.Normal(loc=t, scale=1),
reinterpreted_batch_ndims=1)),
])
def neg_log_likelihood(y_true, y_pred, sigma=1.0):
dist = tfp.distributions.Normal(loc=y_pred, scale=sigma)
return K.sum(-dist.log_prob(y_true))
#neg_log_likelihood = lambda y, p_y: -p_y.log_prob(y)
kl_loss_weight = 1.0 / STEPS_PER_EPOCH
histories = {}
fold_no = 1
for train, test in kfold.split(X, y):
model = tf.keras.Sequential([
keras.layers.InputLayer(input_shape=(INPUT_SHAPE,)),
tfp.layers.DenseVariational(units=32,
make_posterior_fn=posterior_mean_field,
make_prior_fn=prior_trainable,
kl_weight=kl_loss_weight,
activation='sigmoid'),
tfp.layers.DenseVariational(units=1,
make_posterior_fn=posterior_mean_field,
make_prior_fn=prior_trainable,
kl_weight=kl_loss_weight,
activation='sigmoid'),
tfp.layers.DistributionLambda(lambda t: tfd.Normal(loc=t, scale=1)),
])
model.compile(loss=neg_log_likelihood, optimizer=tf.keras.optimizers.Adam(lr=0.0001), metrics=['mse'])
history = model.fit(X[train], y[train],
validation_split = 0.2,
steps_per_epoch = STEPS_PER_EPOCH,
epochs=MAX_EPOCHS,
callbacks=get_callbacks(),
verbose=0)
histories['BNN2_'+str(fold_no)] = history
y_pred_list = []
for i in range(500):
y_pred = model.predict(X[test])
y_pred_list.append(y_pred)
y_preds = np.concatenate(y_pred_list, axis=1)
y_mean = np.mean(y_preds, axis=1)
m_err = MSE(y[test],y_mean)
r2_acc = r2_score(y[test],y_mean)
df = df.append({'MSE':m_err, 'R2':r2_acc, 'method':'BNN2'}, ignore_index=True)
fold_no = fold_no + 1
df.to_csv("results.csv")
I wrote a small
"Linear Regression Neural Network Tensorflow Keras Python program"
Input dataset is
y = mx + c straight line data.
Predicted y values are not correct and are giving horizontal line kind of
values, instead of a line with some slope.
I ran this program on Windows laptop with tensorflow, Keras and
Jupyter notebook.
What to do to fix this program please?
Thanks and best regards,
SSJ
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
n2 = 50
count = 20
n4 = n2 + count
p = 100
m = 10
c = 5
x = np.linspace(n2, n4, p)
y = m * x + c
x
y
plt.scatter(x,y)
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers.experimental import preprocessing
x_normalizer = preprocessing.Normalization(input_shape=[1,])
x_normalizer.adapt(x)
x_normalized = x_normalizer(x)
y_normalizer = preprocessing.Normalization(input_shape=[1,])
y_normalizer.adapt(y)
y_normalized = x_normalizer(y)
y_model = tf.keras.Sequential([
y_normalizer,
layers.Dense(1)
])
y_model.compile(optimizer='rmsprop', loss='mse', metrics = ['mae'])
y_hist = y_model.fit(x, y, epochs=100, verbose=0, validation_split = 0.2)
hist = pd.DataFrame(y_hist.history)
hist['epoch'] = y_hist.epoch
hist.head()
hist.tail()
xin = [51,53,59,64]
ypred = y_model.predict(xin)
ypred
plt.scatter(x, y)
plt.scatter(xin, ypred, color = 'r')
plt.grid(linestyle = '--')
Use StandardScaler instead of Normalization
Normalizer acts row-wise and StandardScaler column-wise.
Normalizer does not remove the mean and scale by deviation but scales
the whole row to unit norm.
Found here: Difference between StandardScaler and Normalizer
This is how you can process the data:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
from sklearn.preprocessing import StandardScaler
x = np.linspace(50, 70, 100).reshape(-1, 1)
y = 10 * x + 5
x_standard_scaler = StandardScaler().fit(x)
y_standard_scaler = StandardScaler().fit(y)
x_scaled = x_standard_scaler.transform(x)
y_scaled = y_standard_scaler.transform(y)
Remember that you need two separate scalers for x and y so don't use the same object for that. Also if you want to use that scaler to process new data for testing, save the scaler in some variable. A good practice is to not refit the scaler again on test data.
model = Sequential([
Dense(1, input_dim=1, activation='linear'),
])
model.compile(optimizer='rmsprop', loss='mse')
history = model.fit(x_scaled, y_scaled, epochs=1000, verbose=0, validation_split = 0.2).history
pd.DataFrame(history).plot()
plt.show()
As you can see the model is converging. Its worth to plot the loss history which helps to tell if your model is learning or not.
x_test = np.linspace(20, 100, 10).reshape(-1, 1)
y_test = 10 * x_test + 5
x_test_scaled = x_standard_scaler.transform(x_test)
y_test_scaled = y_standard_scaler.transform(y_test)
If you have a test data that you want to use for validation or just predict it, remember to use standard scaler again, but without fitting. It should be fitted on train data only in most cases.
y_test_pred_scaled = model.predict(x_test_scaled)
y_test_pred = y_standard_scaler.inverse_transform(y_test_pred_scaled)
plt.scatter(x_test, y_test, s=30, label='true')
plt.scatter(x_test, y_test_pred, s=15, label='pred')
plt.legend()
plt.show()
If you want to get your prediction rescaled back to its original range use inverse_transform. Notice that prediction on x_test after rescaling is very close to y_test.
If I feed the model five Setosa flowers, I cannot get my model to predict that they are indeed Setosas.
Here is my code setup:
# Load libraries
import numpy as np
import pandas as pd
from keras import models
from keras import layers
from keras.models import Sequential
from keras.layers import Dense
from sklearn.utils import shuffle
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
# Set random seed
np.random.seed(0)
# Step 1: Load data
iris = pd.read_csv("iris.csv")
X = iris.drop('species', axis=1)
y = pd.get_dummies(iris['species']).values
# Step 2: Preprocess data
scaler = preprocessing.StandardScaler()
X = scaler.fit_transform(X)
X, y = shuffle(X, y)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
network = models.Sequential()
network.add(layers.Dense(units=8, activation="relu", input_shape=(4,)))
network.add(layers.Dense(units=3, activation="softmax"))
# Compile neural network
network.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
# Train neural network
history = network.fit(X_train, # Features
y_train, # Target
epochs= 200,
verbose= 1,
batch_size=10, # Number of observations per batch
validation_data=(X_test, y_test)) # Test data
The model trained well, here is the last epoch:
Epoch 200/200
112/112 [==============================] - 0s 910us/step - loss: 0.0740 - acc: 0.9911 - val_loss: 0.1172 - val_acc: 0.9737
Now, let's pull some predictions.
new_iris = iris.iloc[0:5, 0:4] # pull out the first five Setosas from original iris dataset;
# prediction should give me Setosa since I am feeding it Setosas
np.around(network.predict(new_iris), decimals = 2) # predicts versicolor with high probability
array([[0. , 0.95, 0.04],
[0. , 0.94, 0.06],
[0. , 0.96, 0.04],
[0. , 0.91, 0.09],
[0. , 0.96, 0.04]], dtype=float32)\
Any ideas as to why this is the case?
You need to apply the transformation learned during training at test time.
new_iris = iris.iloc[0:5, 0:4] # pull out the first five Setosas from original iris dataset;
new_iris = scaler.transform(new_iris)
np.around(network.predict(new_iris), decimals = 2)
Outputs
array([[1. , 0. , 0. ],
[0.99, 0.01, 0. ],
[1. , 0. , 0. ],
[0.99, 0.01, 0. ],
[1. , 0. , 0. ]], dtype=float32)