Related
I am trying to train a model using Gaussian Process regression in Python, and I have some problems in predicting. The training set is of size 265, and after having trained the model, I am trying to predict on the test set.
I get the error "RuntimeError: shape '[265, 265]' is invalid for input of size 265" and I don't really understand why.
I also get a vector as loss while training (instead than a scalar). I solved the loss problem by doing loss.mean() (even if I still don't understand why the loss is returned as a vector), but I have not managed to solve the other problem.
265 is the size of the training set (and not of the test set), so I imagine the error is related to that and not to the test set that I give in input.
Thank you really much for your help! I send all my code, the problem is in the function predict. PS: I am sorry but I don't know how to upload the training and test data here.
'''
import os
import typing
#from sklearn.gaussian_process.kernels import *
import numpy as np
#from sklearn.gaussian_process import GaussianProcessRegressor
#from sklearn.model_selection import cross_val_score
#from sklearn.model_selection import KFold
import matplotlib.pyplot as plt
from matplotlib import cm
import gpytorch
import matplotlib.pyplot as plt
from matplotlib import rcParams
import numpy as np
import torch
import pandas as pd
class Model(object): # were we allowed to change here?
"""
Model for this task.
You need to implement the fit_model and predict methods
without changing their signatures, but are allowed to create additional methods.
"""
def __init__(self): # are we allowed to change here?
"""
Initialize your model here.
We already provide a random number generator for reproducibility.
"""
self.likelihood = gpytorch.likelihoods.GaussianLikelihood()
# self.mean_module = gpytorch.means.ZeroMean() #prior mean
# self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel()) #covariance (which kernel to use?)
self.model = None
self.rng = np.random.default_rng(seed=0) # it was already present
# TODO: Add custom initialization for your model here if necessary
def predict(self, x: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Predict the pollution concentration for a given set of locations.
:param x: Locations as a 2d NumPy float array of shape (NUM_SAMPLES, 2)
#TODO: for us it is 1-d!!!
:return:
Tuple of three 1d NumPy float arrays, each of shape (NUM_SAMPLES,),
containing your predictions, the GP posterior mean, and the GP posterior stddev (in that order)
"""
# TODO: Use your GP to estimate the posterior mean and stddev for each location here
gp_mean = np.zeros(x.shape[0], dtype=float)
gp_std = np.zeros(x.shape[0], dtype=float)
x = torch.Tensor(x)
# Turn the model into evaluation mode
self.model.eval()
self.likelihood.eval()
# The gpytorch.settings.fast_pred_var flag activates LOVE (for fast variances)
# See https://arxiv.org/abs/1803.06058
with torch.no_grad(), gpytorch.settings.fast_pred_var():
# Obtain the predictive mean and covariance matrix
#print(x.size())
f_preds = self.model(x) #TODO: check the input!!!
print(type(f_preds))
gp_mean = np.array(f_preds.mean)
gp_std = torch.sqrt(f_preds.variance) # check if it is right
gp_std = np.array(gp_std)
# Make predictions by feeding model through likelihood
# observed_pred = self.likelihood(f_preds) #I think it is useless
# TODO: Use the GP posterior to form your predictions here
predictions = gp_mean
# check if we want to change the predictions for the asymmetric cost
'''for i in range(0, len(predictions)):
if (predictions[i] + 3 * gp_std[i] > 35.5) and (predictions[i] < 35.5):
predictions[i] = 35.51
elif ((predictions[i] - gp_std[i]) > 35.5):
predictions[i] = predictions[i] - gp_std[i]'''
return predictions, gp_mean, gp_std
def fit_model(self, train_x: np.ndarray, train_y: np.ndarray):
"""
Fit your model on the given training data.
:param train_x: Training features as a 2d NumPy float array of shape (NUM_SAMPLES, 2) #TODO: 1-d for us!!!
:param train_y: Training pollution concentrations as a 1d NumPy float array of shape (NUM_SAMPLES,)
"""
# TODO: Fit your model here
# Put the model into training mode
# initialize likelihood and model (necessary?)
train_x = torch.Tensor(train_x)
train_y = torch.Tensor(train_y)
# torch.Tensor prende in input matrici di numpy e le trasforma in elementi di torch,
# dentro pytorch hai di default un algoritmo di propagation
# clustering
# samplesize=5000
# train_x = train_x[0:samplesize, :]
# train_y = train_y[0:samplesize]
#
self.model = ExactGPModel(train_x, train_y, self.likelihood)
self.model.train()
self.likelihood.train()
# understand how to manage these last 4 rows
# Use the SGD optimizer
optimizer = torch.optim.Adam([
{'params': self.model.parameters()}, # Includes GaussianLikelihood parameters
],
lr=0.1) # if instead of "Adam" we put "SGD" it uses stochastic gradient descent! Oppure minibatch gradient descent
#TODO: choose which optimizer to use
# we can always use stochastic, minibatch or standard(batch)
# we can use gridsearch, change parameters like lr,beta1,beta2,... in order to have a better approximation
# "Loss" for GPs - the marginal log likelihood
mll = gpytorch.mlls.ExactMarginalLogLikelihood(self.likelihood, self.model)
training_iter = 1200 # numero di epochs
for i in range(training_iter):
# Zero gradients from previous iteration
optimizer.zero_grad()
# Output from model
output = self.model(train_x)
# Calc loss and backprop gradients
loss = -mll(output, train_y)
loss = torch.mean(loss) #TODO: understand how to deal with this loss, we shouldn't do the mean!!!
#print(loss)
#print(type(loss))
loss.backward()
print('Iter %d/%d - Loss: %.3f lengthscale: %.3f noise: %.3f' % (
i + 1, training_iter, loss.item(),
self.model.covar_module.base_kernel.lengthscale.item(),
self.model.likelihood.noise.item()))
optimizer.step()
# Auxiliary class
class ExactGPModel(gpytorch.models.ExactGP):
# every object of ExactGPmodel will have every module of models.ExactGP
def __init__(self, train_x, train_y, likelihood):
super(ExactGPModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = gpytorch.means.ZeroMean()
self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.MaternKernel(nu=1 / 2))
def forward(self, x):
mean_x = self.mean_module(x)
covar_x = self.covar_module(x)
return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
# Load the training dateset and test features
trainfilename="TrainingData.txt"
testfilename="TestingData.txt"
'''train_set = pd.read_csv(trainfilename)
test_set = pd.read_csv(testfilename)
# Convert Pandas DataFrame to NumpyArray
train_set = train_set.to_numpy()
test_set = test_set.to_numpy()
meant = np.mean(train_set[:,0:1])
sdt = np.std(train_set[:,0:1])
train_set[:,0:1] = (train_set[:,0:1]-meant)/sdt
train_set = torch.from_numpy(train_set)
meant = np.mean(test_set[:,0:1])
sdt = np.std(test_set[:,0:1])
test_set[:,0:1] = (test_set[:,0:1]-meant)/sdt
test_set = torch.from_numpy(test_set)
train_x = train_set[:,0:1]
train_y = train_set[:,1:2]
test_x = test_set[:,0:1]'''
train_set = np.loadtxt(trainfilename, delimiter=',', skiprows=1)
test_x = np.loadtxt(testfilename, delimiter=',', skiprows=1)
train_x = train_set[:,0:1]
train_y = train_set[:,1:2]
#print(type(train_x)) #the type looks fine
# Fit the model
print('Fitting model')
model = Model()
model.fit_model(train_x, train_y) #TODO: how to give the input here???
# Predict on the test features
print('Predicting on test features')
#print(test_x.size())
print(test_x.shape)
predicted_y = model.predict(test_x) #TODO: how to give the input here???
print(predicted_y)
#TODO: RuntimeError: shape '[265, 265]' is invalid for input of size 265
#TODO: I think it does not get the covariance matrix
'''
I'm currently trying to build a "simple" LSTM model that takes historical Bitcoin data, learns from that and then tries to predict the future X steps in advance.
I've build it on the idea that A + B + C = D so B + C + D should be E. (I think that's a very simple idea behind an LSTM model. I might be wrong however i'm pretty new to it.)
I managed to build the basics in python (I'm fairly new to python) but something seems off by the prediction. For some reason many of the predictions i test / make end up flatlining. I have a theory on why but we have no idea if it's correct and even less idea on how to solve it.
My theory is that within a sequence the model learns to put more importance / weight on the last digit in the sequence because with Bitcoin prices the future price (in 1 minute) is probably pretty close to the price now. That's try the predicted values keeps getting closer to the real value eventually being equal and thus flatlining in a graph. (I don't know if that makes sense but thats what i tought anyway.)
I've also added a screenshot of my graph from a few days ago. Almost all predictions however end similar to this graph. This is just a more extreme example as demonstration.
Here is my code, can someone please explain why it flatlines and what i did wrong?
import numpy as np
from matplotlib import pyplot
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
import yfinance as yf
from sklearn.preprocessing import MinMaxScaler
from math import sqrt
from sklearn.metrics import mean_squared_error
# Create output sets X + Y from given input-set
# with inputset : a 1-dimensional list of floats
# with N : the number of lookback values to use for X
# with Gap : the number of point skipped between X and Y
# Y: is equal to input, (although the first N are missing)
# X: for each y of Y a corresponding set of size N is created
# composed of the N values preceeding y.
def create_lookback(inputset, n=1, gap=0):
print("create_lookback with n=%d gap=%d" % (n,gap))
print(" - length of inputset = %d" % len(inputset))
dataX, dataY = [], []
for i in range(len(inputset) - (n+gap)):
a = inputset[i:(i + n), 0]
dataX.append(a)
dataY.append(inputset[i + n+gap, 0])
print(" - length of dataY = %d" % len(dataY))
data_x = np.array(dataX)
xret = data_x.reshape(data_x.shape[0], 1, data_x.shape[1])
return xret, np.array(dataY)
# Train model based on given training-set + Test-set
def create_model(trainX,trainY,testX,testY):
model = Sequential()
model.add(LSTM(units = 100, input_shape=(trainX.shape[1], trainX.shape[2], )))
model.add(Dropout(0.2))
#model.add(LSTM(30, return_sequences=True))
#model.add(Dropout(0.1))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
history = model.fit(trainX, trainY, epochs=100, batch_size=5, validation_data=(testX, testY), verbose=1, shuffle=False)
return model
# Evaluate given X / Y set.
# - Calculate RMSE
# - Generate visual line-plot to screen
def show_result(scaler,yhat,setY,txt):
print("Show %s result" % txt)
yhat_inverse = scaler.inverse_transform(yhat.reshape(-1, 1))
testY_inverse = scaler.inverse_transform(setY.reshape(-1, 1))
if len(testY_inverse) == len(yhat_inverse):
rmse = sqrt(mean_squared_error(testY_inverse, yhat_inverse))
print(' RMSE %s : %.3f' % (txt,rmse))
pyplot.plot(yhat_inverse, label='predict '+txt)
pyplot.plot(testY_inverse, label='actual '+txt, alpha=0.5)
pyplot.legend()
pyplot.show()
# Extrapoleer is dutch for Extrapolate
def extrapoleer(i,model,tup,toekomst):
if(i == 0):
return
setX = np.array([[tup]])
y = model.predict(setX)
y_float = y[0][0]
tup_new = np.append(tup[1:], y_float)
toekomst.append(y_float)
extrapoleer(i-1, model, tup_new,toekomst)
# --- end of defined functions
# -- start of main flow
data_grid_1 = yf.download('BTC-USD', start="2021-04-14",end="2021-04-15", interval="1m");
data_grid_2 = yf.download('BTC-USD', period="12h", interval="1m");
dataset_1 = data_grid_1.iloc[:, 1:2].values
dataset_2 = data_grid_2.iloc[:, 1:2].values
scaler = MinMaxScaler(feature_range = (0, 1))
scaled = scaler.fit_transform(dataset_1)
# 70% of dataset_1 is used to train ; 30% to test
train_size = int(len(scaled) * 0.7)
test_size = len(scaled) - train_size
train, test = scaled[0:train_size,:], scaled[train_size:len(scaled),:]
print("train: %d test: %d" % (len(train), len(test)))
scaled_2 = scaler.fit_transform(dataset_2)
look_back_n = 3
look_back_gap = 0
trainX, trainY = create_lookback(train, look_back_n, look_back_gap)
testX, testY = create_lookback(test, look_back_n, look_back_gap)
testX_2, testY_2 = create_lookback(scaled_2, look_back_n, look_back_gap)
model = create_model(trainX,trainY,testX,testY)
yhat_1 = model.predict(testX)
yhat_2 = model.predict(testX_2)
show_result(scaler,yhat_1,testY,"test")
show_result(scaler,yhat_2,testY_2,"test2")
last_n = testY_2[-look_back_n:]
#toekomst = Future in dutch
toekomst = []
#aantal = Amount in Dutch, this indicates the amount if steps you want to future predict
aantal = 30
extrapoleer(aantal, model, last_n, toekomst)
print("Resultaat van %d voorspelde punten in de toekomst: " % aantal)
print(toekomst)
yhat_2_plus = np.append(yhat_2,toekomst)
show_result(scaler,yhat_2_plus,testY_2,"test2-plus")
I want to convert the code written in Python into Matlab code. May I know is it possible to do that. l am wonder, how can we use the python libraries in Matlab. Share the procedure to do the conversion
Here is the Data I used:
https://drive.google.com/open?id=1GLm87-5E_6YhUIPZ_CtQLV9F9wcGaTj2
Here is my code in Python:
# imports libraries
import numpy as np
import pandas as pd
import os
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from scipy import signal
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.models import Sequential
from tensorflow import set_random_seed
from tensorflow.keras.initializers import glorot_uniform
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from importlib import reload
# useful pandas display settings
pd.options.display.float_format = '{:.3f}'.format
# useful functions
def plot_history(history, metrics_to_plot):
"""
Function plots history of selected metrics for fitted neural net.
"""
# plot
for metric in metrics_to_plot:
plt.plot(history.history[metric])
# name X axis informatively
plt.xlabel('epoch')
# name Y axis informatively
plt.ylabel('metric')
# add informative legend
plt.legend(metrics_to_plot)
# plot
plt.show()
def plot_fit(y_true, y_pred, title='title'):
"""
Function plots true values and predicted values, sorted in increase order by true values.
"""
# create one dataframe with true values and predicted values
results = y_true.reset_index(drop=True).merge(pd.DataFrame(y_pred), left_index=True, right_index=True)
# rename columns informartively
results.columns = ['true', 'prediction']
# sort for clarity of visualization
results = results.sort_values(by=['true']).reset_index(drop=True)
# plot true values vs predicted values
results.plot()
# adding scatter on line plots
plt.scatter(results.index, results.true, s=5)
plt.scatter(results.index, results.prediction, s=5)
# name X axis informatively
plt.xlabel('obs sorted in ascending order with respect to true values')
# add customizable title
plt.title(title)
# plot
plt.show();
def reset_all_randomness():
"""
Function assures reproducibility of NN estimation results.
"""
# reloads
reload(tf)
reload(np)
reload(random)
# seeds - for reproducibility
os.environ['PYTHONHASHSEED']=str(984797)
random.seed(984797)
set_random_seed(984797)
np.random.seed(984797)
my_init = glorot_uniform(seed=984797)
return my_init
def give_me_mse(true, prediction):
"""
This function returns mse for 2 vectors: true and predicted values.
"""
return np.mean((true-prediction)**2)
# Importing the dataset
X = pd.read_excel(r"C:\filelocation\Data.xlsx","Sheet1").values
y = pd.read_excel(r"C:\filelocation\Data.xlsx","Sheet2").values
# Importing the experiment data
Data = pd.read_excel(r"C:\filelocation\Data.xlsx","Sheet1")
v = pd.DataFrame(Data, columns= ['v']).values
c = pd.DataFrame(Data, columns= ['c']).values
ird = pd.DataFrame(Data, columns= ['ird']).values
tmp = pd.DataFrame(Data, columns= ['tmp']).values
#Data Prepration
ird = ird.ravel()
tmp = tmp.ravel()
ir = np.nanmax(ird)
tp = np.nanmax(tmp)
p = v*c
p = p.ravel()
peaks, _ = signal.find_peaks(p)
nop = len(peaks)
pv = p.max()
#Experimental Data for testing
E_data = np.array([[ir,tp,pv,nop]])
#importing some more libraries
from sklearn.preprocessing import LabelEncoder
from keras.utils import np_utils
# encode class values as integers
encoder = LabelEncoder()
encoder.fit(np.ravel(y))
y_encoded = encoder.transform(np.ravel(y))
# convert integers to dummy variables (i.e. one hot encoded)
y_dummy = np_utils.to_categorical(y_encoded)
# reset_all_randomness - for reproducibility
my_init = reset_all_randomness()
# Splitting the dataset into the Training set and Test set
X_train, X_test, y_train, y_test, y_train_dummy, y_test_dummy = train_test_split(X, y, y_dummy, test_size = 0.3, random_state = 20)
# Feature Scaling
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
E_data = sc.transform(E_data)
# Initialising the ANN
model0 = Sequential()
# Adding 1 hidden layer: the input layer and the first hidden layer
model0.add(Dense(units = 160, activation = 'tanh', input_dim = 4, kernel_initializer=my_init))
# Adding 2 hidden layer
model0.add(Dense(units = 49, activation = 'tanh', kernel_initializer=my_init))
# Adding 3 hidden layer
model0.add(Dense(units = 24, activation = 'tanh', kernel_initializer=my_init))
# Adding 4 hidden layer
model0.add(Dense(units = 15, activation = 'tanh', kernel_initializer=my_init))
# Adding output layer
model0.add(Dense(units = 6, activation = 'softmax', kernel_initializer=my_init))
# Set up Optimizer
Optimizer = tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.99)
# Compiling the ANN
model0.compile(optimizer = Optimizer, loss = 'categorical_crossentropy', metrics=['accuracy','categorical_crossentropy','mse'])
# Fitting the ANN to the Train set, at the same time observing quality on Valid set
history = model0.fit(X_train, y_train_dummy, validation_data=(X_test, y_test_dummy), batch_size = 100, epochs = 1500)
# Generate prediction for all Train, Valid set and Experimental set
y_train_pred_model0 = model0.predict(X_train)
y_test_pred_model0 = model0.predict(X_test)
y_exp_pred_model0 = model0.predict(E_data)
# find final prediction by taking class with highest probability
y_train_pred_model0 = np.array([[list(x).index(max(list(x))) + 1] for x in y_train_pred_model0])
y_test_pred_model0 = np.array([[list(x).index(max(list(x))) + 1] for x in y_test_pred_model0])
y_exp_pred_model0 = np.array([[list(x).index(max(list(x))) + 1] for x in y_exp_pred_model0])
# check what metrics are in fact available in history
history.history.keys()
# Inverse scaling
X_train_inverse = sc.inverse_transform(X_train)
X_test_inverse = sc.inverse_transform(X_test)
E_data_inverse = sc.inverse_transform(E_data)
#Plots
print('#######################################################################')
# look at model fitting history
plot_history(history, ['mean_squared_error', 'val_mean_squared_error'])
plot_history(history, ['categorical_crossentropy', 'val_categorical_crossentropy'])
plot_history(history, ['acc', 'val_acc'])
# look at model fit quality
plot_fit(pd.DataFrame(y_train), y_train_pred_model0, 'Fit on train data')
plot_fit(pd.DataFrame(y_test), y_test_pred_model0, 'Fit on test data')
#Results
print('#######################################################################')
print('=============Mean Squared Error============')
print('MSE on train data is: {}'.format(give_me_mse(y_train, y_train_pred_model0)))
print('MSE on test data is: {}'.format(give_me_mse(y_test, y_test_pred_model0)))
print('#######################################################################')
print('================Accuracy===================')
print('Accuracy of ANN is: {} Percentage'.format((accuracy_score(y_test, y_test_pred_model0))*100))
print('#######################################################################')
print('========Result of Test Data set is=========')
for i in range(len(y_test)):
print('%s => %d (expected %s)' % (X_test_inverse[i].tolist(), y_test_pred_model0[i], y_test[i].tolist()))
print('#######################################################################')
print('====Result of Experimental Data set is=====')
print('%s => %d' % (E_data_inverse, y_exp_pred_model0))
There is no "direct" way to convert Python code to MATLAB code.
What you can do is directly translate the approach (the algorithm) and write the code from scratch.
or what I think would be more preferable to you is to directly call python script in MATLAB using their API
here is the link for further reading: https://in.mathworks.com/help/matlab/call-python-libraries.html
for example:
>> py.math.sqrt(4)
ans =
1
To run your own function, you can create a file in your current MATLAB working directory. here is the file ‘hello.py’ that contained these two lines:
def world():
return 'hello world'
Then in MATLAB:
>> py.hello.world();
Hello world!
if you run into errors make sure you're using the supported version of Python and add
pyversion <path_to_executable>
to the start of your MATLAB file.
Although I'm not sure how well it will work considering all the Python libraries you're importing (Scipy, Tensorflow etc)
Below is a simple example in numpy of what I would like to do:
import numpy as np
y_true = np.array([0,0,1])
y_pred = np.array([0.1,0.2,0.7])
yc = (1-y_true).astype('bool')
desired = y_pred[yc]
>>> desired
>>> array([0.1, 0.2])
So the prediction corresponding to the ground truth is 0.7, I want to operate on an array containing all the elements of y_pred, except for the ground truth element.
I am unsure of how to make this work within Keras. Here is a working example of the problem in the loss function. Right now 'desired' isn't accomplishing anything, but that is what I need to work with:
# using tensorflow 2.0.0 and keras 2.3.1
import tensorflow.keras.backend as K
import tensorflow as tf
from tensorflow.keras.layers import Input,Dense,Flatten
from tensorflow.keras.models import Model
from keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# Normalize data.
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
# Convert class vectors to binary class matrices.
y_train = tf.keras.utils.to_categorical(y_train, 10)
y_test = tf.keras.utils.to_categorical(y_test, 10)
input_shape = x_train.shape[1:]
x_in = Input((input_shape))
x = Flatten()(x_in)
x = Dense(256,'relu')(x)
x = Dense(256,'relu')(x)
x = Dense(256,'relu')(x)
out = Dense(10,'softmax')(x)
def loss(y_true,y_pred):
yc = tf.math.logical_not(kb.cast(y_true, 'bool'))
desired = tf.boolean_mask(y_pred,yc,axis = 1) #Remove and it runs
CE = tf.keras.losses.categorical_crossentropy(
y_true,
y_pred)
L = CE
return L
model = Model(x_in,out)
model.compile('adam',loss = loss,metrics = ['accuracy'])
model.fit(x_train,y_train)
I end up getting an error
ValueError: Shapes (10,) and (None, None) are incompatible
Where 10 is the number of categories. The end purpose is to implement this: ComplementEntropy
in Keras, where my issue seems to be lines 26-28.
You can remove axis=1 from the Boolean_mask and it will run. And frankly, I don't see why you need axis=1 here.
def loss(y_true,y_pred):
yc = tf.math.logical_not(K.cast(y_true, 'bool'))
print(yc.shape)
desired = tf.boolean_mask(y_pred, yc) #Remove axis=1 and it runs
CE = tf.keras.losses.categorical_crossentropy(
y_true,
y_pred)
L = CE
return L
This is probably what happens. You have y_pred which is a 2D tensor (N=2). Then you have a 2D mask (K=2). But there's this condition K + axis <= N. And if you pass axis=1 this fails.
Using thushv89's answer, here is the full code for how I implemented COT on LeNet from the referenced paper. The one trick is I am not actually flipping back and forth between the two objectives, instead there is just a random weight that flips s.
# using tensorflow 2.0.0 and keras 2.3.1
import tensorflow.keras.backend as kb
import tensorflow as tf
from tensorflow.keras.layers import Conv2D, Input, Dense,Flatten,AveragePooling2D,GlobalAveragePooling2D
from tensorflow.keras.models import Model
from keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# Normalize data.
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
#exapnd dims to fit chn format
x_train = np.expand_dims(x_train,axis=3)
x_test = np.expand_dims(x_test,axis=3)
# Convert class vectors to binary class matrices.
y_train = tf.keras.utils.to_categorical(y_train, 10)
y_test = tf.keras.utils.to_categorical(y_test, 10)
input_shape = x_train.shape[1:]
x_in = Input((input_shape))
act = 'tanh'
x = Conv2D(32, (5, 5), activation=act, padding='same',strides = 1)(x_in)
x = AveragePooling2D((2, 2),strides = (2,2))(x)
x = Conv2D(16, (5, 5), activation=act)(x)
x = AveragePooling2D((2, 2),strides = (2,2))(x)
conv_out = Flatten()(x)
z = Dense(120,activation = act)(conv_out)#120
z = Dense(84,activation = act)(z)#84
last = Dense(10,activation = 'softmax')(z)
model = Model(x_in,last)
def loss(y_true,y_pred, axis=-1):
s = kb.round(tf.random.uniform( (1,), minval=0, maxval=1, dtype=tf.dtypes.float32))
s_ = 1 - s
y_pred = y_pred + 1e-8
yg = kb.max(y_pred,axis=1)
yc = tf.math.logical_not(kb.cast(y_true, 'bool'))
yp_c = tf.boolean_mask(y_pred, yc)
ygc_ = 1/(1-yg+1e-8)
ygc_ = kb.expand_dims(ygc_,axis=1)
Px = yp_c*ygc_ +1e-8
COT = kb.mean(Px*kb.log(Px),axis=1)
CE = -kb.mean(y_true*kb.log(y_pred),axis=1)
L = s*CE +s_*(1/(10-1))*COT
return L
model.compile(loss=loss,
optimizer='adam', metrics=['accuracy'])
model.fit(x_train,y_train,epochs=20,batch_size = 128,validation_data= (x_test,y_test))
pred = model.predict(x_test)
pred_label = np.argmax(pred,axis=1)
label = np.argmax(y_test,axis=1)
cor = (pred_label == label).sum()
acc = print('acc:',cor/label.shape[0])
Consider the following example problem:
# dummy data for a SO question
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')
from keras.models import Model
from keras.layers import Input, Conv1D, Dense
from keras.optimizers import Adam, SGD
time = np.array(range(100))
brk = np.array((time>40) & (time < 60)).reshape(100,1)
B = np.array([5, -5]).reshape(1,2)
np.dot(brk, B)
y = np.c_[np.sin(time), np.sin(time)] + np.random.normal(scale = .2, size=(100,2))+ np.dot(brk, B)
plt.clf()
plt.plot(time, y[:,0])
plt.plot(time, y[:,1])
You've got N time series, and they've got one component that follows a common process, and another component that is idiosyncratic to the series itself. Assume for simplicity that you know a priori that the bump is between 40 and 60, and you want to model it simultaneously with the sinusoidal component.
A TCN does a good job on the common component, but it can't get the series-idiosyncratic component:
# time series model
n_filters = 10
filter_width = 3
dilation_rates = [2**i for i in range(7)]
inp = Input(shape=(None, 1))
x = inp
for dilation_rate in dilation_rates:
x = Conv1D(filters=n_filters,
kernel_size=filter_width,
padding='causal',
activation = "relu",
dilation_rate=dilation_rate)(x)
x = Dense(1)(x)
model = Model(inputs = inp, outputs = x)
model.compile(optimizer = Adam(), loss='mean_squared_error')
model.summary()
X_train = np.transpose(np.c_[time, time]).reshape(2,100,1)
y_train = np.transpose(y).reshape(2,100,1)
history = model.fit(X_train, y_train,
batch_size=2,
epochs=1000,
verbose = 0)
yhat = model.predict(X_train)
plt.clf()
plt.plot(time, y[:,0])
plt.plot(time, y[:,1])
plt.plot(time, yhat[0,:,:])
plt.plot(time, yhat[1,:,:])
On the other hand, a basic linear regression with N outputs (here implemented in Keras) is perfect for the idiosyncratic component:
inp1 = Input((1,))
x1 = inp1
x1 = Dense(2)(x1)
model1 = Model(inputs = inp1, outputs = x1)
model1.compile(optimizer = Adam(), loss='mean_squared_error')
model1.summary()
brk_train = brk
y_train = y
history = model1.fit(brk_train, y_train,
batch_size=100,
epochs=6000, verbose = 0)
yhat1 = model1.predict(brk_train)
plt.clf()
plt.plot(time, y[:,0])
plt.plot(time, y[:,1])
plt.plot(time, yhat1[:,0])
plt.plot(time, yhat1[:,1])
I want to use keras to jointly estimate the time series component and the idiosyncratic component. The major problem is that feed-forward networks (which linear regression is a special case of) take shape batch_size x dims while time series networks take dimension batch_size x time_steps x dims.
Because I want to jointly estimate the idiosyncratic part of the model (the linear regression part) together with the time series part, I'm only ever going to batch-sample whole time-series. Which is why I specified batch_size = time_steps for model 1.
But in the static model, what I'm really doing is modeling my data as time_steps x dims.
I have tried to re-cast the feed-forward model as a time-series model, without success. Here's the non-working approach:
inp3 = Input(shape = (None, 1))
x3 = inp3
x3 = Dense(2)(x3)
model3 = Model(inputs = inp3, outputs = x3)
model3.compile(optimizer = Adam(), loss='mean_squared_error')
model3.summary()
brk_train = brk.reshape(1, 100, 1)
y_train = np.transpose(y).reshape(2,100,1)
history = model3.fit(brk_train, y_train,
batch_size=1,
epochs=1000, verbose = 1)
ValueError: Error when checking target: expected dense_40 to have shape (None, 2) but got array with shape (100, 1)
I am trying to fit the same model as model1, but with a different shape, so that it is compatible with the TCN model -- and importantly so that it will have the same batching structure.
The output should ultimately have the shape (2, 100, 1) in this example. Basically I want the model to do the following algorithm:
ingest X of shape (N, time_steps, dims)
Lose the first dimension, because the design matrix is going to be identical for every series, yielding X1 of shape (time_steps, dims)
Forward step: np.dot(X1, W), where W is of dimension (dims, N), yielding X2 of dimension (time_steps, N)
Reshape X2 to (N, time_steps, 1). Then I can add it to the output of the other part of the model.
Backwards step: since this is just a linear model, the gradient of W with respect to the output is just X1
How can I implement this? Do I need a custom layer?
I'm building off of ideas in this paper, in case you're curious about the motivation behind all of this.
EDIT: After posting, I noticed that I used only the time variable, rather than the time series itself. A TCN fit with the lagged series fits the idiosyncratic part of the series just fine (in-sample anyway). But my basic question still stands -- I want to merge the two types of networks.
So, I solved my own problem. The answer is to create dummy interactions (and a thus a really sparse design matrix) and then reshape the data.
###########################
# interaction model
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')
from keras.models import Model
from keras.layers import Input, Conv1D, Dense
from keras.optimizers import Adam, SGD
from patsy import dmatrix
def shift5(arr, num, fill_value=np.nan):
result = np.empty_like(arr)
if num > 0:
result[:num] = fill_value
result[num:] = arr[:-num]
elif num < 0:
result[num:] = fill_value
result[:num] = arr[-num:]
else:
result = arr
return result
time = np.array(range(100))
brk = np.array((time>40) & (time < 60)).reshape(100,1)
B = np.array([5, -5]).reshape(1,2)
np.dot(brk, B)
y = np.c_[np.sin(time), np.sin(time)] + np.random.normal(scale = .2, size=(100,2))+ np.dot(brk, B)
plt.clf()
plt.plot(time, y[:,0])
plt.plot(time, y[:,1])
# define interaction model
inp = Input(shape=(None, 2))
x = inp
x = Dense(1)(x)
model = Model(inputs = inp, outputs = x)
model.compile(optimizer = Adam(), loss='mean_squared_error')
model.summary()
from patsy import dmatrix
df = pd.DataFrame(data = {"fips": np.concatenate((np.zeros(100), np.ones(100))),
"brk": np.concatenate((brk.reshape(100), brk.squeeze()))})
df.brk = df.brk.astype(int)
tm = np.asarray(dmatrix("brk:C(fips)-1", data = df))
brkint = np.concatenate(( \
tm[:100,:].reshape(1,100,2),
tm[100:200,:].reshape(1,100,2)
), axis = 0)
y_train = np.transpose(y).reshape(2,100,1)
history = model.fit(brkint, y_train,
batch_size=2,
epochs=1000,
verbose = 1)
yhat = model.predict(brkint)
plt.clf()
plt.plot(time, y[:,0])
plt.plot(time, y[:,1])
plt.plot(time, yhat[0,:,:])
plt.plot(time, yhat[1,:,:])
The output shape is the same as for the TCN, and can simply be added element-wise.