I've a corpus and I divided it into 3 parts.
Training set 80%
Dev set 10%
Testing set 10%
I've the below CNN model trained on Training set and Evaluated against Dev set
model.add(SpatialDropout1D(0.1))
model.add(Conv1D(128, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.25))
model.add(Conv1D(64, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.25))
model.add(Flatten())
# Fully connected (Dense layer)
model.add(Dense(64, activation='relu'))
# Output layer with sigmoid activation function
model.add(Dense(8, activation='sigmoid'))
I've saved this model using model.save('model.h5')
Now, I'd like to do the hyper parameter optimization on the saved trained model, providing my dev set as train set and test set to evaluate.
My Values are
Filters 32/64/128/192/256/512 128/64
Kernel size 2/3/4/5/7 3
Dropout rate 0.1/0.2/0.3/0.4/0.5 0.1/0.25
Dense layer size 16/32/64/128/256 32
Batch size 32/50/64/100 32
Learning rate 0.1/0.01/0.001
Any pointers how to achieve this using any library like Talos loading existing model?
Following your last comment, and from Keras documentation:
(look for "grid", the scikit-learn grid search for hyper-parameters fine tuning. The following code should be fully running as is. You can use the same method with your saved/loaded model, using the datasets you wish)
from __future__ import print_function
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.wrappers.scikit_learn import KerasClassifier
from keras import backend as K
from sklearn.model_selection import GridSearchCV
num_classes = 10
# input image dimensions
img_rows, img_cols = 28, 28
# load training data and do basic data normalization
(x_train, y_train), (x_test, y_test) = mnist.load_data()
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
def make_model(dense_layer_sizes, filters, kernel_size, pool_size):
'''Creates model comprised of 2 convolutional layers followed by dense layers
dense_layer_sizes: List of layer sizes.
This list has one number for each layer
filters: Number of convolutional filters in each convolutional layer
kernel_size: Convolutional kernel size
pool_size: Size of pooling area for max pooling
'''
model = Sequential()
model.add(Conv2D(filters, kernel_size,
padding='valid',
input_shape=input_shape))
model.add(Activation('relu'))
model.add(Conv2D(filters, kernel_size))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=pool_size))
model.add(Dropout(0.25))
model.add(Flatten())
for layer_size in dense_layer_sizes:
model.add(Dense(layer_size))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
return model
dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
my_classifier = KerasClassifier(make_model, batch_size=32)
validator = GridSearchCV(my_classifier,
param_grid={'dense_layer_sizes': dense_size_candidates,
# epochs is avail for tuning even when not
# an argument to model building function
'epochs': [3, 6],
'filters': [8],
'kernel_size': [3],
'pool_size': [2]},
scoring='neg_log_loss',
n_jobs=1)
validator.fit(x_train, y_train)
print('The parameters of the best model are: ')
print(validator.best_params_)
# validator.best_estimator_ returns sklearn-wrapped version of best model.
# validator.best_estimator_.model returns the (unwrapped) keras model
best_model = validator.best_estimator_.model
metric_names = best_model.metrics_names
metric_values = best_model.evaluate(x_test, y_test)
for metric, value in zip(metric_names, metric_values):
print(metric, ': ', value)
Related
I'm trying to make a binary Classification by combining CNN (con1D) with GRU. my dataset dataset is like that :
X_train shape : (223461, 5)
y_train shape :(223461,)
the X_train is like that and the Y_train is a labels (0,1) like that
first I convert that train dataset :
dataset = X_train.values
dataset=dataset[1:]
dataset = dataset.astype('float32')
dataset
the same for y-train:
dataset_target = y_train.values
dataset_target=dataset_target[1:]
dataset_target = dataset_target.astype('float32')
dataset_target
now the shapes are dataset.shape =(223460, 5)
, dataset_target.shape = (223460,)
than my model structure is :
verbose, epochs, batch_size = 0, 100, 64
n_timesteps, n_features, n_outputs = dataset.shape[0], dataset.shape[1], dataset_target.shape[0]
model = Sequential()
model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape= (n_timesteps,n_features)))
model.add(MaxPooling1D(pool_size=2))
model.add(GRU(64))
model.add(Dropout(0.4))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(1, activation='sigmoid'))
opt = Adam(learning_rate=0.01)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(), optimizer=opt , metrics=['accuracy'])
model.summary()
and when I want to fit dataset to my model:
# fit network
model.fit(dataset, dataset_target, epochs=epochs, batch_size=batch_size, verbose=1)
# evaluate model
_, accuracy = model.evaluate(X_test, y_test, batch_size=batch_size, verbose=1)
#accuracy
I get an error Input 0 of layer "sequential" is incompatible with the layer: expected shape=(None, 223460, 5), found shape=(64, 5)
Is the first axis of the dataset (233460 samples) actually time steps, and do you have 5 'channels' of data? In that case, it would help if you slice the dataset along the first axis and then assign to each 'slice' single label, for example, the last value related to the slice from the y_train. In that case, n_timesteps would be the length of the slice, and the shape of the dataset something like (n_samples, n_timesteps, 5). Basically, Conv1D expects each training sample to be 2D, but in you case it's 1D, because the first dimension is just a number of samples.
I might have interpreted the dataset the wrong way. In that case, please clarify how it works so I would fix my suggestion.
Here's the example:
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.layers import Conv1D, MaxPooling1D, GRU, \
Dropout, Flatten, Dense
from tensorflow.keras import Sequential
from tensorflow.keras.optimizers import Adam
import numpy as np
X_train = np.random.normal(0, 1, (223461, 5))
y_train = np.random.randint(0, 2, 223461)
dataset = X_train[1:]
dataset_target = y_train[1:]
n_timesteps = 10
# Slice dataset and target
dataset = np.stack(np.split(dataset, n_timesteps)[:-1])
dataset_target = np.stack([y[-1] for y in np.split(dataset_target, n_timesteps)[:-1]])
Define and train the model:
def get_model(dataset, n_timesteps):
verbose, epochs, batch_size = 0, 100, 64
n_timesteps, n_features = dataset.shape[1], dataset.shape[2]
model = Sequential()
model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape = (n_timesteps, n_features)))
model.add(MaxPooling1D(pool_size=2))
model.add(GRU(64))
model.add(Dropout(0.4))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(1, activation='sigmoid'))
opt = Adam(learning_rate=0.01)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(), optimizer=opt , metrics=['accuracy'])
model.summary()
return model
verbose, epochs, batch_size = 0, 1, 64
model = get_model(dataset, n_timesteps)
model.fit(dataset, dataset_target, epochs=epochs, batch_size=batch_size, verbose=1)
Hope it helps!
I am getting into Convolutional Neural Networks and want to create one for MNIST data. Whenever I add a convolutional Layer to my CNN, I get an error:
Input 0 is incompatible with layer conv2d_4: expected ndim=4, found ndim=5
I have attemped to reshape X_Train data set but was not successful
I tried to add a flatten layer first but that returns this error:
Input 0 is incompatible with layer conv2d_5: expected ndim=4, found ndim=2
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Convolution2D
from keras.layers import Flatten, Dense, Dropout
img_width, img_height = 28, 28
mnist = keras.datasets.mnist
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = keras.utils.normalize(X_train, axis=1) #Normalizes from 0-1 (originally each pixel is valued 0-255)
X_test = keras.utils.normalize(X_test, axis=1) #Normalizes from 0-1 (originally each pixel is valued 0-255)
Y_train = keras.utils.to_categorical(Y_train) #Reshapes to allow ytrain to work with x train
Y_test = keras.utils.to_categorical(Y_test)
from sklearn import preprocessing
lb = preprocessing.LabelBinarizer()
Y_train = lb.fit_transform(Y_train)
Y_test = lb.fit_transform(Y_test)
#Model
model = Sequential()
model.add(Flatten())
model.add(Convolution2D(16, 5, 5, activation='relu', input_shape=(1,img_width, img_height, 1)))
model.add(Dense(128, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dropout(.2))
model.add(Dense(64, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer = 'adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(X_train, Y_train, epochs=3, verbose=2)
val_loss, val_acc = model.evaluate(X_test, Y_test) #Check to see if model fits test
print(val_loss, val_acc)
If I comment out the Convolutional layer, it works very well (accuracy>95%), but I am planning on making a more complex neural network that requires Convolution in the future and this is my starting point
Keras is looking for a tensor of dimension 4 but it's getting ndim as number of dimension as 2.
first make sure you kernel size in Conv2D layer is in parenthesis
model.add(Convolution2D(32, (3, 3), activation='relu', input_shape=(img_height, img_height, 1)))
Second you need to reshape the X_train, X_test variable as Conv2D layer is expecting a tensor input.
X_train = X_train.reshape(-1,28, 28, 1) #Reshape for CNN - should work!!
X_test = X_test.reshape(-1,28, 28, 1)
model.fit(X_train, Y_train, epochs=3, verbose=2)
For more information about Conv2D you can look into Keras Documentation here
Hope this helps.
There are two issues in your code.
You are encoding your labels two times, once using to_categorical, and another time using LabelBinarizer. The latter is no needed here, so just encode your labels into categorical once, using to_categorical.
2.- Your input shape is incorrect, it should be (28, 28, 1).
Also you should add a Flatten layer after the convolutional layers so the Dense layer works properly.
I use the following code (courtesy to here) which runs CNN for training MNIST images:
from __future__ import print_function
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K
batch_size = 128
num_classes = 10
epochs = 1
# input image dimensions
img_rows, img_cols = 28, 28
# the data, split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
print(model.save_weights('file.txt')) # <<<<<----
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
My goal is to use CNN model to extract MNIST features into a dataset that I can use as an input for another classifier. In this example, I don't care about the classification operation since all I need is the features of the trained images. The only method I found is save_weights as:
print(model.save_weights('file.txt'))
How can I extract features into a dataset from keras model?
After training or loading the existing trained model, you can create another model:
extract = Model(model.inputs, model.layers[-3].output) # Dense(128,...)
features = extract.predict(data)
and use the .predict method to return the vectors from a specific layer, in this case every image will become (128,), the output of the Dense(128, ...) layer.
You can also train these networks jointly with 2 outputs using the functional API. Follow the guide and you'll see that you can chain models together and have multiple outputs each possibly with a separate loss. This will allow your model to learn shared features that is useful for both classifying the MNIST image and your task at the same time.
I am making a simple application to implement digit recognition. The problem is that it works wonderfully on the mnist dataset but predicts horribly on random images downloaded from google. What should I do to improve the accuracy of my model?
Another question : Someone suggested to add more layers in the model. If so, how do I add more layers to my model?
File in which model is trained:
from __future__ import print_function
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K
import cv2
import matplotlib.pyplot as plt
from keras.models import load_model
import pickle
import h5py
import numpy as np
from keras.callbacks import ModelCheckpoint
batch_size = 128
num_classes = 10
epochs = 12
# input image dimensions
img_rows, img_cols = 28, 28
model=load_model('my_model.h5')
the data, split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
print('initial shape')
print('x_test ',x_test.shape)
print('y_test ',y_test.shape)
# print(x_test);
# print(y_test.shape)
if K.image_data_format() == 'channels_first':
print('reshape1')
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
print('reshape2')
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print('x_test shape: ' , x_test.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
print('x_test final : ')
print(x_test)
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.10))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.20))
model.add(Dense(256,activation='relu'))
model.add(Dropout(0.40))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
model.save('my_model.h5')
score = model.evaluate(x_test, y_test, verbose=0)
# print(x_test.shape)
# print('\n')
# print(y_test.shape)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
The below code is used for testing my model with my own image
image = cv2.imread("2.jpg")
img_rows, img_cols = 28, 28
x_test1 = cv2.resize(image, (28, 28))
x_test1 = cv2.cvtColor(x_test1,cv2.COLOR_RGB2GRAY)
print(x_test1.shape)
if K.image_data_format() == 'channels_first':
print('reshape1')
x_test1 = x_test1.reshape(1, 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
print('reshape2')
x_test1 = x_test1.reshape(1, img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_test1 = x_test1.astype('float32')
x_test1 /= 255
y_test1 = np.array([[ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.]])
score = model.evaluate(x_test1, y_test1, verbose=0)
print('done')
print('score of image = ')
print(score[1])
print(score[0])
score=model.predict_classes(x_test1)
print(score)
The below code is used to load a previous trained model and continue training it from the previous checkpoint. If there are any mistakes please do suggest them.
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
filepath="my_model.h5"
checkpoint = ModelCheckpoint(filepath, monitor='acc', verbose=1, save_best_only=True, mode='max')
callbacks_list = [checkpoint]
score=model.fit(x_test1, y_test1,epochs=12, batch_size=128, callbacks=callbacks_list, verbose=0)
new_model = load_model("my_model.h5")
np.testing.assert_allclose(model.predict(x_test1),
new_model.predict(x_test1),
1e-5)
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')
callbacks_list = [checkpoint]
new_model.fit(x_train, y_train, epochs=12, batch_size=128, callbacks=callbacks_list)
Does the above code cause over-fitting of the model ? If yes how do I make it more efficient so that it is able to predict any kind of data ? Help Required!!
You should not be surprised by these results because you train your model on one data domain but test it on a different one. Yes, you may get slightly better performance if you have a better-designed network, but this gap still exists.
To close this gap, you can improve the following things :
improve your training data: (a) train your model with more real-looking digit data, for example, the street view house number (SVHN) dataset, or the digits in the char74K dataset, and (b) train your model with better data augmentation techniques, for example, blend a digit sample with a random background image.
adapt your testing data: preprocess testing samples such that they look similar to those in your training data, for example, binzarize a testing sample and let it look similar to that of MNIST, before you feed it to your network for prediction.
improve your model by explicitly considering the variations in real data: for example, if you think it is rotation in testing data degrades your model performance, then you can consider this factor in your model training by adding a side task to prediction digit rotation; or you believe a good model should only predict digits but not the dataset that this training sample belongs to, then you can add an adversarial task to force the network forget this information.
I am doing figure extractor from scanned documents. using 1100x850 images. we use 44x34 grids of image. so that last layer will be 1496 fully connected layer.
label is 44x34 BINARY array which is 1 for figure rigion and 0 for non figure region. i.e if figure falls within (top right) (x,y)=(0,0) (bottom left) (x,y)=(50,50) then bin array has 1 at (0,0) (0,1) and (1,0) (1,1) these positions and rest 0s. so i have buit a neural network model. following is the structure.
conv(5,2,48)
maxpool(3,2)
conv(5,2,96)
maxpool(3,2)
conv(5,2,96)
maxpool(3,2)
FC-1496
The notation conv(k,d, n) denotes a convolutional layer with n filters, each of size k × k, applied with a shift of d pixels; maxpool(k, d) denotes a downsampling operation over k×k windows, applied with a shift of d pixels. FC-1496 refers to the final fully connected
layer which connects the hidden units from the previous layers to the 1496 output units (we have 1496 units for a 44x34 grid).
So my question is how to feed input ( images and labels (array) ) to this model using keras and tensor flow.
here is the model code
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.layers.core import Activation
from keras.layers.core import Dense
from keras.models import Sequential
from keras.layers import Flatten
xtrain=#image of 850*1100 for 10 images 10 850*1100
xtest=#binary array of size 1496 for 10 images size is 10*1496
# initialize the model
model = Sequential()
model.add(Conv2D(48, 5, 2, input_shape=(1100, 850, 1)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=2))
model.add(Conv2D(96, 5, 2))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=2))
model.add(Conv2D(96, 5, 2))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=2))
model.add(Flatten())
model.add(Dense(1496, activation='sigmoid'))
model.compile(loss='mean_squared_error', optimizer='sgd', metrics=['accuracy'])
here is a working example based on your data (as I assume from your info)
I have using the label as a vector of 1 and 0 e.g [1,0,1,1,...] 1 for figure region and 0 for none figure region, for a total of 1496 regions
from __future__ import print_function
import numpy as np
np.random.seed(1337) # for reproducibility
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten
from keras.utils import np_utils
batch_size = 128
nb_epoch = 10
nb_regions = 1496
# input image dimensions
img_rows, img_cols = 850, 1100
# create random test and train sets
X_train = np.random.randint(256, size=(10, img_rows, img_cols))
Y_train = np.random.randint(2, size=(10, nb_regions))
X_test = np.random.randint(256, size=(10, img_rows, img_cols))
Y_test = np.random.randint(2, size=(10, nb_regions))
X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
model = Sequential([
Dense(32, input_shape=(1, img_rows, img_cols)),
Activation('relu'),
Flatten(),
Dense(nb_regions),
Activation('softmax'),
])
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
model.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch,
verbose=1, validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])