I get this error when train the model:
ValueError: Error when checking target:
expected dropout_5 to have shape (33,) but got array with shape (1,).
I want to store my images into 33 array from folder using path. I have categories the images into different folder which were 1,2,3,4,5...
I have use this code to do it but i dont know how to store it into different array. Can someone help me.
datadir = 'C:/Users/user/Desktop/RESIZE' #path of the folder
categories = ['1','2','3','4','5','6','7','8','9','A','B','C','D','E','F','G','H','I','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y']
img_rows, img_cols = 100, 100
training_data = []
for category in categories:
path = os.path.join(datadir,category)
class_num = categories.index(category)
for img in os.listdir(path):
img_array = cv2.imread(os.path.join(path,img),cv2.IMREAD_GRAYSCALE)
new_array = cv2.resize(img_array,(img_rows,img_cols))
training_data.append([new_array,class_num])
random.shuffle(training_data)
X = []
y = []
for features, label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1,img_rows,img_cols,1)
X = X.astype("float32")
pickle_out = open("X.pickle","wb")
pickle.dump(X,pickle_out)
pickle_out.close()
pickle_out = open("y.pickle","wb")
pickle.dump(y,pickle_out)
pickle_out.close()
After I save the file, then I use this code to train model and I want get 33 output layer but it only can work when my output layer(Dense) set 1.
I got this error:
ValueError: Error when checking target:
expected dropout_5 to have shape (33,) but got array with shape (1,)
Here was my training code.
import tensorflow as tf
from keras import optimizers
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.layers import Activation
import cv2
import os
import numpy as np
import pickle
from sklearn.utils import shuffle
X = pickle.load(open("X.pickle","rb"))
y = pickle.load(open("y.pickle","rb"))
X = X/255.0
model = Sequential()
model.add(Conv2D(32,(3,3), input_shape = X.shape[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.25))
model.add(Conv2D(64,(3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.25))
model.add(Conv2D(128,(3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.4))
model.add(Dense(128))
model.add(Activation("relu"))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(33, activation='softmax'))
model.add(Dropout(0.4))
model.compile(loss = "binary_crossentropy", optimizer = "adam", metrics = ["accuracy"])
model.fit(X, y, batch_size = 2, epochs = 1, validation_split = 0.2)
You need to change your y as one hot encoded data to do the training.
Try this for y,
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
# integer encode
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(y)
print(label_encoder.classes_) # This is your classes.
print(integer_encoded.shape())
# binary encode
onehot_encoder = OneHotEncoder(sparse=False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
print(onehot_encoded.shape())
And one more thing, if you want to classify for 33 class than change your loss to categorical_crossentropy.
Related
I need some help to get an existing python example which works nicely to classify 2 categories and extend it to classify 3 categories.
here's the working example:
import numpy as np
import matplotlib.pyplot as plt
import os
import cv2 as cv
from tqdm import tqdm
import random
import tensorflow as tf
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
DATADIR = "C:/D/tmp/CNN"
CATEGORIES = ["Cat1","Cat2"]
IMG_SIZE = 100
training_data = []
def create_training_data():
for category in CATEGORIES:
path = os.path.join(DATADIR,category)
class_num = CATEGORIES.index(category)
for img in tqdm(os.listdir(path)):
try:
img_array = cv.imread(os.path.join(path,img) ,cv.IMREAD_GRAYSCALE) # convert to array
new_array = cv.resize(img_array, (IMG_SIZE, IMG_SIZE)) # resize to normalize data size
training_data.append([new_array, class_num])
except Exception as e:
pass
#except OSError as e:
# print("OSErrroBad img most likely", e, os.path.join(path,img))
#except Exception as e:
# print("general exception", e, os.path.join(path,img))
create_training_data()
print(len(training_data))
random.shuffle(training_data)
X = []
y = []
for features,label in training_data:
X.append(features)
y.append(label)
#print(X[0].reshape(-1, IMG_SIZE, IMG_SIZE, 1))
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
y = np.array(y)
X = X/255.0
model = Sequential()
model.add(Conv2D(256, (3, 3), input_shape=X.shape[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(256, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors
model.add(Dense(64))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X, y, batch_size=16, epochs=20, validation_split=0.3)
model.save('cat1_2.model')
My idea was to, apart from creating the 3rd folder with the images of the 3rd category to:
1- change to CATEGORIES = ["Cat1","Cat2","Cat3"]
2- change model.add(Dense(1)) to model.add(Dense(3))
3- change model.add(Activation('sigmoid')) to model.add(Activation('softmax'))
4- change from loss='binary_crossentropy' to loss='sparse_categorical_crossentropy'
Unfortunately that, although the model seems to converge to a very good accuracy (above .95) it always predict
[[0. 0. 1.]]
regardless the input.
What am I missing?
solved....rookie mistake....the error was actually in the prediction calling. basically I was passing the image to the model without scaling 0<->1 (i.e. diving by 255)..Once I did that predictions started to make sense...
In the code below, I am getting the error:
raise ValueError('This model has not yet been built. ' ValueError: This model has not yet been built. Build the model first by calling ``build()`` or calling ``fit()`` with some data, or specify an ``input_shape`` argument in the first layer(s) for automatic build.
I am trying to use a spectrogram instead of the Mel feature extraction for this CNN. I think the error may be in the extraction process for the extract_features step.
Any help will be greatly appreciated in figuring out the source of this error.
Here are some sizes of arrays generated before the error:
x_test = {ndarray:(353,128)}
x_train = {ndarray:(1408,128)}
y = {ndarray:(1761,)}
y_test = {ndarray:(353,10)}
y_ttrain = {ndarray:(1408,10)}
CODE
# Sample rate conversion
import librosa
import librosa.display
from scipy.io import wavfile as wav
import numpy as np
import pandas as pd
import os
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
max_pad_len = 174
n_mels = 128
def extract_features(file_name):
try:
audio, sample_rate = librosa.core.load(file_name, res_type='kaiser_fast')
mely = librosa.feature.melspectrogram(y=audio, sr=sample_rate, n_mels=n_mels)
# pad_width = max_pad_len - mely.shape[1]
# mely = np.pad(mely, pad_width=((0, 0), (0, pad_width)), mode='constant')
melyscaled = np.mean(mely.T, axis=0)
except Exception as e:
print("Error encountered while parsing file: ", file_name)
return None
return melyscaled
features = []
# Iterate through each sound file and extract the features
from time import sleep
#from tqdm import tqdm
from alive_progress import alive_bar
items = len(metadata.index)
with alive_bar(items) as bar:
for index, row in metadata.iterrows():
bar()
file_name = os.path.join(os.path.abspath(fulldatasetpath), 'fold' + str(row["fold"]) + '/',
str(row["slice_file_name"]))
class_label = row["class_name"]
data = extract_features(file_name)
features.append([data, class_label])
# Convert into a Panda dataframe
featuresdf = pd.DataFrame(features, columns=['feature', 'class_label'])
print('Finished feature extraction from ', len(featuresdf), ' files')
from sklearn.preprocessing import LabelEncoder
from keras.utils import to_categorical
# Convert features and corresponding classification labels into numpy arrays
X = np.array(featuresdf.feature.tolist())
y = np.array(featuresdf.class_label.tolist())
# Encode the classification labels
le = LabelEncoder()
yy = to_categorical(le.fit_transform(y))
# split the dataset
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(X, yy, test_size=0.2, random_state = 42)
### Convolutional Neural Network (CNN) model architecture
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, Conv2D, MaxPooling2D, GlobalAveragePooling2D
from keras.optimizers import Adam
from keras.utils import np_utils
from sklearn import metrics
num_rows = 40
num_columns = 174
num_channels = 1
#
#x_train = x_train.reshape(x_train.shape[0], num_rows, num_columns, num_channels)
#x_test = x_test.reshape(x_test.shape[0], num_rows, num_columns, num_channels)
num_labels = yy.shape[1]
filter_size = 2
# Construct model
model = Sequential()
#model.add(Conv2D(filters=16, kernel_size=2, input_shape=(num_rows, num_columns, num_channels), activation='relu'))
model.add(Conv2D(filters=16, kernel_size=2, activation='relu'))
model.add(MaxPooling2D(pool_size=2))
model.add(Dropout(0.2))
model.add(Conv2D(filters=32, kernel_size=2, activation='relu'))
model.add(MaxPooling2D(pool_size=2))
model.add(Dropout(0.2))
model.add(Conv2D(filters=64, kernel_size=2, activation='relu'))
model.add(MaxPooling2D(pool_size=2))
model.add(Dropout(0.2))
model.add(Conv2D(filters=128, kernel_size=2, activation='relu'))
model.add(MaxPooling2D(pool_size=2))
model.add(Dropout(0.2))
model.add(GlobalAveragePooling2D())
model.add(Dense(num_labels, activation='softmax'))
model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')
# Display model architecture summary
model.summary()
You have to build your model with your input shape before calling model.summary():
model.build((128,))
Hello I'm trying to make the basic example of tensorflow minst using data from images on my pc.
But I run into this error all the time: "ValueError: Failed to find data adapter that can handle input: , ( containing values of types {""})"
here's how i generate data:
import numpy as np # for array operations
import matplotlib.pyplot as plt # to show image
import os # to move through directories
import cv2 # to make image operations
import random
import pickle
DATADIR=r"C:\Users\...\mnist_png\training"
DIGITS = ["0","1","2","3","4","5","6","7","8","9"]
training_data = []
for digit in DIGITS:
path = os.path.join(DATADIR, digit)
class_num = DIGITS.index(digit)
for img in os.listdir(path):
img_array = cv2.imread(os.path.join(path,img), cv2.IMREAD_GRAYSCALE)
training_data.append([img_array, class_num])
random.shuffle(training_data)
X = []
y = []
for features, label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1, 28, 28, 1)
pickle_out = open("X.pickle", "wb")
pickle.dump(X, pickle_out)
pickle_out.close()
pickle_out = open("y.pickle", "wb")
pickle.dump(y, pickle_out)
pickle_out.close()
and here's the tensorflow model that I get error from:
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
import pickle
X = pickle.load(open("X.pickle", "rb"))
y = pickle.load(open("y.pickle", "rb"))
X=X/255.0
model = Sequential()
model.add(Conv2D(64, (3,3), input_shape = X.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(64, (3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Dense(10))
model.add(Activation('sigmoid'))
model.compile(
loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=['accuracy']
)
model.fit(X, y, batch_size=32, validation_split=0.1)
Please help me
After
for features, label in training_data:
X.append(features)
y.append(label)
you have to add
y = np.array(y)
This is my first time image classification, I have tried to classification the images that have 2 classes. My images dataset are 128*128 and I use RGB so the I thinks the dimension is 128, 128, 3. The code is
import cv2
import glob
import numpy as np
import matplotlib.pyplot as plt
train = []
train_label = []
train_files_1 = glob.glob('/Users/filmer2002/Desktop/real_rash_project/train_images/Atopic_Dermatitis/*.jpg')
for files in train_files_1:
image = cv2.imread(files)
train.append(image)
train_label.append([0., 1.])
train_files_2 = glob.glob('/Users/filmer2002/Desktop/real_rash_project/train_images/Contact_Dermatitis/*.jpg')
for files in train_files_2:
image = cv2.imread(files, )
train.append(image)
train_label.append([1., 0.])
train_array = np.array(train, dtype='int')
train_label_array = np.array(train_label, dtype='int')
test = []
test_label = []
test_files = glob.glob('/Users/filmer2002/Desktop/real_rash_project/test_images/Atopic_Dermatitis/*.jpg')
for files in test_files:
image = cv2.imread(files)
test.append(image)
test_label.append([0., 1.])
test_files_2 = glob.glob('/Users/filmer2002/Desktop/real_rash_project/test_images/Contact_Dermatitis/*.jpg')
for files in test_files_2:
image = cv2.imread(files)
test.append(image)
test_label.append([1., 0.])
test_array = np.array(test, dtype='int')
test_label_array = np.array(test_array, dtype='int')
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten
from keras.utils import to_categorical
x_train = train_array
y_train = train_label_array
x_test = test_array
y_test = test_label_array
x_train = x_train.reshape(373, 128, 128, 3)
x_test = x_test.reshape(95, 128, 128, 3)
model = Sequential()
model.add(Conv2D(64, kernel_size = 3, activation = 'relu', input_shape = (128, 128, 3)))
model.add(Conv2D(32, kernel_size = 3, activation = 'relu'))
model.add(Conv2D(16, kernel_size = 3, activation = 'relu'))
model.add(Flatten())
model.add(Dense(10, activation = 'softmax'))
model.compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
model.fit(x_train, to_categorical(y_train), validation_data = (x_test, to_categorical(y_test)), epochs = 3)
After run the code it show 'ValueError: Error when checking target: expected dense_1 to have 2 dimensions, but got array with shape (373, 2, 2)' and I don't know how to fix it you can see the code in github at https://github.com/filmer2002/real_rash_project/blob/master/images_to_numpy_and_CNN_code.ipynb
You are getting this error because you have labels, with a shape different from your output layer, given to the network to train.
to_categorical() needs 2 parameters, labels and len, labels have to be dtype=int and smaller then len, and make sure that your tensorflow and tensorboard are up to date.
Try
import cv2
import glob
import numpy as np
import matplotlib.pyplot as plt
train = []
train_label = []
train_files_1 = glob.glob('/Users/filmer2002/Desktop/real_rash_project/train_images/Atopic_Dermatitis/*.jpg')
for files in train_files_1:
image = cv2.imread(files)
train.append(image)
train_label.append(1)
train_files_2 = glob.glob('/Users/filmer2002/Desktop/real_rash_project/train_images/Contact_Dermatitis/*.jpg')
for files in train_files_2:
image = cv2.imread(files, )
train.append(image)
train_label.append(0)
train_array = np.array(train, dtype=int)
train_label_array = np.array(train_label, dtype=int)
test = []
test_label = []
test_files = glob.glob('/Users/filmer2002/Desktop/real_rash_project/test_images/Atopic_Dermatitis/*.jpg')
for files in test_files:
image = cv2.imread(files)
test.append(image)
test_label.append(1)
test_files_2 = glob.glob('/Users/filmer2002/Desktop/real_rash_project/test_images/Contact_Dermatitis/*.jpg')
for files in test_files_2:
image = cv2.imread(files)
test.append(image)
test_label.append(0)
test_array = np.array(test, dtype=int)
test_label_array = np.array(test_array, dtype=int)
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten
from keras.utils import to_categorical
x_train = train_array
y_train = train_label_array
x_test = test_array
y_test = test_label_array
x_train = x_train.reshape(373, 128, 128, 3)
x_test = x_test.reshape(95, 128, 128, 3)
model = Sequential()
model.add(Conv2D(64, kernel_size = 3, activation = 'relu', input_shape = (128, 128, 3)))
model.add(Conv2D(32, kernel_size = 3, activation = 'relu'))
model.add(Conv2D(16, kernel_size = 3, activation = 'relu'))
model.add(Flatten())
model.add(Dense(10, activation = 'softmax'))
model.compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
model.fit(x_train, to_categorical(y_train, 10), validation_data = (x_test, to_categorical(y_test, 10)), epochs = 3)
len given to to_categorical in this case is 10 because your output layer has shape == (10,)
Also make sure that you have equal amount of images and labels for them, you cen test that by:
x_train.shape[0] == to_categorical(y_train, 10).shape[0]
True #should be True if everything is ok
And one final note: tensorflow works with tensors i. e. n dimensional arrays, don't use lists when working with training data, numpy arrays should be used most of the time.
EDIT:
Only 1 dimensional arrays or lists consisting of int values are ok for to_categorical().
Don't try to use lists consisting of [0, 1] in to_categorical(), it won't work, just use 1 to_categorical() will convert it for you into [0, 1] elements.
Your mistake was to use lists as labels for to_categorical(), to_categorical() only accepts 1 dimensional int arrays as it's first parameter, the shape you where giving it was (number_of_labels, 2), when it only works with shape == (number_of_labels,).
OR
If you want to use lists/arrays consisting of elements like [0, 1] and your model output shape is 2, then just don't use to_categorical() at all.
This should work, and if it doesn't you have inconsistent data.
import cv2
import glob
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, Flatten
from tensorflow.keras.utils import to_categorical
#assert tf.__version__ == '1.14.0'
train = np.zeros((1, 128, 128, 3))
train_label = []
train_files_1 = glob.glob('/path/to/your/folder/with/data/*.jpg')
for files in train_files_1:
image = cv2.imread(files)
img = np.array(image).astype(int).reshape(128, 128, 3)
train = np.concatenate((train, [img]), axis=0)
train_label.append(1)
train_files_2 = glob.glob('/path/to/your/folder/with/data/*.jpg')
for files in train_files_2:
image = cv2.imread(files)
img = np.array(image).astype(int)
train = np.concatenate((train, [img]), axis=0)
train_label.append(0)
x_train = train[1:]
y_train = to_categorical(train_label, 2)
test = np.zeros((1, 128, 128, 3))
test_label = []
test_files = glob.glob('/path/to/your/folder/with/data/*.jpg')
for files in test_files:
image = cv2.imread(files)
img = np.array(image).astype(int)
test = np.concatenate((test, [img]), axis=0)
test_label.append(1)
test_files_2 = glob.glob('/path/to/your/folder/with/data/*.jpg')
for files in test_files_2:
image = cv2.imread(files)
img = np.array(image).astype(int)
test = np.concatenate((test, [img]), axis=0)
test_label.append(0)
x_test = test[1:]
y_test = to_categorical(test_label, 2)
print ('train', x_train.shape)
print ('test', x_test.shape)
print ('test labels', y_test.shape)
print ('train labels', y_train.shape)
assert x_train.shape == (len(train[1:]), 128, 128, 3)
assert x_test.shape == (len(test[1:]), 128, 128, 3)
assert y_train.shape == (len(train_label), 2)
assert y_test.shape == (len(test_label), 2)
# if you get an error about one of the asserts above => your data is not consistant
model = Sequential()
model.add(Conv2D(64, kernel_size=3, activation='relu', input_shape=(128, 128, 3)))
model.add(Conv2D(32, kernel_size=3, activation='relu'))
model.add(Conv2D(16, kernel_size=3, activation='relu'))
model.add(Flatten())
model.add(Dense(2, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=3)
print ('all good, all done!')
Note: mixing python lists and numpy arrays is very bad idea
I have a model and although I have a flatten layer it keeps giving me the error:
ValueError: Error when checking target: expected dense_53 to have 2 dimensions, but got array with shape (1, 1, 28, 28)
Additionaly, if I remove the dense and flatten layers it gives me the same error for the maxPooling and then for the convolutional. Here is my entire code:
import os
import numpy as np
from keras.preprocessing import image
from keras import backend as K
from keras.callbacks import EarlyStopping
from keras.datasets import cifar10
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Flatten
from keras.layers.convolutional import Conv2D
from keras.optimizers import Adam
from keras.layers.pooling import MaxPooling2D
from keras.utils import to_categorical
PATH = os.getcwd()
train_path = PATH+'/Downloads/KerasStuff/train_data/rice/'
train_batch = os.listdir(train_path)
x_train = np.empty((0,32,32,3))
# if data are in form of images
for sample in train_batch:
img_path = train_path+sample
if img_path == '/Users/23athreyad/Downloads/KerasStuff/train_data/rice/.DS_Store':
print("INVALID")
else:
x = image.load_img(img_path)
np.append(x_train,x)
test_path = PATH+'/Downloads/KerasStuff/test_data/rice/'
test_batch = os.listdir(test_path)
x_test = []
for sample in test_batch:
img_path = test_path+sample
x = image.load_img(img_path)
# preprocessing if required
x_test.append(x)
# Create the model
K.set_image_dim_ordering('tf')
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(32, 32, 3)))
model.add(MaxPooling2D(pool_size = (2,2),data_format="channels_last"))
print(model.output_shape)
model.add(Dropout(0.25))
model.add(Flatten())
print(model.output_shape)
model.add(Dense(7200,activation = 'relu'))
# Compile the model
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=0.0001, decay=1e-6),
metrics=['accuracy'])
model.fit(x_train,y_train,batch_size = 10, epochs = 20,callbacks=[EarlyStopping(min_delta=0.001, patience=3)])
Can someone help me with this? I am new to Keras and have looked this up but not found any solutions.