I need some help to get an existing python example which works nicely to classify 2 categories and extend it to classify 3 categories.
here's the working example:
import numpy as np
import matplotlib.pyplot as plt
import os
import cv2 as cv
from tqdm import tqdm
import random
import tensorflow as tf
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
DATADIR = "C:/D/tmp/CNN"
CATEGORIES = ["Cat1","Cat2"]
IMG_SIZE = 100
training_data = []
def create_training_data():
for category in CATEGORIES:
path = os.path.join(DATADIR,category)
class_num = CATEGORIES.index(category)
for img in tqdm(os.listdir(path)):
try:
img_array = cv.imread(os.path.join(path,img) ,cv.IMREAD_GRAYSCALE) # convert to array
new_array = cv.resize(img_array, (IMG_SIZE, IMG_SIZE)) # resize to normalize data size
training_data.append([new_array, class_num])
except Exception as e:
pass
#except OSError as e:
# print("OSErrroBad img most likely", e, os.path.join(path,img))
#except Exception as e:
# print("general exception", e, os.path.join(path,img))
create_training_data()
print(len(training_data))
random.shuffle(training_data)
X = []
y = []
for features,label in training_data:
X.append(features)
y.append(label)
#print(X[0].reshape(-1, IMG_SIZE, IMG_SIZE, 1))
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
y = np.array(y)
X = X/255.0
model = Sequential()
model.add(Conv2D(256, (3, 3), input_shape=X.shape[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(256, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors
model.add(Dense(64))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X, y, batch_size=16, epochs=20, validation_split=0.3)
model.save('cat1_2.model')
My idea was to, apart from creating the 3rd folder with the images of the 3rd category to:
1- change to CATEGORIES = ["Cat1","Cat2","Cat3"]
2- change model.add(Dense(1)) to model.add(Dense(3))
3- change model.add(Activation('sigmoid')) to model.add(Activation('softmax'))
4- change from loss='binary_crossentropy' to loss='sparse_categorical_crossentropy'
Unfortunately that, although the model seems to converge to a very good accuracy (above .95) it always predict
[[0. 0. 1.]]
regardless the input.
What am I missing?
solved....rookie mistake....the error was actually in the prediction calling. basically I was passing the image to the model without scaling 0<->1 (i.e. diving by 255)..Once I did that predictions started to make sense...
Related
I'm Getting:-
loss: 0.4413 - accuracy: 0.8617 - val_loss: 0.7057 - val_accuracy: 0.8038
Code:
import numpy as np
import pandas as pd
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import os
import random
import cv2
import imutils
import random
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelBinarizer
from keras.utils import np_utils
from tensorflow.keras.models import Sequential
from tensorflow.keras import optimizers
from sklearn.preprocessing import LabelBinarizer
from tensorflow.keras import backend as K
from tensorflow.keras.layers import Dense, Activation, Flatten, Dense,MaxPooling2D, Dropout
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization
DATADIR = "E:\Final Year project\Virtual Robotic Arm controlled using Hand Gesture Reconginse & AI\Traning Data\Dataset\Train"
CATEGORIES = ["0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","a1","b1","c1","d1","e1","f1","g1","h1","i1","j1","k1","l1","m1","n1","o1","p1","q1","r1","s1","t1","u1","v1","w1","x1","y1","z1"]
for category in CATEGORIES: #
path = os.path.join(DATADIR,category) # create path to dogs and cats
for img in os.listdir(path): # iterate over each image per dogs and cats
img_array = cv2.imread(os.path.join(path,img) ,cv2.IMREAD_GRAYSCALE) # convert to array
plt.imshow(img_array, cmap='gray') # graph it
plt.show() # display!
break # we just want one for now so break
break #...and one more!
img_array
print(img_array.shape)
# Image Resize
IMG_SIZE = 32
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE))
plt.imshow(new_array, cmap='gray')
plt.show()
from tqdm import *
training_data = []
def create_training_data():
for category in CATEGORIES: # do dogs and cats
path = os.path.join(DATADIR,category) # create path to dogs and cats
class_num = CATEGORIES.index(category) # get the classification (0 or a 1). 0=dog 1=cat
for img in tqdm(os.listdir(path)): # iterate over each image per dogs and cats
try:
img_array = cv2.imread(os.path.join(path,img) ,cv2.IMREAD_GRAYSCALE) # convert to array
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE)) # resize to normalize data size
training_data.append([new_array, class_num]) # add this to our training_data
except Exception as e: # in the interest in keeping the output clean...
pass
#except OSError as e:
# print("OSErrroBad img most likely", e, os.path.join(path,img))
#except Exception as e:
# print("general exception", e, os.path.join(path,img))
create_training_data()
print(len(training_data))
random.shuffle(training_data)
from sklearn.model_selection import train_test_split
train_data , test_data = train_test_split(training_data,test_size=0.2,random_state=42)
train_data , val_data = train_test_split(train_data,test_size=0.2,random_state=42)
print(len(train_data))--->4012
print(len(test_data))--->1255
print(len(val_data))---> 1004
train_X = []
train_Y = []
for features,label in train_data:
train_X.append(features)
train_Y.append(label)
test_X = []
test_Y = []
for features,label in test_data:
test_X.append(features)
test_Y.append(label)
val_X = []
val_Y = []
for features,label in val_data:
val_X.append(features)
val_Y.append(label)
LB = LabelBinarizer()
train_Y = LB.fit_transform(train_Y)
test_Y = LB.fit_transform(test_Y)
val_Y = LB.fit_transform(val_Y)
train_X = np.array(train_X)/255.0
train_X = train_X.reshape(-1,32,32,1)
train_Y = np.array(train_Y)
test_X = np.array(test_X)/255.0
test_X = test_X.reshape(-1,32,32,1)
test_Y = np.array(test_Y)
val_X = np.array(val_X)/255.0
val_X = val_X.reshape(-1,32,32,1)
val_Y = np.array(val_Y)
print(train_X.shape,test_X.shape,val_X.shape)---> (4012, 32, 32, 1) (1255, 32, 32, 1) (1004,
32, 32, 1)
print(train_Y.shape,test_Y.shape,val_Y.shape)---> (4012, 62) (1255, 62) (1004, 62)
model = Sequential()
model.add(Conv2D(32, (3, 3), padding = "same", activation='relu', input_shape=(32,32,1)))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(62, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy', optimizer="adam",metrics=['accuracy'])
history = model.fit(train_X,train_Y, epochs=25, batch_size=32, validation_data = (val_X, val_Y), verbose=1)
import pickle
# Save the Modle to file in the current working directory
Filename = "Model.pkl"
with open(Filename, 'wb') as file:
pickle.dump(model, file)
TypeError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_13132/1527247688.py in
5
6 with open(Pkl_Filename, 'wb') as file:
----> 7 pickle.dump(model, file)
TypeError: can't pickle _thread.RLock objects
This code executed successfully when I tried replicating the same error in both Google Colab and Jupyter notebook.
You can check the saved .pkl file in the left pane.
Please find the attached gist for your reference.(I have used binary dataset to reproduce the issue).
You can try again executing the same code and let us know if the issue still persists with the details of python and tensorflow version you are using.
Firstly, my level in English is bad, so sorry for that.
So when I start the training of my CNN, it return this error :
ValueError: validation_split is only supported for Tensors or NumPy arrays, found following types in the input: [<class 'int'>, <class 'int'>, <class 'int'>, .... ]
I'm beginner at CNN. I don't know where is the error, so I put my entire code here (written by sentdex YouTube channel) :
create my training data
import numpy as np
import os
import cv2
from tqdm import tqdm
import pickle
import random
DATADIR = "C:/content/datasets/Cats and dogs 2"
CATEGORIES = ["Dog", "Cat"]
IMG_SIZE = 100
training_data = []
def create_training_data():
for category in CATEGORIES: # do dogs and cats
path = os.path.join(DATADIR,category) # create path to dogs and cats
class_num = CATEGORIES.index(category) # get the classification (0 or a 1). 0=dog 1=cat
for img in tqdm(os.listdir(path)): # iterate over each image per dogs and cats
try:
img_array = cv2.imread(os.path.join(path,img)) # convert to array
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE)) # resize to normalize data size
training_data.append([new_array, class_num]) # add this to our training_data
except Exception as e: # in the interest in keeping the output clean...
pass
#except OSError as e:
# print("OSErrroBad img most likely", e, os.path.join(path,img))
#except Exception as e:
# print("general exception", e, os.path.join(path,img))
create_training_data()
print(training_data)
random.shuffle(training_data)
X = []
y = []
for features,label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
print(y)
pickle_out = open("X.pickle","wb")
pickle.dump(X, pickle_out)
pickle_out.close()
pickle_out = open("y.pickle","wb")
pickle.dump(y, pickle_out)
pickle_out.close()
Build and train the neural network
import tensorflow as tf
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
import pickle
pickle_in = open("X.pickle","rb")
X = pickle.load(pickle_in)
pickle_in = open("y.pickle","rb")
y = pickle.load(pickle_in)
X = X/255.0
model = Sequential()
model.add(Conv2D(256, (3, 3), input_shape=X.shape[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(256, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors
model.add(Dense(64))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X, y, batch_size=32, epochs=3, validation_split=0.1)
Perhaps the "reshape at the array X" is strange, pycharm tell me that the two last arguments are unexpected.
Please tell me if you see improvement going for my code,
Thanks for your help
ValueError: validation_split is only supported for Tensors or NumPy
arrays, found following types in the input: [<class 'int'>, <class
'int'>, <class 'int'>, .... ]
for features,label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
Combine your code with the error, y is a list in your code, try to convert it to numpy arrays.
I'm new to keras. This code is working on classifying between MRI images of brain with or without tumor. When I run model.evaluate() to see the accuracy I get very high loss value even it is low when I'm training the model(normal less than 1) and I get the following error:
WARNING:tensorflow:6 out of the last 11 calls to <function Model.make_test_function.<locals>.test_function at 0x00000221AC143AF0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating #tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your #tf.function outside of the loop. For (2), #tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.
The most of the code is copied from this link.
Here is the full code:
import numpy as np
import matplotlib.pyplot as plt
import os
import cv2
import tensorflow as tf
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
def load_data( DATADIR, IMG_SIZE, CATEGORIES ):
data = []
for category in CATEGORIES: # do dogs and cats
path = os.path.join(DATADIR,category) # create path to dogs and cats
class_num = CATEGORIES.index(category) # get the classification (0 or a 1). 0=dog 1=cat
for img in os.listdir(path): # iterate over each image per dogs and cats
try:
img_array = cv2.imread(os.path.join(path,img) ,cv2.IMREAD_GRAYSCALE) # convert to array
img_array = cv2.medianBlur(img_array,5)
img_array = cv2.adaptiveThreshold(img_array,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,11,2)
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE)) # resize to normalize data size
data.append([new_array, class_num]) # add this to our training_data
except Exception as e: # in the interest in keeping the output clean...
pass
#except OSError as e:
# print("OSErrroBad img most likely", e, os.path.join(path,img))
#except Exception as e:
# print("general exception", e, os.path.join(path,img))
return data
TRAIN_DATADIR = "F:\Train"
TEST_DATADIR = "F:\Test"
CATEGORIES = ["no", "yes"]
IMG_SIZE = 128
training_data = load_data(TRAIN_DATADIR, IMG_SIZE, CATEGORIES)
testing_data = load_data(TEST_DATADIR, IMG_SIZE, CATEGORIES)
print(len(training_data))
import random
random.shuffle(training_data)
random.shuffle(testing_data)
X_train = []
y_train = []
for features,label in training_data:
X_train.append(features)
y_train.append(label)
#print(X[0].reshape(-1, IMG_SIZE, IMG_SIZE, 1))
X_train = np.asarray(X_train)
y_train = np.asarray(y_train)
X_train = np.array(X_train).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
X_test = []
y_test = []
for features,label in testing_data:
X_test.append(features)
y_test.append(label)
X_test = np.asarray(X_test)
y_test = np.asarray(y_test)
#print(X[0].reshape(-1, IMG_SIZE, IMG_SIZE, 1))
X_test = np.array(X_test).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
X_train = X_train/255.0
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape = X_train.shape[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X_train, y_train, batch_size=10, epochs=15)
score = model.evaluate(X_test, y_test,verbose=1)
Ignore the warning.
Your low training loss and high evaluation loss means that your model is overfitted. Stop training when your validation accuracy starts to increase.
Hello I'm trying to make the basic example of tensorflow minst using data from images on my pc.
But I run into this error all the time: "ValueError: Failed to find data adapter that can handle input: , ( containing values of types {""})"
here's how i generate data:
import numpy as np # for array operations
import matplotlib.pyplot as plt # to show image
import os # to move through directories
import cv2 # to make image operations
import random
import pickle
DATADIR=r"C:\Users\...\mnist_png\training"
DIGITS = ["0","1","2","3","4","5","6","7","8","9"]
training_data = []
for digit in DIGITS:
path = os.path.join(DATADIR, digit)
class_num = DIGITS.index(digit)
for img in os.listdir(path):
img_array = cv2.imread(os.path.join(path,img), cv2.IMREAD_GRAYSCALE)
training_data.append([img_array, class_num])
random.shuffle(training_data)
X = []
y = []
for features, label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1, 28, 28, 1)
pickle_out = open("X.pickle", "wb")
pickle.dump(X, pickle_out)
pickle_out.close()
pickle_out = open("y.pickle", "wb")
pickle.dump(y, pickle_out)
pickle_out.close()
and here's the tensorflow model that I get error from:
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
import pickle
X = pickle.load(open("X.pickle", "rb"))
y = pickle.load(open("y.pickle", "rb"))
X=X/255.0
model = Sequential()
model.add(Conv2D(64, (3,3), input_shape = X.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(64, (3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Dense(10))
model.add(Activation('sigmoid'))
model.compile(
loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=['accuracy']
)
model.fit(X, y, batch_size=32, validation_split=0.1)
Please help me
After
for features, label in training_data:
X.append(features)
y.append(label)
you have to add
y = np.array(y)
I get this error when train the model:
ValueError: Error when checking target:
expected dropout_5 to have shape (33,) but got array with shape (1,).
I want to store my images into 33 array from folder using path. I have categories the images into different folder which were 1,2,3,4,5...
I have use this code to do it but i dont know how to store it into different array. Can someone help me.
datadir = 'C:/Users/user/Desktop/RESIZE' #path of the folder
categories = ['1','2','3','4','5','6','7','8','9','A','B','C','D','E','F','G','H','I','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y']
img_rows, img_cols = 100, 100
training_data = []
for category in categories:
path = os.path.join(datadir,category)
class_num = categories.index(category)
for img in os.listdir(path):
img_array = cv2.imread(os.path.join(path,img),cv2.IMREAD_GRAYSCALE)
new_array = cv2.resize(img_array,(img_rows,img_cols))
training_data.append([new_array,class_num])
random.shuffle(training_data)
X = []
y = []
for features, label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1,img_rows,img_cols,1)
X = X.astype("float32")
pickle_out = open("X.pickle","wb")
pickle.dump(X,pickle_out)
pickle_out.close()
pickle_out = open("y.pickle","wb")
pickle.dump(y,pickle_out)
pickle_out.close()
After I save the file, then I use this code to train model and I want get 33 output layer but it only can work when my output layer(Dense) set 1.
I got this error:
ValueError: Error when checking target:
expected dropout_5 to have shape (33,) but got array with shape (1,)
Here was my training code.
import tensorflow as tf
from keras import optimizers
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.layers import Activation
import cv2
import os
import numpy as np
import pickle
from sklearn.utils import shuffle
X = pickle.load(open("X.pickle","rb"))
y = pickle.load(open("y.pickle","rb"))
X = X/255.0
model = Sequential()
model.add(Conv2D(32,(3,3), input_shape = X.shape[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.25))
model.add(Conv2D(64,(3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.25))
model.add(Conv2D(128,(3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.4))
model.add(Dense(128))
model.add(Activation("relu"))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(33, activation='softmax'))
model.add(Dropout(0.4))
model.compile(loss = "binary_crossentropy", optimizer = "adam", metrics = ["accuracy"])
model.fit(X, y, batch_size = 2, epochs = 1, validation_split = 0.2)
You need to change your y as one hot encoded data to do the training.
Try this for y,
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
# integer encode
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(y)
print(label_encoder.classes_) # This is your classes.
print(integer_encoded.shape())
# binary encode
onehot_encoder = OneHotEncoder(sparse=False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
print(onehot_encoded.shape())
And one more thing, if you want to classify for 33 class than change your loss to categorical_crossentropy.