After reading this and taking the courses, I am struggling to solve the second problem in assignment 1 (notMnist):
Let's verify that the data still looks good. Displaying a sample of the labels and images from the ndarray. Hint: you can use matplotlib.pyplot.
Here is what I tried:
import random
rand_smpl = [ train_datasets[i] for i in sorted(random.sample(xrange(len(train_datasets)), 1)) ]
print(rand_smpl)
filename = rand_smpl[0]
import pickle
loaded_pickle = pickle.load( open( filename, "r" ) )
image_size = 28 # Pixel width and height.
import numpy as np
dataset = np.ndarray(shape=(len(loaded_pickle), image_size, image_size),
dtype=np.float32)
import matplotlib.pyplot as plt
plt.plot(dataset[2])
plt.ylabel('some numbers')
plt.show()
but this is what I get:
which doesn't make much sense. To be honest my code may too, since I am not really sure how to tackle that problem!
The pickles are created like this:
image_size = 28 # Pixel width and height.
pixel_depth = 255.0 # Number of levels per pixel.
def load_letter(folder, min_num_images):
"""Load the data for a single letter label."""
image_files = os.listdir(folder)
dataset = np.ndarray(shape=(len(image_files), image_size, image_size),
dtype=np.float32)
print(folder)
num_images = 0
for image in image_files:
image_file = os.path.join(folder, image)
try:
image_data = (ndimage.imread(image_file).astype(float) -
pixel_depth / 2) / pixel_depth
if image_data.shape != (image_size, image_size):
raise Exception('Unexpected image shape: %s' % str(image_data.shape))
dataset[num_images, :, :] = image_data
num_images = num_images + 1
except IOError as e:
print('Could not read:', image_file, ':', e, '- it\'s ok, skipping.')
dataset = dataset[0:num_images, :, :]
if num_images < min_num_images:
raise Exception('Many fewer images than expected: %d < %d' %
(num_images, min_num_images))
print('Full dataset tensor:', dataset.shape)
print('Mean:', np.mean(dataset))
print('Standard deviation:', np.std(dataset))
return dataset
where that function is called like this:
dataset = load_letter(folder, min_num_images_per_class)
try:
with open(set_filename, 'wb') as f:
pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
The idea here is:
Now let's load the data in a more manageable format. Since, depending on your computer setup you might not be able to fit it all in memory, we'll load each class into a separate dataset, store them on disk and curate them independently. Later we'll merge them into a single dataset of manageable size.
We'll convert the entire dataset into a 3D array (image index, x, y) of floating point values, normalized to have approximately zero mean and standard deviation ~0.5 to make training easier down the road.
Do this as below:
#define a function to conver label to letter
def letter(i):
return 'abcdefghij'[i]
# you need a matplotlib inline to be able to show images in python notebook
%matplotlib inline
#some random number in range 0 - length of dataset
sample_idx = np.random.randint(0, len(train_dataset))
#now we show it
plt.imshow(train_dataset[sample_idx])
plt.title("Char " + letter(train_labels[sample_idx]))
Your code changed the type of dataset actually, it is not an ndarray of size (220000, 28,28)
In general, pickle is a file which holds some objects, not the array itself. You should use the object from pickle directly to get your train dataset (using the notation from your code snippet):
#will give you train_dataset and labels
train_dataset = loaded_pickle['train_dataset']
train_labels = loaded_pickle['train_labels']
UPDATED:
Per request from #gsarmas the link to my solution for whole Assignment1 lies here.
The code is commented and mostly self-explanatory, but in case of any questions feel free to contact via any way you prefer on github
Please check with this code
pickle_file = train_datasets[0]
with open(pickle_file, 'rb') as f:
# unpickle
letter_set = pickle.load(f)
# pick a random image index
sample_idx = np.random.randint(len(letter_set))
# extract a 2D slice
sample_image = letter_set[sample_idx, :, :]
plt.figure()
# display it
plt.imshow(sample_image)
Use this code:
#random select a letter
i = np.random.randint( len(train_datasets) )
plt.title( "abcdefghij"[i] )
#read the file of selected letter
f = open( train_datasets[i], "rb" )
f = pickle.load(f)
#random select an image in the file
j = np.random.randint( len(f) )
#show image
plt.axis('off')
img = plt.imshow( f[ j, :, : ] )
enter image description here
Related
Well, I am curious to understand how to generate a function call which could return two arrays(images = [Batch of images in terms of .npy], labels =[ Image labels for a batch of images]), without using generator type or any other ML frameworks(TensorFlow -> flow from directory etc.).
Also, in case the batch size is unevenly dividable by a number of images, I would like the last batch to inherit the remaining set of images from the first batch. Is it possible ?, I am thinking in terms of Stack/ Queue but not sure if that's the required structure here
def next(self):
# Converting JSON to DICTIONARY
with open(label_path, 'r') as label_json:
label_dict = json.load(label_json)
number_of_images = len((os.listdir(file_path))) # Number of images = Length of the directory
images = os.listdir(file_path) # As a list
images_name = [each_image.strip(".npy") for each_image in images]
labels = label_path
number_of_batches = number_of_images / batch_size
# Sort the label dictionary
label_dict = dict(sorted(label_dict.items()))
# Match label and Image
batch_images = []
label_names = []
for i in images_name:
if i in label_dict.keys():
# batch_images.append(file_path + '/' + i + '.npy') #Image Location
img_array = np.load(file_path + '/' + i + '.npy') # Fetching file location
batch_images.append(img_array)
label_names.append(label_dict.get(i)) # Fetching Label name.
return batch_images, label_names
I am working on a multiclass image classification problem and there are over 120 folders inside my training folder and I am trying to load all the images from the 120 folders along with their label in the numpy arrays both X and Y.I am trying the below code But Getting the following error
ValueError: all input arrays must have the same shape
The code that I am using
def loadimgs(path,n = 0):
'''
path => Path of train directory or test directory
'''
X=[]
y = []
cat_dict = {}
fruit_dict = {}
curr_y = n
# we load every alphabet seperately so we can isolate them later
for fruit in os.listdir(path):
print("loading Fruit: " + fruit)
fruit_dict[fruit] = [curr_y,None]
fruit_path = os.path.join(path,fruit)
# every letter/category has it's own column in the array, so load seperately
for img in os.listdir(fruit_path):
cat_dict[curr_y] = (fruit, img)
category_images=[]
img_path = os.path.join(fruit_path, img)
# read all the images in the current category
for filename in img_path:
image = cv2.imread(filename)
category_images.append(image)
y.append(curr_y)
try:
X.append(np.stack(category_images))
# edge case - last one
except ValueError as e:
print(e)
print("error - category_images:", category_images)
curr_y += 1
fruit_dict[fruit][1] = curr_y - 1
y = np.vstack(y)
X = np.stack(X)
return X,y,fruit_dict
My training folder is as follows
Training Folder
Note: I donot want to use the keras ImageDataGenerator for doing this task
I'm trying to create an array of images using Numpy to feed it into an image classification neural network. When I put the image into an array, it comes 3 dimensions,but when I use np.append to append it into my array of all of the images the shape is 631800003. Why is this happening and how do I fix this? Or should I be loading the images some other way?
Here is my code for the variable definition cell:
normal = np.array([])
normalSet = np.array([])
badSet = np.array([])
Labels = np.array([])
Training_data = np.array([])
validationSet = []
process_data = True
ramCheck = 0
And the image loading:
if process_data:
for image in os.listdir('train/NORMAL/'):
normal = imread('train/NORMAL/'+image)
normalSet = np.append(normal, normalSet)
Labels = np.append(Labels, 0)
validationSet.append(normal)
for image in os.listdir('train/PNEUMONIA/'):
bad = imread('train/PNEUMONIA/'+image)
badSet = np.append(badSet, bad)
Labels = np.append(Labels, 1)
validationSet.append(bad)
print("done!")
Training_Data = np.append(badSet, normalSet)
np.save("TrainingData.npy", Training_data)
np.save("TrainingLabels.npy", Labels)
else:
Training_data = np.load("TrainingData.npy")
Labels = np.load("TrainingLabels.npy")
You can append into a list easier than change the list to a numpy array which will have the correct dimensions you need to feed it to a neural network.
normalSet = []
labels = []
for image in os.listdir('train/NORMAL/'):
normal = imread('train/NORMAL/'+ image)
normalSet.append(normal)
validationSet.append(normal)
labels.append(0)
normalSet = np.array(normalSet)
labels = np.array(labels)
Hello I am using Python to try to read the digit data provided by MNIST into a data structure I can use to train a neural network. I am testing to ensure the data was read properly by creating an image using PIL. The image that is being created is horribly wrong, and I am not sure if it is because I am using PIL incorrectly or my data structures and methods are not right.
The format of the two data files is described here:
http://yann.lecun.com/exdb/mnist/
Here are the applicable functions:
read_image_data reads the pixel data organizing it into a list of 2D array numpy arrays
def read_image_data():
fd = open("train-images.idx3-ubyte", "rb")
images_bin_string = fd.read()
num_images = struct.unpack(">i", images_bin_string[4:8])[0]
image_data_bank = []
uint32_num_bytes = 4
current_index = 8
num_rows = struct.unpack(">I", \
images_bin_string[current_index: current_index + uint32_num_bytes])[0]
num_cols = struct.unpack(">I", \
images_bin_string[current_index + uint32_num_bytes: \
current_index + uint32_num_bytes * 2])[0]
current_index += 8
i = 0
while i < num_images:
image_data = np.zeros([num_rows, num_cols])
for j in range(num_rows - 1):
for k in range(num_cols - 1):
image_data[j][k] = images_bin_string[current_index + j * k]
current_index += num_rows * num_cols
i += 1
image_data_bank.append(image_data)
return image_data_bank
read_label_data reads the corresponding labels into a list
def read_label_data():
fd = open("train-labels.idx1-ubyte", "rb")
images_bin_string = fd.read()
num_images = struct.unpack(">i", images_bin_string[4:8])[0]
image_data_bank = []
current_index = 8
i = 0
while i < num_images:
image_data_bank.append(images_bin_string[current_index])
current_index += 1
i += 1
return image_data_bank
collect_data zips the structures together
def collect_data():
print("Reading image data...")
image_data = read_image_data()
print("Reading label data...")
label_data = read_label_data()
print("Zipping data sets...")
all_data = np.array(list(zip(image_data, label_data)))
return all_data
lastly run_test uses PIL to print the pixels from the first 28x28 np structure created by read_image_data
def run_test(data):
example = data[0]
pixel_data = example[0]
number = example[1]
print(number)
im = Image.fromarray(pixel_data)
im.show()
When I run the script:
Collecting data... Reading image data... Reading label data... Zipping
data sets... 5
I must be messing something up with the PIL library, but I do not know what.
That is a really weird looking 5. I am guessing that I went wrong somewhere in my organization of the data. The directions did say "Pixels are organized row-wise.", but I think I covered that by having my outer loop as the row loop then the inner as the column loop
UPDATE
I reversed the order of the row and column index in the np.arrays in read_image_data and it is making no difference.
image_data[k][j] = images_bin_string[current_index + j * k]
UPDATE
Ran quick test with matplotlib
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
imgplot = plt.imshow(pixel_data)
plt.show()
Here is what I got from matplotlib
That means it is definitely a problem with my code and not the library. The question is if it is the way I am passing the pixels to the imaging libraries or how I structured the data. If anyone can find the mistake, I would greatly appreciate.
This is a program for face recognition using pca logic. Everything went fine except for the index error that came up at the end of the program.
When I run the code I get an index error at the fourth last line of my program.
distances.append((dist, y[i]))
IndexError: list index out of range
can anyone just help in this. I am newbie into python, so am I not so expert in solving.
Here is my code :
from sklearn.decomposition import RandomizedPCA
import numpy as np
import glob
import cv2
import math
import os.path
import string
#function to get ID from filename
def ID_from_filename(filename):
part = string.split(filename, '/')
return part[1].replace("s", "")
#function to convert image to right format
def prepare_image(filename):
img_color = cv2.imread(filename)
img_gray = cv2.cvtColor(img_color, cv2.cv.CV_RGB2GRAY)
img_gray = cv2.equalizeHist(img_gray)
return img_gray.flat
IMG_RES = 92 * 112 # img resolution
NUM_EIGENFACES = 10 # images per train person
NUM_TRAINIMAGES = 110 # total images in training set
#loading training set from folder train_faces
folders = glob.glob('train_faces/*')
# Create an array with flattened images X
# and an array with ID of the people on each image y
X = np.zeros([NUM_TRAINIMAGES, IMG_RES], dtype='int8')
y = []
# Populate training array with flattened imags from subfolders of
train_faces and names
c = 0
for x, folder in enumerate(folders):
train_faces = glob.glob(folder + '/*')
for i, face in enumerate(train_faces):
X[c,:] = prepare_image(face)
y.append(ID_from_filename(face))
c = c + 1
# perform principal component analysis on the images
pca = RandomizedPCA(n_components=NUM_EIGENFACES, whiten=True).fit(X)
X_pca = pca.transform(X)
# load test faces (usually one), located in folder test_faces
test_faces = glob.glob('test_faces/*')
# Create an array with flattened images X
X = np.zeros([len(test_faces), IMG_RES], dtype='int8')
# Populate test array with flattened imags from subfolders of train_faces
for i, face in enumerate(test_faces):
X[i,:] = prepare_image(face)
# run through test images (usually one)
for j, ref_pca in enumerate(pca.transform(X)):
distances = []
# Calculate euclidian distance from test image to each of the known
images and save distances
for i, test_pca in enumerate(X_pca):
dist = math.sqrt(sum([diff**2 for diff in (ref_pca - test_pca)]))
distances.append((dist, y[i]))
found_ID = min(distances)[1]
print "Identified (result: "+ str(found_ID) +" - dist - " +
str(min(distances)[0]) + ")"
Your i in the loop below goes up to the length of X_pca - 1
for i, test_pca in enumerate(X_pca):
dist = math.sqrt(sum([diff**2 for diff in (ref_pca - test_pca)]))
distances.append((dist, y[i]))
However, your y is not built to have that length necessarily:
for x, folder in enumerate(folders):
train_faces = glob.glob(folder + '/*')
for i, face in enumerate(train_faces):
X[c,:] = prepare_image(face)
y.append(ID_from_filename(face))
So you are using an index i which is greater than the bounds of your list y.