This is how the dataset was created
def createDataset1(
outputPath,
imagePathList,
labelList,
lexiconList=None,
validset_percent=10,
testset_percent=0,
random_seed=1111,
checkValid=True,
):
"""
Create LMDB dataset for CRNN training.
ARGS:
outputPath : LMDB output path
imagePathList : list of image path
labelList : list of corresponding groundtruth texts
lexiconList : (optional) list of lexicon lists
checkValid : if true, check the validity of every image
"""
train_path = os.path.join(outputPath, "training", "9M")
valid_path = os.path.join(outputPath, "validation", "9M")
# CAUTION: if train_path (lmdb) already exists, this function add dataset
# into it. so remove former one and re-create lmdb.
if os.path.exists(train_path):
os.system(f"rm -r {train_path}")
if os.path.exists(valid_path):
os.system(f"rm -r {valid_path}")
os.makedirs(train_path, exist_ok=True)
os.makedirs(valid_path, exist_ok=True)
gt_train_path = gt_file.replace(".txt", "_train.txt")
gt_valid_path = gt_file.replace(".txt", "_valid.txt")
data_log = open(gt_train_path, "w", encoding="utf-8")
if testset_percent != 0:
test_path = os.path.join(outputPath, "evaluation", dataset_name)
if os.path.exists(test_path):
os.system(f"rm -r {test_path}")
os.makedirs(test_path, exist_ok=True)
gt_test_path = gtFile.replace(".txt", "_test.txt")
assert(len(imagePathList) == len(labelList))
nSamples = len(imagePathList)
num_valid_dataset = int(nSamples * validset_percent / 100.0)
num_test_dataset = int(nSamples * testset_percent / 100.0)
num_train_dataset = nSamples - num_valid_dataset - num_test_dataset
print("validation datasets: ",num_valid_dataset,"\n", "test datasets: ", num_test_dataset, " \n training datasets: ", num_train_dataset)
env = lmdb.open(outputPath, map_size=1099511627776)
cache = {}
cnt = 1
random.seed(random_seed)
random.shuffle(imagePathList)
for i in tqdm(range(nSamples)):
data_log.write(imagePathList[i])
imagePath = imagePathList[i]
label = labelList[i]
if len(label) == 0:
continue
if not os.path.exists(imagePath):
print('%s does not exist' % imagePath)
continue
with open(imagePath, 'rb') as f:
imageBin = f.read()
if checkValid:
if not checkImageIsValid(imageBin):
print('%s is not a valid image' % imagePath)
continue
embed_vec = fasttext_model[label]
imageKey = 'image-%09d' % cnt
labelKey = 'label-%09d' % cnt
embedKey = 'embed-%09d' % cnt
cache[imageKey] = imageBin
cache[labelKey] = label.encode()
cache[embedKey] = ' '.join(str(v) for v in embed_vec.tolist()).encode()
if lexiconList:
lexiconKey = 'lexicon-%09d' % cnt
cache[lexiconKey] = ' '.join(lexiconList[i])
if cnt % 1000 == 0:
writeCache(env, cache)
cache = {}
print('Written %d / %d' % (cnt, nSamples))
#finish train dataset and start validation dataset
if i + 1 == num_train_dataset:
print(f"# Train dataset: {num_train_dataset} is finished")
cache["num-samples".encode()] = str(num_train_dataset).encode()
writeCache(env, cache)
data_log.close()
#start validation set
env = lmdb.open(valid_path, map_size=30 * 2 ** 30)
cache = {}
cnt = 0
data_log = open(gt_valid_path, "w", encoding="utf-8")
# Finish train/valid dataset and Start test dataset
if (i + 1 == num_train_dataset + num_valid_dataset) and num_test_dataset != 0:
print(f"# Valid dataset: {num_valid_dataset} is finished")
cache["num-samples".encode()] = str(num_valid_dataset).encode()
writeCache(env, cache)
data_log.close()
# start test set
env = lmdb.open(test_path, map_size=30 * 2 ** 30)
cache = {}
cnt = 0 # not 1 at this time
data_log = open(gt_test_path, "w", encoding="utf-8")
cnt += 1
if testset_percent == 0:
cache["num-samples".encode()] = str(num_valid_dataset).encode()
writeCache(env, cache)
print(f"# Valid datast: {num_valid_dataset} is finished")
else:
cache["num-samples".encode()] = str(num_test_dataset).encode()
writeCache(env, cache)
print(f"# Test datast: {num_test_dataset} is finished")
This is how i am trying to retrieve the data
class LmdbDataset(data.Dataset):
def __init__(self, root, voc_type, max_len, num_samples, transform=None):
super(LmdbDataset, self).__init__()
if global_args.run_on_remote:
dataset_name = os.path.basename(root)
data_cache_url = "/cache/%s" % dataset_name
if not os.path.exists(data_cache_url):
os.makedirs(data_cache_url)
if mox.file.exists(root):
mox.file.copy_parallel(root, data_cache_url)
else:
raise ValueError("%s not exists!" % root)
self.env = lmdb.open(data_cache_url, max_readers=32, readonly=True)
else:
self.env = lmdb.open(root, max_readers=32, readonly=True)
assert self.env is not None, "cannot create lmdb from %s" % root
self.txn = self.env.begin()
self.voc_type = voc_type
self.transform = transform
self.max_len = max_len
# nums = b"num-samples"
# print('NUM SAMPLES ------ \n',nums)
nSamples = self.txn.get('num-samples'.encode())
print("STRING nSamples :", nSamples)
self.nSamples = int(self.txn.get(b"num-samples"))
self.nSamples = min(self.nSamples, num_samples)
assert voc_type in ['LOWERCASE', 'ALLCASES', 'ALLCASES_SYMBOLS']
self.EOS = 'EOS'
self.PADDING = 'PADDING'
self.UNKNOWN = 'UNKNOWN'
self.voc = get_vocabulary(voc_type, EOS=self.EOS, PADDING=self.PADDING, UNKNOWN=self.UNKNOWN)
self.char2id = dict(zip(self.voc, range(len(self.voc))))
self.id2char = dict(zip(range(len(self.voc)), self.voc))
self.rec_num_classes = len(self.voc)
self.lowercase = (voc_type == 'LOWERCASE')
I am getting the error below whenever the code tries to call elf.txn.get(b"num-samples")
Traceback (most recent call last):
File "main.py", line 268, in <module>
main(args)
File "main.py", line 157, in main
train_dataset, train_loader = get_data_lmdb(args.synthetic_train_data_dir, args.voc_type, args.max_len, args.num_train,
File "main.py", line 66, in get_data_lmdb
dataset_list.append(LmdbDataset(data_dir_, voc_type, max_len, num_samples))
File "/Users/SEED/lib/datasets/dataset.py", line 189, in __init__
self.nSamples = int(self.txn.get(b"num-samples"))
TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'
I have tried many different suggestions online and some stackoverflow threads but could not figure out what is wrong.
What is causing this error and how can I fix this?
Your code is dense, convoluted and hard to follow the flow of data. It mixes lots of computation with IO, side effects, and even system calls (os.system(f"rm -r {test_path}"), you should use shutil.rmtree instead for example).
Try breaking up each action that you wish to perform so that it primarily does one specific thing:
logical operations but no side effects
input (read from file or network)
output (write to file, generate a result)
file system operations/cleanup
At each stage, you should perform verifications, and use the rule of least power. If you expect self.txn to always have 'num-samples, then you should use self.txn[b'num-samples']rather.get, which defaults to None`. This makes it easier to catch errors earlier in the chain.
Also I have no idea what the lmbd module is. Is that a library, or another file in your codebase? You should link to any libraries you are using if they aren't well known. I see several python packages pertaining to lmbd.
From the code example, I can dissect for you. May be it will help you to close the issue.
The log says..
self.nSamples = int(self.txn.get(b"num-samples"))
TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'
self.txn should be a dictionary to use get method there. As you are trying to get it, there can be two scenarios
num-samples can be '' in the data means there is no value
there is no variable called num-samples in the data.
Both of them can cause a NoneType which cannot be handled by type conversion(to int). So, you have to check whether that field is present in the data or not, by using self.txn.keys() and see if that key is present
Furthermore, if
self.nSamples = int(self.txn.get(b"num-samples"))
is failing and not the above statement
nSamples = self.txn.get('num-samples'.encode())
then you can simply decode the variable and use it there
like
Samples = nSamples.decode("utf-8")
try:
int_samples = int(Samples)
except TypeError:
print(Samples)
Related
I am running a keras/tensorflow model, when I try to save and process data to TFrecords as following the keras model tutorial, then it generates an error two errors as listed below
TypeError: encode() missing 1 required positional argument: 'encoding' and when I comment this line then below line generates another error NotFoundError: 519_IM-2131-2001.dcm.png; No such file or directory [Op:ReadFile]
The code which generates these errors is following
`train_size = 30000
valid_size = 5000
captions_per_image = 2
images_per_file = 2000
train_image_paths = image_paths[:train_size]
num_train_files = int(np.ceil(train_size / images_per_file))
train_files_prefix = os.path.join(tfrecords_dir, "train")
valid_image_paths = image_paths[-valid_size:]
num_valid_files = int(np.ceil(valid_size / images_per_file))
valid_files_prefix = os.path.join(tfrecords_dir, "valid")
tf.io.gfile.makedirs(tfrecords_dir)
def bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def create_example(image_path, caption):
feature = {
"caption": bytes_feature(caption.str.encode()),
"raw_image": bytes_feature(tf.io.read_file(image_path).numpy()),
}
return tf.train.Example(features=tf.train.Features(feature=feature))
def write_tfrecords(file_name, image_paths):
caption_list = []
image_path_list = []
for image_path in image_paths:
captions = image_path_to_caption[image_path][:captions_per_image]
caption_list.extend(captions)
image_path_list.extend([image_path] * len(captions))
with tf.io.TFRecordWriter(file_name) as writer:
for example_idx in range(len(image_path_list)):
example = create_example(
image_path_list[example_idx], caption_list[example_idx]
)
writer.write(example.SerializeToString())
return example_idx + 1
def write_data(image_paths, num_files, files_prefix):
example_counter = 0
for file_idx in tqdm(range(num_files)):
file_name = files_prefix + "-%02d.tfrecord" % (file_idx)
start_idx = images_per_file * file_idx
end_idx = start_idx + images_per_file
example_counter += write_tfrecords(file_name, image_paths[start_idx:end_idx])
return example_counter
train_example_count = write_data(train_image_paths, num_train_files, train_files_prefix)
print(f"{train_example_count} training examples were written to tfrecord files.")
valid_example_count = write_data(valid_image_paths, num_valid_files, valid_files_prefix)
print(f"{valid_example_count} evaluation examples were written to tfrecord files.")`
Error are in below two lines
"caption": bytes_feature(caption.str.encode()), "raw_image": bytes_feature(tf.io.read_file(image_path).numpy()),
Complete code is in this Colab file.
I install the Faste-RCNN script from here, this version is modified to fit python3. After following all the instructions I still got an error as below:
Loading initial model weights from ./data/imagenet_weights/vgg16.ckpt
Unable to open table file .\data\imagenet_weights\vgg16.ckpt: Data loss: not an sstable (bad magic number): perhaps your file is in a different file format and you need to use a different restore operator?
2022-02-25 09:30:16.917233: W C:\tf_jenkins\workspace\rel-win\M\windows-gpu\PY\36\tensorflow\core\util\tensor_slice_reader.cc:95] Could not open .\data\imagenet_weights\vgg16.ckpt: Data loss: not an sstable (bad magic number): perhaps your file is in a different file format and you need to use a different restore operator? Traceback (most recent call last):
File "D:/Object detection/FRCNNPY3.6/train.py", line 218, in <module>
train.train()
File "D:/Object detection/FRCNNPY3.6/train.py", line 123, in train
variables_to_restore = self.net.get_variables_to_restore(variables, var_keep_dic)
File "D:\Object detection\FRCNNPY3.6\lib\nets\vgg16.py", line 66, in get_variables_to_restore
if v.name.split(':')[0] in var_keep_dic:
TypeError: argument of type 'NoneType' is not iterable
I was trying to solve the " 'NoneType' is not iterable " error, but still no progress.
Of course, the file be stored as vgg16.ckpt type and the path is D:\Object detection\FRCNNPY3.6\data\imagenet_weights.
From the error message I find out this line "Loading initial model weights from ./data/imagenet_weights/vgg16.ckpt" shows the path where it loading, the direction of slash is wrong for windows, this form is used in Linux, but I don't know where can I modify it.
The complete program as below:
import time
import numpy as np
import tensorflow as tf
from tensorflow.python import pywrap_tensorflow
import lib.config.config as cfg
from lib.datasets import roidb as rdl_roidb
from lib.datasets.factory import get_imdb
from lib.datasets.imdb import imdb as imdb2
from lib.layer_utils.roi_data_layer import RoIDataLayer
from lib.nets.vgg16 import vgg16
from lib.utils.timer import Timer
try:
import cPickle as pickle
except ImportError:
import pickle
import os
def get_training_roidb(imdb):
"""Returns a roidb (Region of Interest database) for use in training."""
if True:
print('Appending horizontally-flipped training examples...')
imdb.append_flipped_images()
print('done')
print('Preparing training data...')
rdl_roidb.prepare_roidb(imdb)
print('done')
return imdb.roidb
def combined_roidb(imdb_names):
"""
Combine multiple roidbs
"""
def get_roidb(imdb_name):
imdb = get_imdb(imdb_name)
print('Loaded dataset `{:s}` for training'.format(imdb.name))
imdb.set_proposal_method("gt")
print('Set proposal method: {:s}'.format("gt"))
roidb = get_training_roidb(imdb)
return roidb
roidbs = [get_roidb(s) for s in imdb_names.split('+')]
roidb = roidbs[0]
if len(roidbs) > 1:
for r in roidbs[1:]:
roidb.extend(r)
tmp = get_imdb(imdb_names.split('+')[1])
imdb = imdb2(imdb_names, tmp.classes)
else:
imdb = get_imdb(imdb_names)
return imdb, roidb
class Train:
def __init__(self):
# Create network
if cfg.FLAGS.network == 'vgg16':
self.net = vgg16(batch_size=cfg.FLAGS.ims_per_batch)
else:
raise NotImplementedError
self.imdb, self.roidb = combined_roidb("voc_2007_trainval")
self.data_layer = RoIDataLayer(self.roidb, self.imdb.num_classes)
self.output_dir = cfg.get_output_dir(self.imdb, 'default')
def train(self):
# Create session
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True
sess = tf.Session(config=tfconfig)
with sess.graph.as_default():
tf.set_random_seed(cfg.FLAGS.rng_seed)
layers = self.net.create_architecture(sess, "TRAIN", self.imdb.num_classes, tag='default')
loss = layers['total_loss']
lr = tf.Variable(cfg.FLAGS.learning_rate, trainable=False)
momentum = cfg.FLAGS.momentum
optimizer = tf.train.MomentumOptimizer(lr, momentum)
gvs = optimizer.compute_gradients(loss)
# Double bias
# Double the gradient of the bias if set
if cfg.FLAGS.double_bias:
final_gvs = []
with tf.variable_scope('Gradient_Mult'):
for grad, var in gvs:
scale = 1.
if cfg.FLAGS.double_bias and '/biases:' in var.name:
scale *= 2.
if not np.allclose(scale, 1.0):
grad = tf.multiply(grad, scale)
final_gvs.append((grad, var))
train_op = optimizer.apply_gradients(final_gvs)
else:
train_op = optimizer.apply_gradients(gvs)
# We will handle the snapshots ourselves
self.saver = tf.train.Saver(max_to_keep=100000)
# Write the train and validation information to tensorboard
# writer = tf.summary.FileWriter(self.tbdir, sess.graph)
# valwriter = tf.summary.FileWriter(self.tbvaldir)
# Load weights
# Fresh train directly from ImageNet weights
print('Loading initial model weights from {:s}'.format(cfg.FLAGS.pretrained_model))
variables = tf.global_variables()
# Initialize all variables first
sess.run(tf.variables_initializer(variables, name='init'))
var_keep_dic = self.get_variables_in_checkpoint_file(cfg.FLAGS.pretrained_model)
# Get the variables to restore, ignorizing the variables to fix
variables_to_restore = self.net.get_variables_to_restore(variables, var_keep_dic)
restorer = tf.train.Saver(variables_to_restore)
restorer.restore(sess, cfg.FLAGS.pretrained_model)
print('Loaded.')
# Need to fix the variables before loading, so that the RGB weights are changed to BGR
# For VGG16 it also changes the convolutional weights fc6 and fc7 to
# fully connected weights
self.net.fix_variables(sess, cfg.FLAGS.pretrained_model)
print('Fixed.')
sess.run(tf.assign(lr, cfg.FLAGS.learning_rate))
last_snapshot_iter = 0
timer = Timer()
iter = last_snapshot_iter + 1
last_summary_time = time.time()
while iter < cfg.FLAGS.max_iters + 1:
# Learning rate
if iter == cfg.FLAGS.step_size + 1:
# Add snapshot here before reducing the learning rate
# self.snapshot(sess, iter)
sess.run(tf.assign(lr, cfg.FLAGS.learning_rate * cfg.FLAGS.gamma))
timer.tic()
# Get training data, one batch at a time
blobs = self.data_layer.forward()
# Compute the graph without summary
try:
rpn_loss_cls, rpn_loss_box, loss_cls, loss_box, total_loss = self.net.train_step(sess, blobs, train_op)
except Exception:
# if some errors were encountered image is skipped without increasing iterations
print('image invalid, skipping')
continue
timer.toc()
iter += 1
# Display training information
if iter % (cfg.FLAGS.display) == 0:
print('iter: %d / %d, total loss: %.6f\n >>> rpn_loss_cls: %.6f\n '
'>>> rpn_loss_box: %.6f\n >>> loss_cls: %.6f\n >>> loss_box: %.6f\n ' % \
(iter, cfg.FLAGS.max_iters, total_loss, rpn_loss_cls, rpn_loss_box, loss_cls, loss_box))
print('speed: {:.3f}s / iter'.format(timer.average_time))
if iter % cfg.FLAGS.snapshot_iterations == 0:
self.snapshot(sess, iter )
def get_variables_in_checkpoint_file(self, file_name):
try:
reader = pywrap_tensorflow.NewCheckpointReader(file_name)
var_to_shape_map = reader.get_variable_to_shape_map()
return var_to_shape_map
except Exception as e: # pylint: disable=broad-except
print(str(e))
if "corrupted compressed block contents" in str(e):
print("It's likely that your checkpoint file has been compressed "
"with SNAPPY.")
def snapshot(self, sess, iter):
net = self.net
if not os.path.exists(self.output_dir):
os.makedirs(self.output_dir)
# Store the model snapshot
filename = 'vgg16_faster_rcnn_iter_{:d}'.format(iter) + '.ckpt'
# filename = os.path.join(self.output_dir, filename)
filename = os.path.join(self.output_dir, filename)
self.saver.save(sess, filename)
print('Wrote snapshot to: {:s}'.format(filename))
# Also store some meta information, random state, etc.
nfilename = 'vgg16_faster_rcnn_iter_{:d}'.format(iter) + '.pkl'
nfilename = os.path.join(self.output_dir, nfilename)
# current state of numpy random
st0 = np.random.get_state()
# current position in the database
cur = self.data_layer._cur
# current shuffled indeces of the database
perm = self.data_layer._perm
# Dump the meta info
with open(nfilename, 'wb') as fid:
pickle.dump(st0, fid, pickle.HIGHEST_PROTOCOL)
pickle.dump(cur, fid, pickle.HIGHEST_PROTOCOL)
pickle.dump(perm, fid, pickle.HIGHEST_PROTOCOL)
pickle.dump(iter, fid, pickle.HIGHEST_PROTOCOL)
return filename, nfilename
if __name__ == '__main__':
train = Train()
train.train()
I am trying to work with this yolov4 in tensorflow 2.0 model it requires the dataset to be in Tensorflow Object Detection API format. Since I had no prior experience with this api so I was following Adrian Rosebrock's tutorial in his ImageNet bundle book.
I'm working with crowdHuman dataset, and I've simplified its annotations a bit. Like in this format below,
/content/gdrive/MyDrive/Datasets/CrowdHuman/crowdHuman_2671.jpg,head,126,243,133,251
/content/gdrive/MyDrive/Datasets/CrowdHuman/crowdHuman_2671.jpg,person,-20,241,19,343
/content/gdrive/MyDrive/Datasets/CrowdHuman/crowdHuman_2671.jpg,head,-12,242,0,254
/content/gdrive/MyDrive/Datasets/CrowdHuman/crowdHuman_1343.jpg,person,-10,281,203,825
/content/gdrive/MyDrive/Datasets/CrowdHuman/crowdHuman_1343.jpg,head,65,293,127,374
I barely changed the code in tutorial other than changing paths and file names, the code files I used are as follows
1 Config.py
import os
BASE_PATH = "/content/gdrive/MyDrive/Datasets/CrowdHuman/"
ANNOT_PATH = BASE_PATH + "annot_train_final.txt"
# Output paths
TRAIN_RECORD = BASE_PATH + "records/training.record"
TEST_RECORD = BASE_PATH + "records/testing.record"
CLASSES_FILE = BASE_PATH + "records/classes.pbtxt"
TEST_SIZE = 0.10
CLASSES = {"person": 1, "head": 2}
2 tfAnnotations.py
from object_detection.utils.dataset_util import bytes_list_feature
from object_detection.utils.dataset_util import float_list_feature
from object_detection.utils.dataset_util import int64_list_feature
from object_detection.utils.dataset_util import int64_feature
from object_detection.utils.dataset_util import bytes_feature
class TFAnnotation:
def __init__(self):
# initializing bbox and labels lists
self.xMins = []
self.xMaxs = []
self.yMins = []
self.yMaxs = []
self.textLabels = []
self.classes = []
self.difficult = []
# initializing additional variables including image itself
# spatial dims, encoding and filename
self.image = None
self.width = None
self.height = None
self.encoding = None
self.filename = None
def build(self):
# now we encode the attributes using their respective
# Tensorflow encoding function
w = int64_feature(self.width)
h = int64_feature(self.height)
filename = bytes_feature(self.filename.encode("utf8"))
encoding = bytes_feature(self.encoding.encode("utf8"))
image = bytes_feature(self.image)
xMins = float_list_feature(self.xMins)
xMaxs = float_list_feature(self.xMaxs)
yMins = float_list_feature(self.yMins)
yMaxs = float_list_feature(self.yMaxs)
textLabels = bytes_list_feature(self.textLabels)
classes = int64_list_feature(self.classes)
difficult = int64_list_feature(self.difficult)
# Construct the TensorFlow compatible data dict
data = {
"image/height": h,
"image/width": w,
"image/filename": filename,
"image/source_id": filename,
"image/encode": image,
"image/format": encoding,
"image/object/bbox/xmin": xMins,
"image/object/bbox/xmax": xMaxs,
"image/object/bbox/ymin": yMins,
"image/object/bbox/ymax": yMaxs,
"image/object/class/text": textLabels,
"image/object/class/label": classes,
"image/object/difficult": difficult,
}
return data
3 records_builder.py
import crowdHuman_config as config
from tfAnnotation import TFAnnotation
from sklearn.model_selection import train_test_split
from PIL import Image
import tensorflow.compat.v1 as tf
import os
def main(_):
# open classes output file
f = open(config.CLASSES_FILE, "w")
# loop over the classes
for (k, v) in config.CLASSES.items():
# construct the class info and write to file
item = ("item {\n"
"\tid: " + str(v) + "\n"
"\tname: '" + k + "'\n"
"}\n")
f.write(item)
f.close()
# initialize a data dict used to map each image filename to
# all bboxes associated with the image and then load the
# contents of the annot file
D = {}
rows = open(config.ANNOT_PATH).read().strip().split('\n')
# loop over the individual rows
for row in rows:
# break the ewo into compenents
row = row.split(",")
(imagePath, label, startX, startY, endX, endY) = row
(startX, startY) = (float(startX), float(startY))
(endX, endY) = (float(endX), float(endY))
# if we are not interested in the label, ignore it
if label not in config.CLASSES:
continue
# build the path to the input image and then grap any
# other boxes and labels associated with the image
# path, labels and bbx lists respectively
#p = os.path.sep.join([config.BASE_PATH, imagePath])
p = imagePath
b = D.get(p,[])
# build a tuple consisting of the label and bbox
# then update the list and store it in the dict
b.append((label, (startX, startY, endX, endY)))
D[p] = b
#print("[INFO] list(D.keys()): {}".format(list(D.keys())))
# create training and testing splits from our data dict
(trainKeys, testKeys) = train_test_split(list(D.keys()),
test_size=config.TEST_SIZE, random_state=42)
# initialize the data split files
datasets = [
("train", trainKeys, config.TRAIN_RECORD),
("test", testKeys, config.TEST_RECORD)
]
# loop over the datasets
for (dType, keys, outputPath) in datasets:
# intialize the Tensorflow writer and initialize the
# total number of examples written to file
print("[INFO] processing '{}'...".format(dType))
writer = tf.python_io.TFRecordWriter(outputPath)
total = 0
# loop over all the kerys in the current set
for k in keys:
# load the input image from disk as a tensorflow obj
encoded = tf.gfile.GFile(k, "rb").read()
encoded = bytes(encoded)
# load the image from disk again, this time as a PIL
# object
pilImage = Image.open(k)
(w, h) = pilImage.size[:2]
# parse the filename and encoding from the input path
filename = k.split(os.path.sep)[-1]
encoding = filename[filename.rfind(".")+1:]
# intialize the annotattion object used to store
# info regarding the bbox and labels
tfAnnot = TFAnnotation()
tfAnnot.image = encoded
tfAnnot.encoding = encoding
tfAnnot.filename = filename
tfAnnot.width = w
tfAnnot.height = h
#loop over the bbox and labels associated with the img
for (label, (startX, startY, endX, endY)) in D[k]:
# tensorflow assumes all bbox are in the range
# [0,1] so we need to scale them
xMin = startX / w
xMax = endX / w
yMin = startY / h
yMax = endY / h
# update the bbox and labels lists
tfAnnot.xMins.append(xMin)
tfAnnot.xMaxs.append(xMax)
tfAnnot.yMins.append(yMin)
tfAnnot.yMaxs.append(yMax)
tfAnnot.textLabels.append(label.encode("utf8"))
tfAnnot.classes.append(config.CLASSES[label])
tfAnnot.difficult.append(0)
# increment the total number of examples
total += 1
# encode the data point attributes using the Tensorflow
# helper functions
features = tf.train.Features(feature=tfAnnot.build())
example = tf.train.Example(features=features)
# add the example to the writer
writer.write(example.SerializeToString())
# close the writer and print the diagnostic info to the user
writer.close()
print("[INFO] {} examples saved for '{}'".format(total, dType))
# check to see if the main thread should be started
if __name__ == "__main__":
tf.app.run()
All these scripts ran successfully and produced the desired files but when I tried using this dataset in the model mentioned above, it gave me this error. There's no really relatable solution that I could find. So I'd really appreciate the help.
ERROR:
Traceback (most recent call last):
File "train.py", line 195, in <module>
app.run(main)
File "/usr/local/lib/python3.7/dist-packages/absl/app.py", line 303, in run
_run_main(main, args)
File "/usr/local/lib/python3.7/dist-packages/absl/app.py", line 251, in _run_main
sys.exit(main(argv))
File "train.py", line 64, in main
FLAGS.dataset, FLAGS.classes, FLAGS.size)
File "/content/yolov3-tf2/yolov3_tf2/dataset.py", line 124, in load_tfrecord_dataset
class_file, tf.string, 0, tf.int64, LINE_NUMBER, delimiter="\n"), -1)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/lookup_ops.py", line 314, in __init__
super(StaticHashTable, self).__init__(default_value, initializer)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/lookup_ops.py", line 185, in __init__
self._init_op = self._initialize()
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/lookup_ops.py", line 188, in _initialize
return self._initializer.initialize(self)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/lookup_ops.py", line 744, in initialize
-1 if self._vocab_size is None else self._vocab_size, self._delimiter)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/gen_lookup_ops.py", line 363, in initialize_table_from_text_file_v2
_ops.raise_from_not_ok_status(e, name)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py", line 6862, in raise_from_not_ok_status
six.raise_from(core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.FailedPreconditionError: HashTable has different value for same key. Key item { has 0 and trying to add value 4 [Op:InitializeTableFromTextFileV2]
I've viewed some questions regarding the same issue but none of them could help me. The problem is as it says, I'm unable to fit data into learning model.
This is the main file, which calls out the class regarding the data i use to fit in the model:
def main():
action = input(
"Choose an action:\n A - Create LinearSVC classifier\n B - Create Random Forest Classifier\n C - Create K Nearest Neighbor classifier\n -> ").upper()
loader = ImageLoader()
if action == "A":
lsvc = LinearSVC(random_state=0, tol=1e-5)
lsvc.fit(loader.hogArray(), loader.labelArray())
joblib.dump(lsvc, './LSVCmodel.pkl')
elif action == "B":
rfc = RandomForestClassifier(n_estimators=100)
rfc.fit(loader.hogArray(), loader.labelArray())
joblib.dump(rfc, './RFmodel.pkl')
elif action == "C":
knc = KNeighborsClassifier(n_neighbors=3)
knc.fit(loader.hogArray(), loader.labelArray())
joblib.dump(knc, './KNCmodel.pkl')
else:
print("That's not a valid answer")
main()
The same error occurs with all 3 models. The class that retrieves the data is written as following:
class ImageProcess:
def __init__(self, image, hog_data=None):
self.hog_data = hog_data
self.image = image
def hog_data_extractor(self):
self.hog_data = feature.hog(self.image) / 255.0
return self.hog_data
def normalize(self):
imageRead = cv2.resize(cv2.imread(self.image), (150, 150))
gaussImage = cv2.fastNlMeansDenoisingColored(imageRead, None, 10, 10, 7, 21)
self.image = cv2.Canny(gaussImage, 100, 200)
self.image = cv2.cvtColor(self.image, cv2.COLOR_GRAY2RGB)
self.image *= np.array((0, 0, 1), np.uint8)
return self.image
class ImageLoader:
def __init__(self):
self.sourcePath = "dataset/seg_train/"
self.labels = ['Buildings', 'Forest', 'Glacier', 'Mountain', 'Sea', 'Street']
self.x_train = []
self.y_train = []
def fillArray(self):
label_train = []
le = LabelEncoder()
run_time = time.time()
for scene in self.labels:
scene_path = os.path.join(self.sourcePath, scene.lower())
fileNumber = 0
scene_length = len([image for image in os.listdir(scene_path)])
for img in os.listdir(scene_path):
per = (file_number / scene_length)
arrow = '-' * int(round(per * 100) - 1) + '>'
spaces = ' ' * (100 - len(arrow))
sys.stdout.write(
"\rProgress: [{0}] {1}% -Ellapsed time: {2}".format(arrow + spaces, int(round(per * 100, 2)),
(int(time.time() - run_time))))
file_number += 1
img_path = os.path.join(scene_path, img)
process = ImageProcess(img_path)
self.x_train.append(process.hog_data_extractor())
label_train.append(str(scene_type))
self.y_train = le.fit_transform(label_train)
def hogArray(self):
return self.x_train
def labelArray(self):
return self.y_train
A side note: Previously I didn't have this ImageLoader class, and simply had the method fillArray() under main() on the previous code, and it didn't give back this error, all was working well. But due to some restrictions I have to follow I tried to transferred it into a class to be use in more other files.
Traceback (most recent call last):
File "main.py", line 35, in <module>
main()
File "main.py", line 19, in main
lsvc.fit(loader.hogArray(), loader.labelArray())
File "/home/filipe/Documents/NovaPasta/2019_20/LP_recuperacao/Trabalho_recuperacao/venv/lib/python3.6/site-packages/sklearn/svm/classes.py", line 229, in fit
accept_large_sparse=False)
File "/home/filipe/Documents/NovaPasta/2019_20/LP_recuperacao/Trabalho_recuperacao/venv/lib/python3.6/site-packages/sklearn/utils/validation.py", line 756, in check_X_y
estimator=estimator)
File "/home/filipe/Documents/NovaPasta/2019_20/LP_recuperacao/Trabalho_recuperacao/venv/lib/python3.6/site-packages/sklearn/utils/validation.py", line 552, in check_array
"if it contains a single sample.".format(array))
ValueError: Expected 2D array, got 1D array instead:
array=[].
Reshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample.
I've tried reshaping as its recommended in the error but it retrieves("AttributeError: 'list' object has no attribute 'reshape'") and since I didn't needed this reshape before I assumed this wasn't the solution.
Sorry if its poor coding but I'm not that much of an expert(not even close) and the time period I had to do this was very short so I just focused on getting it to work properly.
You are not calling fillArray. so the lists are empty. Try doing it at end of init function.
array=[] in error shows this.
In the following code, I am appending targets to the Chunk_Obj variable "forprop_list". When I iterate through the for loop on line 111, the variable "forprop_list" seems to remember the values appended previously in that loop.
I feel this is a simple fix, but I have been unable to see it.
Does this have to do with assigning an object to a variable? What is causing the loop to continue appending instead of using the new empty "forprop_list"?
import os, pprint
import Data_Obj, ChunkObj
# Takes an index from the txt files for parameters. Takes a data index for the data input.
class Network:
data_obj = None
net_index = int
data_index = int
hparams = []
# Both of the following use the same referenced object:
# Holds all the chunk objects by POSITION
nn_chunks_dict = {}
# Holds all the chunk objects by LAYER
nn_layers_dict = {}
# Generates the neural network configuration and chunks required for the current network.
def __init__(self, nn_index=1, data_index=1):
self.net_index = nn_index
self.data_index = data_index
self.data_obj = Data_Obj.DataObj(data_index)
# Reads from txt file.
# Instantiates the chunks, adds those to the dictionaries.
self.configure_from_txt()
print("\n", "The ./NetConfigs txt file was successfully read to 'nn_coords_dict'. \n")
# Iterates the layers in the nn_layers_dict.
# Generates forward propagation dictionaries via prop_connect.
# i = ex. layer1, layer2, layer3...
print("nn_chunks_dict: ",self.nn_chunks_dict, "\n")
print("nn_chunks dict keys", self.nn_chunks_dict.keys())
for i in self.nn_chunks_dict:
print("i", i)
self.chunk_prop(i)
print("\nThis network is initialized!")
# Reads from the index txt file.
# Instatiates each chunk from those parameters and adds to the dictionary.
def configure_from_txt(self):
param_txt = open(os.path.join('./NetConfigs/%s.txt' % self.net_index)).readlines()
# Extracts hyperparameters.
for i in param_txt[0].strip('\n').split(';'):
self.hparams.append(i.split(','))
# Removes the hyperparameters and '\n' for configuring the network.
del param_txt[0:2]
# Extracts the position of the chunk from the first 2
# indexes of the attribs to a string list.
for i in param_txt:
attribs_targs = i.strip('\n').split(':')
attribs = attribs_targs[0].split(',')
position = [str(attribs[0]), str(attribs[1])]
if len(attribs_targs) > 1:
temp_targets = attribs_targs[1]
temp_targets = temp_targets.split(';')
targets = []
for i in temp_targets:
targets.append(i)
else:
targets = None
temp_config_dict = {"activation": attribs[2],
"num_layers": int(attribs[3]),
"width": int(attribs[4]),
"targets": targets,
"position": position}
temp_obj = ChunkObj.ChunkObj(self, temp_config_dict)
print("TEMP OBJECT KEY NAME:", temp_obj.key_name)
# Assigns the current iterative chunk attributes(temp_config_dict) to the POSITION dictionary.
self.nn_chunks_dict[temp_obj.key_name] = temp_obj
# Assigns the chunk obj to a list of the layers.
if ("layer%s" % position[0]) in self.nn_layers_dict:
print("LAYER: layer%s" % position[0])
self.nn_layers_dict["layer%s" % position[0]].append(temp_obj)
else:
self.nn_layers_dict["layer%s" % position[0]] = [temp_obj]
print("LAYER: layer%s" % position[0])
print("\n This networks configuration dictionary: \n")
pprint.pprint(self.nn_chunks_dict)
pprint.pprint(self.nn_layers_dict)
def chunk_prop(self, chunk_name):
x = self.nn_chunks_dict[chunk_name]
# else, targets are the next layer chunks
if x.targets is None:
# self.placehold_obj.forprop_list = []
print("\nchunk: ", x.key_name, "\nTargets are NOT specified")
print("Sequential layer: ", str(int(x.position[0]) + 1))
if x.backprop:
backpropvar = True
# if a next layer exists...
next_layer = str(int(x.position[0]) + 1)
if ("layer%s" % next_layer) in self.nn_layers_dict:
for i in self.nn_layers_dict["layer%s" % next_layer]:
x.forprop_list.append(str(i.key_name))
# for later use:
"""
if backpropvar:
i.backprop_list.append(self.placehold_obj.key_name)
print("Passed self to target backprop.")
"""
# else, the chunk is the last layer, forward prop will go to "y".
else:
# Checks to see if this chunk is in the last layer (will require Y for dimensions).
print("Next layer does not exist. Last layer == True")
x.last_layer = True
x.forprop_list = ["y"]
# if targets are specified...
else:
x.forprop_list = x.targets
print("self.placeholder.key_name: ", x.key_name)
print("Targets ARE specified: ", x.forprop_list)
print("forprop list: ", x.forprop_list)
def test():
Network()
print("\n", "Network_Obj test complete!")
if __name__ == '__main__':
test()