I am working with the SketchRNN dataset.
I would like to do the followings:
Parse examples in TFRecord to get labels.
Filter rows with specific labels. There are 345 labels, and I would like to keep only data with label number below 10.
Batch multiple observations for training.
Below is my failed attempt:
from pathlib import Path
quickdraw_dir = Path("/home/long/.keras/./datasets/quickdraw/quickdraw_tutorial_dataset_v1.tar.gz").parent
train_files = sorted([str(path) for path in quickdraw_dir.glob("training.tfrecord-*")])
eval_files = sorted([str(path) for path in quickdraw_dir.glob("eval.tfrecord-*")])
def parse_single_example(data):
feature_descriptions = {
"ink": tf.io.VarLenFeature(dtype=tf.float32),
"shape": tf.io.FixedLenFeature([2], dtype=tf.int64),
"class_index": tf.io.FixedLenFeature([1], dtype=tf.int64)
}
example = tf.io.parse_single_example(data, feature_descriptions)
flat_sketch = tf.sparse.to_dense(example["ink"])
sketch = tf.reshape(flat_sketch, shape=[-1, 3])
length = example["shape"][0]
label = example["class_index"][0]
return sketch, length, label
def filter_cond(dataset):
return dataset.filter(lambda x, y, z: tf.math.less(z, 10))
def get_dataset(filepaths, batch_size=32, shuffle_buffer_size=None, n_parse_threads=5, n_read_threads=5, cache=False):
dataset = tf.data.TFRecordDataset(filepaths, num_parallel_reads=n_read_threads)
if cache:
dataset.cache()
if shuffle_buffer_size:
dataset.shuffle(shuffle_buffer_size)
dataset = dataset.map(parse_single_example, num_parallel_calls=n_parse_threads)
dataset = dataset.apply(filter_cond)
dataset = dataset.batch(batch_size=batch_size)
return dataset.prefetch(1)
train_dataset = get_dataset(train_files, shuffle_buffer_size=10000)
train_dataset.take(1)
Below is the error:
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
Input In [31], in <cell line: 1>()
----> 1 for sketches, lengths, labels in train_dataset.take(1):
2 print("sketches =", sketches)
3 print("sketches shape=", sketches.shape)
File ~/anaconda3/envs/experiment/lib/python3.9/site-packages/tensorflow/python/data/ops/iterator_ops.py:800, in OwnedIterator.__next__(self)
798 def __next__(self):
799 try:
--> 800 return self._next_internal()
801 except errors.OutOfRangeError:
802 raise StopIteration
File ~/anaconda3/envs/experiment/lib/python3.9/site-packages/tensorflow/python/data/ops/iterator_ops.py:783, in OwnedIterator._next_internal(self)
780 # TODO(b/77291417): This runs in sync mode as iterators use an error status
781 # to communicate that there is no more data to iterate over.
782 with context.execution_mode(context.SYNC):
--> 783 ret = gen_dataset_ops.iterator_get_next(
784 self._iterator_resource,
785 output_types=self._flat_output_types,
786 output_shapes=self._flat_output_shapes)
788 try:
789 # Fast path for the case `self._structure` is not a nested structure.
790 return self._element_spec._from_compatible_tensor_list(ret) # pylint: disable=protected-access
File ~/anaconda3/envs/experiment/lib/python3.9/site-packages/tensorflow/python/ops/gen_dataset_ops.py:2845, in iterator_get_next(iterator, output_types, output_shapes, name)
2843 return _result
2844 except _core._NotOkStatusException as e:
-> 2845 _ops.raise_from_not_ok_status(e, name)
2846 except _core._FallbackException:
2847 pass
File ~/anaconda3/envs/experiment/lib/python3.9/site-packages/tensorflow/python/framework/ops.py:7107, in raise_from_not_ok_status(e, name)
7105 def raise_from_not_ok_status(e, name):
7106 e.message += (" name: " + name if name is not None else "")
-> 7107 raise core._status_to_exception(e) from None
InvalidArgumentError: Cannot batch tensors with different shapes in component 0. First element had shape [55,3] and element 1 had shape [42,3]. [Op:IteratorGetNext]
From what I understand this is because tensorflow is unable to group together sequence data with variable length.
Below code works with processing and batching, but I am unable to filter my data (to reduce classes to predict):
def parse_example(data_batch):
feature_descriptions = {
"ink": tf.io.VarLenFeature(dtype=tf.float32),
"shape": tf.io.FixedLenFeature([2], dtype=tf.int64),
"class_index": tf.io.FixedLenFeature([1], dtype=tf.int64)
}
examples = tf.io.parse_example(data_batch, feature_descriptions)
flat_sketches = tf.sparse.to_dense(examples["ink"])
sketches = tf.reshape(flat_sketches, shape=[tf.size(data_batch), -1, 3])
lengths = examples["shape"][:, 0]
labels = examples["class_index"][:, 0]
return sketches, lengths, labels
def get_dataset(filepaths, batch_size=32, shuffle_buffer_size=None, n_parse_threads=5, n_read_threads=5, cache=False):
dataset = tf.data.TFRecordDataset(filepaths, num_parallel_reads=n_read_threads)
if cache:
dataset.cache()
if shuffle_buffer_size:
dataset.shuffle(shuffle_buffer_size)
dataset = dataset.batch(batch_size=batch_size)
dataset = dataset.map(parse_example, num_parallel_calls=n_parse_threads)
# dataset = dataset.apply(filter_cond)
return dataset.prefetch(1)
Related
I am trying to use a custom CNN to classify spectrogram images generated for 3s audio segments. I am using GroupShuffleSplit to divide the training dataset into a training set and a validation set and to ensure that each participant is only included in one set (to prevent data leakage). I am using a Custom Class, to load the audio files and augment the training set.
If I generate a SoundDS object using the training set, randomly split the object into a training and validation subset, and pass these subsets into two respective data loaders, my model trains without any issues.
However, if I initially use GroupShuffleSplit on the dataframe train_df (grouping by the column addressfname), then generate two SoundDS objects and pass the train_DS and validation_DS into two respective data loaders, I encounter the error message shown at the bottom of this post when I try to run for i, data in enumerate(train_dl) in my train function. Does anyone know where the issue lies?
from torch.utils.data import DataLoader, Dataset, random_split
import torchaudio
# ----------------------------
# Sound Dataset
# ----------------------------
class SoundDS(Dataset):
def __init__(self, df):
self.df = df
self.duration = 3000
self.sr = 44100
self.channel = 2
self.shift_pct = 0.4
# ----------------------------
# Number of items in dataset
# ----------------------------
def __len__(self):
return len(self.df)
# ----------------------------
# Get i'th item in dataset
# ----------------------------
def __getitem__(self, idx):
audio_file = self.df.loc[idx, "relative_path"]
class_id = self.df.loc[idx, "dx"]
aud = AudioUtil.open(audio_file)
reaud = AudioUtil.resample(aud, self.sr)
rechan = AudioUtil.rechannel(reaud, self.channel)
dur_aud = AudioUtil.pad_trunc(rechan, self.duration)
shift_aud = AudioUtil.time_shift(dur_aud, self.shift_pct)
sgram = AudioUtil.spectro_gram(shift_aud, n_mels=64, n_fft=1024, hop_len=None)
aug_sgram = AudioUtil.spectro_augment(sgram, max_mask_pct=0.1, n_freq_masks=2, n_time_masks=2)
return aug_sgram, class_id
This method which randomly splits the training set into a training and validation set works.
from torch.utils.data import random_split
myds = SoundDS(train_df)
# Random split of 80:20 between training and validation
num_items = len(myds)
num_train = round(num_items * 0.8)
num_val = num_items - num_train
train_ds, val_ds = random_split(myds, [num_train, num_val])
# Create training and validation data loaders
train_dl = torch.utils.data.DataLoader(train_ds, batch_size=16, shuffle=True)
val_dl = torch.utils.data.DataLoader(val_ds, batch_size=16, shuffle=False)
This method which uses GroupShuffleSplit to splits the training set into a training and validation set and then generates two SoundDS objects does not work.
from sklearn.model_selection import GroupShuffleSplit
splitter = GroupShuffleSplit(test_size=0.15, n_splits=1, random_state = 7)
split = splitter.split(train_df, groups=train_df['adressfname'])
train_inds, valid_inds = next(split)
train_data_df = train_df.iloc[train_inds]
valid_data_df = train_df.iloc[valid_inds]
train_dataset_DS = SoundDS(train_data_df)
valid_dataset_DS = SoundDS(valid_data_df)
train_dl = torch.utils.data.DataLoader(train_dataset_DS, batch_size=16, shuffle=True)
val_dl = torch.utils.data.DataLoader(valid_dataset_DS, batch_size=16, shuffle=False)
Function to train the model
def training(model, train_dl, num_epochs):
# Loss Function, Optimizer and Scheduler
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),lr=0.001)
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.001,
steps_per_epoch=int(len(train_dl)),
epochs=num_epochs,
anneal_strategy='linear')
# Repeat for each epoch
for epoch in range(num_epochs):
running_loss = 0.0
correct_prediction = 0
total_prediction = 0
# Repeat for each batch in the training set
for i, data in enumerate(train_dl): ---- This is where the issue occurs
# Get the input features and target labels, and put them on the GPU
inputs, labels = data[0].to(device), data[1].to(device)
# Normalize the inputs
inputs_m, inputs_s = inputs.mean(), inputs.std()
inputs = (inputs - inputs_m) / inputs_s
This is the error message:
KeyError
Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
3360 try:
-> 3361 return self._engine.get_loc(casted_key)
3362 except KeyError as err:
15 frames
/usr/local/lib/python3.8/dist-packages/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
/usr/local/lib/python3.8/dist-packages/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.Int64HashTable.get_item()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.Int64HashTable.get_item()
KeyError: 3170
The above exception was the direct cause of the following exception:
KeyError Traceback (most recent call last)
<ipython-input-201-b60077dcefb4> in <module>
54
55 num_epochs=2 # Just for demo, adjust this higher.
---> 56 training(myModel, train_dl, num_epochs)
<ipython-input-201-b60077dcefb4> in training(model, train_dl, num_epochs)
15
16 # Repeat for each batch in the training set
---> 17 for i, data in enumerate(train_dl):
18 # Get the input features and target labels, and put them on the GPU
19 inputs, labels = data[0].to(device), data[1].to(device)
/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py in __next__(self)
626 # TODO(https://github.com/pytorch/pytorch/issues/76750)
627 self._reset() # type: ignore[call-arg]
--> 628 data = self._next_data()
629 self._num_yielded += 1
630 if self._dataset_kind == _DatasetKind.Iterable and \
/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py in _next_data(self)
669 def _next_data(self):
670 index = self._next_index() # may raise StopIteration
--> 671 data = self._dataset_fetcher.fetch(index) # may raise StopIteration
672 if self._pin_memory:
673 data = _utils.pin_memory.pin_memory(data, self._pin_memory_device)
/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/fetch.py in fetch(self, possibly_batched_index)
56 data = self.dataset.__getitems__(possibly_batched_index)
57 else:
---> 58 data = [self.dataset[idx] for idx in possibly_batched_index]
59 else:
60 data = self.dataset[possibly_batched_index]
/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/fetch.py in <listcomp>(.0)
56 data = self.dataset.__getitems__(possibly_batched_index)
57 else:
---> 58 data = [self.dataset[idx] for idx in possibly_batched_index]
59 else:
60 data = self.dataset[possibly_batched_index]
<ipython-input-191-4c84224a9983> in __getitem__(self, idx)
27
28 # print(self.df.loc[idx, 'relative_path'])
---> 29 audio_file = self.df.loc[idx, "relative_path"]
30 class_id = self.df.loc[idx, "dx"]
31 # participant_id = self.df.loc[idx, 'adressfname']
/usr/local/lib/python3.8/dist-packages/pandas/core/indexing.py in __getitem__(self, key)
923 with suppress(KeyError, IndexError):
924 return self.obj._get_value(*key, takeable=self._takeable)
--> 925 return self._getitem_tuple(key)
926 else:
927 # we by definition only have the 0th axis
/usr/local/lib/python3.8/dist-packages/pandas/core/indexing.py in _getitem_tuple(self, tup)
1098 def _getitem_tuple(self, tup: tuple):
1099 with suppress(IndexingError):
-> 1100 return self._getitem_lowerdim(tup)
1101
1102 # no multi-index, so validate all of the indexers
/usr/local/lib/python3.8/dist-packages/pandas/core/indexing.py in _getitem_lowerdim(self, tup)
836 # We don't need to check for tuples here because those are
837 # caught by the _is_nested_tuple_indexer check above.
--> 838 section = self._getitem_axis(key, axis=i)
839
840 # We should never have a scalar section here, because
/usr/local/lib/python3.8/dist-packages/pandas/core/indexing.py in _getitem_axis(self, key, axis)
1162 # fall thru to straight lookup
1163 self._validate_key(key, axis)
-> 1164 return self._get_label(key, axis=axis)
1165
1166 def _get_slice_axis(self, slice_obj: slice, axis: int):
/usr/local/lib/python3.8/dist-packages/pandas/core/indexing.py in _get_label(self, label, axis)
1111 def _get_label(self, label, axis: int):
1112 # GH#5667 this will fail if the label is not present in the axis.
-> 1113 return self.obj.xs(label, axis=axis)
1114
1115 def _handle_lowerdim_multi_index_axis0(self, tup: tuple):
/usr/local/lib/python3.8/dist-packages/pandas/core/generic.py in xs(self, key, axis, level, drop_level)
3774 raise TypeError(f"Expected label or tuple of labels, got {key}") from e
3775 else:
-> 3776 loc = index.get_loc(key)
3777
3778 if isinstance(loc, np.ndarray):
/usr/local/lib/python3.8/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
3361 return self._engine.get_loc(casted_key)
3362 except KeyError as err:
-> 3363 raise KeyError(key) from err
3364
3365 if is_scalar(key) and isna(key) and not self.hasnans:
KeyError: 3170
Does anyone have any idea why one approach works without any issues, but the other does not?
I am trying to use the "visheratin/t5-efficient-mini-grammar-correction" pre-trained model for grammar correction and I would like to add my own training examples.
I've loaded the model:
model = AutoModelForSeq2SeqLM.from_pretrained("visheratin/t5-efficient-mini-grammar-correction")
tokenizer = AutoTokenizer.from_pretrained("visheratin/t5-efficient-mini-grammar-correction")
set the training arguments:
training_args = TrainingArguments(
output_dir="./models",
num_train_epochs=3,
per_device_train_batch_size=8,
learning_rate=3e-5,
weight_decay=0.01,
)
and created the training data:
training_examples = [ ('input text 1', 'output text 1'), ('input text 2', 'output text 2') ]
train_data = []
for input_text, target_text in training_examples:
input_ids = tokenizer.encode(input_text, return_tensors="pt", truncation=True, padding=True)
target_ids = tokenizer.encode(target_text, return_tensors="pt", truncation=True, padding=True)
train_data.append({
'input_ids': input_ids,
'attention_mask': torch.ones_like(input_ids),
'labels': target_ids,
})
but when I go to train:
trainer = Trainer(
model=model,
tokenizer=tokenizer,
args=training_args,
train_dataset=train_data,
)
trainer.train()
I get this error:
ValueError: Unable to create tensor, you should probably activate truncation and/or
padding with 'padding=True' 'truncation=True' to have batched tensors with the same
length. Perhaps your features (`input_ids` in this case) have excessive nesting (inputs
type `list` where type `int` is expected).
I already have 'padding=True' and 'truncation=True' to tokenizer.encode() and as far as I can tell I do not have any excessive nesting in my features.
This is the full Traceback:
ValueError Traceback (most recent call last)
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:715, in BatchEncoding.convert_to_tensors(self, tensor_type, prepend_batch_axis)
714 if not is_tensor(value):
--> 715 tensor = as_tensor(value)
717 # Removing this for now in favor of controlling the shape with `prepend_batch_axis`
718 # # at-least2d
719 # if tensor.ndim > 2:
720 # tensor = tensor.squeeze(0)
721 # elif tensor.ndim < 2:
722 # tensor = tensor[None, :]
ValueError: expected sequence of length 37 at dim 2 (got 44)
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
Cell In[104], line 1
----> 1 trainer.train()
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/transformers/trainer.py:1501, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
1496 self.model_wrapped = self.model
1498 inner_training_loop = find_executable_batch_size(
1499 self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size
1500 )
-> 1501 return inner_training_loop(
1502 args=args,
1503 resume_from_checkpoint=resume_from_checkpoint,
1504 trial=trial,
1505 ignore_keys_for_eval=ignore_keys_for_eval,
1506 )
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/transformers/trainer.py:1723, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
1720 self._load_rng_state(resume_from_checkpoint)
1722 step = -1
-> 1723 for step, inputs in enumerate(epoch_iterator):
1724
1725 # Skip past any already trained steps if resuming training
1726 if steps_trained_in_current_epoch > 0:
1727 steps_trained_in_current_epoch -= 1
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/torch/utils/data/dataloader.py:628, in _BaseDataLoaderIter.__next__(self)
625 if self._sampler_iter is None:
626 # TODO(https://github.com/pytorch/pytorch/issues/76750)
627 self._reset() # type: ignore[call-arg]
--> 628 data = self._next_data()
629 self._num_yielded += 1
630 if self._dataset_kind == _DatasetKind.Iterable and \
631 self._IterableDataset_len_called is not None and \
632 self._num_yielded > self._IterableDataset_len_called:
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/torch/utils/data/dataloader.py:671, in _SingleProcessDataLoaderIter._next_data(self)
669 def _next_data(self):
670 index = self._next_index() # may raise StopIteration
--> 671 data = self._dataset_fetcher.fetch(index) # may raise StopIteration
672 if self._pin_memory:
673 data = _utils.pin_memory.pin_memory(data, self._pin_memory_device)
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py:61, in _MapDatasetFetcher.fetch(self, possibly_batched_index)
59 else:
60 data = self.dataset[possibly_batched_index]
---> 61 return self.collate_fn(data)
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/transformers/trainer_utils.py:696, in RemoveColumnsCollator.__call__(self, features)
694 def __call__(self, features: List[dict]):
695 features = [self._remove_columns(feature) for feature in features]
--> 696 return self.data_collator(features)
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/transformers/data/data_collator.py:249, in DataCollatorWithPadding.__call__(self, features)
248 def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
--> 249 batch = self.tokenizer.pad(
250 features,
251 padding=self.padding,
252 max_length=self.max_length,
253 pad_to_multiple_of=self.pad_to_multiple_of,
254 return_tensors=self.return_tensors,
255 )
256 if "label" in batch:
257 batch["labels"] = batch["label"]
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2985, in PreTrainedTokenizerBase.pad(self, encoded_inputs, padding, max_length, pad_to_multiple_of, return_attention_mask, return_tensors, verbose)
2982 batch_outputs[key] = []
2983 batch_outputs[key].append(value)
-> 2985 return BatchEncoding(batch_outputs, tensor_type=return_tensors)
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:210, in BatchEncoding.__init__(self, data, encoding, tensor_type, prepend_batch_axis, n_sequences)
206 n_sequences = encoding[0].n_sequences
208 self._n_sequences = n_sequences
--> 210 self.convert_to_tensors(tensor_type=tensor_type, prepend_batch_axis=prepend_batch_axis)
File ~/opt/miniconda3/envs/ub_nbdev2/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:731, in BatchEncoding.convert_to_tensors(self, tensor_type, prepend_batch_axis)
726 if key == "overflowing_tokens":
727 raise ValueError(
728 "Unable to create tensor returning overflowing tokens of different lengths. "
729 "Please see if a fast version of this tokenizer is available to have this feature available."
730 )
--> 731 raise ValueError(
732 "Unable to create tensor, you should probably activate truncation and/or padding with"
733 " 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your"
734 f" features (`{key}` in this case) have excessive nesting (inputs type `list` where type `int` is"
735 " expected)."
736 )
738 return self
ValueError: Unable to create tensor, you should probably activate truncation and/or padding with 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your features (`input_ids` in this case) have excessive nesting (inputs type `list` where type `int` is expected).
Could someone please help me understand what could be causing this error?
so I'm running into the error that my class ESC50Data does not have any length.
from torch.utils.data import Dataset, DataLoader
class ESC50Data(Dataset):
def __init__(self, base, df, in_col, out_col):
self.df = df
self.data = []
self.labels = []
self.c2i={}
self.i2c={}
self.categories = sorted(df[out_col].unique())
for i, category in enumerate(self.categories):
self.c2i[category]=i
self.i2c[i]=category
for ind in tqdm(range(len(df))):
row = df.iloc[ind]
file_path = os.path.join(base,row[in_col])
self.data.append(spec_to_image(get_melspectrogram(file_path))[np.newaxis,...])
self.labels.append(self.c2i[row['category']])
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
return self.data[idx], self.labels[idx]
train_data = ESC50Data('audio', train, 'filename', 'category')
valid_data = ESC50Data('audio', valid, 'filename', 'category')
train_loader = DataLoader(train_data, batch_size=16, shuffle=True)
valid_loader = DataLoader(valid_data, batch_size=16, shuffle=True)
This is the point at which I get my error. Using Jypter Notebooks as a sidenote.
TypeError Traceback (most recent call last)
Input In [47], in <cell line: 1>()
----> 1 train_loader = DataLoader(train_data, batch_size=16, shuffle=True)
2 valid_loader = DataLoader(valid_data, batch_size=16, shuffle=True)
File ~/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py:353, in DataLoader.__init__(self, dataset, batch_size, shuffle, sampler, batch_sampler, num_workers, collate_fn, pin_memory, drop_last, timeout, worker_init_fn, multiprocessing_context, generator, prefetch_factor, persistent_workers, pin_memory_device)
351 else: # map-style
352 if shuffle:
--> 353 sampler = RandomSampler(dataset, generator=generator) # type: ignore[arg-type]
354 else:
355 sampler = SequentialSampler(dataset) # type: ignore[arg-type]
File ~/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/sampler.py:106, in RandomSampler.__init__(self, data_source, replacement, num_samples, generator)
102 if not isinstance(self.replacement, bool):
103 raise TypeError("replacement should be a boolean value, but got "
104 "replacement={}".format(self.replacement))
--> 106 if not isinstance(self.num_samples, int) or self.num_samples <= 0:
107 raise ValueError("num_samples should be a positive integer "
108 "value, but got num_samples={}".format(self.num_samples))
File ~/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/sampler.py:114, in RandomSampler.num_samples(self)
110 #property
111 def num_samples(self) -> int:
112 # dataset size might change at runtime
113 if self._num_samples is None:
--> 114 return len(self.data_source)
115 return self._num_samples
TypeError: object of type 'ESC50Data' has no len()
Any ideas as to what could be happening? I created the class ESC50Data and then I gave it the child class called Dataset that will inherent the properties of ESC50Data. I also loaded the data into pytorch with train and valid data.
Check the indentation of __len__(self) and __getitem__(self, idx) methods in your class ESC50Data code. Right now, it seems like these methods are defined inside the __init__ method, and not under the class itself.
See, e.g., this answer.
I got an error of ValueError: Input tensors to a Functional must come from tf.keras.Input. Received: 0 (missing previous layer metadata) and i cant find the cause
this is my error trace and my code
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-15-8058f3a2fd50> in <module>()
6 test_loss, test_accuracy = eg.test(dg.user_test)
7 print('Test set: Loss=%.4f ; Accuracy=%.1f%%' % (test_loss, test_accuracy * 100))
----> 8 eg.save_embeddings('embeddings.csv')
7 frames
<ipython-input-5-54ff9897b1c3> in save_embeddings(self, file_name)
66 inp = self.m.input # input placeholder
67 outputs = [layer.output for layer in self.m.layers] # all layer outputs
---> 68 functor = K.function([inp, K.learning_phase()], outputs ) # evaluation function
69
70 #append embeddings to vectors
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py in function(inputs, outputs, updates, name, **kwargs)
3934 from tensorflow.python.keras import models # pylint: disable=g-import-not-at-top
3935 from tensorflow.python.keras.utils import tf_utils # pylint: disable=g-import-not-at-top
-> 3936 model = models.Model(inputs=inputs, outputs=outputs)
3937
3938 wrap_outputs = isinstance(outputs, list) and len(outputs) == 1
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in __new__(cls, *args, **kwargs)
240 # Functional model
241 from tensorflow.python.keras.engine import functional # pylint: disable=g-import-not-at-top
--> 242 return functional.Functional(*args, **kwargs)
243 else:
244 return super(Model, cls).__new__(cls, *args, **kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, *args, **kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py in __init__(self, inputs, outputs, name, trainable)
113 # 'arguments during initialization. Got an unexpected argument:')
114 super(Functional, self).__init__(name=name, trainable=trainable)
--> 115 self._init_graph_network(inputs, outputs)
116
117 #trackable.no_automatic_dependency_tracking
/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, *args, **kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py in _init_graph_network(self, inputs, outputs)
142 base_layer_utils.create_keras_history(self._nested_outputs)
143
--> 144 self._validate_graph_inputs_and_outputs()
145
146 # A Network does not create weights of its own, thus it is already
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py in _validate_graph_inputs_and_outputs(self)
637 'must come from `tf.keras.Input`. '
638 'Received: ' + str(x) +
--> 639 ' (missing previous layer metadata).')
640 # Check that x is an input tensor.
641 # pylint: disable=protected-access
ValueError: Input tensors to a Functional must come from `tf.keras.Input`. Received: 0 (missing previous layer metadata).
and this is my snipped code:
class EmbeddingsGenerator:
def __init__(self, train_users, data):
self.train_users = train_users
#preprocess
self.data = data.sort_values(by=['timestamp'])
#make them start at 0
self.data['userId'] = self.data['userId'] - 1
self.data['itemId'] = self.data['itemId'] - 1
self.user_count = self.data['userId'].max() + 1
self.movie_count = self.data['itemId'].max() + 1
self.user_movies = {} #list of rated movies by each user
for userId in range(self.user_count):
self.user_movies[userId] = self.data[self.data.userId == userId]['itemId'].tolist()
self.m = self.model()
def model(self, hidden_layer_size=100):
m = Sequential()
m.add(Dense(hidden_layer_size, input_shape=(1, self.movie_count)))
m.add(Dropout(0.2))
m.add(Dense(self.movie_count, activation='softmax'))
m.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
return m
def generate_input(self, user_id):
'''
Returns a context and a target for the user_id
context: user's history with one random movie removed
target: id of random removed movie
'''
user_movies_count = len(self.user_movies[user_id])
#picking random movie
random_index = np.random.randint(0, user_movies_count-1) # -1 avoids taking the last movie
#setting target
target = np.zeros((1, self.movie_count))
target[0][self.user_movies[user_id][random_index]] = 1
#setting context
context = np.zeros((1, self.movie_count))
context[0][self.user_movies[user_id][:random_index] + self.user_movies[user_id][random_index+1:]] = 1
return context, target
def train(self, nb_epochs = 300, batch_size = 10000):
'''
Trains the model from train_users's history
'''
for i in range(nb_epochs):
print('%d/%d' % (i+1, nb_epochs))
batch = [self.generate_input(user_id=np.random.choice(self.train_users) - 1) for _ in range(batch_size)]
X_train = np.array([b[0] for b in batch])
y_train = np.array([b[1] for b in batch])
self.m.fit(X_train, y_train, epochs=1, validation_split=0.5)
def test(self, test_users, batch_size = 100000):
'''
Returns [loss, accuracy] on the test set
'''
batch_test = [self.generate_input(user_id=np.random.choice(test_users) - 1) for _ in range(batch_size)]
X_test = np.array([b[0] for b in batch_test])
y_test = np.array([b[1] for b in batch_test])
return self.m.evaluate(X_test, y_test)
def save_embeddings(self, file_name):
'''
Generates a csv file containg the vector embedding for each movie.
'''
inp = self.m.input # input placeholder
outputs = [layer.output for layer in self.m.layers] # all layer outputs
functor = K.function([inp, K.learning_phase()], outputs ) # evaluation function
#append embeddings to vectors
vectors = []
for movie_id in range(self.movie_count):
movie = np.zeros((1, 1, self.movie_count))
movie[0][0][movie_id] = 1
layer_outs = functor([movie])
vector = [str(v) for v in layer_outs[0][0][0]]
vector = '|'.join(vector)
vectors.append([movie_id, vector])
#saves as a csv file
embeddings = pd.DataFrame(vectors, columns=['item_id', 'vectors']).astype({'item_id': 'int32'})
embeddings.to_csv(file_name, sep=';', index=False)
files.download(file_name)
this is the part of code which call the save_embeddings method
if True: # Generate embeddings?
eg = EmbeddingsGenerator(dg.user_train, pd.read_csv('ml-100k/u.data', sep='\t', names=['userId', 'itemId', 'rating', 'timestamp']))
eg.train(nb_epochs=300)
train_loss, train_accuracy = eg.test(dg.user_train)
print('Train set: Loss=%.4f ; Accuracy=%.1f%%' % (train_loss, train_accuracy * 100))
test_loss, test_accuracy = eg.test(dg.user_test)
print('Test set: Loss=%.4f ; Accuracy=%.1f%%' % (test_loss, test_accuracy * 100))
eg.save_embeddings('embeddings.csv')
I am trying to convert my Tensorflow graph to use a custom tensorflow estimator, but I am getting stuck defining the function for input_fn ; I am currently getting an error.
This is the function I use to generate my input data and labels
data_index = 0
epoch_index = 0
recEpoch_indexA = 0 #Used to help keep store of the total number of epoches with the models
def generate_batch(batch_size, inputCount):
global data_index, epoch_index
batch = np.ndarray(shape=(batch_size, inputCount), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
n=0
while n < batch_size:
if len( set(my_data[data_index, 1]) ) >= inputCount:
labels[n,0] = my_data[data_index, 0]
batch[n] = random.sample( set(my_data[data_index, 1]), inputCount)
n = n+1
data_index = (data_index + 1) % len(my_data) #may have to do something like len my_data[:]
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
else:
data_index = (data_index + 1) % len(my_data)
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
return batch, labels
This is where I define my Estimator and attempt to do training
#Define the estimator
word2vecEstimator = tf.estimator.Estimator(
model_fn=my_model,
params={
'batch_size': 1024,
'embedding_size': 50,
'num_inputs': 5,
'num_sampled':128
})
batch_size = 16
num_inputs = 3
#Train with Estimator
word2vecEstimator.train(
input_fn=generate_batch(batch_size, num_inputs),
steps=10)
This is the error message that I get
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
/usr/lib/python3.6/inspect.py in getfullargspec(func)
1118 skip_bound_arg=False,
-> 1119 sigcls=Signature)
1120 except Exception as ex:
/usr/lib/python3.6/inspect.py in _signature_from_callable(obj, follow_wrapper_chains, skip_bound_arg, sigcls)
2185 if not callable(obj):
-> 2186 raise TypeError('{!r} is not a callable object'.format(obj))
2187
TypeError: (array([[1851833, 670357, 343012],
[ 993526, 431296, 935528],
[ 938067, 1155719, 2277388],
[ 534965, 1125669, 1665716],
[1412657, 2152211, 1176177],
[ 268114, 2097642, 2707258],
[1280762, 1516464, 453615],
[2545980, 2302607, 2421182],
[1706260, 2735027, 292652],
[1802025, 2949676, 653015],
[ 854228, 2626773, 225486],
[1747135, 1608478, 2503487],
[1326661, 272883, 2089444],
[3082922, 1359481, 621031],
[2636832, 1842777, 1979638],
[2512269, 1617986, 389356]], dtype=int32), array([[1175598],
[2528125],
[1870906],
[ 643521],
[2349752],
[ 754986],
[2277570],
[2121120],
[2384306],
[1881398],
[3046987],
[2505729],
[2908573],
[2438025],
[ 441422],
[2355625]], dtype=int32)) is not a callable object
The above exception was the direct cause of the following exception:
TypeError Traceback (most recent call last)
<ipython-input-15-7acc939af001> in <module>()
5 word2vecEstimator.train(
6 input_fn=generate_batch(batch_size, num_inputs),
----> 7 steps=10)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in train(self, input_fn, hooks, steps, max_steps, saving_listeners)
352
353 saving_listeners = _check_listeners_type(saving_listeners)
--> 354 loss = self._train_model(input_fn, hooks, saving_listeners)
355 logging.info('Loss for final step: %s.', loss)
356 return self
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _train_model(self, input_fn, hooks, saving_listeners)
1205 return self._train_model_distributed(input_fn, hooks, saving_listeners)
1206 else:
-> 1207 return self._train_model_default(input_fn, hooks, saving_listeners)
1208
1209 def _train_model_default(self, input_fn, hooks, saving_listeners):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _train_model_default(self, input_fn, hooks, saving_listeners)
1232 features, labels, input_hooks = (
1233 self._get_features_and_labels_from_input_fn(
-> 1234 input_fn, model_fn_lib.ModeKeys.TRAIN))
1235 worker_hooks.extend(input_hooks)
1236 estimator_spec = self._call_model_fn(
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _get_features_and_labels_from_input_fn(self, input_fn, mode)
1073 """Extracts the `features` and labels from return values of `input_fn`."""
1074 return estimator_util.parse_input_fn_result(
-> 1075 self._call_input_fn(input_fn, mode))
1076
1077 def _extract_batch_length(self, preds_evaluated):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _call_input_fn(self, input_fn, mode)
1151 ValueError: if `input_fn` takes invalid arguments.
1152 """
-> 1153 input_fn_args = function_utils.fn_args(input_fn)
1154 kwargs = {}
1155 if 'mode' in input_fn_args:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/function_utils.py in fn_args(fn)
54 if _is_callable_object(fn):
55 fn = fn.__call__
---> 56 args = tf_inspect.getfullargspec(fn).args
57 if _is_bounded_method(fn):
58 args.remove('self')
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/tf_inspect.py in getfullargspec(obj)
214 return next((d.decorator_argspec
215 for d in decorators
--> 216 if d.decorator_argspec is not None), _getfullargspec(target))
217
218
/usr/lib/python3.6/inspect.py in getfullargspec(func)
1123 # else. So to be fully backwards compatible, we catch all
1124 # possible exceptions here, and reraise a TypeError.
-> 1125 raise TypeError('unsupported callable') from ex
1126
1127 args = []
TypeError: unsupported callable
Here is a link to the Google Colab notebook for people to run on their own. For anyone looking to execute this, this will download a data file that is ~500 mbs.
https://colab.research.google.com/drive/1LjIz04xhRi5Fsw_Q3IzoG_5KkkXI3WFE
And here is the full code, from the notebook.
import math
import numpy as np
import random
import zipfile
import shutil
from collections import namedtuple
import os
import pprint
import tensorflow as tf
import pandas as pd
import pickle
from numpy import genfromtxt
!pip install -U -q PyDrive
from google.colab import files
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
vocabulary_size = 3096637 #updated 10-25-18 3096636
import gc
dl_id = '19yha9Scxq4zOdfPcw5s6L2lkYQWenApC' #updated 10-22-18
myDownload = drive.CreateFile({'id': dl_id})
myDownload.GetContentFile('Data.npy')
my_data = np.load('Data.npy')
#os.remove('Data.npy')
np.random.shuffle(my_data)
print(my_data[0:15])
data_index = 0
epoch_index = 0
recEpoch_indexA = 0 #Used to help keep store of the total number of epoches with the models
def generate_batch(batch_size, inputCount):
global data_index, epoch_index
batch = np.ndarray(shape=(batch_size, inputCount), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
n=0
while n < batch_size:
if len( set(my_data[data_index, 1]) ) >= inputCount:
labels[n,0] = my_data[data_index, 0]
batch[n] = random.sample( set(my_data[data_index, 1]), inputCount)
n = n+1
data_index = (data_index + 1) % len(my_data) #may have to do something like len my_data[:]
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
else:
data_index = (data_index + 1) % len(my_data)
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
return batch, labels
def my_model( features, labels, mode, params):
# train_dataset = tf.placeholder(tf.int32, shape=[batch_size, num_inputs ])
# train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
train_dataset = features
train_labels = labels
batch_sizeE=params["batch_size"]
embedding_sizeE=params["embedding_size"]
num_inputsE=params["num_inputs"]
num_sampledE=params["num_sampled"]
epochCount = tf.get_variable( 'epochCount', initializer= 0) #to store epoch count to total # of epochs are known
update_epoch = tf.assign(epochCount, epochCount + 1)
embeddings = tf.get_variable( 'embeddings', dtype=tf.float32,
initializer= tf.random_uniform([vocabulary_size, embedding_sizeE], -1.0, 1.0, dtype=tf.float32) )
softmax_weights = tf.get_variable( 'softmax_weights', dtype=tf.float32,
initializer= tf.truncated_normal([vocabulary_size, embedding_sizeE],
stddev=1.0 / math.sqrt(embedding_sizeE), dtype=tf.float32 ) )
softmax_biases = tf.get_variable('softmax_biases', dtype=tf.float32,
initializer= tf.zeros([vocabulary_size], dtype=tf.float32), trainable=False )
embed = tf.nn.embedding_lookup(embeddings, train_dataset) #train data set is
embed_reshaped = tf.reshape( embed, [batch_sizeE*num_inputs, embedding_sizeE] )
segments= np.arange(batch_size).repeat(num_inputs)
averaged_embeds = tf.segment_mean(embed_reshaped, segments, name=None)
loss = tf.reduce_mean(
tf.nn.sampled_softmax_loss(weights=softmax_weights, biases=softmax_biases, inputs=averaged_embeds,
sampled_values=tf.nn.uniform_candidate_sampler(true_classes=tf.cast(train_labels, tf.int64), num_sampled=num_sampled, num_true=1, unique=True, range_max=vocabulary_size, seed=None),
labels=train_labels, num_sampled=num_sampled, num_classes=vocabulary_size))
optimizer = tf.train.AdagradOptimizer(1.0).minimize(loss)
saver = tf.train.Saver()
#Define the estimator
word2vecEstimator = tf.estimator.Estimator(
model_fn=my_model,
params={
'batch_size': 1024,
'embedding_size': 50,
'num_inputs': 5,
'num_sampled':128
})
batch_size = 16
num_inputs = 3
#Train with Estimator
word2vecEstimator.train(
input_fn=generate_batch(batch_size, num_inputs),
steps=10)
There's no way of correcting the function, cuz it can never be implemented using Tensorflow. The input_fn() function must return Tensors, not numpy arrays, cuz the input_fn() is a function constructing the graph, and it maybe is just called once when building the graph. In this context, the numpy array is just a constant value. It may seem to be weird, But it's the truth. You need to understand the mechanism of Tensorflow: the STATIC COMPUTE GRAPH!
Answer here
Tensorflow error : unsupported callable
train method accepts the input function, so it should be input_fn, not input_fn().