Develop Reference

"TypeError: 'NoneType' object is not iterable" Keras Sequential Model not working

I'm trying to create a binary classification neural network using Keras Sequential Model.
This is my code:
# IMPORTED DATASET AND LIBRARIES
test_df = data.sample(frac=0.2, random_state=1337)
train_df = data.drop(test_df.index)
train_df = train_df.reindex(np.random.permutation(train_df.index)) # shuffle the training set
train_df.head()
train_df_mean = train_df.mean()
train_df_std = train_df.std()
train_df_norm = (train_df - train_df_mean)/train_df_std
test_df_mean = test_df.mean()
test_df_std = test_df.std()
test_df_norm = (test_df - test_df_mean)/test_df_std
# Create an empty list that will eventually hold all created feature columns.
feature_columns = []
# Create numerical feature columns.
p1_p2 = tf.feature_column.numeric_column("p1_p2")
p1_p3 = tf.feature_column.numeric_column("p1_p3")
p1_p4 = tf.feature_column.numeric_column("p1_p4")
ves_R3 = tf.feature_column.numeric_column("CalcESR_R3_refitted")
feature_columns.append(p1_p2)
feature_columns.append(p1_p3)
feature_columns.append(p1_p4)
feature_columns.append(ves_R3)
# Convert the list of feature columns into a layer that will later be fed into the model.
feature_layer = layers.DenseFeatures(feature_columns)
def create_model(my_learning_rate, feature_layer, my_metrics):
"""Create and compile a simple classification model."""
# Most simple tf.keras models are sequential.
model = tf.keras.models.Sequential()
# Add the feature layer (the list of features and how they are represented) to the model.
model.add(feature_layer)
# Funnel the regression value through a sigmoid function.
model.add(tf.keras.layers.Dense(units=1, input_shape=(1,),
activation=tf.sigmoid),)
# Call the compile method to construct the layers into a model that
# TensorFlow can execute. Notice that we're using a different loss
# function for classification than for regression.
model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=my_learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=my_metrics)
return model
def train_model(model, dataset, epochs, label_name, batch_size=None, shuffle=True):
"""Feed a dataset into the model in order to train it."""
# The x parameter of tf.keras.Model.fit can be a list of arrays, where
# each array contains the data for one feature. Here, we're passing
# every column in the dataset. Note that the feature_layer will filter
# away most of those columns, leaving only the desired columns and their
# representations as features.
features = {name:np.array(value) for name, value in dataset.items()}
label = np.array(features.pop(label_name))
history = model.fit(x=features, y=label, batch_size=batch_size,
epochs=epochs, shuffle=shuffle)
# The list of epochs is stored separately from the rest of history.
epochs = history.epoch
# Isolate the classification metric for each epoch.
hist = pd.DataFrame(history.history)
return epochs, hist
The following code cell calls specify the hyperparameters, and then invokes the functions to create and train the model, and then to plot the results.
learning_rate = 0.001
epochs = 20
batch_size = 100
label_name = "target"
classification_threshold = 0.35
# Establish the metrics the model will measure.
METRICS = [
tf.keras.metrics.BinaryAccuracy(name='accuracy',
threshold=classification_threshold),
]
# Establish the model's topography.
my_model = create_model(learning_rate, feature_layer, METRICS)
[print(i.shape, i.dtype) for i in my_model.inputs]
[print(o.shape, o.dtype) for o in my_model.outputs]
[print(l.name, l.input_shape, l.dtype) for l in my_model.layers]
But it displays this error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-112-444849d4a214> in <module>
16
17
---> 18 [print(i.shape, i.dtype) for i in my_model.inputs]
19 [print(o.shape, o.dtype) for o in my_model.outputs]
20 [print(l.name, l.input_shape, l.dtype) for l in my_model.layers]
TypeError: 'NoneType' object is not iterable
I think it means that the model inputs are NoneType, but i can't figure out why.

How to use DistilBERT Huggingface NLP model to perform sentiment analysis on new data?

I am using DistilBERT to do sentiment analysis on my dataset. The dataset contains text and a label for each row which identifies whether the text is a positive or negative movie review (eg: 1 = positive and 0 = negative). Here is the code from the huggingface documentation (https://huggingface.co/transformers/custom_datasets.html?highlight=imdb)
#This dataset can be explored in the Hugging Face model hub (IMDb), and can be alternatively downloaded with the 🤗 Datasets library with load_dataset("imdb").
wget http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz
tar -xf aclImdb_v1.tar.gz
#This data is organized into pos and neg folders with one text file per example. Let’s write a function that can read this in.
from pathlib import Path
def read_imdb_split(split_dir):
split_dir = Path(split_dir)
texts = []
labels = []
for label_dir in ["pos", "neg"]:
for text_file in (split_dir/label_dir).iterdir():
texts.append(text_file.read_text())
labels.append(0 if label_dir is "neg" else 1)
return texts, labels
train_texts, train_labels = read_imdb_split('aclImdb/train')
test_texts, test_labels = read_imdb_split('aclImdb/test')
from sklearn.model_selection import train_test_split
train_texts, val_texts, train_labels, val_labels = train_test_split(train_texts, train_labels, test_size=.2)
from transformers import DistilBertTokenizerFast
tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
train_encodings = tokenizer(train_texts, truncation=True, padding=True)
val_encodings = tokenizer(val_texts, truncation=True, padding=True)
test_encodings = tokenizer(test_texts, truncation=True, padding=True)
import torch
class IMDbDataset(torch.utils.data.Dataset):
def __init__(self, encodings, labels):
self.encodings = encodings
self.labels = labels
def __getitem__(self, idx):
item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
item['labels'] = torch.tensor(self.labels[idx])
return item
def __len__(self):
return len(self.labels)
train_dataset = IMDbDataset(train_encodings, train_labels)
val_dataset = IMDbDataset(val_encodings, val_labels)
test_dataset = IMDbDataset(test_encodings, test_labels)
#Now that our datasets our ready, we can fine-tune a model either #with the 🤗 Trainer/TFTrainer or with native PyTorch/TensorFlow. See #training.
#Fine-tuning with Trainer
#The steps above prepared the datasets in the way that the trainer is #expected. Now all we need to do is create a model to fine-tune, #define the TrainingArguments/TFTrainingArguments and instantiate a #Trainer/TFTrainer.
from transformers import DistilBertForSequenceClassification, Trainer, TrainingArguments
training_args = TrainingArguments(
output_dir='./results', # output directory
num_train_epochs=3, # total number of training epochs
per_device_train_batch_size=16, # batch size per device during training
per_device_eval_batch_size=64, # batch size for evaluation
warmup_steps=500, # number of warmup steps for learning rate scheduler
weight_decay=0.01, # strength of weight decay
logging_dir='./logs', # directory for storing logs
logging_steps=10,
)
model = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased")
trainer = Trainer(
model=model, # the instantiated 🤗 Transformers model to be trained
args=training_args, # training arguments, defined above
train_dataset=train_dataset, # training dataset
eval_dataset=val_dataset # evaluation dataset
)
trainer.train()
#We can also train with Pytorch/Tensorflow
from torch.utils.data import DataLoader
from transformers import DistilBertForSequenceClassification, AdamW
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
model.to(device)
model.train()
train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
optim = AdamW(model.parameters(), lr=5e-5)
for epoch in range(3):
for batch in train_loader:
optim.zero_grad()
input_ids = batch['input_ids'].to(device)
attention_mask = batch['attention_mask'].to(device)
labels = batch['labels'].to(device)
outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
loss = outputs[0]
loss.backward()
optim.step()
model.eval()
I want to know test this model on a new piece of data. So, I have a dataframe which contains a piece of text/review for each row, and I want to predict the label. Does anyone know how I would go about doing that? I apologize, I am very new to this and would greatly appreciate any help! I tried taking in text, cleaning it, and then doing
prediction = model.predict(text)
and I got an error saying DistilBERT has no attribute .predict.

If you just want to use the model, you can use the corresponding pipeline:
from transformers import pipeline
classifier = pipeline('sentiment-analysis')
Then you can use it:
classifier("I hate this book")

The code that you've shared from the documentation essentially covers the training and evaluation loop. Beware that your shared code contains two ways of fine-tuning, once with the trainer, which also includes evaluation, and once with native Pytorch/TF, which contains just the training portion and not the evaluation portion.
Here is how the native method can be tweaked to generate predictions on the test set:
# Put model in evaluation mode
model.eval()
# Tracking variables for storing ground truth and predictions
predictions , true_labels = [], []
# Prediction Loop
for batch in test_dataset:
# Unpack the inputs from our dataloader and move to GPU/accelerator
input_ids = batch['input_ids'].to(device)
attention_mask = batch['attention_mask'].to(device)
labels = batch['labels'].to(device)
# Telling the model not to compute or store gradients, saving memory and
# speeding up prediction
with torch.no_grad():
# Forward pass, calculate logit predictions
outputs = model(input_ids, attention_mask=attention_mask,
labels=labels)
logits = outputs[0]
# Move logits and labels to CPU
logits = logits.detach().cpu().numpy()
label_ids = labels.to('cpu').numpy()
# Store predictions and true labels
predictions.append(logits)
true_labels.append(label_ids)
After the execution of this loop, predictions will contain logits, i.e., the probability distribution from the model before any form of normalization.
You can use the following to pick the label with the maximum score from the logits, and produce a classification report
from sklearn.metrics import classification_report, accuracy_score
# Combine the results across all batches.
flat_predictions = np.concatenate(predictions, axis=0)
# For each sample, pick the label (0 or 1) with the higher score.
flat_predictions = np.argmax(flat_predictions, axis=1).flatten()
# Combine the correct labels for each batch into a single list.
flat_true_labels = np.concatenate(true_labels, axis=0)
# Accuracy
print(accuracy_score(flat_true_labels, flat_predictions))
# Classification Report
report = classification_report(flat_true_labels, flat_predictions)
For a more elegant way of performing predictions, you can create a BERTModel Class that would contain different methods and variables for handling the tokenization, creation of dataloader, running the predictions, etc.

You can try code like this example: Link-BERT
You'll arrange the dataset according to the BERT model. D Section in this link, you can just change the model name and your dataset.

Keras model.fit() with tf.dataset fails while using tf.train works fine

Summary: according to the documentation, Keras model.fit() should accept tf.dataset as input (I am using TF version 1.12.0). I can train my model if I manually do the training steps but using model.fit() on the same model, I get an error I cannot resolve.
Here is a sketch of what I did: my dataset, which is too big to fit in the memory, consists of many files each with different number of rows of (100 features, label). I'd like to use tf.data to build my data pipeline:
def data_loader(filename):
'''load a single data file with many rows'''
features, labels = load_hdf5(filename)
...
return features, labels
def make_dataset(filenames, batch_size):
'''read files one by one, pick individual rows, batch them and repeat'''
dataset = tf.data.Dataset.from_tensor_slices(filenames)
dataset = dataset.map( # Problem here! See edit for solution
lambda filename: tuple(tf.py_func(data_loader, [filename], [float32, tf.float32])))
dataset = dataset.flat_map(
lambda features, labels: tf.data.Dataset.from_tensor_slices((features, labels)))
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()
dataset = dataset.prefetch(1000)
return dataset
_BATCH_SIZE = 128
training_set = make_dataset(training_files, batch_size=_BATCH_SIZE)
I'd like to try a very basic logistic regression model:
inputs = tf.keras.layers.Input(shape=(100,))
outputs = tf.keras.layers.Dense(1, activation='softmax')(inputs)
model = tf.keras.Model(inputs, outputs)
If I train it manually everything works fine, e.g.:
labels = tf.placeholder(tf.float32)
loss = tf.reduce_mean(tf.keras.backend.categorical_crossentropy(labels, outputs))
train_step = tf.train.GradientDescentOptimizer(.05).minimize(loss)
iterator = training_set.make_one_shot_iterator()
next_element = iterator.get_next()
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
for i in range(training_size // _BATCH_SIZE):
x, y = sess.run(next_element)
train_step.run(feed_dict={inputs: x, labels: y})
However, if I instead try to use model.fit like this:
model.compile('adam', 'categorical_crossentropy', metrics=['acc'])
model.fit(training_set.make_one_shot_iterator(),
steps_per_epoch=training_size // _BATCH_SIZE,
epochs=1,
verbose=1)
I get an error message ValueError: Cannot take the length of Shape with unknown rank. inside the keras'es _standardize_user_data function.
I have tried quite a few things but could not resolve the issue. Any ideas?
Edit: based on #kvish's answer, the solution was to change the map from a lambda to a function that would specify the correct tensor dimensions, e.g.:
def data_loader(filename):
def loader_impl(filename):
features, labels, _ = load_hdf5(filename)
...
return features, labels
features, labels = tf.py_func(loader_impl, [filename], [tf.float32, tf.float32])
features.set_shape((None, 100))
labels.set_shape((None, 1))
return features, labels
and now, all needed to do is to call this function from map:
dataset = dataset.map(data_loader)

Probably tf.py_func produces an unknown shape which Keras cannot infer. We can set the shape of the tensor returned by it using set_shape(your_shape) method and that would help Keras infer the shape of the result.

Stuck trying to make this tensorflow script work

I'm trying to move my scikit-learn python script into tensorflow code. Keep getting stuck with errors. Please help!
import pandas as pd
import numpy as np
import tensorflow as tf
# read csv
df = pd.read_csv("/Downloads/iris-2.csv", header=0)
# get header names as array
features = list(df.columns.values)
label = features.pop()
classes = len(df[label].unique())
# encode target
X = df[features]
y = df[label]
# convert feature headers into tf
for index,value in enumerate(features):
features[index] = tf.feature_column.numeric_column(value)
# initialize classifier
classifier = tf.estimator.DNNClassifier(
feature_columns=features,
hidden_units=[10, 10],
n_classes=classes)
# train the classifier
dataset = tf.data.Dataset.from_tensor_slices((dict(X), y))
dataset = dataset.shuffle(1000).repeat().batch(0)
data = dataset.make_one_shot_iterator().get_next()
classifier.train(input_fn=lambda:data,steps=3)
predictions = classifier.predict([5.1,3.0,4.2,1.2])
print(predictions)
Latest error I'm stuck on is:
ValueError: Passed Tensor("dnn/head/weighted_loss/Sum:0", shape=(), dtype=float32) should have graph attribute that is equal to current graph <tensorflow.python.framework.ops.Graph object at 0x10dd9a190>.
Here's the dataset I'm using: https://gist.githubusercontent.com/curran/a08a1080b88344b0c8a7/raw/d546eaee765268bf2f487608c537c05e22e4b221/iris.csv

The input tensor (variables data and dataset) cannot be precomputed. They need to be computed inside the function passed to input_fn in the call to train so that the tensors are in the graph that the Estimator (classifier) creates during the call to train(). So for your last block you could use:
# train the classifier
def my_input_fn():
dataset = tf.data.Dataset.from_tensor_slices((dict(X), y))
dataset = dataset.shuffle(1000).repeat().batch(0)
return dataset.make_one_shot_iterator().get_next()
classifier.train(input_fn=my_input_fn, steps=3)
predictions = classifier.predict([5.1,3.0,4.2,1.2])
print(predictions)

TypeError when training Tensorflow Random Forest using TensorForestEstimator

I get a TypeError when attempting to train an Tensorflow Random Forest using TensorForestEstimator.
TypeError: Input 'input_data' of 'CountExtremelyRandomStats' Op has type float64 that does not match expected type of float32.
I've tried using Python 2.7 and Python 3, and I've tried using tf.cast() to put everything in float32 but it doesn't help. I have checked the data type on execution and it's float32. The problem doesn't seem to be the data I provide (csv of all floats), so I'm not sure where to go from here.
Any suggestions of things I can try would be much appreciated.
Code:
# Build an estimator.
def build_estimator(model_dir):
params = tensor_forest.ForestHParams(
num_classes=2, num_features=760,
num_trees=FLAGS.num_trees, max_nodes=FLAGS.max_nodes)
graph_builder_class = tensor_forest.RandomForestGraphs
if FLAGS.use_training_loss:
graph_builder_class = tensor_forest.TrainingLossForest
# Use the SKCompat wrapper, which gives us a convenient way to split in-memory data into batches.
return estimator.SKCompat(random_forest.TensorForestEstimator(params, graph_builder_class=graph_builder_class, model_dir=model_dir))
# Train and evaluate the model.
def train_and_eval():
# load datasets
training_set = pd.read_csv('/Users/carl/Dropbox/Docs/Python/randomforest_balanced_train.csv', dtype=np.float32, header=None)
test_set = pd.read_csv('/Users/carl/Dropbox/Docs/Python/randomforest_balanced_test.csv', dtype=np.float32, header=None)
print('###########')
print(training_set.loc[:,1].dtype) # this prints float32
# load labels
training_labels = pd.read_csv('/Users/carl/Dropbox/Docs/Python/randomforest_balanced_train_class.csv', dtype=np.int32, names=LABEL, header=None)
test_labels = pd.read_csv('/Users/carl/Dropbox/Docs/Python/randomforest_balanced_test_class.csv', dtype=np.int32, names=LABEL, header=None)
# define the path where the model will be stored - default is current directory
model_dir = tempfile.mkdtemp() if not FLAGS.model_dir else FLAGS.model_dir
print('model directory = %s' % model_dir)
# build the random forest estimator
est = build_estimator(model_dir)
tf.cast(training_set, tf.float32) #error occurs with/without casts
tf.cast(test_set, tf.float32)
# train the forest to fit the training data
est.fit(x=training_set, y=training_labels) #this line throws the error

You are using tf.cast in incorrect manner
tf.cast(training_set, tf.float32) #error occurs with/without casts
should be
training_set = tf.cast(training_set, tf.float32)
tf.cast is not in-place method, it is a tensor flow op, as any other operation, and needs to be assigned and run.