I am trying to run the code regarding CNN using the dataset of Deep Learning and Beamforming.
During the process, I found an error at the local variable 'btch_index'. According to information by the Internet, there is a problem in Keras.
there is a problem in 'if batch_index == len(batches) - 1: # Last batch.'
UnboundLocalError: local variable 'batch_index' referenced before assignment
I tried to declare 'batch_index' as a global variable, but it also has another error.
"""Part of the training engine related to plain array data (e.g. Numpy).
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
from scipy.sparse import issparse
from .training_utils import batch_shuffle
from .training_utils import check_num_samples
from .training_utils import make_batches
from .training_utils import should_run_validation
from .. import backend as K
from .. import callbacks as cbks
from ..utils.generic_utils import Progbar
from ..utils.generic_utils import slice_arrays
from ..utils.generic_utils import to_list
from ..utils.generic_utils import unpack_singleton
def fit_loop(model, fit_function, fit_inputs,
out_labels=None,
batch_size=None,
epochs=100,
verbose=1,
callbacks=None,
val_function=None,
val_inputs=None,
shuffle=True,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None,
validation_freq=1):
"""Abstract fit function for `fit_function(fit_inputs)`.
Assumes that fit_function returns a list, labeled by out_labels.
# Arguments
model: Keras model instance.
fit_function: Keras function returning a list of tensors
fit_inputs: List of tensors to be fed to `fit_function`
out_labels: List of strings, display names of
the outputs of `fit_function`
batch_size: Integer batch size or None if unknown.
epochs: Number of times to iterate over the data
verbose: Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training and validation
(if `val_function` and `val_inputs` are not `None`).
val_function: Keras function to call for validation
val_inputs: List of tensors to be fed to `val_function`
shuffle: Whether to shuffle the data at the beginning of each epoch
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for
(only if doing validation from data tensors).
Ignored with the default value of `None`.
validation_freq: Only relevant if validation data is provided. Integer
or list/tuple/set. If an integer, specifies how many training
epochs to run before a new validation run is performed, e.g.
validation_freq=2` runs validation every 2 epochs. If a list,
tuple, or set, specifies the epochs on which to run validation,
e.g. `validation_freq=[1, 2, 10]` runs validation at the end
of the 1st, 2nd, and 10th epochs.
# Returns
`History` object.
"""
do_validation = False
if val_function and val_inputs:
do_validation = True
if (verbose and fit_inputs and
hasattr(fit_inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')):
print('Train on %d samples, validate on %d samples' %
(fit_inputs[0].shape[0], val_inputs[0].shape[0]))
if validation_steps:
do_validation = True
if steps_per_epoch is None:
raise ValueError('Can only use `validation_steps` '
'when doing step-wise '
'training, i.e. `steps_per_epoch` '
'must be set.')
elif do_validation:
if steps_per_epoch:
raise ValueError('Must specify `validation_steps` '
'to perform validation '
'when doing step-wise training.')
num_train_samples = check_num_samples(fit_inputs,
batch_size=batch_size,
steps=steps_per_epoch,
steps_name='steps_per_epoch')
if num_train_samples is not None:
index_array = np.arange(num_train_samples)
model.history = cbks.History()
_callbacks = [cbks.BaseLogger(stateful_metrics=model.metrics_names[1:])]
if verbose:
if steps_per_epoch is not None:
count_mode = 'steps'
else:
count_mode = 'samples'
_callbacks.append(
cbks.ProgbarLogger(count_mode, stateful_metrics=model.metrics_names[1:]))
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than itself
# (used by Sequential models)
callback_model = model._get_callback_model()
callback_metrics = list(model.metrics_names)
if do_validation:
callback_metrics += ['val_' + n for n in model.metrics_names]
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks._call_begin_hook('train')
callbacks.model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_inputs
# To prevent a slowdown,
# we find beforehand the arrays that need conversion.
feed = (model._feed_inputs +
model._feed_targets +
model._feed_sample_weights)
indices_for_conversion_to_dense = []
for i in range(len(feed)):
if issparse(fit_inputs[i]) and not K.is_sparse(feed[i]):
indices_for_conversion_to_dense.append(i)
for epoch in range(initial_epoch, epochs):
model.reset_metrics()
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
if steps_per_epoch is not None:
for step_index in range(steps_per_epoch):
batch_logs = {'batch': step_index, 'size': 1}
callbacks._call_batch_hook('train', 'begin', step_index, batch_logs)
outs = fit_function(fit_inputs)
outs = to_list(outs)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks._call_batch_hook('train', 'end', step_index, batch_logs)
if callback_model.stop_training:
break
if do_validation and should_run_validation(validation_freq, epoch):
val_outs = test_loop(model, val_function, val_inputs,
steps=validation_steps,
callbacks=callbacks,
verbose=0)
val_outs = to_list(val_outs)
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
else:
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(fit_inputs[-1], int):
# Do not slice the training phase flag.
ins_batch = slice_arrays(
fit_inputs[:-1], batch_ids) + [fit_inputs[-1]]
else:
ins_batch = slice_arrays(fit_inputs, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {'batch': batch_index, 'size': len(batch_ids)}
callbacks._call_batch_hook('train', 'begin', batch_index, batch_logs)
for i in indices_for_conversion_to_dense:
ins_batch[i] = ins_batch[i].toarray()
outs = fit_function(ins_batch)
outs = to_list(outs)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks._call_batch_hook('train', 'end', batch_index, batch_logs)
if callbacks.model.stop_training:
break
if batch_index == len(batches) - 1: # Last batch.
if do_validation and should_run_validation(validation_freq, epoch):
val_outs = test_loop(model, val_function, val_inputs,
batch_size=batch_size,
callbacks=callbacks,
verbose=0)
val_outs = to_list(val_outs)
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callbacks.model.stop_training:
break
callbacks._call_end_hook('train')
return model.history
def predict_loop(model, f, ins,
batch_size=32,
verbose=0,
steps=None,
callbacks=None):
"""Abstract method to loop over some data in batches.
# Arguments
model: Keras model instance.
f: Keras function returning a list of tensors.
ins: list of tensors to be fed to `f`.
batch_size: integer batch size.
verbose: verbosity mode.
steps: Total number of steps (batches of samples)
before declaring `predict_loop` finished.
Ignored with the default value of `None`.
callbacks: List of callbacks or an instance of
`keras.callbacks.CallbackList` to be called during prediction.
# Returns
Array of predictions (if the model has a single output)
or list of arrays of predictions
(if the model has multiple outputs).
"""
num_samples = check_num_samples(ins,
batch_size=batch_size,
steps=steps,
steps_name='steps')
# Check if callbacks have not been already configured
if not isinstance(callbacks, cbks.CallbackList):
callbacks = cbks.CallbackList(callbacks)
callback_model = model._get_callback_model()
callbacks.set_model(callback_model)
callback_params = {
'batch_size': batch_size,
'steps': steps,
'samples': num_samples,
'verbose': verbose,
}
callbacks.set_params(callback_params)
if verbose == 1:
if steps is not None:
progbar = Progbar(target=steps)
else:
progbar = Progbar(target=num_samples)
indices_for_conversion_to_dense = []
for i in range(len(model._feed_inputs)):
if issparse(ins[i]) and not K.is_sparse(model._feed_inputs[i]):
indices_for_conversion_to_dense.append(i)
callbacks.model.stop_training = False
callbacks._call_begin_hook('predict')
if steps is not None:
# Step-based predictions.
# Since we do not know how many samples
# we will see, we cannot pre-allocate
# the returned Numpy arrays.
# Instead, we store one array per batch seen
# and concatenate them upon returning.
unconcatenated_outs = []
for step in range(steps):
batch_logs = {'batch': step, 'size': 1}
callbacks._call_batch_hook('predict', 'begin', step, batch_logs)
batch_outs = f(ins)
batch_outs = to_list(batch_outs)
if step == 0:
for batch_out in batch_outs:
unconcatenated_outs.append([])
for i, batch_out in enumerate(batch_outs):
unconcatenated_outs[i].append(batch_out)
batch_logs['outputs'] = batch_outs
callbacks._call_batch_hook('predict', 'end', step, batch_logs)
if verbose == 1:
progbar.update(step + 1)
callbacks.on_predict_end()
if len(unconcatenated_outs) == 1:
return np.concatenate(unconcatenated_outs[0], axis=0)
return [np.concatenate(unconcatenated_outs[i], axis=0)
for i in range(len(unconcatenated_outs))]
else:
# Sample-based predictions.
outs = []
batches = make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
#global batch_index
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if ins and isinstance(ins[-1], int):
# Do not slice the training phase flag.
ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_arrays(ins, batch_ids)
for i in indices_for_conversion_to_dense:
ins_batch[i] = ins_batch[i].toarray()
batch_logs = {'batch': batch_index, 'size': len(batch_ids)}
callbacks._call_batch_hook('predict', 'begin', batch_index, batch_logs)
batch_outs = f(ins_batch)
batch_outs = to_list(batch_outs)
if batch_index == 0:
# Pre-allocate the results arrays.
for batch_out in batch_outs:
shape = (num_samples,) + batch_out.shape[1:]
outs.append(np.zeros(shape, dtype=batch_out.dtype))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
batch_logs['outputs'] = batch_outs
callbacks._call_batch_hook('predict', 'end', batch_index, batch_logs)
if verbose == 1:
progbar.update(batch_end)
callbacks._call_end_hook('predict')
return unpack_singleton(outs)
def test_loop(model, f, ins,
batch_size=None,
verbose=0,
steps=None,
callbacks=None):
"""Abstract method to loop over some data in batches.
# Arguments
model: Keras model instance.
f: Keras function returning a list of tensors.
ins: list of tensors to be fed to `f`.
batch_size: integer batch size or `None`.
verbose: verbosity mode.
steps: Total number of steps (batches of samples)
before declaring predictions finished.
Ignored with the default value of `None`.
callbacks: List of callbacks or an instance of
`keras.callbacks.CallbackList` to be called during evaluation.
# Returns
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
"""
model.reset_metrics()
num_samples = check_num_samples(ins,
batch_size=batch_size,
steps=steps,
steps_name='steps')
# Check if callbacks have not been already configured
if not isinstance(callbacks, cbks.CallbackList):
callbacks = cbks.CallbackList(callbacks)
callback_model = model._get_callback_model()
callbacks.set_model(callback_model)
callback_metrics = list(model.metrics_names)
callback_params = {
'batch_size': batch_size,
'steps': steps,
'samples': num_samples,
'verbose': verbose,
'metrics': callback_metrics,
}
callbacks.set_params(callback_params)
outs = []
if verbose == 1:
if steps is not None:
progbar = Progbar(target=steps)
else:
progbar = Progbar(target=num_samples)
# To prevent a slowdown,
# we find beforehand the arrays that need conversion.
feed = (model._feed_inputs +
model._feed_targets +
model._feed_sample_weights)
indices_for_conversion_to_dense = []
for i in range(len(feed)):
if issparse(ins[i]) and not K.is_sparse(feed[i]):
indices_for_conversion_to_dense.append(i)
callbacks.model.stop_training = False
callbacks._call_begin_hook('test')
if steps is not None:
for step in range(steps):
batch_logs = {'batch': step, 'size': 1}
callbacks._call_batch_hook('test', 'begin', step, batch_logs)
batch_outs = f(ins)
if isinstance(batch_outs, list):
if step == 0:
outs.extend([0.] * len(batch_outs))
for i, batch_out in enumerate(batch_outs):
if i == 0: # Index 0 == `Loss`
outs[i] = float(batch_out)
else:
outs[i] += float(batch_out)
else:
if step == 0:
outs.append(0.)
outs[0] += float(batch_outs)
for l, o in zip(model.metrics_names, batch_outs):
batch_logs[l] = o
callbacks._call_batch_hook('test', 'end', step, batch_logs)
if verbose == 1:
progbar.update(step + 1)
outs[0] /= steps # Index 0 == `Loss`
else:
batches = make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(ins[-1], int):
# Do not slice the training phase flag.
ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_arrays(ins, batch_ids)
for i in indices_for_conversion_to_dense:
ins_batch[i] = ins_batch[i].toarray()
batch_logs = {'batch': batch_index, 'size': len(batch_ids)}
callbacks._call_batch_hook('test', 'begin', batch_index, batch_logs)
batch_outs = f(ins_batch)
if isinstance(batch_outs, list):
if batch_index == 0:
outs.extend([0.] * len(batch_outs))
for i, batch_out in enumerate(batch_outs):
if i == 0: # Index 0 == `Loss`
outs[i] += float(batch_out) * len(batch_ids)
else:
outs[i] = float(batch_out)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += float(batch_outs) * len(batch_ids)
for l, o in zip(model.metrics_names, batch_outs):
batch_logs[l] = float(o)
callbacks._call_batch_hook('test', 'end', batch_index, batch_logs)
if verbose == 1:
progbar.update(batch_end)
outs[0] /= num_samples # Index 0 == `Loss`
callbacks._call_end_hook('test')
return unpack_singleton(outs)
I use the Windows OS, Python 3.6.8, Keras 2.3.1 and Tensorflow 2.0.0.
I search the net and I don't realize how I should deal with it.
Double check your indentation, you are getting this error because you are referencing the variable batch_index outside of the loop in which it was defined in, so it is out of scope.
Related
Can any one help me
I am facing this error "h5py objects cannot be pickled" while running (train.py) on https://github.com/RoyalSkye/Image-Caption
(my OS is Window 10)
train.py
#!/usr/bin/env python3
import h5py
import time
import torch.backends.cudnn as cudnn
import torch.optim
import torch.utils.data
import torchvision.transforms as transforms
from torch import nn
from torch.nn.utils.rnn import pack_padded_sequence
from models import *
from transformer import *
from datasets import *
from utils import *
from nltk.translate.bleu_score import corpus_bleu
import argparse
import codecs
import numpy as np
def train(args, train_loader, encoder, decoder, criterion, encoder_optimizer, decoder_optimizer, epoch):
"""
Performs one epoch's training.
:param train_loader: DataLoader for training data
:param encoder: encoder model
:param decoder: decoder model
:param criterion: loss layer
:param encoder_optimizer: optimizer to update encoder's weights (if fine-tuning)
:param decoder_optimizer: optimizer to update decoder's weights
:param epoch: epoch number
"""
decoder.train() # train mode (dropout and batchnorm is used)
encoder.train()
batch_time = AverageMeter() # forward prop. + back prop. time
data_time = AverageMeter() # data loading time
losses = AverageMeter() # loss (per word decoded)
top5accs = AverageMeter() # top5 accuracy
start = time.time()
# Batches
for i, (imgs, caps, caplens) in enumerate(train_loader):
data_time.update(time.time() - start)
# Move to GPU, if available
imgs = imgs.to(device)
caps = caps.to(device)
caplens = caplens.to(device)
# Forward prop.
imgs = encoder(imgs)
# imgs: [batch_size, 14, 14, 2048]
# caps: [batch_size, 52]
# caplens: [batch_size, 1]
scores, caps_sorted, decode_lengths, alphas, sort_ind = decoder(imgs, caps, caplens)
# Since we decoded starting with <start>, the targets are all words after <start>, up to <end>
targets = caps_sorted[:, 1:]
# Remove timesteps that we didn't decode at, or are pads
# pack_padded_sequence is an easy trick to do this
scores = pack_padded_sequence(scores, decode_lengths, batch_first=True).data
targets = pack_padded_sequence(targets, decode_lengths, batch_first=True).data
# print(scores.size())
# print(targets.size())
# Calculate loss
loss = criterion(scores, targets)
# Add doubly stochastic attention regularization
# Second loss, mentioned in paper "Show, Attend and Tell: Neural Image Caption Generation with Visual Attention"
# https://arxiv.org/abs/1502.03044
# In section 4.2.1 Doubly stochastic attention regularization: We know the weights sum to 1 at a given timestep.
# But we also encourage the weights at a single pixel p to sum to 1 across all timesteps T.
# This means we want the model to attend to every pixel over the course of generating the entire sequence.
# Therefore, we want to minimize the difference between 1 and the sum of a pixel's weights across all timesteps.
if args.decoder_mode == "lstm":
loss += args.alpha_c * ((1. - alphas.sum(dim=1)) ** 2).mean()
elif args.decoder_mode == "transformer":
dec_alphas = alphas["dec_enc_attns"]
alpha_trans_c = args.alpha_c / (args.n_heads * args.decoder_layers)
for layer in range(args.decoder_layers): # args.decoder_layers = len(dec_alphas)
cur_layer_alphas = dec_alphas[layer] # [batch_size, n_heads, 52, 196]
for h in range(args.n_heads):
cur_head_alpha = cur_layer_alphas[:, h, :, :]
loss += alpha_trans_c * ((1. - cur_head_alpha.sum(dim=1)) ** 2).mean()
# Back prop.
decoder_optimizer.zero_grad()
if encoder_optimizer is not None:
encoder_optimizer.zero_grad()
loss.backward()
# Clip gradients
if args.grad_clip is not None:
clip_gradient(decoder_optimizer, args.grad_clip)
if encoder_optimizer is not None:
clip_gradient(encoder_optimizer, args.grad_clip)
# Update weights
decoder_optimizer.step()
if encoder_optimizer is not None:
encoder_optimizer.step()
# Keep track of metrics
top5 = accuracy(scores, targets, 5)
losses.update(loss.item(), sum(decode_lengths))
top5accs.update(top5, sum(decode_lengths))
batch_time.update(time.time() - start)
start = time.time()
if i % args.print_freq == 0:
print("Epoch: {}/{} step: {}/{} Loss: {} AVG_Loss: {} Top-5 Accuracy: {} Batch_time: {}s".format(epoch+1, args.epochs, i+1, len(train_loader), losses.val, losses.avg, top5accs.val, batch_time.val))
def validate(args, val_loader, encoder, decoder, criterion):
"""
Performs one epoch's validation.
:param val_loader: DataLoader for validation data.
:param encoder: encoder model
:param decoder: decoder model
:param criterion: loss layer
:return: score_dict {'Bleu_1': 0., 'Bleu_2': 0., 'Bleu_3': 0., 'Bleu_4': 0., 'METEOR': 0., 'ROUGE_L': 0., 'CIDEr': 1.}
"""
decoder.eval() # eval mode (no dropout or batchnorm)
if encoder is not None:
encoder.eval()
batch_time = AverageMeter()
losses = AverageMeter()
top5accs = AverageMeter()
start = time.time()
references = list() # references (true captions) for calculating BLEU-4 score
hypotheses = list() # hypotheses (predictions)
# explicitly disable gradient calculation to avoid CUDA memory error
with torch.no_grad():
# Batches
for i, (imgs, caps, caplens, allcaps) in enumerate(val_loader):
# Move to device, if available
imgs = imgs.to(device)
caps = caps.to(device)
caplens = caplens.to(device)
# Forward prop.
if encoder is not None:
imgs = encoder(imgs)
scores, caps_sorted, decode_lengths, alphas, sort_ind = decoder(imgs, caps, caplens)
# Since we decoded starting with <start>, the targets are all words after <start>, up to <end>
targets = caps_sorted[:, 1:]
# Remove timesteps that we didn't decode at, or are pads
# pack_padded_sequence is an easy trick to do this
scores_copy = scores.clone()
scores = pack_padded_sequence(scores, decode_lengths, batch_first=True).data
targets = pack_padded_sequence(targets, decode_lengths, batch_first=True).data
# Calculate loss
loss = criterion(scores, targets)
# Add doubly stochastic attention regularization
if args.decoder_mode == "lstm":
loss += args.alpha_c * ((1. - alphas.sum(dim=1)) ** 2).mean()
elif args.decoder_mode == "transformer":
dec_alphas = alphas["dec_enc_attns"]
alpha_trans_c = args.alpha_c / (args.n_heads * args.decoder_layers)
for layer in range(args.decoder_layers): # args.decoder_layers = len(dec_alphas)
cur_layer_alphas = dec_alphas[layer] # [batch_size, n_heads, 52, 196]
for h in range(args.n_heads):
cur_head_alpha = cur_layer_alphas[:, h, :, :]
loss += alpha_trans_c * ((1. - cur_head_alpha.sum(dim=1)) ** 2).mean()
# Keep track of metrics
losses.update(loss.item(), sum(decode_lengths))
top5 = accuracy(scores, targets, 5)
top5accs.update(top5, sum(decode_lengths))
batch_time.update(time.time() - start)
start = time.time()
# Store references (true captions), and hypothesis (prediction) for each image
# If for n images, we have n hypotheses, and references a, b, c... for each image, we need -
# references = [[ref1a, ref1b, ref1c], [ref2a, ref2b], ...], hypotheses = [hyp1, hyp2, ...]
# References
allcaps = allcaps[sort_ind] # because images were sorted in the decoder
for j in range(allcaps.shape[0]):
img_caps = allcaps[j].tolist()
img_captions = list(
map(lambda c: [w for w in c if w not in {word_map['<start>'], word_map['<pad>']}],
img_caps)) # remove <start> and pads
references.append(img_captions)
# Hypotheses
_, preds = torch.max(scores_copy, dim=2)
preds = preds.tolist()
temp_preds = list()
for j, p in enumerate(preds):
temp_preds.append(preds[j][:decode_lengths[j]]) # remove pads
preds = temp_preds
hypotheses.extend(preds)
assert len(references) == len(hypotheses)
# Calculate BLEU-1~4 scores
# metrics = {}
# weights = (1.0 / 1.0,)
# metrics["bleu1"] = corpus_bleu(references, hypotheses, weights)
# weights = (1.0/2.0, 1.0/2.0,)
# metrics["bleu2"] = corpus_bleu(references, hypotheses, weights)
# weights = (1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0,)
# metrics["bleu3"] = corpus_bleu(references, hypotheses, weights)
# metrics["bleu4"] = corpus_bleu(references, hypotheses)
# Calculate BLEU1~4, METEOR, ROUGE_L, CIDEr scores
metrics = get_eval_score(references, hypotheses)
print("EVA LOSS: {} TOP-5 Accuracy {} BLEU-1 {} BLEU2 {} BLEU3 {} BLEU-4 {} METEOR {} ROUGE_L {} CIDEr {}".format
(losses.avg, top5accs.avg, metrics["Bleu_1"], metrics["Bleu_2"], metrics["Bleu_3"], metrics["Bleu_4"],
metrics["METEOR"], metrics["ROUGE_L"], metrics["CIDEr"]))
return metrics
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Image_Captioning')
# Data parameters
parser.add_argument('--data_folder', default="./dataset/generated_data",
help='folder with data files saved by create_input_files.py.')
parser.add_argument('--data_name', default="coco_5_cap_per_img_5_min_word_freq",
help='base name shared by data files.')
# Model parameters
parser.add_argument('--emb_dim', type=int, default=300, help='dimension of word embeddings.')
parser.add_argument('--attention_dim', type=int, default=512, help='dimension of attention linear layers.')
parser.add_argument('--decoder_dim', type=int, default=512, help='dimension of decoder RNN.')
parser.add_argument('--n_heads', type=int, default=8, help='Multi-head attention.')
parser.add_argument('--dropout', type=float, default=0.1, help='dropout')
parser.add_argument('--decoder_mode', default="transformer", help='which model does decoder use?') # lstm or transformer
parser.add_argument('--attention_method', default="ByPixel", help='which attention method to use?') # ByPixel or ByChannel
parser.add_argument('--encoder_layers', type=int, default=2, help='the number of layers of encoder in Transformer.')
parser.add_argument('--decoder_layers', type=int, default=6, help='the number of layers of decoder in Transformer.')
# Training parameters
parser.add_argument('--epochs', type=int, default=100,
help='number of epochs to train for (if early stopping is not triggered).')
parser.add_argument('--stop_criteria', type=int, default=25, help='training stop if epochs_since_improvement == stop_criteria')
parser.add_argument('--batch_size', type=int, default=32, help='batch_size')
parser.add_argument('--print_freq', type=int, default=100, help='print training/validation stats every __ batches.')
parser.add_argument('--workers', type=int, default=1, help='for data-loading; right now, only 1 works with h5pys.')
parser.add_argument('--encoder_lr', type=float, default=1e-4, help='learning rate for encoder if fine-tuning.')
parser.add_argument('--decoder_lr', type=float, default=1e-4, help='learning rate for decoder.')
parser.add_argument('--grad_clip', type=float, default=5., help='clip gradients at an absolute value of.')
parser.add_argument('--alpha_c', type=float, default=1.,
help='regularization parameter for doubly stochastic attention, as in the paper.')
parser.add_argument('--fine_tune_encoder', type=bool, default=False, help='whether fine-tune encoder or not')
parser.add_argument('--fine_tune_embedding', type=bool, default=False, help='whether fine-tune word embeddings or not')
parser.add_argument('--checkpoint', default=None, help='path to checkpoint, None if none.')
parser.add_argument('--embedding_path', default=None, help='path to pre-trained word Embedding.')
args = parser.parse_args()
# load checkpoint, these parameters can't be modified
final_args = {"emb_dim": args.emb_dim,
"attention_dim": args.attention_dim,
"decoder_dim": args.decoder_dim,
"n_heads": args.n_heads,
"dropout": args.dropout,
"decoder_mode": args.decoder_mode,
"attention_method": args.attention_method,
"encoder_layers": args.encoder_layers,
"decoder_layers": args.decoder_layers}
start_epoch = 0
best_bleu4 = 0. # BLEU-4 score right now
epochs_since_improvement = 0 # keeps track of number of epochs since there's been an improvement in validation BLEU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # sets device for model and PyTorch tensors
cudnn.benchmark = True # set to true only if inputs to model are fixed size; otherwise lot of computational overhead
print(device)
# Read word map
word_map_file = os.path.join(args.data_folder, 'WORDMAP_' + args.data_name + '.json')
with open(word_map_file, 'r') as j:
word_map = json.load(j)
# Initialize / load checkpoint
if args.checkpoint is None:
encoder = CNN_Encoder(attention_method=args.attention_method)
encoder.fine_tune(args.fine_tune_encoder)
encoder_optimizer = torch.optim.Adam(params=filter(lambda p: p.requires_grad, encoder.parameters()),
lr=args.encoder_lr) if args.fine_tune_encoder else None
if args.decoder_mode == "lstm":
decoder = DecoderWithAttention(attention_dim=args.attention_dim,
embed_dim=args.emb_dim,
decoder_dim=args.decoder_dim,
vocab_size=len(word_map),
dropout=args.dropout)
elif args.decoder_mode == "transformer":
decoder = Transformer(vocab_size=len(word_map), embed_dim=args.emb_dim, encoder_layers=args.encoder_layers,
decoder_layers=args.decoder_layers, dropout=args.dropout,
attention_method=args.attention_method, n_heads=args.n_heads)
decoder_optimizer = torch.optim.Adam(params=filter(lambda p: p.requires_grad, decoder.parameters()),
lr=args.decoder_lr)
# load pre-trained word embedding
if args.embedding_path is not None:
all_word_embeds = {}
for i, line in enumerate(codecs.open(args.embedding_path, 'r', 'utf-8')):
s = line.strip().split()
all_word_embeds[s[0]] = np.array([float(i) for i in s[1:]])
# change emb_dim
args.emb_dim = list(all_word_embeds.values())[-1].size
word_embeds = np.random.uniform(-np.sqrt(0.06), np.sqrt(0.06), (len(word_map), args.emb_dim))
for w in word_map:
if w in all_word_embeds:
word_embeds[word_map[w]] = all_word_embeds[w]
elif w.lower() in all_word_embeds:
word_embeds[word_map[w]] = all_word_embeds[w.lower()]
else:
# <pad> <start> <end> <unk>
embedding_i = torch.ones(1, args.emb_dim)
torch.nn.init.xavier_uniform_(embedding_i)
word_embeds[word_map[w]] = embedding_i
word_embeds = torch.FloatTensor(word_embeds).to(device)
decoder.load_pretrained_embeddings(word_embeds)
decoder.fine_tune_embeddings(args.fine_tune_embedding)
print('Loaded {} pre-trained word embeddings.'.format(len(word_embeds)))
else:
checkpoint = torch.load(args.checkpoint, map_location=str(device))
start_epoch = checkpoint['epoch'] + 1
epochs_since_improvement = checkpoint['epochs_since_improvement']
best_bleu4 = checkpoint['metrics']["Bleu_4"]
encoder = checkpoint['encoder']
encoder_optimizer = checkpoint['encoder_optimizer']
decoder = checkpoint['decoder']
decoder_optimizer = checkpoint['decoder_optimizer']
decoder.fine_tune_embeddings(args.fine_tune_embedding)
# load final_args from checkpoint
final_args = checkpoint['final_args']
for key in final_args.keys():
args.__setattr__(key, final_args[key])
if args.fine_tune_encoder is True and encoder_optimizer is None:
print("Encoder_Optimizer is None, Creating new Encoder_Optimizer!")
encoder.fine_tune(args.fine_tune_encoder)
encoder_optimizer = torch.optim.Adam(params=filter(lambda p: p.requires_grad, encoder.parameters()),
lr=args.encoder_lr)
# Move to GPU, if available
decoder = decoder.to(device)
encoder = encoder.to(device)
print("encoder_layers {} decoder_layers {} n_heads {} dropout {} attention_method {} encoder_lr {} "
"decoder_lr {} alpha_c {}".format(args.encoder_layers, args.decoder_layers, args.n_heads, args.dropout,
args.attention_method, args.encoder_lr, args.decoder_lr, args.alpha_c))
print(encoder)
print(decoder)
# Loss function
criterion = nn.CrossEntropyLoss().to(device)
# Custom dataloaders
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
# normalize = transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
# pin_memory: If True, the data loader will copy Tensors into CUDA pinned memory before returning them.
# If your data elements are a custom type, or your collate_fn returns a batch that is a custom type.
train_loader = torch.utils.data.DataLoader(
CaptionDataset(args.data_folder, args.data_name, 'TRAIN', transform=transforms.Compose([normalize])),
batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
CaptionDataset(args.data_folder, args.data_name, 'VAL', transform=transforms.Compose([normalize])),
batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True)
# Epochs
for epoch in range(start_epoch, args.epochs):
# Decay learning rate if there is no improvement for 5 consecutive epochs, and terminate training after 25
# 8 20
if epochs_since_improvement == args.stop_criteria:
print("the model has not improved in the last {} epochs".format(args.stop_criteria))
break
if epochs_since_improvement > 0 and epochs_since_improvement % 5 == 0:
adjust_learning_rate(decoder_optimizer, 0.8)
if args.fine_tune_encoder and encoder_optimizer is not None:
print(encoder_optimizer)
adjust_learning_rate(encoder_optimizer, 0.8)
# One epoch's training
train(args, train_loader=train_loader, encoder=encoder, decoder=decoder, criterion=criterion,
encoder_optimizer=encoder_optimizer, decoder_optimizer=decoder_optimizer, epoch=epoch)
# One epoch's validation
metrics = validate(args, val_loader=val_loader, encoder=encoder, decoder=decoder, criterion=criterion)
recent_bleu4 = metrics["Bleu_4"]
# Check if there was an improvement
is_best = recent_bleu4 > best_bleu4
best_bleu4 = max(recent_bleu4, best_bleu4)
if not is_best:
epochs_since_improvement += 1
print("\nEpochs since last improvement: %d\n" % (epochs_since_improvement,))
else:
epochs_since_improvement = 0
# Save checkpoint
save_checkpoint(args.data_name, epoch, epochs_since_improvement, encoder, decoder, encoder_optimizer,
decoder_optimizer, metrics, is_best, final_args)
Traceback (most recent call last):
File D:\COCO\imge_captioning_transform_github\3\Image-Caption-master\train.py:394 in
train(args, train_loader=train_loader, encoder=encoder, decoder=decoder, criterion=criterion,
File D:\COCO\imge_captioning_transform_github\3\Image-Caption-master\train.py:44 in train
for i, (imgs, caps, caplens) in enumerate(train_loader):
File ~\anaconda3\envs\my_envir_gpu\lib\site-packages\torch\utils\data\dataloader.py:368 in iter
return self._get_iterator()
File ~\anaconda3\envs\my_envir_gpu\lib\site-packages\torch\utils\data\dataloader.py:314 in _get_iterator
return _MultiProcessingDataLoaderIter(self)
File ~\anaconda3\envs\my_envir_gpu\lib\site-packages\torch\utils\data\dataloader.py:927 in init
w.start()
File ~\anaconda3\envs\my_envir_gpu\lib\multiprocessing\process.py:121 in start
self._popen = self._Popen(self)
File ~\anaconda3\envs\my_envir_gpu\lib\multiprocessing\context.py:224 in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File ~\anaconda3\envs\my_envir_gpu\lib\multiprocessing\context.py:327 in _Popen
return Popen(process_obj)
File ~\anaconda3\envs\my_envir_gpu\lib\multiprocessing\popen_spawn_win32.py:93 in init
reduction.dump(process_obj, to_child)
File ~\anaconda3\envs\my_envir_gpu\lib\multiprocessing\reduction.py:60 in dump
ForkingPickler(file, protocol).dump(obj)
File ~\anaconda3\envs\my_envir_gpu\lib\site-packages\h5py_hl\base.py:368 in getnewargs
raise TypeError("h5py objects cannot be pickled")
TypeError: h5py objects cannot be pickled
2022-06-30 17:24:41.206091: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX AVX2
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2022-06-30 17:24:41.525476: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1532] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 3497 MB memory: -> device: 0, name: NVIDIA GeForce RTX 3060 Laptop GPU, pci bus id: 0000:01:00.0, compute capability: 8.6
2022-06-30 17:24:44.486920: W tensorflow/core/common_runtime/forward_type_inference.cc:231] Type inference failed. This indicates an invalid graph that escaped type checking. Error message: INVALID_ARGUMENT: expected compatible input types, but input 1:
type_id: TFT_OPTIONAL
args {
type_id: TFT_PRODUCT
args {
type_id: TFT_TENSOR
args {
type_id: TFT_LEGACY_VARIANT
}
}
}
is neither a subtype nor a supertype of the combined inputs preceding it:
type_id: TFT_OPTIONAL
args {
type_id: TFT_PRODUCT
args {
type_id: TFT_TENSOR
args {
type_id: TFT_INT32
}
}
}
while inferring type of node 'cond_40/output/_25'
2022-06-30 17:24:45.077383: I tensorflow/stream_executor/cuda/cuda_blas.cc:1786] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
Traceback (most recent call last):
File "", line 1, in
File "C:\Users\MSI\anaconda3\envs\my_envir_gpu\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Users\MSI\anaconda3\envs\my_envir_gpu\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
I am using Python 3.9, PyTorch 1.10 with Cuda 11.3 (WINDOWS 10)
Thanks,
I am trying ( num_workers=0 ) , but still same error
I have a following neural network code, and I get "Expected device cuda:0 but got device cpu in PyTorch" error and I can't figure out why. I assign the device to be cuda and the print line returns cuda. I've tried assign the device as device = cuda:0 as well just in case but that had no effect. Here's the code:
def run():
torch.multiprocessing.freeze_support()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
metabolites = pd.read_excel("testmetabolitedata.xlsx")
subject_metadata = pd.read_excel("testsubj.xlsx")
metabolitesdf = pd.DataFrame(data=metabolites)
metabolitesdf = metabolitesdf.iloc[:, 1:9153]
subjectsdf = pd.DataFrame(data=subject_metadata)
n_samples, n_metabolites = metabolitesdf.shape
print(n_samples)
#genotypes of the target gene
print(subjectsdf['SLCO1B1_rs4149056'])
genotypes = subjectsdf['SLCO1B1_rs4149056']
print(genotypes)
# print('{} unique genotypes'.format(len(set(genotypes))))
labels = [1 if g == 1 else 0 for g in genotypes]
print('{} samples with genotype 1 out of {} samples ({:.1%})'.format(sum(labels), len(labels),
sum(labels) / len(labels)))
#Insert 0 into index 0 (first) into the list for the first row with column names
labels.insert(0, 0)
#log transform
log_metabol = np.log10(metabolitesdf + 1)
#Split data into training and validation 70% / 30%
data = torch.utils.data.TensorDataset(torch.Tensor(np.array(log_metabol)),
torch.Tensor(labels))
train, val = torch.utils.data.random_split(data, [int(0.7 * len(data)),
len(data) - int(0.7 * len(data))])
print('{:.0f}/{} training/total ({:.1%}) in training set, {:.0f}/{} val/total ({:.1%}) in validation set'.format(\
train[:][1].sum(), len(train), train[:][1].sum() / len(train),
val[:][1].sum(), len(val), val[:][1].sum() / len(val)))
class MultiLayerPredictor(torch.nn.Module):
def __init__(self, input_shape, output_shape=1, hidden_dim=1024, **kwargs):
super().__init__()
self.fc1 = torch.nn.Linear(in_features=input_shape, out_features=hidden_dim)
self.bn1 = torch.nn.BatchNorm1d(hidden_dim)
self.fc2 = torch.nn.Linear(in_features=hidden_dim, out_features=hidden_dim)
self.bn2 = torch.nn.BatchNorm1d(hidden_dim)
self.fc3 = torch.nn.Linear(in_features=hidden_dim, out_features=output_shape)
def forward(self, x):
l1 = torch.relu(self.bn1(self.fc1(x)))
l2 = torch.relu(self.bn2(self.fc2(l1)))
return torch.sigmoid(self.fc3(l2)).reshape(-1)
#load the training and validation sets
print("Load training and validation data ")
train_loader = torch.utils.data.DataLoader(train, batch_size=128,
shuffle=True, num_workers=10, pin_memory=True)
val_loader = torch.utils.data.DataLoader(val, batch_size=128,
shuffle=False, num_workers=10, pin_memory=True)
print("Loading complete, create model")
model3 = MultiLayerPredictor(input_shape=n_metabolites).to(device)
print("Model created! Moving to optimizer")
optimizer3 = torch.optim.SGD(model3.parameters(), lr=1e-2)
print("Optimizer done")
objective3 = torch.nn.BCELoss()
epochs = 30
print_stats_interval = 10
log3 = []
print("Moving to training loop")
for epoch in range(epochs):
loss = n_correct = 0
model3.train()
for batch, target in train_loader:
batch = batch.view(-1, n_metabolites).to(device)
optimizer3.zero_grad()
outputs = model3(batch) # stack trace shows the issue being either on this line
train_loss = objective3(outputs, target) # or this line
loss += train_loss.item()
n_correct += (target == (outputs.reshape(-1) > 0.5).float()).sum()
train_loss.backward()
optimizer3.step()
loss = loss / len(train_loader)
acc = (n_correct.float() / len(train)).numpy()
epoch += 1
model3.eval();
val_loss = val_n_correct = 0
with torch.no_grad():
for batch, target in val_loader:
batch = batch.view(-1, n_metabolites).to(device)
outputs = model3(batch)
val_loss += objective3(outputs, target)
val_n_correct += (target == (outputs.reshape(-1) > 0.5).float()).sum()
val_loss = (val_loss / len(val_loader)).numpy()
val_acc = (val_n_correct.float() / len(val)).numpy()
if (epoch % print_stats_interval) == 0 or epoch == epochs:
print(f'epoch={epoch:.0f}, loss={loss:.5f}, val_loss={np.round(val_loss,5):.5f}, acc={np.round(acc,5):.5f}, val_acc={np.round(val_acc,5):.5f}')
log3.append((epoch, loss, val_loss, acc, val_acc))
log3 = pd.DataFrame(log3, columns=['epoch', 'loss', 'val_loss', 'acc', 'val_acc'])
plt.figure(figsize=(6, 3))
plt.plot(log3['epoch'], log3['loss'], label='Training');
plt.plot(log3['epoch'], log3['val_loss'], label='Validation');
plt.xlabel('Epoch'); plt.ylabel('Loss')
plt.legend();
val_log_mutations = val_hcc[:][0].numpy().reshape(-1)
val_true_labels = val_hcc[:][1].numpy() + 0
res = model3(val_hcc[:][0])
predictions = (res.detach().numpy().reshape(-1) > 0.5) + 0
correct = (val_true_labels == predictions) + 0
n_correct = correct.sum()
print('{}/{} ({:.1%}) in the validation set'.format(n_correct, len(correct), n_correct / len(correct)))
print('Majority classifier accuracy: {:.1%}'.format((len(correct) - val_true_labels.sum()) / len(correct)))
if __name__ == '__main__':
run()
What's going on here? The stack trace here:
Traceback (most recent call last):
File "//ad..fi/home/h/h/Desktop/neuralnet/neuralnet_train.py", line 142, in <module>
run()
File "//ad..fi/home/h/h/Desktop/neuralnet/neuralnet_train.py", line 99, in run
train_loss = objective3(outputs, target)
File "C:\Users\h\AppData\Roaming\Python\Python38\site-packages\torch\nn\modules\module.py", line 550, in __call__
result = self.forward(*input, **kwargs)
File "C:\Users\h\AppData\Roaming\Python\Python38\site-packages\torch\nn\modules\loss.py", line 516, in forward
return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)
File "C:\Users\h\AppData\Roaming\Python\Python38\site-packages\torch\nn\functional.py", line 2378, in binary_cross_entropy
return torch._C._nn.binary_cross_entropy(
RuntimeError: expected device cuda:0 but got device cpu
PS Microsoft.PowerShell.Core\FileSystem::\\ad..fi\home\h\h\Desktop\neuralnet>
Also move the targets to CUDA in both the training and validation for loops.
for batch, target in train_loader:
batch,target = batch.view(-1, n_metabolites).to(device),target.to(device)
.
.
.
for batch, target in val_loader:
batch,target = batch.view(-1, n_metabolites).to(device),target.to(device)``
.
.
.
I'm trying to save my GAN models. I didn't find much information on the internet and pretty confused about how to save— generator, discriminator, and GAN. Here is an issue I found, of which I coded accordingly. But even after saving and loading the models in this way, I get an error message You must compile a model before training/testing. Note that they have asked to use tensorflow.keras and not keras, but I've not understood that well.
How to save and load GAN so that intentional keyboard interruption can be made between epochs, and run again later?
Function to save GAN models:
def save_model_to_file(gan, generator, discriminator, epoch):
discriminator.trainable = False
gan.save('facegan-gannet-epoch:%02d.h5' % epoch)
discriminator.trainable = True
generator.save('facegan-generator-epoch:%02d.h5' % epoch)
discriminator.save('facegan-discriminator-epoch:%02d.h5' % epoch)
Have loaded the models it this way: (All params found trainable)
discriminator = load_model(models_paths["discriminator"])
discriminator.trainable = False
generator = load_model(models_paths["generator"])
gan = load_model(models_paths["gan"])
gan.summary()
discriminator.summary()
generator.summary()
Main train part:
generator = get_generator()
discriminator = get_discriminator()
gan = get_gan_network(discriminator, LATENT_DIM, generator, optimizer)
for epoch in range(1, epochs + 1):
print('\n', '\t' * 3, '-' * 4, 'Epoch %d' % epoch, '-' * 4)
for batch_count, image_batch in tqdm(enumerate(datagen)):
if batch_count == len(datagen): # len(datagen)
break
# Get a random set of input noise and images
noise = np.random.normal(0, 1, size=[BATCH_SIZE, LATENT_DIM])
# Generate fake images
generated_images = generator.predict(noise)
X = np.concatenate([image_batch, generated_images])
# Labels for generated and real data
y_dis = np.zeros(2 * BATCH_SIZE)
# One-sided label smoothing
y_dis[:BATCH_SIZE] = 0.9
# Train discriminator
discriminator.trainable = True
discriminator_loss = discriminator.train_on_batch(X, y_dis)
# Train generator
noise = np.random.normal(0, 1, size=[BATCH_SIZE, LATENT_DIM])
y_gen = np.ones(BATCH_SIZE)
discriminator.trainable = False
gannet_loss = gan.train_on_batch(noise, y_gen)
save_model_to_file(gan, generator, discriminator, epoch)
Full Error Message:
RuntimeError
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-29-740ccd719c22> in <module>
245
246 print("[INFO] Training started...")
--> 247 train(3, BATCH_SIZE, model_paths)
248 print("[INFO] Training completed.")
<ipython-input-29-740ccd719c22> in train(epochs, batch_size, models_paths)
225 # Train discriminator
226 discriminator.trainable = True
--> 227 discriminator_loss = discriminator.train_on_batch(X, y_dis)
228
229 # Train generator
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight, reset_metrics)
1506 x, y,
1507 sample_weight=sample_weight,
-> 1508 class_weight=class_weight)
1509 if self._uses_dynamic_learning_phase():
1510 ins = x + y + sample_weights + [1]
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
506 if y is not None:
507 if not self.optimizer:
--> 508 raise RuntimeError('You must compile a model before '
509 'training/testing. '
510 'Use `model.compile(optimizer, loss)`.')
RuntimeError: You must compile a model before training/testing. Use `model.compile(optimizer, loss)`.
You must compile a model before training/testing. Use `model.compile(optimizer, loss)`.
If you want to save and continue later, I recommend using save_weights and load_weights instead, which means that you create the model and compile it always, and then check wether there is a weights file, if there is such a file, load weights from it.
Something like this:
import os
g_file_name = 'your generator.h5'
d_file_name = 'your descriminator.h5'
(generator, descriminator) = create_your_models()
compile_your_models(generator, descriminator)
if os.path.isfile(g_file_name):
generator.load_weights(g_file_name)
if os.path.isfile(d_file_name):
descriminator.load_weights(d_file_name)
for epoch in range(num_epochs):
gan_training()
generator.save_weights(g_file_name)
descriminator.save_weights(g_file_name)
I'm working on a classification problem. The dataset has 9 classes, so I want to get the probabilities of 9 classes, but the model.predict() in keras looks like only output the biggest one. I think the shape of yprobs should be (3014,9), then I can get the log_loss() work. hoping for some help! Thanks!!
'''model code'''
class AdaBNModel:
def __init__(self, nfeatures=50, arch=[8, 'abn', 'act'], activations='relu',
droprate=0.0, noise=0.0, optimizer=None, val_data=None,
validate_every=1, epochs=5000, batch_size=128, verbose=False):
self.epochs = epochs
self.batch_size = batch_size
self.noise = noise
self.verbose = verbose
self.validate_every = validate_every
if val_data is None:
self.validate_every = 0
self.Xval = None
self.yval = None
else:
self.Xval = val_data[0]
self.yval = val_data[1]
self._build_model(arch, activations, nfeatures, droprate, noise, optimizer)
def _build_model(self, arch, activations, nfeatures, droprate, noise, optimizer):
self.layers = [Input(shape=(nfeatures,))]
for i, nunits in enumerate(arch):
if isinstance(nunits, int):
self.layers += [Dense(nunits, activation='linear')(self.layers[-1])]
elif nunits == 'noise':
self.layers += [GaussianNoise(noise)(self.layers[-1])]
elif nunits == 'bn':
self.layers += [BatchNormalization()(self.layers[-1])]
elif nunits == 'abn':
self.layers += [AdaBN()(self.layers[-1])]
elif nunits == 'drop':
self.layers += [Dropout(droprate)(self.layers[-1])]
elif nunits == 'act':
if activations == 'prelu':
self.layers += [PReLU()(self.layers[-1])]
elif activations == 'elu':
self.layers += [ELU()(self.layers[-1])]
elif activations == 'leakyrelu':
self.layers += [LeakyReLU()(self.layers[-1])]
else:
self.layers += [Activation(activations)(self.layers[-1])]
else:
print('Unrecognised layer {}, type: {}'.format(nunits, type(nunits)))
self.layers += [Dense(1, activation='sigmoid')(self.layers[-1])]
self.model = Model(self.layers[0], self.layers[-1])
self.model.compile(loss='sparse_categorical_crossentropy', optimizer=optimizer)
def _fit(self, Xs, ys, Xt, yt, domains, Xval=None, yval=None, epochs=None, batch_size=None, verbose=None):
if epochs is None: epochs = self.epochs
if batch_size is None: batch_size = self.batch_size
if Xval is None:
Xval = self.Xval
yval = self.yval
if verbose is None: verbose = self.verbose
# batch generator that ensures that samples are always from the same domain
S_batches = domain_batch_gen(Xs, ys, domains, batch_size=batch_size)
# self.history = {'source_loss': [], 'target_loss': [], 'val_loss': []}
self.history = {'source_loss': [], 'target_loss': []}
for i in range(epochs):
Xsource, ysource = S_batches.__next__()
self.model.fit(Xsource, ysource, epochs=1, batch_size=batch_size, verbose=0)
#if self.validate_every > 0 and i % self.validate_every == 0:
if True:
if i == 0:
print('Epoch sloss tloss vloss')
self.history['source_loss'] += [self.evaluate(Xs, ys)]
self.history['target_loss'] += [self.evaluate(Xt, yt)]
#self.history['val_loss'] += [self.evaluate(Xval, yval)]
print('{:04d} {:.5f} {:.5f} '.format(i,
self.history['source_loss'][-1],
self.history['target_loss'][-1]))
# print('{:04d} {:.5f} {:.5f} {:.5f} '.format(i,
# self.history['source_loss'][-1], self.history['target_loss'][-1], self.history['val_loss'][-1]))
def fit(self, Xs, ys, Xt, yt, domains=None, Xval=None, yval=None, epochs=None, batch_size=None, verbose=None):
if epochs is None: epochs = self.epochs
if batch_size is None: batch_size = self.batch_size
if Xval is None:
Xval = self.Xval
yval = self.yval
if verbose is None: verbose = self.verbose
if domains is None: domains = np.ones_like(ys, dtype=int)
self._fit(Xs, ys, Xt, yt, domains, Xval, yval, epochs, batch_size, verbose)
def predict_proba(self, X, domains=None, batch_size=None):
if batch_size is None: batch_size = self.batch_size
if domains is None:
return self.model.predict(X,batch_size,verbose=1)
else:
ypreds = np.zeros(X.shape[0])
udomains = np.unique(domains)#[1]
#print(udomains.shape)
for i in range(udomains.shape[0]):
idx = (domains == udomains[i])
# print(idx)
# print(idx.shape) #(3014,)
print('domain')
thisX = X[idx]
# print(thisX)
# print(thisX.shape) #(3014, 145)
# ypreds[idx] = self.model.predict(thisX, batch_size=batch_size).flatten()
ypreds[idx] = self.model.predict(thisX, batch_size=batch_size).flatten()
return ypreds
def evaluate(self, X, y, domains=None, batch_size=None):
#yprobs = self.predict_proba(X, domains, batch_size)
yprobs = self.model.predict(X,batch_size,verbose=1)
print(yprobs)
print(yprobs.shape)#(3014,1)
print(y.shape) #(3014,)
return log_loss(y, yprobs)
Traceback (most recent call last):
32/3014 [..............................] - ETA: 1s
3014/3014 [==============================] - 0s 5us/step
[[0.99279547]**strong text**
File "D:/cmWorks/AdaBN-1d/example_adabn.py", line 33, in <module>
model.fit(Xs, ys, Xt, yt)
File "D:\cmWorks\AdaBN-1d\adabn.py", line 124, in fit
self._fit(Xs, ys, Xt, yt, domains, Xval, yval, epochs, batch_size, verbose)
File "D:\cmWorks\AdaBN-1d\adabn.py", line 103, in _fit
[0.9899932 ]
self.history['source_loss'] += [self.evaluate(Xs, ys)]
[0.9922549 ]
File "D:\cmWorks\AdaBN-1d\adabn.py", line 154, in evaluate
return log_loss(y, yprobs)
...
[0.9604445 ]
[0.9100603 ]
[0.95761013]]
(3014, 1)
(3014,)
[8. 8. 8. ... 6. 6. 6.]
File "E:\Anaconda3\envs\keras\lib\site-packages\sklearn\metrics\classification.py", line 1809, in log_loss
lb.classes_))
ValueError: y_true and y_pred contain different number of classes 9, 2. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [0. 1. 2. 3. 4. 5. 6. 7. 8.]
Occam's razor is always your best initial debugging tool, look at this line for the last layer of your model:
self.layers += [Dense(1, activation='sigmoid')(self.layers[-1])]
Your model only has one output neuron, not nine as it should be for 9 classes. This is why keras predicts only one probability per sample.
Also note that for multi-class classification you should use a softmax activation at the last layer, unless you want to do multi-label classification, for which sigmoid is used.
I am trying to build tensorflow input function with tf.train.batch(). I have dataframe for train, eval and prediction. So input_fn should take argument for df, batch_size. In df there are continuous and categorical columns.
Revised code:
COLUMNS = ['atemp', 'holiday', 'humidity', 'season', 'temp', 'weather', 'windspeed', 'workingday', 'hour', 'weekday', 'month', 'label']
CONTINUOUS_COLUMNS = ['atemp', 'humidity', 'temp', 'windspeed',]
CATEGORICAL_COLUMNS =[ 'holiday', 'season', 'weather',
'workingday', 'weekday', 'month', 'hour' ]
LEARNING_RATE = 0.1
LABEL_COLUMN = 'label'
batch_size = 128
data_set = pd.read_excel('bike_str.xlsx')
# Split the data into a training set, an eval set and a pred set.
train_set = data_set[:9500]
eval_set = data_set[9500:10800]
pred_set = data_set[10800:]
## Eval and Prediction labels:
eval_label = eval_set['label']
pred_label = pred_set['label']
Input_fn:
def batch_input_fn(df, batch_size):
def input_fn():
"""Input builder function."""
# Creates a dictionary mapping from each continuous feature column name (k) to
# the values of that column stored in a constant Tensor.
continuous_cols = {k: tf.constant(df[k].values) for k in CONTINUOUS_COLUMNS}
# Creates a dictionary mapping from each categorical feature column name (k)
# to the values of that column stored in a tf.SparseTensor.
categorical_cols = {
k: tf.SparseTensor(
indices=[[i, 0] for i in range(df[k].size)],
values=df[k].values,
dense_shape=[df[k].size, 1])
for k in CATEGORICAL_COLUMNS}
# Merges the two dictionaries into one.
x = dict(continuous_cols)
x.update(categorical_cols)
# Converts the label column into a constant Tensor.
y = tf.constant(df[LABEL_COLUMN].values)
# Returns the feature columns and the label.
sliced_input = tf.train.slice_input_producer([x, y], shuffle = shuffle)
# So i'm trying to shuffle data for train and not shuffle for eval and pred
return tf.train.batch(sliced_input, batch_size=batch_size, num_threads= 3)
return input_fn
## Continuous base columns.
atemp = tf.contrib.layers.real_valued_column('atemp')
humidity = tf.contrib.layers.real_valued_column('humidity')
temp = tf.contrib.layers.real_valued_column('temp')
windspeed = tf.contrib.layers.real_valued_column('windspeed')
## Categoric base columns:
### To define a feature column for a categorical feature, we can create a SparseColumn
holiday = tf.contrib.layers.sparse_column_with_keys(column_name="holiday", keys=["no", "yes"])
season = tf.contrib.layers.sparse_column_with_keys(column_name="season", keys=["spring", "summer", "fall","winter"])
feat_dnn = [atemp_b, humidity_b, windspeed_b, temp_b,
tf.contrib.layers.embedding_column(holiday, dimension=1)
]
dnnregressor = tf.contrib.learn.DNNRegressor(feature_columns= feat_dnn,
hidden_units=[512,256, 512],
optimizer=tf.train.FtrlOptimizer(
learning_rate=0.250, l1_regularization_strength=0.8, l2_regularization_strength=0.8),
activation_fn =tf.nn.relu, dropout = 0.08)
dnnregressor.fit(input_fn= lambda: batch_input_fn(train_set, batch_size, shuffle = True), steps=1000 )
When calling directly batch_input_fn, error is:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-8-9c356159093d> in <module>()
----> 1 dnnregressor.fit(input_fn= lambda: batch_input_fn(train_set, batch_size), steps=15000 )
C:\Python\Anaconda\lib\site-packages\tensorflow\python\util\deprecation.py in new_func(*args, **kwargs)
287 'in a future version' if date is None else ('after %s' % date),
288 instructions)
--> 289 return func(*args, **kwargs)
290 return tf_decorator.make_decorator(func, new_func, 'deprecated',
291 _add_deprecated_arg_notice_to_docstring(
C:\Python\Anaconda\lib\site-packages\tensorflow\contrib\learn\python\learn\estimators\estimator.py in fit(self, x, y, input_fn, steps, batch_size, monitors, max_steps)
453 hooks.append(basic_session_run_hooks.StopAtStepHook(steps, max_steps))
454
--> 455 loss = self._train_model(input_fn=input_fn, hooks=hooks)
456 logging.info('Loss for final step: %s.', loss)
457 return self
C:\Python\Anaconda\lib\site-packages\tensorflow\contrib\learn\python\learn\estimators\estimator.py in _train_model(self, input_fn, hooks)
951 random_seed.set_random_seed(self._config.tf_random_seed)
952 global_step = contrib_framework.create_global_step(g)
--> 953 features, labels = input_fn()
954 self._check_inputs(features, labels)
955 model_fn_ops = self._get_train_ops(features, labels)
TypeError: 'function' object is not iterable
From this code it's seems to work but here tensors are not list of dict:
def batched_input_fn(dataset_x, dataset_y, batch_size):
def _input_fn():
all_x = tf.constant(dataset_x, shape=dataset_x.shape, dtype=tf.float32)
all_y = tf.constant(dataset_y, shape=dataset_y.shape, dtype=tf.float32)
sliced_input = tf.train.slice_input_producer([all_x, all_y])
return tf.train.batch(sliced_input, batch_size=batch_size)
return _input_fn