Develop Reference

training multiple sequence alignment in autoencoder

I have some code that I am trying to run. But I have a problem with the code. The code is from the internet page. My data is multiple sequence alignment of a specific protein. Firstly I applied one-hot encoding and then tried to put it into a variational autoencoder for training. But I failed. when I ran this code, I got the error message about ''' Input 0 of layer "encoder" is incompatible with the layer: expected shape=(None, 28, 28, 1), found shape=(None, 13, 1)''''
As I am a beginner in this field, I don't know where I made a mistake. would you please help with this issue? Or how to train it with multiple sequence alignment data?
###################################
datax = pd.DataFrame({'sequence':['ACKIENIKYKGKEVESKLGSQLIDIFNDLDRAKEEYDKLSSPEFIAKFGDWINDEVERNVNEDGEPLLIQDVRQDSSKHYFFILKNGERFDLLTR','---------TGKEVQSSLFDQILELVKDPQLAQDAYAQIRTPEFIEKFGDWINDPYESKLDDNGEPILIE-------------------------','-------PNTGKEVQSSLFDQIIELIKDPKLAQDAYEQVRTPEFIEKFGDWINDPYAAKLDDNGEPILVERD-----------------------','-------PNQGKEVESKLYNDLVNLTGSEKAADEAYEQVHHPNFLRWFGDWINNKVSNVVDENGEPLIV--------------------------','ACRIIISLNKGKEVKSKLFDSLLDLTQSEAKAEEAYKKVFSEEFINWFGDWINTPASKVVDENGEPLMVYRFTQEE-------------------','-------PNKGKEVRSKLYDDLLELTGDDNAALEAYKKVHHPNFLRWFGDWINNKVSKVVDENGEPLIVY-------------------------','-------PNKGKEVESKLYNDLVNLTGSEKAADEAYTKVHHPNFLRWFGDWMTNPSSKVVDENGEPKIV--------------------------','-------PNKGQRVDSILYNDLLSLTGNEKSADKAYTKAHTSSFLNWFGDWINTNIQDNVDQNGEPKV---------------------------','-----------------------------NLTSEQYKLVRTPEFIAWFGNWMDDPNASVIDENGEPLVCFH-GTDNSFHIF--------------','---------------------------------KQWVQVRTPAFIEWFGDWMNDPASKGVDENGEPLVVYHGTENKFTQYDFD------------','-------------------------------TPKQYKLVRTLEFKAWFGDWENDPASKVVDENGEPLVVYH--GTDSKHNVFSYEQ---------','--------------GSSLYEQYVVIIDSSNLTTEQYKLVRTPEFKKWFGDWENNPSEAVVDENGEPLVVYH------------------------','------------------------------LTPEQYKLVRTPEFKAWFGDWENNPSSKVVDDNGEPMVVY---HGSRKKAFTEFK----------']})
from sklearn.preprocessing import OneHotEncoder
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import pandas as pd
encoder = OneHotEncoder(sparse=False)
onehot = encoder.fit_transform(datax)
print(onehot)
class Sampling(layers.Layer):
"""Uses (z_mean, z_log_var) to sample z, the vector encoding a digit."""
def call(self, inputs):
z_mean, z_log_var = inputs
batch = tf.shape(z_mean)[0]
dim = tf.shape(z_mean)[1]
epsilon = tf.keras.backend.random_normal(shape=(batch, dim))
return z_mean + tf.exp(0.5 * z_log_var) * epsilon
latent_dim = 10
encoder_inputs = keras.Input(shape=(28, 28, 1))
x = layers.Conv2D(32, 3, activation="relu", strides=2, padding="same")(encoder_inputs)
x = layers.Conv2D(64, 3, activation="relu", strides=2, padding="same")(x)
x = layers.Flatten()(x)
x = layers.Dense(16, activation="relu")(x)
z_mean = layers.Dense(latent_dim, name="z_mean")(x)
z_log_var = layers.Dense(latent_dim, name="z_log_var")(x)
z = Sampling()([z_mean, z_log_var])
encoder = keras.Model(encoder_inputs, [z_mean, z_log_var, z], name="encoder")
encoder.summary()
latent_inputs = keras.Input(shape=(latent_dim,))
x = layers.Dense(7 * 7 * 64, activation="relu")(latent_inputs)
x = layers.Reshape((7, 7, 64))(x)
x = layers.Conv2DTranspose(64, 3, activation="relu", strides=2, padding="same")(x)
x = layers.Conv2DTranspose(32, 3, activation="relu", strides=2, padding="same")(x)
decoder_outputs = layers.Conv2DTranspose(1, 3, activation="sigmoid", padding="same")(x)
decoder = keras.Model(latent_inputs, decoder_outputs, name="decoder")
decoder.summary()
class VAE(keras.Model):
def __init__(self, encoder, decoder, **kwargs):
super(VAE, self).__init__(**kwargs)
self.encoder = encoder
self.decoder = decoder
self.total_loss_tracker = keras.metrics.Mean(name="total_loss")
self.reconstruction_loss_tracker = keras.metrics.Mean(
name="reconstruction_loss"
)
self.kl_loss_tracker = keras.metrics.Mean(name="kl_loss")
#property
def metrics(self):
return [
self.total_loss_tracker,
self.reconstruction_loss_tracker,
self.kl_loss_tracker,
]
def train_step(self, data):
with tf.GradientTape() as tape:
z_mean, z_log_var, z = self.encoder(data)
reconstruction = self.decoder(z)
reconstruction_loss = tf.reduce_mean(
tf.reduce_sum(
keras.losses.binary_crossentropy(datax, reconstruction), axis=(1, 2)
)
)
kl_loss = -0.5 * (1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var))
kl_loss = tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1))
total_loss = reconstruction_loss + kl_loss
grads = tape.gradient(total_loss, self.trainable_weights)
self.optimizer.apply_gradients(zip(grads, self.trainable_weights))
self.total_loss_tracker.update_state(total_loss)
self.reconstruction_loss_tracker.update_state(reconstruction_loss)
self.kl_loss_tracker.update_state(kl_loss)
return {
"loss": self.total_loss_tracker.result(),
"reconstruction_loss": self.reconstruction_loss_tracker.result(),
"kl_loss": self.kl_loss_tracker.result(),
}
data_digits = np.concatenate([train, test], axis=0)
data_digits = np.expand_dims(data_digits, -1).astype("float32") / 255
vae = VAE(encoder, decoder)
vae.compile(optimizer=keras.optimizers.Adam())
vae.fit(mnist_digits, epochs=1, batch_size=128)
##########################################

ValueError: Layer "vq_vae" expects 1 input(s), but it received 2 input tensors on a VQVAE

I am training a VQVAE with this dataset (64x64x3). I have downloaded it locally and loaded it with keras in Jupyter notebook. The problem is that when I ran fit() to train the model I get this error: ValueError: Layer "vq_vae" expects 1 input(s), but it received 2 input tensors. Inputs received: [<tf.Tensor 'IteratorGetNext:0' shape=(None, 64, 64, 3) dtype=float32>, <tf.Tensor 'IteratorGetNext:1' shape=(None,) dtype=int32>] . I have taken most of the code from here and adapted it myself. But for some reason I can't make it work for other datasets. You can ignore most of the code here and check it in the page, help is much appreciated.
The code I have so far:
img_height = 64
img_width = 64
dataset = tf.keras.utils.image_dataset_from_directory(directory="PATH",
image_size=(64, 64), batch_size=64, shuffle=True)
normalization_layer = tf.keras.layers.Rescaling(1./255)
normalized_ds = dataset.map(lambda x, y: (normalization_layer(x), y))
AUTOTUNE = tf.data.AUTOTUNE
train_ds = normalized_ds.cache().prefetch(buffer_size=AUTOTUNE)
VQVAE code:
class VectorQuantizer(layers.Layer):
def __init__(self, num_embeddings, embedding_dim, beta=0.25, **kwargs):
super().__init__(**kwargs)
self.embedding_dim = embedding_dim
self.num_embeddings = num_embeddings
self.beta = (beta)
w_init = tf.random_uniform_initializer()
self.embeddings = tf.Variable(initial_value=w_init(shape=(self.embedding_dim, self.num_embeddings), dtype="float32"),
trainable=True,
name="embeddings_vqvae",)
def call(self, x):
input_shape = tf.shape(x)
flattened = tf.reshape(x, [-1, self.embedding_dim])
# Quantization.
encoding_indices = self.get_code_indices(flattened)
encodings = tf.one_hot(encoding_indices, self.num_embeddings)
quantized = tf.matmul(encodings, self.embeddings, transpose_b=True)
quantized = tf.reshape(quantized, input_shape)
commitment_loss = self.beta * tf.reduce_mean((tf.stop_gradient(quantized) - x) ** 2)
codebook_loss = tf.reduce_mean((quantized - tf.stop_gradient(x)) ** 2)
self.add_loss(commitment_loss + codebook_loss)
# Straight-through estimator.
quantized = x + tf.stop_gradient(quantized - x)
return quantized
def get_code_indices(self, flattened_inputs):
# Calculate L2-normalized distance between the inputs and the codes.
similarity = tf.matmul(flattened_inputs, self.embeddings)
distances = (
tf.reduce_sum(flattened_inputs ** 2, axis=1, keepdims=True)
+ tf.reduce_sum(self.embeddings ** 2, axis=0)
- 2 * similarity
)
# Derive the indices for minimum distances.
encoding_indices = tf.argmin(distances, axis=1)
return encoding_indices
def get_vqvae(latent_dim=16, num_embeddings=64):
vq_layer = VectorQuantizer(num_embeddings, latent_dim,
name="vector_quantizer")
encoder = get_encoder(latent_dim)
decoder = get_decoder(latent_dim)
inputs = keras.Input(shape=(64, 64, 3))
encoder_outputs = encoder(inputs)
quantized_latents = vq_layer(encoder_outputs)
reconstructions = decoder(quantized_latents)
return keras.Model(inputs, reconstructions, name="vq_vae")
class VQVAETrainer(keras.models.Model):
def __init__(self, train_variance, latent_dim=32, num_embeddings=128, **kwargs):
super(VQVAETrainer, self).__init__(**kwargs)
self.train_variance = train_variance
self.latent_dim = latent_dim
self.num_embeddings = num_embeddings
self.vqvae = get_vqvae(self.latent_dim, self.num_embeddings)
self.total_loss_tracker = keras.metrics.Mean(name="total_loss")
self.reconstruction_loss_tracker = keras.metrics.Mean(
name="reconstruction_loss"
)
self.vq_loss_tracker = keras.metrics.Mean(name="vq_loss")
#property
def metrics(self):
return [
self.total_loss_tracker,
self.reconstruction_loss_tracker,
self.vq_loss_tracker,]
def train_step(self, x):
with tf.GradientTape() as tape:
# Outputs from the VQ-VAE.
reconstructions = self.vqvae(x)
# Calculate the losses.
reconstruction_loss = (
tf.reduce_mean((x - reconstructions) ** 2) / self.train_variance
)
total_loss = reconstruction_loss + sum(self.vqvae.losses)
# Backpropagation.
grads = tape.gradient(total_loss, self.vqvae.trainable_variables)
self.optimizer.apply_gradients(zip(grads, self.vqvae.trainable_variables))
# Loss tracking.
self.total_loss_tracker.update_state(total_loss)
self.reconstruction_loss_tracker.update_state(reconstruction_loss)
self.vq_loss_tracker.update_state(sum(self.vqvae.losses))
# Log results.
return {
"loss": self.total_loss_tracker.result(),
"reconstruction_loss": self.reconstruction_loss_tracker.result(),
"vqvae_loss": self.vq_loss_tracker.result(),
}
get_vqvae().summary()
Encoder and Decoder (I have made changes here but I dont think this is the problem):
def get_encoder(latent_dim=16):
encoder_inputs = keras.Input(shape=(64, 64, 3))
x = layers.Conv2D(64, 3, padding="same")(encoder_inputs)
x = layers.Dropout(0.25)(x)
x = layers.Activation("relu")(x)
x = layers.UpSampling2D()(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Conv2D(64, 3, strides=2, padding="same")(x)
x = layers.Dropout(0.25)(x)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Conv2D(128, 3, strides=2, padding="same")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2D(128, 3, strides=2, padding="same")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2D(256, 3, strides=2, padding="same")(x)
x = layers.Activation("relu")(x)
x = layers.UpSampling2D()(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Dense(4*4*128)(x)
encoder_outputs = layers.Conv2D(latent_dim, 1, padding="same")(x)
return keras.Model(encoder_inputs, encoder_outputs, name="encoder")
get_encoder().summary()
def get_decoder(latent_dim=16):
latent_inputs = keras.Input(shape=get_encoder().output.shape[1:])
x = layers.Conv2DTranspose(32, 3, padding="same")(latent_inputs)
x = layers.Dropout(0.25)(x)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Conv2DTranspose(32, 3, padding="same")(latent_inputs)
x = layers.Dropout(0.25)(x)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Conv2DTranspose(64, 3, padding="same")(x)
x = layers.Dropout(0.25)(x)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Conv2DTranspose(128, 3, strides=2, padding="same")(x)
x = layers.Dropout(0.25)(x)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Conv2DTranspose(256, 3, padding="same")(x)
x = layers.Dropout(0.25)(x)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
x = layers.Conv2DTranspose(512, 3, strides=2, padding="same")(x)
x = layers.Dropout(0.25)(x)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization(momentum=0.8)(x)
decoder_outputs = layers.Conv2DTranspose(1, 3, padding="same")(x)
return keras.Model(latent_inputs, decoder_outputs, name="decoder")
get_decoder().summary()
I get the error here:
vqvae_trainer = VQVAETrainer(data_variance, latent_dim=16,
num_embeddings=128)
vqvae_trainer.compile(optimizer=tf.keras.optimizers.Adam())
vqvae_trainer.fit(train_ds, epochs=30, batch_size=128)

This kind of model does not work with labels. Try running:
normalized_ds = dataset.map(lambda x, y: normalization_layer(x))
to discard the labels, since you are actually only interested in x.

'int' object is not subscriptable in vae.fit() function

I am developing a VAE using this: dataset
I have used keras tutorial code and I have developed my own encoder and decoder, the problem is that when I run vae.fit() I get 'int' object is not subscriptable. What am I doing wrong?
df = pd.read_csv('local path')
xtrain, xtest = train_test_split(df, test_size=0.2)
encoder:
def encoder(input_shape):
inputs = keras.Input(shape=input_shape)
x = layers.Dense(128, activation='relu')(inputs)
x = layers.Dense(128, activation='relu')(x)
z_mean = layers.Dense(2, name='z_mean')(x)
z_log_var = layers.Dense(2, name='z_log_var')(x)
z = Sampling()([z_mean, z_log_var])
encoder = keras.Model(inputs, [z_mean, z_log_var, z], name='encoder')
encoder.summary()
return encoder
decoder:
def decoder(input_shape):
inputs = keras.Input(shape=input_shape)
x = layers.Dense(128, activation='relu')(inputs)
x = layers.Dense(128, activation='relu')(x)
outputs = layers.Dense(input_shape[0], activation='sigmoid')(x)
decoder = keras.Model(inputs, outputs, name='decoder')
decoder.summary()
return decoder
VAE class:
class VAE(keras.Model):
def __init__(self, encoder, decoder, **kwargs):
super(VAE, self).__init__(**kwargs)
self.encoder = encoder
self.decoder = decoder
self.total_loss_tracker = keras.metrics.Mean(name="total_loss")
self.reconstruction_loss_tracker = keras.metrics.Mean(
name="reconstruction_loss"
)
self.kl_loss_tracker = keras.metrics.Mean(name="kl_loss")
#property
def metrics(self):
return [
self.total_loss_tracker,
self.reconstruction_loss_tracker,
self.kl_loss_tracker,
]
def train_step(self, data):
with tf.GradientTape() as tape:
z_mean, z_log_var, z = self.encoder(data)
reconstruction = self.decoder(z)
reconstruction_loss = tf.reduce_mean(
tf.reduce_sum(
keras.losses.binary_crossentropy(data, reconstruction), axis=(1, 2)
)
)
kl_loss = -0.5 * (1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var))
kl_loss = tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1))
total_loss = reconstruction_loss + kl_loss
grads = tape.gradient(total_loss, self.trainable_weights)
self.optimizer.apply_gradients(zip(grads, self.trainable_weights))
self.total_loss_tracker.update_state(total_loss)
self.reconstruction_loss_tracker.update_state(reconstruction_loss)
self.kl_loss_tracker.update_state(kl_loss)
return {
"loss": self.total_loss_tracker.result(),
"reconstruction_loss": self.reconstruction_loss_tracker.result(),
"kl_loss": self.kl_loss_tracker.result(),
}
This is where I get the error:
data = np.concatenate([xtrain.values, xtest.values])
vae = VAE(encoder(data.shape[1]),
decoder(data.shape[1]))
vae.compile(optimizer="adam",
loss="binary_crossentropy")
vae.fit(data, epochs=10, batch_size=32,
validation_split=0.2)
Full traceback:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
c:\Users\User\Documents\Github\Generative-Models\TFG\VAE.ipynb Cell 9' in <cell line: 3>()
1 data = np.concatenate([xtrain.values, xtest.values])
----> 3 vae = VAE(encoder(data.shape[1]), decoder(data.shape[1]))
4 vae.compile(optimizer="adam", loss="binary_crossentropy")
5 vae.fit(data, epochs=10, batch_size=32, validation_split=0.2)
c:\Users\User\Documents\Github\Generative-Models\TFG\VAE.ipynb Cell 7' in
decoder(input_shape)
3 x = layers.Dense(128, activation='relu')(inputs)
4 x = layers.Dense(128, activation='relu')(x)
----> 5 outputs = layers.Dense(input_shape[0], activation='sigmoid')(x)
6 decoder = keras.Model(inputs, outputs, name='decoder')
7 decoder.summary()
TypeError: 'int' object is not subscriptable
What should I change? help is much appreciated.

The encoder and decoder functions expect an input_shape sequence. But with
vae = VAE(
encoder(data.shape[1]),
decoder(data.shape[1])
)
you are passing int values.
You can fix this by passing in a sequence of int values. For example with
vae = VAE(
encoder(data.shape[1:]),
decoder(data.shape[1:])
)
This assumes that the shape of data is (samples, features). Then your input_shape will be (features,).

Conditional Variational AutoEncoder

I tried to implement conditional variational auto encoder, using variational auto encoder at the Keras website : https://keras.io/examples/generative/vae/
I added the second input to the model but I don't know how to fit two inputs to encoder. I think custom train_step() function has some problem and should be updated according to encoder that has two inputs.
I got this error:
ValueError: Layer decoder expects 2 input(s), but it received 1 input tensors. Inputs received: [<tf.Tensor 'encoder/12_enc_sampling/add:0' shape=(8, 8) dtype=float32>]
and here is my script:
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import matplotlib.pyplot as plt
from tensorflow.keras.utils import plot_model
from VAEfunctions import track_loss, sample_decoding, test, latent_space_mds
# Add Graphviz to windows PATH for plot_model
import os
# os.environ["PATH"] += os.pathsep + 'C:/Program Files/Graphviz/bin/'
print("tensorflow version:",tf.__version__)
# =====================================Set initial parameters ==============================
epoch = 3
batch_size = 8
latent_dim = 8
random_sample_points = 3 #Number of random sample points from latent space
# ==========================================================================
# for seed_value in range(7, 8): # change the seed to find the best seed_value
# Seed value
seed_value = 7 #7 #8good
# 1. Set `PYTHONHASHSEED` environment variable at a fixed value
import os
os.environ['PYTHONHASHSEED'] = str(seed_value)
# 2. Set `python` built-in pseudo-random generator at a fixed value
import random
random.seed(seed_value)
# 3. Set `numpy` pseudo-random generator at a fixed value
import numpy as np
np.random.seed(seed_value)
# 4. Set the `tensorflow` pseudo-random generator at a fixed value
import tensorflow as tf
tf.random.set_seed(seed_value)
# for later versions:
# tf.compat.v1.set_random_seed(seed_value)
# 5. Configure a new global `tensorflow` session
session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf)
tf.compat.v1.keras.backend.set_session(sess)
# ========================================Load data===============================
x_train = np.load('data_x_train_3D_all_par_3D_800_64_64_64_1.npy') # load data, #shape:(800,64,64,64,1)
x_test = np.load('data_x_test_3D_all_par_3D_200_64_64_64_1.npy')
x = np.load('data_x_3D_all_par_3D_1000_64_64_64_1.npy')
input_num = x_train.shape[0]
input_d1 = x_train.shape[1]
input_d2 = x_train.shape[2]
input_d3 = x_train.shape[3]
input_channels = x_train.shape[4]
# ================================== Load one_hot_vector (Parameters) ===================
one_hot_vec_train = np.load('one_hot_vec.npy')[:800]
print(one_hot_vec_train.shape)
# ======================View a few images=======================
print("min of x_train is:", np.min(x_train), "max of x_train is:", np.max(x_train))
print("min of x is:", np.min(x), "max of x is:", np.max(x))
plt.figure(10)
fig, axes = plt.subplots(2, 2)
# fig.tight_layout()
for ax in axes.flat:
i = 1
im = ax.imshow(x_train[100*i][31][:][:], vmin=0, vmax=1)
i = i+1
cbar_ax = fig.add_axes([0.9, 0.15, 0.03, 0.70])
fig.colorbar(im, cax=cbar_ax)
plt.suptitle('Slide[31] of four input samples')
plt.show()
""""****************************** Variational Auto Encoder Model *************************"""
# ======================================= sampling layer ===============================================
class Sampling(layers.Layer):
"""Uses (z_mean, z_log_var) to sample z, the vector encoding a digit."""
# tf.random.set_seed(0)
def call(self, inputs):
z_mean, z_log_var = inputs
batch = tf.shape(z_mean)[0]
dim = tf.shape(z_mean)[1]
epsilon = tf.keras.backend.random_normal(shape=(batch, dim))
return z_mean + tf.exp(0.5 * z_log_var) * epsilon
# ===================== Define the VAE as a `Model` with a custom `train_step` ==============================
class CVAE(keras.Model):
# tf.random.set_seed(0)
def __init__(self, encoder, decoder, **kwargs):
super(CVAE, self).__init__(**kwargs)
self.encoder = encoder
self.decoder = decoder
self.total_loss_tracker = keras.metrics.Mean(name="total_loss")
self.reconstruction_loss_tracker = keras.metrics.Mean(
name="reconstruction_loss"
)
self.kl_loss_tracker = keras.metrics.Mean(name="kl_loss")
self.rec_tracker = keras.metrics.Mean(name="rec") ####
#property
def metrics(self):
return [
self.total_loss_tracker,
self.reconstruction_loss_tracker,
self.kl_loss_tracker,
self.rec_tracker, ####
]
def train_step(self, data):
with tf.GradientTape() as tape:
z_mean, z_log_var, z = self.encoder(data)
reconstruction = self.decoder(z)
reconstruction_loss = tf.reduce_mean(
tf.reduce_sum(
keras.losses.mean_squared_error(data, reconstruction), axis=(1, 2, 3)
)
)
rec = tf.reduce_mean( #####
tf.reduce_mean(
keras.losses.mean_squared_error(data, reconstruction), axis=(1, 2, 3)
)
)
kl_loss = -0.5 * (1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var))
kl_loss = tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1))
total_loss = reconstruction_loss + kl_loss
grads = tape.gradient(total_loss, self.trainable_weights)
self.optimizer.apply_gradients(zip(grads, self.trainable_weights))
self.total_loss_tracker.update_state(total_loss)
self.reconstruction_loss_tracker.update_state(reconstruction_loss)
self.kl_loss_tracker.update_state(kl_loss)
self.rec_tracker.update_state(rec) ####
return {
"loss": self.total_loss_tracker.result(),
"reconstruction_loss": self.reconstruction_loss_tracker.result(),
"kl_loss": self.kl_loss_tracker.result(),
"rec": self.rec_tracker.result() #####
}
# ===================================== Build the encoder ====================================
input_shape = (input_d1, input_d2, input_d3, input_channels) # input_shape=(64, 64, 64, 1)
encoder_input = keras.Input(shape=input_shape, name="enc_input")
input_one_hot_vec = keras.Input(shape=(27, ), name="one_hot_vec_input") # one_hot_vect input layer shape=(27)
x = layers.Conv3D(32, 3, activation="relu", padding="same", name="1_enc_conv")(encoder_input) # delete strides=2
x = layers.MaxPooling3D(name="2_enc_MaxPool")(x)
x = layers.Dropout(0.2)(x)
x = layers.Conv3D(32, 3, activation="relu", padding="same", name="3_enc_conv")(x)
x = layers.MaxPooling3D(name="4_enc_MaxPool")(x)
x = layers.Dropout(0.2)(x)
x = layers.Conv3D(32, 3, activation="relu", padding="same", name="5_enc_conv")(x)
x = layers.MaxPooling3D(name="6_enc_MaxPool")(x)
x = layers.Dropout(0.2)(x)
x = layers.Conv3D(32, 3, activation="relu", padding="same", name="7_en_conv")(x)
enc_last_conv = x.shape
x = layers.Flatten(name="8_enc_flat")(x)
# ============Add one_hot_vector here: ==============
concat_layer_enc = layers.concatenate([x, input_one_hot_vec], axis=1) # ====== concat one_hot_vec here
x = layers.Dense(16, activation="relu", name="9_enc_dense")(concat_layer_enc)
z_mean = layers.Dense(latent_dim, name="10_enc_z_mean")(x)
z_log_var = layers.Dense(latent_dim, name="11_enc_z_log_var")(x)
z = Sampling(name="12_enc_sampling")([z_mean, z_log_var])
encoder = keras.Model([encoder_input, input_one_hot_vec], [z_mean, z_log_var, z], name="encoder")
print(encoder.summary())
plot_model(encoder, to_file='Plot_Model/CVAE/encoder.jpg', show_shapes=True)
print("encoder model created")
# ===================================== Build the decoder ====================
latent_input = keras.Input(shape=(latent_dim,), name="dec_input")
concat_layer_dec = layers.concatenate([latent_input, input_one_hot_vec], axis=1) # ======= concat one_hot_vec here
x = layers.Dense(enc_last_conv[1] * enc_last_conv[2] * enc_last_conv[3] * enc_last_conv[4], activation="relu", name="1_dec_dense")(concat_layer_dec)
x = layers.Reshape((enc_last_conv[1], enc_last_conv[2], enc_last_conv[3], enc_last_conv[4]), name="2_dec")(x)
x = layers.Conv3DTranspose(32, 3, activation="relu", padding="same", name="3_dec_conv")(x)
x = layers.UpSampling3D(name="4_dec_MaxPool")(x)
x = layers.Conv3DTranspose(32, 3, activation="relu", padding="same", name="5_dec_conv")(x)
x = layers.UpSampling3D(name="6_dec_MaxPool")(x)
x = layers.Conv3DTranspose(32, 3, activation="relu", padding="same", name="7_dec_conv")(x)
x = layers.UpSampling3D(name="8_dec_MaxPool")(x)
x = layers.Conv3DTranspose(32, 3, activation="relu", padding="same", name="9_dec_conv")(x)
decoder_outputs = layers.Conv3DTranspose(1, 3, activation="sigmoid", padding="same", name="10_dec_conv_out")(x)
decoder = keras.Model([latent_input, input_one_hot_vec], decoder_outputs, name="decoder")
print(decoder.summary())
plot_model(decoder, to_file='Plot_Model/CVAE/decoder.jpg', show_shapes=True)
# =========================================== **Train the CVAE** ==================================================
random.seed(seed_value)
cvae = CVAE(encoder, decoder)
cvae.compile(optimizer=keras.optimizers.Adam())
history = cvae.fit([x_train, one_hot_vec_train], epochs=epoch, batch_size=batch_size)
track_loss(history, epoch, batch_size, latent_dim, random_sample_points, seed_value)
random_sample_list = latent_space_mds(cvae, x_train, epoch, latent_dim, random_sample_points, seed_value)
sample_decoding(cvae, epoch, latent_dim, random_sample_points, random_sample_list, seed_value)
test(cvae, x_test, 4, epoch, latent_dim)
# end of seed loop
print("finish")
Here is the model that I created:
Encoder model:
and
Here is the decoder model:
Thanks!

Convert from keras to pytorch

I am training a multilabel classifier in keras ,i want to convert that to pytorch , i am mostly confused about how to handle the loss ,this is what the code looks like
model = keras.applications.densenet.DenseNet121(include_top=False, input_shape=(224, 224, 3))
x = model.output
x = Flatten()(x)
x = Dense(512)(x)
x = Activation('relu')(x)
x = Dropout(0.5)(x)
output1 = Dense(1, activation = 'sigmoid')(x)
output2 = Dense(1, activation = 'sigmoid')(x)
output3 = Dense(1, activation = 'sigmoid')(x)
output4 = Dense(1, activation = 'sigmoid')(x)
output5 = Dense(1, activation = 'sigmoid')(x)
output6 = Dense(1, activation = 'sigmoid')(x)
output7 = Dense(1, activation = 'sigmoid')(x)
output8 = Dense(1, activation = 'sigmoid')(x)
model = Model(model.inputs,[output1,output2,output3,output4,output5, output6, output7, output8])
# print(model.summary())
model.compile(optimizers.rmsprop(lr = 0.0001, decay = 1e-6),
loss = ["binary_crossentropy","binary_crossentropy","binary_crossentropy","binary_crossentropy", "binary_crossentropy","binary_crossentropy","binary_crossentropy","binary_crossentropy"],metrics = ["accuracy"])
How can i do this in pytorch ,this is what i have till now
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 32, kernel_size=3)
self.conv2 = nn.Conv2d(32, 64, kernel_size=3)
self.conv2_drop = nn.Dropout2d()
self.fc1 = nn.Linear(2304, 256)
self.fc2 = nn.Linear(256, 8)
def forward(self, x):
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(x.size(0), -1) # Flatten layer
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return F.sigmoid(x)
model = Net().cuda()
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)
def train(epoch):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
# data, target = data.cuda(async=True), target.cuda(async=True) # On GPU
data, target = Variable(data).cuda(), Variable(target).float().cuda()
optimizer.zero_grad()
output = model(data)
loss = F.binary_cross_entropy(output, target)
loss.backward()
optimizer.step()
if batch_idx % 10 == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.item()))
for epoch in range(1, 10):
train(epoch)
Thanks in advance, any suggestions will be very helpful