Deploy model to the web with flask - python
I am trying to deploy the low light image enhancement to the web using Flask. The main thing that I want to do is, run a webpage on localhost, and when I upload a low light image the model should enhance the image and show the result on same webpage.
Here is my test.py codes:
from glob import glob
import numpy as np
import scipy
import keras
import os
import Network
import utls
import time
import cv2
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--input", "-i", type=str, default='../input', help='test image folder')
parser.add_argument("--result", "-r", type=str, default='../result', help='result folder')
parser.add_argument("--model", "-m", type=str, default='Syn_img_lowlight_withnoise', help='model name')
parser.add_argument("--com", "-c", type=int, default=1, help='output with/without origional image and mid result')
parser.add_argument("--highpercent", "-hp", type=int, default=95, help='should be in [85,100], linear amplification')
parser.add_argument("--lowpercent", "-lp", type=int, default=5, help='should be in [0,15], rescale the range [p%,1] to [0, 1]')
parser.add_argument("--gamma", "-g", type=int, default=8, help='should be in [6,10], increase the saturability')
parser.add_argument("--maxrange", "-mr", type=int, default=8, help='linear amplification range')
arg = parser.parse_args()
result_folder = arg.result
if not os.path.isdir(result_folder):
os.makedirs(result_folder)
input_folder = arg.input
path = glob(input_folder+'/*.*')
model_name = arg.model
mbllen = Network.build_mbllen((None, None, 3))
mbllen.load_weights('../models/'+model_name+'.h5')
opt = keras.optimizers.Adam(lr=2 * 1e-04, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
mbllen.compile(loss='mse', optimizer=opt)
flag = arg.com
lowpercent = arg.lowpercent
highpercent = arg.highpercent
maxrange = arg.maxrange/10.
hsvgamma = arg.gamma/10.
for i in range(len(path)):
img_A_path = path[i]
img_A = utls.imread_color(img_A_path)
img_A = img_A[np.newaxis, :]
starttime = time.process_time()
out_pred = mbllen.predict(img_A)
endtime = time.process_time()
print('The ' + str(i+1)+'th image\'s Time:' +str(endtime-starttime)+'s.')
fake_B = out_pred[0, :, :, :3]
fake_B_o = fake_B
gray_fake_B = fake_B[:, :, 0] * 0.299 + fake_B[:, :, 1] * 0.587 + fake_B[:, :, 1] * 0.114
percent_max = sum(sum(gray_fake_B >= maxrange))/sum(sum(gray_fake_B <= 1.0))
# print(percent_max)
max_value = np.percentile(gray_fake_B[:], highpercent)
if percent_max < (100-highpercent)/100.:
scale = maxrange / max_value
fake_B = fake_B * scale
fake_B = np.minimum(fake_B, 1.0)
gray_fake_B = fake_B[:,:,0]*0.299 + fake_B[:,:,1]*0.587 + fake_B[:,:,1]*0.114
sub_value = np.percentile(gray_fake_B[:], lowpercent)
fake_B = (fake_B - sub_value)*(1./(1-sub_value))
imgHSV = cv2.cvtColor(fake_B, cv2.COLOR_RGB2HSV)
H, S, V = cv2.split(imgHSV)
S = np.power(S, hsvgamma)
imgHSV = cv2.merge([H, S, V])
fake_B = cv2.cvtColor(imgHSV, cv2.COLOR_HSV2RGB)
fake_B = np.minimum(fake_B, 1.0)
if flag:
outputs = np.concatenate([img_A[0,:,:,:], fake_B_o, fake_B], axis=1)
else:
outputs = fake_B
filename = os.path.basename(path[i])
img_name = result_folder+'/' + filename
# scipy.misc.toimage(outputs * 255, high=255, low=0, cmin=0, cmax=255).save(img_name)
outputs = np.minimum(outputs, 1.0)
outputs = np.maximum(outputs, 0.0)
utls.imwrite(img_name, outputs)
The data_load.py codes:
from glob import glob
import numpy as np
import random
import scipy
import os
import cv2 as cv
class Dataloader():
def __init__(self, dataset_name, crop_shape=(256, 256)):
self.dataset_name = dataset_name
self.crop_shape = crop_shape
def imread_color(self, path):
img = cv.imread(path, cv.IMREAD_COLOR | cv.IMREAD_ANYDEPTH)/255.
b, g, r = cv.split(img)
img_rgb = cv.merge([r, g, b])
return img_rgb
def imwrite(self, path, img):
r, g, b = cv.split(img)
img_rgb = cv.merge([b, g, r])
cv.imwrite(path, img_rgb)
def load_data(self, batch_size=16):
path = glob('../dataset/train/*.jpg')
self.n_batches = int(len(path) / batch_size)
while 1:
random.shuffle(path)
for i in range(self.n_batches - 1):
batch_path = path[i * batch_size:(i + 1) * batch_size]
input_imgs = np.empty((batch_size, self.crop_shape[0], self.crop_shape[1], 6), dtype="float32")
gt = np.empty((batch_size, self.crop_shape[0], self.crop_shape[1], 3), dtype="float32")
number = 0
for img_B_path in batch_path:
img_B = self.imread_color(img_B_path)
path_mid = os.path.split(img_B_path)
path_A_1 = path_mid[0] + '_' + self.dataset_name
path_A = os.path.join(path_A_1, path_mid[1])
img_A = self.imread_color(path_A)
nw = random.randint(0, img_B.shape[0] - self.crop_shape[0])
nh = random.randint(0, img_B.shape[1] - self.crop_shape[1])
crop_img_A = img_A[nw:nw + self.crop_shape[0], nh:nh + self.crop_shape[1], :]
crop_img_B = img_B[nw:nw + self.crop_shape[0], nh:nh + self.crop_shape[1], :]
if np.random.randint(2, size=1)[0] == 1: # random flip
crop_img_A = np.flipud(crop_img_A)
crop_img_B = np.flipud(crop_img_B)
if np.random.randint(2, size=1)[0] == 1:
crop_img_A = np.fliplr(crop_img_A)
crop_img_B = np.fliplr(crop_img_B)
if np.random.randint(2, size=1)[0] == 1: # random transpose
crop_img_A = np.transpose(crop_img_A, (1, 0, 2))
crop_img_B = np.transpose(crop_img_B, (1, 0, 2))
input_imgs[number, :, :, :] = np.concatenate([crop_img_A, crop_img_B], axis=-1)
gt[number, :, :, :] = crop_img_B
number += 1
yield input_imgs, gt
The utls.py codes:
import tensorflow as tf
from keras import backend as K
import numpy as np
import scipy
import os
import cv2 as cv
def bright_mae(y_true, y_pred):
return K.mean(K.abs(y_pred[:,:,:,:3] - y_true[:,:,:,:3]))
def bright_mse(y_true, y_pred):
return K.mean((y_pred[:,:,:,:3] - y_true[:,:,:,:3])**2)
def bright_AB(y_true, y_pred):
return K.abs(K.mean(y_true[:,:,:,:3])-K.mean(y_pred[:,:,:,:3]))
def log10(x):
numerator = K.log(x)
denominator = K.log(K.constant(10, dtype=numerator.dtype))
return numerator / denominator
def bright_psnr(y_true, y_pred):
mse = K.mean((K.abs(y_pred[:,:,:,:3] - y_true[:,:,:,:3])) ** 2)
max_num = 1.0
psnr = 10 * log10(max_num ** 2 / mse)
return psnr
def _tf_fspecial_gauss(size, sigma):
"""Function to mimic the 'fspecial' gaussian MATLAB function
"""
x_data, y_data = np.mgrid[-size//2 + 1:size//2 + 1, -size//2 + 1:size//2 + 1]
x_data = np.expand_dims(x_data, axis=-1)
x_data = np.expand_dims(x_data, axis=-1)
y_data = np.expand_dims(y_data, axis=-1)
y_data = np.expand_dims(y_data, axis=-1)
x = tf.constant(x_data, dtype=tf.float32)
y = tf.constant(y_data, dtype=tf.float32)
g = tf.exp(-((x**2 + y**2)/(2.0*sigma**2)))
return g / tf.reduce_sum(g)
def tf_ssim(img1, img2, cs_map=False, mean_metric=True, size=11, sigma=1.5):
window = _tf_fspecial_gauss(size, sigma) # window shape [size, size]
K1 = 0.01
K2 = 0.03
L = 1 # depth of image (255 in case the image has a differnt scale)
C1 = (K1*L)**2
C2 = (K2*L)**2
mu1 = tf.nn.conv2d(img1, window, strides=[1,1,1,1], padding='VALID')
mu2 = tf.nn.conv2d(img2, window, strides=[1,1,1,1],padding='VALID')
mu1_sq = mu1*mu1
mu2_sq = mu2*mu2
mu1_mu2 = mu1*mu2
sigma1_sq = tf.nn.conv2d(img1*img1, window, strides=[1,1,1,1],padding='VALID') - mu1_sq
sigma2_sq = tf.nn.conv2d(img2*img2, window, strides=[1,1,1,1],padding='VALID') - mu2_sq
sigma12 = tf.nn.conv2d(img1*img2, window, strides=[1,1,1,1],padding='VALID') - mu1_mu2
if cs_map:
value = (((2*mu1_mu2 + C1)*(2*sigma12 + C2))/((mu1_sq + mu2_sq + C1)*
(sigma1_sq + sigma2_sq + C2)),
(2.0*sigma12 + C2)/(sigma1_sq + sigma2_sq + C2))
else:
value = ((2*mu1_mu2 + C1)*(2*sigma12 + C2))/((mu1_sq + mu2_sq + C1)*
(sigma1_sq + sigma2_sq + C2))
if mean_metric:
value = tf.reduce_mean(value)
return value
def tf_ms_ssim(img1, img2, mean_metric=True, level=5):
weight = tf.constant([0.0448, 0.2856, 0.3001, 0.2363, 0.1333], dtype=tf.float32)
mssim = []
mcs = []
for l in range(level):
ssim_map, cs_map = tf_ssim(img1, img2, cs_map=True, mean_metric=False)
mssim.append(tf.reduce_mean(ssim_map))
mcs.append(tf.reduce_mean(cs_map))
filtered_im1 = tf.nn.avg_pool(img1, [1,2,2,1], [1,2,2,1], padding='SAME')
filtered_im2 = tf.nn.avg_pool(img2, [1,2,2,1], [1,2,2,1], padding='SAME')
img1 = filtered_im1
img2 = filtered_im2
# list to tensor of dim D+1
mssim = tf.stack(mssim, axis=0)
mcs = tf.stack(mcs, axis=0)
value = (tf.reduce_prod(mcs[0:level-1]**weight[0:level-1])*
(mssim[level-1]**weight[level-1]))
if mean_metric:
value = tf.reduce_mean(value)
return value
def bright_SSIM(y_true, y_pred):
SSIM_loss = tf_ssim(tf.expand_dims(y_pred[:,:,:,0], -1), tf.expand_dims(y_true[:,:,:,0], -1))+tf_ssim(tf.expand_dims(y_pred[:,:,:,1], -1), tf.expand_dims(y_true[:,:,:,1], -1)) + tf_ssim(tf.expand_dims(y_pred[:,:,:,2], -1), tf.expand_dims(y_true[:,:,:,2], -1))
return SSIM_loss/3
def psnr_cau(y_true, y_pred):
mse = np.mean((np.abs(y_pred - y_true)) ** 2)
max_num = 1.0
psnr = 10 * np.log10(max_num ** 2 / mse)
return psnr
def save_model(model, name, epoch, batch_i):
modelname = './Res_models/' + str(epoch) + '_' + str(batch_i) + name + '.h5'
model.save_weights(modelname)
def imread_color(path):
img = cv.imread(path, cv.IMREAD_COLOR | cv.IMREAD_ANYDEPTH) / 255.
b, g, r = cv.split(img)
img_rgb = cv.merge([r, g, b])
return img_rgb
# return scipy.misc.imread(path, mode='RGB').astype(np.float) / 255.
def imwrite(path, img):
r, g, b = cv.split(img*255)
img_rgb = cv.merge([b, g, r])
cv.imwrite(path, img_rgb)
# scipy.misc.toimage(img * 255, high=255, low=0, cmin=0, cmax=255).save(path)
def range_scale(x):
return x * 2 - 1.
In the below codes, I wanted to make a webpage and create an upload button to take low light images from users to enhance it by using the model. But I could not deploy the model as well.
main.py:
from flask import Flask, request
from werkzeug.utils import secure_filename
import torch
def update_image():
app = Flask(
__name__,
static_url_path='',
static_folder=''
)
def get_form():
return """
<form action="/" enctype="multipart/form-data" method="POST">
<input type="file" id="file" name="file">
<input type="submit">
</form>
"""
#app.route("/", methods = ['GET', 'POST'])
def hello_world():
if request.method == 'GET':
return get_form()
elif request.method == 'POST':
f = request.files['file']
print(f.filename)
f.save(secure_filename(f.filename))
update_image(f.filename)
return f'{get_form()} <img src="{f.filename}"/>'
Related
TensorFlow/Keras multi-threaded model prediction
I am trying to call my Face Recognition model implemented in keras, using flask API. I am unable to call the model using different cam urls as a parameter.
I am getting the following error:
TypeError: Cannot interpret feed_dict key as Tensor: Tensor Tensor("Placeholder_50:0", shape=(3, 3, 3, 32), dtype=float32) is not an element of this graph.
127.0.0.1 - - [23/Nov/2022 13:39:49] "GET /api/recognise?url=rtsp://admin:inndata123#10.10.5.202:554/cam/realmonitor?channel=1&subtype=0 HTTP/1.1" 500 -
I found that creating a new session for each thread, but I don't have any idea where to place those lines in my code.
# running db and email functions in background and parallalized action and bbox dist loops
import json
import os
import pickle
import cv2
import imutils
import dlib
import torch
import time
import numpy as np
import datetime
from pathlib import Path
import matplotlib.pyplot as plt
from PIL import Image, ImageFont, ImageDraw
from script.fx import prewhiten, l2_normalize
from keras.models import load_model
from scipy.spatial import distance
from mtcnn.mtcnn import MTCNN
from script.generate_data import generate_embeddings
import mysql.connector
from mysql.connector import (connection)
import smtplib
import mimetypes
from email.message import EmailMessage
message = EmailMessage()
import tensorflow as tf
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=2)
from flask import Flask, jsonify, request,render_template,Response
app = Flask(__name__)
global graph
graph = tf.get_default_graph()
sess = tf.Session(graph=graph, config=session_conf)
model_path = './data/model/facenet_keras.h5'
font_path = './data/font/Calibri Regular.ttf'
embedding_path = './data/arrays/embeddings.npz'
vars_path = './data/arrays/vars.npz'
curr_time = datetime.datetime.now()
time_date = curr_time.strftime('%Y-%m-%d %H:%M:%S')
only_date= curr_time.strftime('%Y-%m-%d')
login_time = curr_time.replace(hour=8, minute=0, second=0, microsecond=0)
logout_time = curr_time.replace(hour=17, minute=15, second=0, microsecond=0)
if os.path.exists(embedding_path) == True:
print('Loadings embeddings...')
loaded_embeddings = np.load(embedding_path)
embeddings, names = loaded_embeddings['a'], loaded_embeddings['b']
loaded_vars = np.load(vars_path)
slope, intercept = loaded_vars['a'], loaded_vars['b']
else:
print('Creatings embeddings...')
generate_embeddings()
loaded_embeddings = np.load(embedding_path)
embeddings, names = loaded_embeddings['a'], loaded_embeddings['b']
loaded_vars = np.load(vars_path)
slope, intercept = loaded_vars['a'], loaded_vars['b']
location='IE'
cam_id='Entrance-Cam'
frame_count = 0
frame_number = 0
bbox_centers = []
log_in = []
log_out = []
date_list = []
mins_lst = []
#app.route('/api/recognise')
def recognise():
url = request.args.get('url')
if url!=str(0):
subtype=request.args.get('subtype')
url=url+'&'+'subtype='+subtype
print(url)
else:url=int(url)
video_sources = cv2.VideoCapture(url)
detector = MTCNN()
model = load_model(model_path, compile=False)
graph = tf.get_default_graph()
def inner():
frame_count = 0
frame_number = 0
while 1:
start= time.time()
var, frame = video_sources.read()
if frame is not None:
small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
rgb_small_frame = small_frame[:, :, ::-1]
# frame = cv2.resize(frame, (1500, 1000))
if frame_count % 10 == 0 and rgb_small_frame is not None:
faces = detector.detect_faces(rgb_small_frame) # result
#print(faces)
print('faces :',len(faces))
for result in faces:
x_face, y_face, w_face, h_face = result['box']
x_face = x_face * 4
y_face = y_face * 4
w_face = w_face * 4
h_face = h_face * 4
x_face2=w_face+x_face
y_face2=h_face+y_face
#face bbox tuples
face_tuple1=(x_face,y_face)
face_tuple2=(x_face2,y_face2)
#zone bbox tuples
zone_tuple1 = (950, 700)
zone_tuple2 = (2000, 1050)
# Margins for Face box
dw = 0.1 * w_face
dh = 0.2 * h_face
#center = (x_face + w_face // 2, y_face + h_face // 2)
#cv2.rectangle(frame, zone_tuple1, zone_tuple2, (255, 0, 0), 2)
#if (all(x > y for x, y in zip(face_tuple1, zone_tuple1)))==True and (all(x < y for x, y in zip(face_tuple2, zone_tuple2)))==True:
# radius=2
with graph.as_default():
dist = []
for i in range(len(embeddings)):
dist.append(distance.euclidean(l2_normalize(model.predict(prewhiten(
cv2.resize(frame[y_face:y_face + h_face, x_face:x_face + w_face], (160, 160)).reshape(
-1, 160,
160,
3)))),
embeddings[i].reshape(1, 128)))
dist = np.array(dist)
if os.path.exists(only_date + '.txt') == False:
f = open(only_date + '.txt', "a+")
log_in.clear()
log_out.clear()
else:
if dist.min() > 1.20:
log = 'Unauthorized Entry'
emp_id = 'None'
f1 = open("unauthorised.txt", "a")
f1.writelines(f"\n{cam_id},{time_date},{log}")
elif dist.min() <= 1:
emp_id = names[dist.argmin()]
if int(emp_id) not in log_in and curr_time >= login_time:
log = 'punch-in'
f2 = open(only_date + '.txt', "a")
f2.writelines(f"\n{cam_id},{emp_id},{time_date},{log}")
f2.close()
log_in.append(int(emp_id))
print(log_in)
if int(emp_id) in log_in and curr_time >= logout_time and int(emp_id) not in log_out:
# and center[0] > 750 and center[0] > 960:
log = 'punch-out'
f2 = open(only_date + '.txt', "a")
f2.writelines(f"\n{cam_id},{emp_id},{time_date},{log}")
f2.close()
log_out.append(int(emp_id))
else:
emp_id = 'None'
log = 'unidentified'
if emp_id != 'unauthorized' and emp_id != 'unidentified':
font_size = int(
slope[dist.argmin()] * ((w_face + 2 * dw) // 3) * 2 + intercept[dist.argmin()])
color = (0, 255, 0)
elif emp_id == 'unauthorized':
font_size = int(
slope[dist.argmin()] * ((w_face + 2 * dw) // 3) * 2 + intercept[dist.argmin()])
color = (0, 0, 255)
else:
font_size = int(0.1974311 * ((w_face + 2 * dw) // 3) * 2 + 0.03397702412218706)
color = (0, 255, 0)
font = ImageFont.truetype(font_path, font_size)
size = font.getbbox(emp_id)
cv2.rectangle(frame,
pt1=(x_face - int(np.floor(dw)), (y_face - int(np.floor(dh)))),
pt2=(
(x_face + w_face + int(np.ceil(dw))), (y_face + h_face + int(np.ceil(dh)))),
color=(0, 255, 0),
thickness=2) # Face Rectangle
cv2.rectangle(frame,
pt1=(x_face - int(np.floor(dw)), y_face - int(np.floor(dh)) - size[1]),
pt2=(x_face + size[0], y_face - int(np.floor(dh))),
color=(0, 255, 0),
thickness=-1)
img = Image.fromarray(frame)
draw = ImageDraw.Draw(img)
draw.text((x_face - int(np.floor(dw)), y_face - int(np.floor(dh)) - size[1]), emp_id,
font=font,
fill=color)
frame = np.array(img)
if emp_id == 'unauthorized':
frame_name = f'{emp_id}_{frame_number}.jpg'
cv2.imwrite(f'data/unauthorized_faces/{frame_name}',
cv2.resize(frame[y_face:y_face + h_face, x_face:x_face + w_face],
(250, 250)))
elif emp_id != 'unauthorised' and emp_id != 'unidentified':
frame_name = f'{emp_id}_{frame_number}.jpg'
cv2.imwrite(f'data/detected_faces/{frame_name}',
cv2.resize(frame[y_face:y_face + h_face, x_face:x_face + w_face],
(250, 250)))
# add_attachment(frame_name)
frame_number += 1
end = time.time()
print(end-start)
print(emp_id)
if log != 'unidentified':
data = {'emp_id': emp_id, 'date': time_date, 'log': log}
yield json.dumps(data) + "\n"
# cv2.imshow('Frame', cv2.resize(frame, (950, 950)))
if cv2.waitKey(15) & 255 == ord('q'):
break
else:
continue
return Response(inner(), mimetype='application/json')
if __name__=='__main__':
app.run(host="0.0.0.0",threaded=True)
This is my face recognition model integrated in flask.
parameter estimation using tensorflow
I am trying to create a nueral network using tensor flow. I am not using keras api. I have some parameter estimation(weight,bias and some other parameters) to do. The code is working but the parameter estimation is really bad and error percentage is very high what is the problem here? I tried so many ways still no improvement. the loss fn is less.
I tried creating my own optimizer but the process is slow and the error is large. Is there any way to apply optimizers parameter.
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import scipy.io
from scipy.interpolate import griddata
from pyDOE import lhs
import math as ma
class PhysicsInformedNN:
def __init__(self,X_n,v,layers,lb,ub):
self.lb = lb
self.ub = ub
self.layers = layers
self.dx_n = tf.convert_to_tensor(X_n[:,0:1],dtype = 'float32')
self.t_n = tf.convert_to_tensor(X_n[:,1:2],dtype = 'float32')
self.v_r = tf.convert_to_tensor(v,dtype = 'float32')
self.lambda_1 = tf.Variable(0,dtype = 'float32')#1.5
self.lambda_2 = tf.Variable(-6,dtype = 'float32')
self.para =[self.lambda_1,self.lambda_2]
self.weights, self.biases = self.initialize_NN(layers)
def initialize_NN(self,layers):
weights = []
biases = []
num_layers = len(layers)
for l in range(0,num_layers-1):
W = self.xavier_init(size=[layers[l], layers[l+1]])
b = tf.Variable(tf.zeros([1,layers[l+1]], dtype='float32'), dtype='float32')
weights.append(W)
biases.append(b)
return weights, biases
def xavier_init(self, size):
in_dim = size[0]
out_dim = size[1]
xavier_stddev = np.sqrt(2/(in_dim + out_dim))
return tf.Variable(tf.random.truncated_normal([in_dim, out_dim], stddev=xavier_stddev), dtype='float32')
def neural_net(self, X, weights, biases):
num_layers = len(weights) + 1
H = 2.0*(X - self.lb)/(self.ub - self.lb) - 1.0
for l in range(0,num_layers-2):
W = weights[l]
b = biases[l]
H = tf.math.tanh(tf.math.add(tf.linalg.matmul(H, W), b))
W = weights[-1]
b = biases[-1]
Y = tf.math.add(tf.linalg.matmul(H, W), b)
return Y
def net_u(self, x, t):
v = self.neural_net(tf.concat([x,t],1), self.weights, self.biases)
return v
def net_f(self, x, t):
lambda_1 = self.para[0]
lambda_2 = tf.exp(self.para[1])
with tf.GradientTape(persistent=True) as tape :
tape.watch(t)
tape.watch(x)
u = self.net_u(x,t)
u_x = tape.gradient(u,x)
u_t = tape.gradient(u,t)
u_xx = tape.gradient(u_x,x)
f = u_t + lambda_1*u*u_x - lambda_2*u_xx
del tape
return f
def callback(self, loss,n):
print('Loss:', loss, ' Epoch : ', n)
def train(self,epoch):
for i in range(epoch):
with tf.GradientTape(persistent=True) as tape :
tape.watch(self.weights)
tape.watch(self.biases)
tape.watch(self.para)
f_pred = self.net_f(self.dx_n, self.t_n)
v_pred = self.net_u(self.dx_n, self.t_n)
loss = tf.reduce_mean(tf.square(self.v_r - v_pred)) + tf.reduce_mean(tf.square(f_pred))
dw = tape.gradient(loss,self.weights)
db = tape.gradient(loss,self.biases)
dp = tape.gradient(loss,self.para)
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=1e-2,
decay_steps=10000,
decay_rate=0.9)
optimizer1 = tf.keras.optimizers.Adam(learning_rate=0.0001)
optimizer1.apply_gradients(zip(dw, self.weights))
optimizer1.apply_gradients(zip(db, self.biases))
optimizer2 = tf.keras.optimizers.Adam(learning_rate=0.001)
optimizer2.apply_gradients(zip(dp, self.para))
del tape
self.callback(loss,i)
def predict(self, X_star):
v_star = self.net_u(X_star[:,0:1], X_star[:,1:2])
f_star = f_pred = self.net_f(X_star[:,0:1], X_star[:,1:2])
para_last = self.para
return v_star, f_star, para_last
if __name__ == '__main__':
#PARAMETERS for the problem
np.random.seed(123)
nu =0.01/np.pi
layers = [2, 20, 20, 20, 20, 1]
N_u = 2000
data = scipy.io.loadmat('burgers_shock.mat')
t = data['t'].flatten()[:,None]
x = data['x'].flatten()[:,None]
Exact = np.real(data['usol']).T
X, T = np.meshgrid(x,t)
X_star = np.hstack((X.flatten()[:,None], T.flatten()[:,None]))
u_star = Exact.flatten()[:,None]
lb = X_star.min(0)
ub = X_star.max(0)
idx = np.random.choice(X_star.shape[0], N_u, replace=False)
X_u_train = X_star[idx,:]
u_train = u_star[idx,:]
model = PhysicsInformedNN(X_u_train, u_train, layers, lb, ub)
model.train(1000)
X_star = tf.convert_to_tensor(X_star,dtype='float32')
u_pred, f_pred, param = model.predict(X_star)
error_lambda_1 = np.abs(param[0] - 1.0)*100
error_lambda_2 = np.abs( np.exp(param[1])- nu)/nu * 100
print(error_lambda_1,error_lambda_2)
How to reduce model size in Pytorch post training
I have created a pytorch model and I want to reduce the model size.
Defining Model Architecture :-
import torch
import torch.quantization
import torch.nn as nn
import copy
import os
import time
import numpy as np
import torch.autograd as autograd
from torch.autograd import Variable
import torch.nn.utils.prune as prune
import torch.nn.functional as F
import os
import random
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sys
import time
import codecs
import pickle
import torch
from torch.autograd import Variable
import torch.nn.utils.prune as prune
from config import Config
from loader import *
from utils import *
from model import BiLSTM_CRF
START_TAG = '<START>'
STOP_TAG = '<STOP>'
def init_embedding(input_embedding):
"""
Initialize embedding
"""
bias = np.sqrt(3.0 / input_embedding.size(1))
nn.init.uniform(input_embedding, -bias, bias)
def init_linear(input_linear):
"""
Initialize linear transformation
"""
bias = np.sqrt(6.0 / (input_linear.weight.size(0) + input_linear.weight.size(1)))
nn.init.uniform(input_linear.weight, -bias, bias)
if input_linear.bias is not None:
input_linear.bias.data.zero_()
def init_lstm(input_lstm):
"""
Initialize lstm
"""
for ind in range(0, input_lstm.num_layers):
weight = eval('input_lstm.weight_ih_l' + str(ind))
bias = np.sqrt(6.0 / (weight.size(0) / 4 + weight.size(1)))
nn.init.uniform(weight, -bias, bias)
weight = eval('input_lstm.weight_hh_l' + str(ind))
bias = np.sqrt(6.0 / (weight.size(0) / 4 + weight.size(1)))
nn.init.uniform(weight, -bias, bias)
if input_lstm.bidirectional:
for ind in range(0, input_lstm.num_layers):
weight = eval('input_lstm.weight_ih_l' + str(ind) + '_reverse')
bias = np.sqrt(6.0 / (weight.size(0) / 4 + weight.size(1)))
nn.init.uniform(weight, -bias, bias)
weight = eval('input_lstm.weight_hh_l' + str(ind) + '_reverse')
bias = np.sqrt(6.0 / (weight.size(0) / 4 + weight.size(1)))
nn.init.uniform(weight, -bias, bias)
if input_lstm.bias:
for ind in range(0, input_lstm.num_layers):
weight = eval('input_lstm.bias_ih_l' + str(ind))
weight.data.zero_()
weight.data[input_lstm.hidden_size: 2 * input_lstm.hidden_size] = 1
weight = eval('input_lstm.bias_hh_l' + str(ind))
weight.data.zero_()
weight.data[input_lstm.hidden_size: 2 * input_lstm.hidden_size] = 1
if input_lstm.bidirectional:
for ind in range(0, input_lstm.num_layers):
weight = eval('input_lstm.bias_ih_l' + str(ind) + '_reverse')
weight.data.zero_()
weight.data[input_lstm.hidden_size: 2 * input_lstm.hidden_size] = 1
weight = eval('input_lstm.bias_hh_l' + str(ind) + '_reverse')
weight.data.zero_()
weight.data[input_lstm.hidden_size: 2 * input_lstm.hidden_size] = 1
def to_scalar(var):
return var.view(-1).data.tolist()[0]
def argmax(vec):
_, idx = torch.max(vec, 1)
return to_scalar(idx)
def log_sum_exp(vec):
# vec 2D: 1 * tagset_size
max_score = vec[0, argmax(vec)]
max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])
return max_score + \
torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))
class BiLSTM_CRF(nn.Module):
def __init__(self, vocab_size, tag_to_ix, embedding_dim, hidden_dim, char_lstm_dim=25,
char_to_ix=None, pre_word_embeds=None, char_embedding_dim=25, use_gpu=False,
n_cap=None, cap_embedding_dim=None, use_crf=True, char_mode='CNN'):
super(BiLSTM_CRF, self).__init__()
self.use_gpu = use_gpu
self.embedding_dim = embedding_dim #100
self.hidden_dim = hidden_dim #200
self.vocab_size = vocab_size
self.tag_to_ix = tag_to_ix
self.n_cap = n_cap
self.cap_embedding_dim = cap_embedding_dim
self.use_crf = use_crf
self.tagset_size = len(tag_to_ix)
self.out_channels = char_lstm_dim #25
self.char_mode = char_mode
print('char_mode: %s, out_channels: %d, hidden_dim: %d, ' % (char_mode, char_lstm_dim, hidden_dim))
if self.n_cap and self.cap_embedding_dim:
self.cap_embeds = nn.Embedding(self.n_cap, self.cap_embedding_dim)
# print("self.cap_embeds.weight------",self.cap_embeds.weight)
init_embedding(self.cap_embeds.weight)
if char_embedding_dim is not None:
self.char_lstm_dim = char_lstm_dim
self.char_embeds = nn.Embedding(len(char_to_ix), char_embedding_dim)
# print("self.char_embeds.weight-------", self.char_embeds.weight)
init_embedding(self.char_embeds.weight)
if self.char_mode == 'LSTM':
self.char_lstm = nn.LSTM(char_embedding_dim, char_lstm_dim, num_layers=1, bidirectional=True)
init_lstm(self.char_lstm)
if self.char_mode == 'CNN':
self.char_cnn3 = nn.Conv2d(in_channels=1, out_channels=self.out_channels, kernel_size=(3, char_embedding_dim), padding=(2,0))
self.word_embeds = nn.Embedding(vocab_size, embedding_dim)
if pre_word_embeds is not None:
self.pre_word_embeds = True
self.word_embeds.weight = nn.Parameter(torch.FloatTensor(pre_word_embeds))
else:
self.pre_word_embeds = False
self.dropout = nn.Dropout(0.5)
if self.n_cap and self.cap_embedding_dim:
if self.char_mode == 'LSTM':
self.lstm = nn.LSTM(embedding_dim+char_lstm_dim*2+cap_embedding_dim, hidden_dim, bidirectional=True)
if self.char_mode == 'CNN':
self.lstm = nn.LSTM(embedding_dim+self.out_channels+cap_embedding_dim, hidden_dim, bidirectional=True)
else:
if self.char_mode == 'LSTM':
self.lstm = nn.LSTM(embedding_dim+char_lstm_dim*2, hidden_dim, bidirectional=True)
if self.char_mode == 'CNN':
self.lstm = nn.LSTM(embedding_dim+self.out_channels, hidden_dim, bidirectional=True)
init_lstm(self.lstm)
self.hw_trans = nn.Linear(self.out_channels, self.out_channels)
self.hw_gate = nn.Linear(self.out_channels, self.out_channels)
self.h2_h1 = nn.Linear(hidden_dim*2, hidden_dim)
self.tanh = nn.Tanh()
self.hidden2tag = nn.Linear(hidden_dim*2, self.tagset_size)
init_linear(self.h2_h1)
init_linear(self.hidden2tag)
init_linear(self.hw_gate)
init_linear(self.hw_trans)
if self.use_crf:
self.transitions = nn.Parameter(
torch.zeros(self.tagset_size, self.tagset_size))
self.transitions.data[tag_to_ix[START_TAG], :] = -10000
self.transitions.data[:, tag_to_ix[STOP_TAG]] = -10000
def _score_sentence(self, feats, tags):
# tags is ground_truth, a list of ints, length is len(sentence)
# feats is a 2D tensor, len(sentence) * tagset_size
r = torch.LongTensor(range(feats.size()[0]))
if self.use_gpu:
r = r.cuda()
pad_start_tags = torch.cat([torch.cuda.LongTensor([self.tag_to_ix[START_TAG]]), tags])
pad_stop_tags = torch.cat([tags, torch.cuda.LongTensor([self.tag_to_ix[STOP_TAG]])])
else:
pad_start_tags = torch.cat([torch.LongTensor([self.tag_to_ix[START_TAG]]), tags])
pad_stop_tags = torch.cat([tags, torch.LongTensor([self.tag_to_ix[STOP_TAG]])])
score = torch.sum(self.transitions[pad_stop_tags, pad_start_tags]) + torch.sum(feats[r, tags])
return score
def _get_lstm_features(self, sentence, chars2, caps, chars2_length, d):
if self.char_mode == 'LSTM':
# self.char_lstm_hidden = self.init_lstm_hidden(dim=self.char_lstm_dim, bidirection=True, batchsize=chars2.size(0))
chars_embeds = self.char_embeds(chars2).transpose(0, 1)
packed = torch.nn.utils.rnn.pack_padded_sequence(chars_embeds, chars2_length)
lstm_out, _ = self.char_lstm(packed)
outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(lstm_out)
outputs = outputs.transpose(0, 1)
chars_embeds_temp = Variable(torch.FloatTensor(torch.zeros((outputs.size(0), outputs.size(2)))))
if self.use_gpu:
chars_embeds_temp = chars_embeds_temp.cuda()
for i, index in enumerate(output_lengths):
chars_embeds_temp[i] = torch.cat((outputs[i, index-1, :self.char_lstm_dim], outputs[i, 0, self.char_lstm_dim:]))
chars_embeds = chars_embeds_temp.clone()
for i in range(chars_embeds.size(0)):
chars_embeds[d[i]] = chars_embeds_temp[i]
if self.char_mode == 'CNN':
chars_embeds = self.char_embeds(chars2).unsqueeze(1)
chars_cnn_out3 = self.char_cnn3(chars_embeds)
chars_embeds = nn.functional.max_pool2d(chars_cnn_out3, kernel_size=(chars_cnn_out3.size(2), 1)).view(chars_cnn_out3.size(0), self.out_channels)
# t = self.hw_gate(chars_embeds)
# g = nn.functional.sigmoid(t)
# h = nn.functional.relu(self.hw_trans(chars_embeds))
# chars_embeds = g * h + (1 - g) * chars_embeds
embeds = self.word_embeds(sentence)
if self.n_cap and self.cap_embedding_dim:
cap_embedding = self.cap_embeds(caps)
if self.n_cap and self.cap_embedding_dim:
embeds = torch.cat((embeds, chars_embeds, cap_embedding), 1)
else:
embeds = torch.cat((embeds, chars_embeds), 1)
embeds = embeds.unsqueeze(1)
embeds = self.dropout(embeds)
lstm_out, _ = self.lstm(embeds)
lstm_out = lstm_out.view(len(sentence), self.hidden_dim*2)
lstm_out = self.dropout(lstm_out)
lstm_feats = self.hidden2tag(lstm_out)
return lstm_feats
def _forward_alg(self, feats):
# calculate in log domain
# feats is len(sentence) * tagset_size
# initialize alpha with a Tensor with values all equal to -10000.
init_alphas = torch.Tensor(1, self.tagset_size).fill_(-10000.)
init_alphas[0][self.tag_to_ix[START_TAG]] = 0.
forward_var = autograd.Variable(init_alphas)
if self.use_gpu:
forward_var = forward_var.cuda()
for feat in feats:
emit_score = feat.view(-1, 1)
tag_var = forward_var + self.transitions + emit_score
max_tag_var, _ = torch.max(tag_var, dim=1)
tag_var = tag_var - max_tag_var.view(-1, 1)
forward_var = max_tag_var + torch.log(torch.sum(torch.exp(tag_var), dim=1)).view(1, -1) # ).view(1, -1)
terminal_var = (forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]).view(1, -1)
alpha = log_sum_exp(terminal_var)
# Z(x)
return alpha
def viterbi_decode(self, feats):
backpointers = []
# analogous to forward
init_vvars = torch.Tensor(1, self.tagset_size).fill_(-10000.)
init_vvars[0][self.tag_to_ix[START_TAG]] = 0
forward_var = Variable(init_vvars)
if self.use_gpu:
forward_var = forward_var.cuda()
for feat in feats:
next_tag_var = forward_var.view(1, -1).expand(self.tagset_size, self.tagset_size) + self.transitions
_, bptrs_t = torch.max(next_tag_var, dim=1)
bptrs_t = bptrs_t.squeeze().data.cpu().numpy()
next_tag_var = next_tag_var.data.cpu().numpy()
viterbivars_t = next_tag_var[range(len(bptrs_t)), bptrs_t]
viterbivars_t = Variable(torch.FloatTensor(viterbivars_t))
if self.use_gpu:
viterbivars_t = viterbivars_t.cuda()
forward_var = viterbivars_t + feat
backpointers.append(bptrs_t)
terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]
terminal_var.data[self.tag_to_ix[STOP_TAG]] = -10000.
terminal_var.data[self.tag_to_ix[START_TAG]] = -10000.
best_tag_id = argmax(terminal_var.unsqueeze(0))
path_score = terminal_var[best_tag_id]
best_path = [best_tag_id]
for bptrs_t in reversed(backpointers):
best_tag_id = bptrs_t[best_tag_id]
best_path.append(best_tag_id)
start = best_path.pop()
assert start == self.tag_to_ix[START_TAG]
best_path.reverse()
return path_score, best_path
def neg_log_likelihood(self, sentence, tags, chars2, caps, chars2_length, d):
# sentence, tags is a list of ints
# features is a 2D tensor, len(sentence) * self.tagset_size
feats = self._get_lstm_features(sentence, chars2, caps, chars2_length, d)
if self.use_crf:
forward_score = self._forward_alg(feats)
gold_score = self._score_sentence(feats, tags)
return forward_score - gold_score
else:
tags = Variable(tags)
scores = nn.functional.cross_entropy(feats, tags)
return scores
def forward(self, sentence, chars, caps, chars2_length, d):
feats = self._get_lstm_features(sentence, chars, caps, chars2_length, d)
# viterbi to get tag_seq
if self.use_crf:
score, tag_seq = self.viterbi_decode(feats)
else:
score, tag_seq = torch.max(feats, 1)
tag_seq = list(tag_seq.cpu().data)
return score, tag_seq
create Model Instance:-
model_fp32 = BiLSTM_CRF(vocab_size=len(word_to_id),
tag_to_ix=tag_to_id,
embedding_dim=parameters['word_dim'],
hidden_dim=parameters['word_lstm_dim'],
use_gpu=parameters['use_gpu'],
char_to_ix=char_to_id,
pre_word_embeds=word_embeds,
use_crf=parameters['crf'],
char_mode=parameters['char_mode'])
Apply Quantization
model_int8 = torch.quantization.quantize_dynamic(
model_fp32, # the original model
{nn.LSTM,nn.Linear}, # a set of layers to dynamically quantize
dtype=torch.qint8)
Checking Quantization Results:
def print_size_of_model(model, label=""):
torch.save(model.state_dict(), "temp.p")
size=os.path.getsize("temp.p")
print("model: ",label,' \t','Size (KB):', size/1e3)
os.remove('temp.p')
return size
compare the sizes
f=print_size_of_model(model_fp32,"model_fp32")
q=print_size_of_model(model_int8,"model_int8")
print("{0:.2f} times smaller".format(f/q))
Results
model: model_fp32 Size (KB): 806494.996
model: model_int8 Size (KB): 804532.412
1.00 times smaller
is there any way to reduce the model size significantly??
Based on Results section of question and vocab_size of approximately 2 million, it's seems reasonable to quantize attribute word_embeds. Expected that quantization of only this module alone will result in significant drop of memory occupation by weights. According to documentation there is no support for dynamic quantization(which is used for nn.Linear and nn.LSTM in snippet above) of nn.Embedding(type of word_embeds), but static quantization can handle this. Default qconfig which is used in some pytorch examples seems not working on nn.Embedding, but there is a hint in issue discussion how to quantize nn.Embedding. After training: from torch.quantization.qconfig import float_qparams_weight_only_qconfig model_fp32.word_embeds.qconfig = float_qparams_weight_only_qconfig torch.quantization.prepare(model_fp32, inplace=True) torch.quantization.convert(model_fp32, inplace=True) And after that word_embeds in model_fp32 will be quantized to torhc.quint8.
RuntimeError ,Error opening ,System error ,when reading wav file
At first,I am a python beginner. I need to read my clean.wav(cut to 50 slices to haha0~haha49 .wav) and noise.wav ,doing a snr [5,10,15]. Then output the before and after's waveform and Spectrogram. But,teriminal always tell me RuntimeError: Error opening 'D:\python\pythonnoisy\add_white\haha0_white_snr5.wav': System error. However, I can see this file in my folder.
import os
import re
import sys
import wave
import librosa
import matplotlib
import numpy as np
import pylab as pl
import soundfile as sf
import matplotlib.pyplot as plt
from scipy.fftpack import fft
wavedir = r"D:\python"
noisydir = wavedir+"\\pythonnoisy"
noisedir = wavedir+"\\pythonnoise"
cleandir = wavedir+"\\pythonclean"
def add_noise(noisydir, noisedir, cleandir, snr): # noisy
noisewav = "white.wav"
noise, fs = sf.read(noisedir+"\\"+noisewav) # 讀取白雜訊.wav
noisy_splitdir = noisydir+"\\"+"add_"+noisewav[:-4]+"\\"
# 迴圈取原始wav檔資料夾裡所有檔案
for cleanwav in os.listdir(cleandir):
clean, Fs = sf.read(cleandir+"\\"+cleanwav) # 讀取原始.wav檔
# 取樣頻率:原始音檔==白雜訊&&時長:原始音檔<白雜訊
if fs == Fs and len(clean) <= len(noise):
cleanenergy = np.sum(np.power(clean, 2)) # 原始音檔Power(=振幅^2)
# 1<隨機生成長度<noise長-clean長+1
ind = np.random.randint(1, len(noise) - len(clean) + 1)
noiselen = noise[ind:len(clean) + ind]
noiseenergy = np.sum(np.power(noiselen, 2)) # 白雜訊Power
ratio2 = np.sqrt(
(cleanenergy / noiseenergy) / (np.power(10, snr * 0.1)))
noisyAudio = clean + noiselen * ratio2 # 混音振幅
# 混音路徑+檔名
noisywavname = noisy_splitdir + \
cleanwav[:-4]+"_"+noisewav[:-4]+"_snr"+str(snr)+".wav"
sf.write(noisywavname, noisyAudio, 44100) # 生成混音.wav
def draw_clean(cleandir, j):
f = wave.open(
r"D:\python\pythonclean\haha" + str(j) + ".wav", "rb")
params = f.getparams()
nchannels, sampwidth, framerate, nframes = params[:4]
str_data = f.readframes(nframes)
f.close()
wave_data = np.fromstring(str_data, dtype=np.short)
wave_data.shape = -1, 2
wave_data = wave_data.T
time = np.arange(0, nframes) * (1.0 / framerate)
plt.subplot(4, 2, 7)
plt.plot(time, wave_data[0])
plt.xlabel("time(s)")
plt.subplot(4, 2, 8)
plt.specgram(wave_data[0], Fs=framerate)
plt.xlabel("time(s)")
def draw_noisy(noisydir, j):
noisydirr = noisydir+"\\add_white\\haha"
for k in range(5, 20, 5):
f = wave.open(r"D:\python\pythonnoisy\add_white\haha" +
str(j)+"_white_snr"+str(k)+".wav", "rb")
params = f.getparams()
nchannels, sampwidth, framerate, nframes = params[:4]
str_data = f.readframes(nframes)
wave_data = np.fromstring(str_data, dtype=np.short)
wave_data.shape = -1, 2
wave_data = wave_data.T
time = np.arange(0, nframes) * (1.0 / framerate)
plt.subplot(4, 2, k/5*2-1)
plt.plot(time, wave_data[0])
plt.subplot(4, 2, k/5*2)
plt.specgram(wave_data[0], Fs=framerate)
wavedir = r"D:\python"
noisydir = wavedir+"\\pythonnoisy"
noisedir = wavedir+"\\pythonnoise"
cleandir = wavedir+"\\pythonclean"
level = [5, 10, 15]
for snr in level:
add_noise(noisydir, noisedir, cleandir, snr)
for j in range(0, 51):
draw_clean(cleandir, j)
draw_noisy(noisydir, j)
picture = noisydir+"\picture\hahawhite"+str(j)+".png"
plt.savefig(picture)
plt.close()
file location
terminal
Heatmapped png's not being created on disk
I'm creating a heatmap for a CNN as per this tutorial.
In the last part:
def create_patiens_cam(case, plane):
patient_id = case['id']
mri = case['mri']
folder_path = f'./CAMS/{plane}/{patient_id}/'
if os.path.isdir(folder_path):
shutil.rmtree(folder_path)
os.makedirs(folder_path)
os.makedirs(folder_path + 'slices/')
os.makedirs(folder_path + 'cams/')
params = list(mrnet.parameters())
weight_softmax = np.squeeze(params[-2].cpu().data.numpy())
num_slices = mri.shape[1]
global feature_blobs
feature_blobs = []
mri = mri.to(device)
logit = mrnet(mri)
size_upsample = (256, 256)
feature_conv = feature_blobs[0]
h_x = F.softmax(logit, dim=1).data.squeeze(0)
probs, idx = h_x.sort(0, True)
probs = probs.cpu().numpy()
idx = idx.cpu().numpy()
slice_cams = returnCAM(feature_blobs[-1], weight_softmax, idx[:1])
for s in tqdm_notebook(range(num_slices), leave=False):
slice_pil = (transforms
.ToPILImage()(mri.cpu()[0][s] / 255))
slice_pil.save(folder_path + f'slices/{s}.png',
dpi=(300, 300))
img = mri[0][s].cpu().numpy()
img = img.transpose(1, 2, 0)
heatmap = (cv2
.cvtColor(cv2.applyColorMap(
cv2.resize(slice_cams[s], (256, 256)),
cv2.COLORMAP_JET),
cv2.COLOR_BGR2RGB)
)
result = heatmap * 0.3 + img * 0.5
pil_img_cam = Image.fromarray(np.uint8(result))
pil_img_cam.save(folder_path + f'cams/{s}.png', dpi=(300, 300))
I have created a folder "CAMS" in my 'mrnet' folder. However when running this last code (in jupyter notebook) I get no errors but no png's are being created. Anyone has any idea what could be wrong or where I could look to see what's wrong as I get no errors?
FULL CODE:
# -*- coding: utf-8 -*-
"""
Created on Sat Mar 13 21:54:40 2021
#author: GlaDOS
"""
import os
import io
import requests
from PIL import Image
from torchvision import models, transforms
from torch.autograd import Variable
from torch.nn import functional as F
import numpy as np
import cv2
import pdb
from matplotlib import pyplot as plt
import sys
sys.path.append('C:/Users/GlaDOS/mrnet')
import shutil
import torch
import model
from dataloader import MRDataset
from tqdm import tqdm_notebook
task = 'acl'
plane = 'sagittal'
prefix = 'sag'
model_name = [name for name in os.listdir('C:/Users/GlaDOS/mrnet/models/')
if (task in name) and
(plane in name) and
(prefix in name)][0]
is_cuda = torch.cuda.is_available()
device = torch.device("cuda" if is_cuda else "cpu")
mrnet = torch.load(f'C:/Users/GlaDOS/mrnet/models/{model_name}')
mrnet = mrnet.to(device)
_ = mrnet.eval()
dataset = MRDataset('C:/Users/GlaDOS/mrnet/data/',
task,
plane,
transform=None,
train=False)
loader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=0,
drop_last=False)
def returnCAM(feature_conv, weight_softmax, class_idx):
size_upsample = (256, 256)
bz, nc, h, w = feature_conv.shape
slice_cams = []
for s in range(bz):
for idx in class_idx:
cam = weight_softmax[idx].dot(feature_conv[s].reshape((nc, h*w)))
cam = cam.reshape(h, w)
cam = cam - np.min(cam)
cam_img = cam / np.max(cam)
cam_img = np.uint8(255 * cam_img)
slice_cams.append(cv2.resize(cam_img, size_upsample))
return slice_cams
patients = []
for i, (image, label, _) in tqdm_notebook(enumerate(loader), total=len(loader)):
patient_data = {}
patient_data['mri'] = image
patient_data['label'] = label[0][0][1].item()
patient_data['id'] = '0' * (4 - len(str(i))) + str(i)
patients.append(patient_data)
acl = list(filter(lambda d: d['label'] == 1, patients))
def create_patiens_cam(case, plane):
patient_id = case['id']
mri = case['mri']
folder_path = f'C:/Users/GlaDOS/mrnet/cams/{plane}/{patient_id}/'
if os.path.isdir(folder_path):
shutil.rmtree(folder_path)
os.makedirs(folder_path)
os.makedirs(folder_path + 'slices/')
os.makedirs(folder_path + 'cams/')
params = list(mrnet.parameters())
weight_softmax = np.squeeze(params[-2].cpu().data.numpy())
num_slices = mri.shape[1]
global feature_blobs
feature_blobs = []
mri = mri.to(device)
logit = mrnet(mri)
size_upsample = (256, 256)
feature_conv = feature_blobs[0]
h_x = F.softmax(logit, dim=1).data.squeeze(0)
probs, idx = h_x.sort(0, True)
probs = probs.cpu().numpy()
idx = idx.cpu().numpy()
slice_cams = returnCAM(feature_blobs[-1], weight_softmax, idx[:1])
for s in tqdm_notebook(range(num_slices), leave=False):
slice_pil = (transforms
.ToPILImage()(mri.cpu()[0][s] / 255))
slice_pil.save(folder_path + f'slices/{s}.png',
dpi=(300, 300))
img = mri[0][s].cpu().numpy()
img = img.transpose(1, 2, 0)
heatmap = (cv2
.cvtColor(cv2.applyColorMap(
cv2.resize(slice_cams[s], (256, 256)),
cv2.COLORMAP_JET),
cv2.COLOR_BGR2RGB)
)
result = heatmap * 0.3 + img * 0.5
pil_img_cam = Image.fromarray(np.uint8(result))
pil_img_cam.save(folder_path + f'cams/{s}.png', dpi=(300, 300))
Use seaborn:
import seaborn as sns
sns_plot.savefig('output.png')