Develop Reference

Running python script with multiple values of command line arguments

I have a python script for pre-processing audio and it has frame length, frame step and fft length as the command line arguments. I am able to run the code if I have single values of these arguments. I wanted to know if there is a way in which I can run the python script with multiple values of the arguments? For example, get the output if values of fft lengths are 128, 256 and 512 instead of just one value.
The code for pre-processing is as follows:
import numpy as np
import pandas as pd
import tensorflow as tf
from scipy.io import wavfile
import os
import time
import pickle
import random
import argparse
import configlib
from configlib import config as C
import mfccwithpaddingandcmd
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.preprocessing import MultiLabelBinarizer
from tensorflow import keras
from tensorflow.python.keras import Sequential
from tensorflow.keras.layers import Dense,Conv2D,MaxPooling2D,Flatten,Dropout,BatchNormalization,LSTM,Lambda,Reshape,Bidirectional,GRU
from tensorflow.keras.callbacks import TensorBoard
start = time.time()
classes = ['blinds','fan','light','music','tv']
#dire = r"/mnt/beegfs/home/gehani/test_speech_command/"
parser = configlib.add_parser("Preprocessing config")
parser.add_argument("-dir","--dire", metavar="", help="Directory for the audio files")
def pp():
data_list=[] #To save paths of all the audio files.....all audio files in list format in data_list
#data_list-->folder-->files in folder
for index,label in enumerate(classes):
class_list=[]
if label=='silence': #creating silence folder and storing 1sec noise audio files
silence_path = os.path.join(C["dire"],'silence')
if not os.path.exists(silence_path):
os.mkdir(silence_path)
silence_stride = 2000
#sample_rate = 16000
folder = os.path.join(C["dire"],'_background_noise_') #all silence are kept in the background_noise folder
for file_ in os.listdir(folder):
if '.wav' in file_:
load_path = os.path.join(folder,file_)
sample_rate,y = wavfile.read(load_path)
for i in range(0,len(y)-sample_rate,silence_stride):
file_path = "silence/{}_{}.wav".format(file_[:-4],i)
y_slice = y[i:i+sample_rate]
wavfile.write(os.path.join(C["dire"],file_path),sample_rate,y_slice)
class_list.append(file_path)
else:
folder = os.path.join(C["dire"],label)
for file_ in os.listdir(folder):
file_path = '{}/{}'.format(label,file_) #Ex: up/c9b653a0_nohash_2.wav
class_list.append(file_path)
random.shuffle(class_list) #To shuffle files
data_list.append(class_list) #if not a silence file then just append to the datalist
X = []
Y = []
preemphasis = 0.985
print("Feature Extraction Started")
for i,class_list in enumerate(data_list): #datalist = all files, class list = folder name in datalist, sample = path to the audio file in that particular class list
for j,samples in enumerate(class_list): #samples are of the form classes_name/audio file
if(samples.endswith('.wav')):
sample_rate,audio = wavfile.read(os.path.join(C["dire"],samples))
if(audio.size<sample_rate):
audio = np.pad(audio,(sample_rate-audio.size,0),mode="constant")
#print("****")
#print(sample_rate)
#print(preemphasis)
#print(audio.shape)
coeff = mfccwithpaddingandcmd.mfcc(audio,sample_rate,preemphasis) # 0.985 = preemphasis
#print("****")
#print(coeff)
#print("****")
X.append(coeff)
#print(X)
if(samples.split('/')[0] in classes):
Y.append(samples.split('/')[0])
elif(samples.split('/')[0]=='_background_noise_'):
Y.append('silence')
#print(len(X))
#print(len(Y))
#X= coefficient array and Y = name of the class
A = np.zeros((len(X),X[0].shape[0],X[0][0].shape[0]),dtype='object')
for i in range(0,len(X)):
A[i] = np.array(X[i]) #Converting list X into array A
end1 = time.time()
print("Time taken for feature extraction:{}sec".format(end1-start))
MLB = MultiLabelBinarizer() # one hot encoding for converting labels into binary form
MLB.fit(pd.Series(Y).fillna("missing").str.split(', '))
Y_MLB = MLB.transform(pd.Series(Y).fillna("missing").str.split(', '))
MLB.classes_ #Same like classes array
print(Y_MLB.shape)
pickle_out = open("A_all.pickle","wb") #Writes array A to a file A.pickle
pickle.dump(A, pickle_out) #pickle is the file containing the extracted features
pickle_out.close()
pickle_out = open("Y_all.pickle","wb")
pickle.dump(Y_MLB, pickle_out)
pickle_out.close()
pickle_in = open("Y_all.pickle","rb")
Y = pickle.load(pickle_in)
X = tf.keras.utils.normalize(X)
X_train,X_valtest,Y_train,Y_valtest = train_test_split(X,Y,test_size=0.2,random_state=37)
X_val,X_test,Y_val,Y_test = train_test_split(X_valtest,Y_valtest,test_size=0.5,random_state=37)
print(X_train.shape,X_val.shape,X_test.shape,Y_train.shape,Y_val.shape,Y_test.shape)
if __name__ == "__main__":
configlib.parse(save_fname="last_arguments.txt")
print("Running with configuration:")
configlib.print_config()
pp()
The code for MFCC is as follows:
import tensorflow as tf
import scipy.io.wavfile as wav
import numpy as np
import matplotlib.pyplot as plt
import pickle
import argparse
import configlib
from configlib import config as C
# Configuration arguments
parser = configlib.add_parser("MFCC config")
parser.add_argument("-fl","--frame_length", type=int, default=400, metavar="", help="Frame Length")
parser.add_argument("-fs","--frame_step", type=int, default=160, metavar="", help="Frame Step")
parser.add_argument("-fft","--fft_length", type=int, default=512, metavar="", help="FFT length")
#args = parser.parse_args()
def Preemphasis(signal,pre_emp):
return np.append(signal[0],signal[1:]-pre_emp*signal[:-1])
def Paddinggg(framelength,framestep,samplerate):
frameStart = np.arange(0,samplerate,framestep)
frameEnd = frameStart + framelength
padding = min(frameEnd[(frameEnd > samplerate)]) - samplerate
return padding
def mfcc(audio,sample_rate,pre_emp):
audio = np.pad(audio,(Paddinggg(C["frame_length"],C["frame_step"],sample_rate),0),mode='reflect')
audio = audio.astype('float32')
#Normalization
audio = tf.keras.utils.normalize(audio)
#Preemphasis
audio = Preemphasis(audio,pre_emp)
stfts = tf.signal.stft(audio,C["frame_length"],C["frame_step"],C["fft_length"],window_fn=tf.signal.hann_window)
spectrograms = tf.abs(stfts)
num_spectrogram_bins = stfts.shape[-1]
lower_edge_hertz, upper_edge_hertz, num_mel_bins = 0.0, sample_rate/2.0, 32
linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(num_mel_bins, num_spectrogram_bins, sample_rate, lower_edge_hertz,upper_edge_hertz)
mel_spectrograms = tf.tensordot(spectrograms, linear_to_mel_weight_matrix, 1)
mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(linear_to_mel_weight_matrix.shape[-1:]))
# Compute a stabilized log to get log-magnitude mel-scale spectrograms.
log_mel_spectrograms = tf.math.log(mel_spectrograms + 1e-6)
# Compute MFCCs from log_mel_spectrograms and take the first 13.
return log_mel_spectrograms
print("End")
And the code for configlib is as follows:
from typing import Dict, Any
import logging
import pprint
import sys
import argparse
# Logging for config library
logger = logging.getLogger(__name__)
# Our global parser that we will collect arguments into
parser = argparse.ArgumentParser(description=__doc__, fromfile_prefix_chars="#")
# Global configuration dictionary that will contain parsed arguments
# It is also this variable that modules use to access parsed arguments
config:Dict[str, Any] = {}
def add_parser(title: str, description: str = ""):
"""Create a new context for arguments and return a handle."""
return parser.add_argument_group(title, description)
def parse(save_fname: str = "") -> Dict[str, Any]:
"""Parse given arguments."""
config.update(vars(parser.parse_args()))
logging.info("Parsed %i arguments.", len(config))
# Optionally save passed arguments
if save_fname:
with open(save_fname, "w") as fout:
fout.write("\n".join(sys.argv[1:]))
logging.info("Saving arguments to %s.", save_fname)
return config
def print_config():
"""Print the current config to stdout."""
pprint.pprint(config)
I use the following command to run my python file:
python3.7 preprocessingwithpaddingandcmd.py -fl 1103 -fs 88 -fft 512 -dir /mnt/beegfs/home/gehani/appliances_audio_one_channel
Should I be writing a shell script or python has some options for it?
EDIT 1
I tried using
parser.add_argument('-fft', '--fft_length', type=int, default=[], nargs=3)
for getting fft length from the command line and used the command
run preprocessingwithpaddingandcmd -dir filepath -fl 1765 -fs 1102 -fft 512 218 64
to run it. But, it gives me this error: ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
Can anyone please help?

I found you can do it by these. mfcc features extraction
You can create your own mfcc features extraction or you can limit window lengths and ceptrums that is enough for simple works except you need logarithms scales where you can use target matrix ( convolution ) or else.
It is logarithms when you use FFT or alternative derivation but mfcc is only extraction where I will provide the sample output in picture.
[ Sample ]:
from python_speech_features import mfcc
from python_speech_features import logfbank
import scipy.io.wavfile as wav
import tensorflow as tf
import matplotlib.pyplot as plt
(rate,sig) = wav.read("F:\\temp\\Python\\Speech\\temple_of_love-sisters_of_mercy.wav")
mfcc_feat = mfcc(signal=sig, samplerate=rate, winlen=0.025, winstep=0.01, numcep=13, nfilt=26, nfft=512, lowfreq=0, highfreq=None, preemph=0.97, ceplifter=22, appendEnergy=True)
fbank_feat = logfbank(sig,rate)
plt.plot( mfcc_feat[50:42000,0] )
plt.xlabel("sample")
plt.show()
plt.close()
input('...')

How do I pass arguments when I import the below .py script as a module?

I know how to execute it via the command line, but looking for input to execute inside another python script. I tried passing the arguments as main(-- score-thr 0.7, --show, vid, config_file, chkpnt_file). Not sure that's how to do it though. Thank you in advance for helping out!
import argparse
import cv2
import mmcv
from mmdet.apis import inference_detector, init_detector
def parse_args():
parser = argparse.ArgumentParser(description='MMDetection video demo')
parser.add_argument('video', help='Video file')
parser.add_argument('config', help='Config file')
parser.add_argument('checkpoint', help='Checkpoint file')
parser.add_argument(
'--device', default='cuda:0', help='Device used for inference')
parser.add_argument(
'--score-thr', type=float, default=0.7, help='Bbox score threshold')
parser.add_argument('--out', type=str, help='Output video file')
parser.add_argument('--show', action='store_true', help='Show video')
parser.add_argument(
'--wait-time',
type=float,
default=1,
help='The interval of show (s), 0 is block')
args = parser.parse_args()
return args
def main():
args = parse_args()
assert args.out or args.show, \
('Please specify at least one operation (save/show the '
'video) with the argument "--out" or "--show"')
model = init_detector(args.config, args.checkpoint, device=args.device)
video_reader = mmcv.VideoReader(args.video)
video_writer = None
if args.out:
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
video_writer = cv2.VideoWriter(
args.out, fourcc, video_reader.fps,
(video_reader.width, video_reader.height))
for frame in mmcv.track_iter_progress(video_reader):
result = inference_detector(model, frame)
frame = model.show_result(frame, result, score_thr=args.score_thr)
if args.show:
cv2.namedWindow('video', 0)
mmcv.imshow(frame, 'video', args.wait_time)
if args.out:
video_writer.write(frame)
if video_writer:
video_writer.release()
cv2.destroyAllWindows()
if __name__ == '__main__':
main()

main() gets its arguments from sys.argv when it uses argparse. So you need to fill that in before calling it.
sys.argv = ['progname.py', '--score-thr', '0.7', '--show', vid, config_file, chkpnt_file]
main()

How can I pass an image into my code from the linux command line?

I am trying to adapt some code to better suit my needs. The code currently takes all the files from a folder and runs image recognition on them, but I need to pass a single image that I specify from the command line.
Here is the code I am using:
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
# import the necessary packages
import json
import os
import random
import cv2 as cv
import keras.backend as K
import numpy as np
import scipy.io
from utils import load_model
if __name__ == '__main__':
img_width, img_height = 224, 224
model = load_model()
model.load_weights('models/model.96-0.89.hdf5')
cars_meta = scipy.io.loadmat('devkit/cars_meta')
class_names = cars_meta['class_names'] # shape=(1, 196)
class_names = np.transpose(class_names)
test_path = 'data/test/'
test_images = [f for f in os.listdir(test_path) if
os.path.isfile(os.path.join(test_path, f)) and f.endswith('.jpg')]
num_samples = 1
samples = random.sample(test_images, num_samples)
results = []
for i, image_name in enumerate(samples):
filename = os.path.join(test_path, image_name)
print('Start processing image: {}'.format(filename))
bgr_img = cv.imread(filename)
bgr_img = cv.resize(bgr_img, (img_width, img_height), cv.INTER_CUBIC)
rgb_img = cv.cvtColor(bgr_img, cv.COLOR_BGR2RGB)
rgb_img = np.expand_dims(rgb_img, 0)
preds = model.predict(rgb_img)
prob = np.max(preds)
class_id = np.argmax(preds)
text = ('Predict: {}, prob: {}'.format(class_names[class_id][0][0], prob))
results.append({'label': class_names[class_id][0][0], 'prob': '{:.4}'.format(prob)})
cv.imwrite('images/{}_out.png'.format(i), bgr_img)
print(results)
with open('results.json', 'w') as file:
json.dump(results, file, indent=4)
K.clear_session()
Is there a way that I can just pass a single image, instead of an entire folder, and do so from the command line?

sys.argv is a list in Python, which contains the command-line arguments passed to the script.
pay attention sys.argv[0] is the name of the script itself!
so the easieaset way to do it is by the built in sys.argv list. for example if my script is called : foo.py, i can pass to it arguments via command line like so:
python foo.py "C:\user\myimg.jpeg"
and in foo.py ican get the passed argument like so:
import sys
if __name__ == '__main__':
for arg in sys.argv[1:]:
#do work with arg! arg is a string type!
with open(arg) as fp:
#.... do work ....
argparse module
The following code is a Python program that takes a list of integers and produces either the sum or the max:
import argparse
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('integers', metavar='N', type=int, nargs='+',
help='an integer for the accumulator')
parser.add_argument('--sum', dest='accumulate', action='store_const',
const=sum, default=max,
help='sum the integers (default: find the max)')
args = parser.parse_args()
print(args.accumulate(args.integers))
using or calling from command line:
python foo.py 1 2 3 4
output: 4
python foo.py 1 2 3 4 --sum
10

Tensorflow ImageAI GPU usage

I use imageai to crop my car dataset for cnn training. My code is quite simple, I have two GPUs and accoding to tensorflow logs and nvidia-smi output, it uses both of my GPUs, but at the same time it doesn't use all available memory
import os
import sys, getopt
# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
from imageai.Detection import ObjectDetection
from time import perf_counter
import shutil
import numpy as np
from io import BytesIO
from PIL import Image
import argparse
import time
from shutil import copyfile
parser = argparse.ArgumentParser(description='Car Images Filter')
parser.add_argument("-i", "--input_folder", action="store", dest="input_folder", help="Folder for filtering", type=str, required=True)
parser.add_argument("-o", "--output_folder", action="store", dest="output_folder", help="Folder for the results of wrong images", type=str, required=False)
parser.add_argument("-start_from_model", "--start_from_model", action="store", dest="start_from_model", help="Start model for exception cases", type=str, default='')
parser.add_argument("-start_from_mark", "--start_from_mark", action="store", dest="start_from_mark", help="Start mark for exception cases", type=str, default='')
parser.add_argument("-start_from_file", "--start_from_file", action="store", dest="start_from_file", help="Start file for exception cases", type=str, default='')
parser.add_argument("-remove", "--remove", action="store_true", dest="remove", help="Folder for the results of wrong images", required=False)
args = parser.parse_args()
execution_path = os.getcwd()
detector = ObjectDetection()
detector.setModelTypeAsRetinaNet()
detector.setModelPath(os.path.join(execution_path , "resnet50_coco_best_v2.0.1.h5"))
detector.loadModel()
directories = os.listdir(args.input_folder)
if args.start_from_mark:
directories = directories[directories.index(args.start_from_mark):]
for mark in directories:
models = os.listdir(os.path.join(args.input_folder, mark))
for model in models:
files = os.listdir(os.path.join(args.input_folder, mark, model))
if model==args.start_from_model and args.start_from_file:
files=files[files.index(args.start_from_file):]
for file in files:
if(file.endswith('.json')):
continue
image = Image.open(os.path.join(args.input_folder, mark, model, file))
detected_image_array, detections = detector.detectCustomObjectsFromImage(custom_objects=detector.CustomObjects(car=True,truck=True), input_image=os.path.join(args.input_folder, mark, model, file), output_type="array")
detected_image = Image.fromarray(detected_image_array, 'RGB')
if len(detections) == 0:
copyfile(os.path.join(args.input_folder, mark, model, file), os.path.join(args.output_folder, mark, model, file))
print(mark + " " + model + " " + file)
And here is my nvidia-smi output:
Image link
How can I increase GPU memory usage to make it faster?

how to pass input in argparse argument in python

this code should ask 2 optional and 1 required argument, 'format' and 'type' argument are optional and 'url' is required argument which will be the address of website from where you wants to download image(either jpg or/and .png type image,depend on what value you give in 'type' argument ) and save it as either as individual image or single JSON file(depend on what value you give in 'format' argument ), i am getting error can you tell where i am going wrong,here is my code..
import argparse
import base64
import json
import os
from bs4 import BeautifulSoup
import requests
def scrape(url,format_,type_):
try:
page = requests.get(url)
except requests.RequestException as rex:
print(str(rex))
else:
soup = BeautifulSoup(url,'html.parser')
images = _fetch_images(soup,url)
images = _filter_images(images,type_)
_save(images,format_)
def _fetch_images(soup,base_url):
images = []
for img in soup.find_all('img'):
src = img.get('src')
img_url = ('{base_url}/{src}'.format(base_url,src))
name = img_url.split('\\')[-1]
images.append(dict(name=name,url=img_url))
return images
def _filter_images(images,type_):
if type == 'all':
return images
ext_map = {'png':['.png'],'jpg':['.jpg','.jpeg']}
return [img for img in images if
_match_extension(img['name'],ext_map(type_))]
def _match_extension(filename,extension_list):
name,extension = os.path.splittext(filename.lower())
return extension in extension_list
def _save(images,format_):
if images:
if format_=='img':
_save_images(images)
else:
_save_json(images)
print('Done!')
else:
print('there are no images!')
def _save_images(images):
for img in images:
img_data = requests.get(img['url']).content
with open(img['name'],'wb') as f:
f.write(img_data)
def _save_json(images):
data = {}
for img in images:
img_data = requests.get(img['url']).content
b64_img_data = base64.b64encode(img_data)
str_img_data = b64_img_data.decode('utf-8')
data[img['name']]=str_img_data
with open('images.json','w') as ijson:
ijson.write(json.dump(data))
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Scrape a webpage.')
parser.add_argument(
'-t',
'--type',
choices=['all', 'png', 'jpg'],
default='all',
help='The image type we want to scrape.')
parser.add_argument(
'-f',
'--format',
choices=['img', 'json'],
default='img',
help='The format images are saved to.')
parser.add_argument(
'url',
help='The URL we want to scrape for images.')
args = parser.parse_args()
args.url = str(input('enter the url\n'))
args.format = str(input('enter img or json \n'))
args.type = str(input('enter all or png or jpg'))
scrape(args.url,args.format,args.type)
i wants to run it in PyCharm and error i got is,
usage: scrape.py [-h] [-t {all,png,jpg}] [-f {img,json}] url
scrape.py: error: the following arguments are required: url

In order to pass parameters to your function in Pycharm, navigate to Run -> Edit Configurations and set the parameters there.
Alternatively, try invoking your program with parameters specified from the command line.