I use imageai to crop my car dataset for cnn training. My code is quite simple, I have two GPUs and accoding to tensorflow logs and nvidia-smi output, it uses both of my GPUs, but at the same time it doesn't use all available memory
import os
import sys, getopt
# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
from imageai.Detection import ObjectDetection
from time import perf_counter
import shutil
import numpy as np
from io import BytesIO
from PIL import Image
import argparse
import time
from shutil import copyfile
parser = argparse.ArgumentParser(description='Car Images Filter')
parser.add_argument("-i", "--input_folder", action="store", dest="input_folder", help="Folder for filtering", type=str, required=True)
parser.add_argument("-o", "--output_folder", action="store", dest="output_folder", help="Folder for the results of wrong images", type=str, required=False)
parser.add_argument("-start_from_model", "--start_from_model", action="store", dest="start_from_model", help="Start model for exception cases", type=str, default='')
parser.add_argument("-start_from_mark", "--start_from_mark", action="store", dest="start_from_mark", help="Start mark for exception cases", type=str, default='')
parser.add_argument("-start_from_file", "--start_from_file", action="store", dest="start_from_file", help="Start file for exception cases", type=str, default='')
parser.add_argument("-remove", "--remove", action="store_true", dest="remove", help="Folder for the results of wrong images", required=False)
args = parser.parse_args()
execution_path = os.getcwd()
detector = ObjectDetection()
detector.setModelTypeAsRetinaNet()
detector.setModelPath(os.path.join(execution_path , "resnet50_coco_best_v2.0.1.h5"))
detector.loadModel()
directories = os.listdir(args.input_folder)
if args.start_from_mark:
directories = directories[directories.index(args.start_from_mark):]
for mark in directories:
models = os.listdir(os.path.join(args.input_folder, mark))
for model in models:
files = os.listdir(os.path.join(args.input_folder, mark, model))
if model==args.start_from_model and args.start_from_file:
files=files[files.index(args.start_from_file):]
for file in files:
if(file.endswith('.json')):
continue
image = Image.open(os.path.join(args.input_folder, mark, model, file))
detected_image_array, detections = detector.detectCustomObjectsFromImage(custom_objects=detector.CustomObjects(car=True,truck=True), input_image=os.path.join(args.input_folder, mark, model, file), output_type="array")
detected_image = Image.fromarray(detected_image_array, 'RGB')
if len(detections) == 0:
copyfile(os.path.join(args.input_folder, mark, model, file), os.path.join(args.output_folder, mark, model, file))
print(mark + " " + model + " " + file)
And here is my nvidia-smi output:
Image link
How can I increase GPU memory usage to make it faster?
Related
I am implementing image morphing from github in python. For that I am using delaunay triangulation, but I getting error importing libraries. I tried installing delaunay but it didn't work.
from face_landmark_detection import generate_face_correspondences
from delaunay_triangulation import make_delaunay
from face_morph import generate_morph_sequence
import subprocess
import argparse
import shutil
import os
import cv2
def doMorphing(img1, img2, duration, frame_rate, output):
[size, img1, img2, points1, points2, list3] = generate_face_correspondences(img1, img2)
if \__name_\_ == "\__main_\_":
parser = argparse.ArgumentParser()
parser.add_argument("--img1", required=True, help="The First Image")
parser.add_argument("--img2", required=True, help="The Second Image")
parser.add_argument("--duration", type=int, default=5, help="The duration")
parser.add_argument("--frame", type=int, default=20, help="The frameame Rate")
parser.add_argument("--output", help="Output Video Path")
args = parser.parse_args()
image1 = cv2.imread(args.img1)
image2 = cv2.imread(args.img2)
doMorphing(image1, image2, args.duration, args.frame, args.output)
I am implementing https://github.com/Azmarie/Face-Morphing/tree/master/code, but I am getting error implementing delaunay triangulation for image morphing.
output)
I'm trying to use IrNet using a github repository. In the instructions it says I need to run a command python run_sample.py to make it work. I have obviously cloned the repo into my google colab but then after running this command line it throws me an error run_sample.py: error: the following arguments are required: --voc12_root. After checking this run_sample.py I can see that the program requires a path parser.add_argument("--voc12_root", required=True, type=str, help="Path to VOC 2012 Devkit, must contain ./JPEGImages as subdirectory."), however I simply do not know how to satisfy this requirement. Where and how to paste it?
I'm trying to use this repo: https://github.com/jiwoon-ahn/irn
Here is the program I'm trying to run:
import argparse
import os
from misc import pyutils
if __name__ == '__main__':
parser = argparse.ArgumentParser()
# Environment
parser.add_argument("--num_workers", default=os.cpu_count()//2, type=int)
parser.add_argument("--voc12_root", required=True, type=str,
help="Path to VOC 2012 Devkit, must contain ./JPEGImages as subdirectory.")
# Dataset
parser.add_argument("--train_list", default="voc12/train_aug.txt", type=str)
parser.add_argument("--val_list", default="voc12/val.txt", type=str)
parser.add_argument("--infer_list", default="voc12/train.txt", type=str,
help="voc12/train_aug.txt to train a fully supervised model, "
"voc12/train.txt or voc12/val.txt to quickly check the quality of the labels.")
parser.add_argument("--chainer_eval_set", default="train", type=str)
# Class Activation Map
parser.add_argument("--cam_network", default="net.resnet50_cam", type=str)
parser.add_argument("--cam_crop_size", default=512, type=int)
parser.add_argument("--cam_batch_size", default=16, type=int)
parser.add_argument("--cam_num_epoches", default=5, type=int)
parser.add_argument("--cam_learning_rate", default=0.1, type=float)
parser.add_argument("--cam_weight_decay", default=1e-4, type=float)
parser.add_argument("--cam_eval_thres", default=0.15, type=float)
parser.add_argument("--cam_scales", default=(1.0, 0.5, 1.5, 2.0),
help="Multi-scale inferences")
# Mining Inter-pixel Relations
parser.add_argument("--conf_fg_thres", default=0.30, type=float)
parser.add_argument("--conf_bg_thres", default=0.05, type=float)
# Inter-pixel Relation Network (IRNet)
parser.add_argument("--irn_network", default="net.resnet50_irn", type=str)
parser.add_argument("--irn_crop_size", default=512, type=int)
parser.add_argument("--irn_batch_size", default=32, type=int)
parser.add_argument("--irn_num_epoches", default=3, type=int)
parser.add_argument("--irn_learning_rate", default=0.1, type=float)
parser.add_argument("--irn_weight_decay", default=1e-4, type=float)
# Random Walk Params
parser.add_argument("--beta", default=10)
parser.add_argument("--exp_times", default=8,
help="Hyper-parameter that controls the number of random walk iterations,"
"The random walk is performed 2^{exp_times}.")
parser.add_argument("--ins_seg_bg_thres", default=0.25)
parser.add_argument("--sem_seg_bg_thres", default=0.25)
# Output Path
parser.add_argument("--log_name", default="sample_train_eval", type=str)
parser.add_argument("--cam_weights_name", default="sess/res50_cam.pth", type=str)
parser.add_argument("--irn_weights_name", default="sess/res50_irn.pth", type=str)
parser.add_argument("--cam_out_dir", default="result/cam", type=str)
parser.add_argument("--ir_label_out_dir", default="result/ir_label", type=str)
parser.add_argument("--sem_seg_out_dir", default="result/sem_seg", type=str)
parser.add_argument("--ins_seg_out_dir", default="result/ins_seg", type=str)
# Step
parser.add_argument("--train_cam_pass", default=True)
parser.add_argument("--make_cam_pass", default=True)
parser.add_argument("--eval_cam_pass", default=True)
parser.add_argument("--cam_to_ir_label_pass", default=True)
parser.add_argument("--train_irn_pass", default=True)
parser.add_argument("--make_ins_seg_pass", default=True)
parser.add_argument("--eval_ins_seg_pass", default=True)
parser.add_argument("--make_sem_seg_pass", default=True)
parser.add_argument("--eval_sem_seg_pass", default=True)
args = parser.parse_args()
os.makedirs("sess", exist_ok=True)
os.makedirs(args.cam_out_dir, exist_ok=True)
os.makedirs(args.ir_label_out_dir, exist_ok=True)
os.makedirs(args.sem_seg_out_dir, exist_ok=True)
os.makedirs(args.ins_seg_out_dir, exist_ok=True)
pyutils.Logger(args.log_name + '.log')
print(vars(args))
if args.train_cam_pass is True:
import step.train_cam
timer = pyutils.Timer('step.train_cam:')
step.train_cam.run(args)
if args.make_cam_pass is True:
import step.make_cam
timer = pyutils.Timer('step.make_cam:')
step.make_cam.run(args)
if args.eval_cam_pass is True:
import step.eval_cam
timer = pyutils.Timer('step.eval_cam:')
step.eval_cam.run(args)
if args.cam_to_ir_label_pass is True:
import step.cam_to_ir_label
timer = pyutils.Timer('step.cam_to_ir_label:')
step.cam_to_ir_label.run(args)
if args.train_irn_pass is True:
import step.train_irn
timer = pyutils.Timer('step.train_irn:')
step.train_irn.run(args)
if args.make_ins_seg_pass is True:
import step.make_ins_seg_labels
timer = pyutils.Timer('step.make_ins_seg_labels:')
step.make_ins_seg_labels.run(args)
if args.eval_ins_seg_pass is True:
import step.eval_ins_seg
timer = pyutils.Timer('step.eval_ins_seg:')
step.eval_ins_seg.run(args)
if args.make_sem_seg_pass is True:
import step.make_sem_seg_labels
timer = pyutils.Timer('step.make_sem_seg_labels:')
step.make_sem_seg_labels.run(args)
if args.eval_sem_seg_pass is True:
import step.eval_sem_seg
timer = pyutils.Timer('step.eval_sem_seg:')
step.eval_sem_seg.run(args)
I'd be grateful on any suggestions how to deal with that issue. It seems like a simple thing "run this and here you go, the program's ready to work", but I've been struggling with it the whole day with no satisfactory result.
Thanks, Joanna
You are sending the parameters when running the program. This is, when running python run_sample.py. The help from that parameter says that you need to include the Path to VOC 2012 Devkit.
Your call should look something like this:
python run_sample.py --voc12_root 'path/to/your/VOC2012'
My command line arguments that use argparse returns AssertionError. I have two scripts and a directory that stores three data files.
Script:
main.py which does a job of loading data
data.py which does pre-processing for language data
Data:
three tokens are stored in C:\Users\archive path
data.py script
**#data.py**
import os
from io import open
import torch
class Dictionary(object):
def __init__(self):
self.word2idx = {}
self.idx2word = []
def add_word(self, word):
if word not in self.word2idx:
self.idx2word.append(word)
self.word2idx[word] = len(self.idx2word) - 1
return self.word2idx[word]
def __len__(self):
return len(self.idx2word)
class Corpus(object):
def __init__(self, path):
self.dictionary = Dictionary()
self.train = self.tokenize(os.path.join(path, 'train.txt'))
self.valid = self.tokenize(os.path.join(path, 'valid.txt'))
self.test = self.tokenize(os.path.join(path, 'test.txt'))
def tokenize(self, path):
"""Tokenizes a text file."""
assert os.path.exists(path)
# Add words to the dictionary
with open(path, 'r', encoding="utf8") as f:
for line in f:
words = line.split() + ['<eos>']
for word in words:
self.dictionary.add_word(word)
# Tokenize file content
with open(path, 'r', encoding="utf8") as f:
idss = []
for line in f:
words = line.split() + ['<eos>']
ids = []
for word in words:
ids.append(self.dictionary.word2idx[word])
idss.append(torch.tensor(ids).type(torch.int64))
ids = torch.cat(idss)
return ids
main.py script
**#main.py**
import argparse
import time
import math
import os
import torch
import torch.nn as nn
import torch.onnx
import data
parser = argparse.ArgumentParser(description='PyTorch Wikitext-2 RNN/LSTM/GRU/Transformer Language Model')
parser.add_argument('--data', type=str, default='./data/wikitext-2',
help='location of the data corpus')
parser.add_argument('--model', type=str, default='LSTM',
help='type of recurrent net (RNN_TANH, RNN_RELU, LSTM, GRU, Transformer)')
parser.add_argument('--emsize', type=int, default=200,
help='size of word embeddings')
parser.add_argument('--nhid', type=int, default=200,
help='number of hidden units per layer')
parser.add_argument('--nlayers', type=int, default=2,
help='number of layers')
parser.add_argument('--lr', type=float, default=20,
help='initial learning rate')
parser.add_argument('--clip', type=float, default=0.25,
help='gradient clipping')
parser.add_argument('--epochs', type=int, default=40,
help='upper epoch limit')
parser.add_argument('--batch_size', type=int, default=20, metavar='N',
help='batch size')
parser.add_argument('--bptt', type=int, default=35,
help='sequence length')
parser.add_argument('--dropout', type=float, default=0.2,
help='dropout applied to layers (0 = no dropout)')
parser.add_argument('--tied', action='store_true',
help='tie the word embedding and softmax weights')
parser.add_argument('--seed', type=int, default=1111,
help='random seed')
parser.add_argument('--cuda', action='store_true',
help='use CUDA')
parser.add_argument('--log-interval', type=int, default=200, metavar='N',
help='report interval')
parser.add_argument('--save', type=str, default='model.pt',
help='path to save the final model')
parser.add_argument('--onnx-export', type=str, default='',
help='path to export the final model in onnx format')
parser.add_argument('--nhead', type=int, default=2,
help='the number of heads in the encoder/decoder of the transformer model')
parser.add_argument('--dry-run', action='store_true',
help='verify the code and the model')
args = parser.parse_args()
# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
if torch.cuda.is_available():
if not args.cuda:
print("WARNING: You have a CUDA device, so you should probably run with --cuda")
device = torch.device("cuda" if args.cuda else "cpu")
###############################################################################
# Load data
###############################################################################
corpus = data.Corpus(args.data)
When I run main.py in terminal:
$ python main.py
The assertionError appears as below:
File "main.py", line 67, in <module>
corpus = data.Corpus(args.data)
File "C:\Users\archive\data.py", line 22, in __init__
self.train = self.tokenize(os.path.join(path, 'train.txt'))
File "C:\Users\archive\data.py", line 28, in tokenize
assert os.path.exists(path)
AssertionError
Opencv-python performs panoramic stitching, and ERR_CAMERA_PARAMS_ADJUST_FAIL = 3 appears
The picture is scaled 6 times to complete the stitching normally
Source image size: 5472*3648
import os
import cv2
import imutils
import traceback
import random
import string
import glob
import numpy as np
import argparse
import sys
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--images", type=str, required=True,help="全景素材目录")
ap.add_argument("-o", "--output", type=str, required=True,help="输出目录")
args = vars(ap.parse_args())
try:
print("[INFO] "+args["images"])
dirname = glob.glob(os.path.join(args["images"], '*'))
names = os.listdir(args["images"])
images = []
for name in dirname:
image = cv2.imread(name)
images.append(image)
filename = ''.join(random.sample('zyxwvutsrqponmlkjihgfedcba',5))
stitcher = cv2.createStitcher(False) if imutils.is_cv3() else cv2.Stitcher_create(False)
status,stitched = stitcher.stitch(images)
# status = 3
cv2.imwrite(args['output']+'/'+filename+'.JPG', stitched)
except:
traceback.print_exc()
New to python,
My professor has given me a piece of code to help process some imagery, however it only works one image at a time due to an input and output needing to be stipulated each time. Usually I would put import os or glob but argparse is something new to me and my usual methods do not work.
I need to edit this in order to create a list of '.hdf' files with the output being the same as the input just with a name change of '_Processed.hdf'
Code below:
# Import the numpy library
import numpy
# Import the GDAL library
from osgeo import gdal
# Import the GDAL/OGR spatial reference library
from osgeo import osr
# Import the HDF4 reader.
import pyhdf.SD
# Import the system library
import sys
# Import the python Argument parser
import argparse
import pprint
import rsgislib
def creatGCPs(lat_arr, lon_arr):
y_size = lat_arr.shape[0]
x_size = lat_arr.shape[1]
print(x_size)
print(y_size)
gcps = []
for y in range(y_size):
for x in range(x_size):
gcps.append([x, y, lon_arr[y,x], lat_arr[y,x]])
return gcps
def run(inputFile, outputFile):
hdfImg = pyhdf.SD.SD(inputFile)
#print("Available Datasets")
pprint.pprint(hdfImg.datasets())
#print("Get Header Attributes")
#attr = hdfImg.attributes(full=1)
#pprint.pprint(attr)
rsgisUtils = rsgislib.RSGISPyUtils()
wktStr = rsgisUtils.getWKTFromEPSGCode(4326)
#print(wktStr)
lat_arr = hdfImg.select('Latitude')[:]
long_arr = hdfImg.select('Longitude')[:]
sel_dataset_arr = hdfImg.select('Optical_Depth_Land_And_Ocean')[:]
gcplst = creatGCPs(lat_arr, long_arr)
y_size = lat_arr.shape[0]
x_size = lat_arr.shape[1]
min_lat = numpy.min(lat_arr)
max_lat = numpy.max(lat_arr)
min_lon = numpy.min(long_arr)
max_lon = numpy.max(long_arr)
lat_res = (max_lat-min_lat)/float(y_size)
lon_res = (max_lon-min_lon)/float(x_size)
driver = gdal.GetDriverByName( "KEA" )
metadata = driver.GetMetadata()
dst_ds = driver.Create( outputFile, x_size, y_size, 1, gdal.GDT_Float32 )
dst_ds.GetRasterBand(1).WriteArray(sel_dataset_arr)
gcp_list = []
for gcp_arr in gcplst:
gcp = gdal.GCP(int(gcp_arr[2]), int(gcp_arr[3]), int(0), gcp_arr[0], gcp_arr[1])
gcp_list.append(gcp)
dst_ds.SetGCPs(gcp_list, wktStr)
dst_ds = None
if __name__ == '__main__':
parser = argparse.ArgumentParser()
# Define the argument for specifying the input file.
parser.add_argument("-i", "--input", type=str, required=True, help="Specify the input image file.")
# Define the argument for specifying the output file.
parser.add_argument("-o", "--output", type=str, required=True, help="Specify the output image file.")
args = parser.parse_args()
run(args.input, args.output)
From the argparse docs here, you can simply add a nargs='*' to the argument definitions. However, be sure to give the input and output files in the same order...
Also, you can use the pathlib.Path object, which is now standard in Python >=3.4, to play with file names.
So with an added from pathlib import Path at the top, the last part of your code becomes:
if __name__ == '__main__':
parser = argparse.ArgumentParser()
# Define the argument for specifying the input file.
parser.add_argument("-i", "--input", nargs='*', type=str, required=True, help="Specify the input image file.")
args = parser.parse_args()
for input in args.input:
output = Path(input).stem + '_Processed.hdf'
run(input, output)
Here, args.input is now a list of strings, so we iterate on it. The .stem attribute returns the file name without any extensions, I find it cleaner than something like input[:-4], which only works for specific extension lengths...
This works well with glob patterns in a standard linux shell (I don't know for other cases).
Ex. calling python this_script.py -i Image_*, processes every file with filenames beginning with "Image_".
You can use the nargs='+' option, and since you're going to have only have one required argument, I'd recommend that you don't use --input as an option, but simply run the script as script_name.py input_file1 input_file2 input_file3 ...:
import os.path
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('input', nargs='+', help="Specify the input image file.")
args = parser.parse_args()
for filename in args.input:
root, ext = os.path.splitext(filename)
run(filename, ''.join((root, '_Processed', ext)))