Develop Reference

How does FFMPEG command works in the following code?

How does FFMPEG working here?
import sys, os, pdb
import numpy as np
import subprocess
from tqdm import tqdm
def extract(vid_dir, frame_dir, start, end, redo=False):
class_list = sorted(os.listdir(vid_dir))[start:end]
print("Classes =", class_list)
for ic,cls in enumerate(class_list):
vlist = sorted(os.listdir(vid_dir + cls))
print("")
print(ic+1, len(class_list), cls, len(vlist))
print("")
for v in tqdm(vlist):
outdir = os.path.join(frame_dir, cls, v[:-4])
# Checking if frames already extracted
if os.path.isfile( os.path.join(outdir, 'done') ) and not redo: continue
try:
os.system('mkdir -p "%s"'%(outdir))
# check if horizontal or vertical scaling factor
o = subprocess.check_output('ffprobe -v error -show_entries stream=width,height -of default=noprint_wrappers=1 "%s"'%(os.path.join(vid_dir, cls, v)), shell=True).decode('utf-8')
lines = o.splitlines()
width = int(lines[0].split('=')[1])
height = int(lines[1].split('=')[1])
resize_str = '-1:256' if width>height else '256:-1'
# extract frames
os.system('ffmpeg -i "%s" -r 25 -q:v 2 -vf "scale=%s" "%s" > /dev/null 2>&1'%( os.path.join(vid_dir, cls, v), resize_str, os.path.join(outdir, '%05d.jpg')))
nframes = len([ fname for fname in os.listdir(outdir) if fname.endswith('.jpg') and len(fname)==9])
if nframes==0: raise Exception
os.system('touch "%s"'%(os.path.join(outdir, 'done') ))
except:
print("ERROR", cls, v)
if __name__ == '__main__':
vid_dir = sys.argv[1]
frame_dir = sys.argv[2]
start = int(sys.argv[3])
end = int(sys.argv[4])
extract(vid_dir, frame_dir, start, end, redo=True)
os.system('ffmpeg -i "%s" -r 25 -q:v 2 -vf "scale=%s" "%s" > /dev/null 2>&1'%( os.path.join(vid_dir, cls, v), resize_str, os.path.join(outdir, '%05d.jpg')))
-i --> Input_file (320x240)
-r --> frame rate (it will extract 25 frame of every video)
-q --> what does this mean?
"scale=%s --> Unable to understand this? how does this work for -1:256 ratio [will it resize the video to 256x256 dimension]?
Can anyone explain me this?

NameError: name 'n' is not defined in python script

I'm trying to run a python script that is optimized to work with python 2.7:
#!/usr/bin/env python
import sys,getopt,os
SplitInput_string = """#!/bin/bash
#BSUB -J SplitInput[1-%numSamples%]
#BSUB -o Logs/SplitInput-Out-%I.out
#BSUB -e Logs/SplitInput-Err-%I.err
#BSUB -q week
#BSUB -W 23:58
echo Date: `date`
t1=`date +%s`
sleep ${LSB_JOBINDEX}
python LSFScripts/array_merge.py -r ${LSB_JOBINDEX} -i %input% -o original_reads/
[ $? -eq 0 ] || echo 'JOB FAILURE: $?'
echo Date: `date`
t2=`date +%s`
tdiff=`echo 'scale=3;('$t2'-'$t1')/3600' | bc`
echo 'Total time: '$tdiff' hours'
"""
help_message = "usage example: python setupDirs.py -i /path/to/reads/ -n numberOfSamples"
if __name__ == "__main__":
try:
opts, args = getopt.getopt(sys.argv[1:], 'hi:n:', ["inputdir="])
except:
print help_message
sys.exit(2)
for opt, arg in opts:
if opt in ('-h', '--help'):
print help_message
sys.exit()
elif opt in ('-i', '--inputdir'):
inputdir = arg
if inputdir[-1] != '/':
inputdir += '/'
elif opt in ('-n'):
n = arg
for dir in ['Logs', 'original_reads', 'hashed_reads', 'cluster_vectors', 'read_partitions']:
os.system('mkdir %s' % (dir))
f = open('LSFScripts/SplitInput_ArrayJob.q', 'w')
f.write(SplitInput_string.replace('%numSamples%', n).replace('%input%', inputdir))
f.close()
But I keep getting this error message:
line 42, in <module>
f.write(SplitInput_string.replace('%numSamples%', n).replace('%input%', inputdir))
NameError: name 'n' is not defined
Any advice would be appreciated!

You must assign a value to n before entering the loop.
if __name__ == "__main__":
try:
opts, args = getopt.getopt(sys.argv[1:],'hi:n:',["inputdir="])
except:
print help_message
sys.exit(2)
n = ""
for opt, arg in opts:
etc...

n is not defined in all code paths. It is only defined if you follow the path:
elif opt in ('-n'):
Define m at a higher scope (i.e. before the for loop) with a default value if you want to use it afterwards.
n = #default value
for opt, arg in opts:
# ...
If you want to parse arguments, I highly recommend using the argparse package. There is a little bit of a learning curve, but you can very robustly generate the necessary usage.

comparing input with a file and minding the sequence

How can I compare the values of input parmeters with a file in such a way that the sequence of the lines in file are "respected". For example:
file sequence.txt has following enteries
aaa
bbb
ccc
ddd
and the input is coming like this (with comas):
./run.py -c migrate -s ddd,bbb
then output is like this:
bbb
ddd
Here is the script I have worked so far
#!/usr/bin/python
import sys
import getopt
import time
import os
def main(argv):
cmd = ''
schemas = ''
script_dir = os.path.dirname(__file__)
seq_file = "system/sequence.txt"
abs_file_path = os.path.join(script_dir, seq_file)
try:
opts, args = getopt.getopt(argv,"h:c:s",["cmd=","schemas="])
except getopt.GetoptError:
print './run.py -c=<command> -s=<schemas> '
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print './run.py -c=<command> -s=<schemas>'
sys.exit()
elif opt in ("-c", "--cmd"):
cmd = arg
elif opt in ("-s", "--schemas"):
schemas = arg
if cmd == "migrate" :
with open(abs_file_path) as z:
for line in z:
print line.rstrip('\n')
if __name__ == "__main__":
main(sys.argv[1:])
I know that I have to do comparisons at position print line.rstrip('\n') but I can't figure out how to do it. Any suggestions?
Also, how can I make -s switch mandatory if -c has "migrate" value?
Thanks in advance.

You need to check whether the current line of the sequence is specified with the -s flag. So you need to modify the schemas value, so that it is a list that contains all schemas and then you can check if the current line is equal to one of the schemas. As for your second question, I'm not familiar with getopt, but you could simply check whether schemas is not empty when -c is migrate and do the approriate error handling.
[...]
schemas = []
[...]
elif opt in ("-s", "--schemas"):
schemas = arg.split(',')
[...]
if cmd == 'migrate':
if not schemas: # list is empty
# do error handling
for line in z:
if line in schemas:
print line

Running a python program, unsure what the argument is written like?

so i want to run this python program but i am unsure how to run it, when i insert in the arguments it get a token error.
First i "import cw2" which is the file name in the terminal after typing python and then i type in the argument to run a individual task but i get and error. Heres the code, can you tell me how to run the individual parts.
Heres what i typed as the argument cw2 -u user_745409913574d4c6 -d doc_140228202800-6ef39a241f35301a9a42cd0ed21e5fb0 -t task_2, But that doesnt work. Heres the code below showing what the argument is.
def main(argv):
user_uuid = ''
doc_uuid = ''
task_id = 0
try:
opts, args = getopt.getopt(argv, "hu:d:t:", ["user_uuid=", "doc_uuid=", "task_id="])
except getopt.GetoptError:
print 'cw2 -u <user_uuid> -d <doc_uuid> -t <task_id>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'cw2.py -u <user_uuid> -d <doc_uuid> -t <task_id>'
sys.exit()
elif opt in ("-u", "--user_uuid"):
user_uuid = arg
elif opt in ("-d", "--doc_uuid"):
doc_uuid = arg
elif opt in ("-t", "--task_id"):
task_id = arg
if(int(task_id) == 1):
with open("../requirements.txt", 'r') as fin:
print("Requirments.txt file content")
print fin.read()
if(int(task_id) == 2):
if(doc_uuid == ''):
print(" No doc_uuid supplied")
else:
task_2(doc_uuid)
print("Histograms for per country beed saved in : static/results/countries_to_book_UUID.png")
print("Histograms for per continent beed saved in : static/results/continent_to_book_UUI.png")
elif(int(task_id) == 3):
task_3()
print("Histograms of browser usage has been seaved in 'static/results/simple_browser_usage.png' ")
print("Histograms of generalised browser usage has been seaved in 'static/results/general_browser_usage.png")
elif(int(task_id) == 4):
print("Data of 10 most active readers")
task_4(10)
elif(int(task_id) == 5):
if((user_uuid == '') | (doc_uuid == '')):
print("Provide user_uuid or/and doc_uuid")
# 938601f24509a9f1 , 110727005030-000000009cca70787e5fba1fda005c85
else:
task_5(user_uuid, doc_uuid)
if __name__ == "__main__":
main(sys.argv[1:])

Don't run it from the Python shell, this is set up to run as a normal program from your terminal.
cd <wherever this script is>
chmod a+x ./cw.py
./cw.py -u user_745409913574d4c6 -d doc_140228202800-6ef39a241f35301a9a42cd0ed21e5fb0 -t task_2
Also, the indentation is all wrong on the script in your question, but I assume that that's a copy-and-paste error.

Getting command line arguments as tuples in python

Here is an example of how I would like to call my script:
python script.py -f file1.txt "string1" "string2" -f file2.txt "string3" "string4"
Every file that goes as input will have 2 strings associated with that file. There can be any number of files.
To simplify, I am trying to get a print like this:
('file1.txt', 'string1', 'string2')
('file2.txt', 'string3', 'string4')
Here is what I have so far:
import sys, os, traceback, optparse
import time
import re
#from pexpect import run, spawn
def main ():
global options, args
print options.filename
#for filename in options.filename:
# print filename
#f = file(filename,'r')
#for line in f:
# print line,
#f.close()
if __name__ == '__main__':
try:
start_time = time.time()
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), usage=globals()['__doc__'], version='$Id$')
parser.add_option ('-f', '--file', dest='filename', help='write report to FILE', metavar='FILE', nargs=3)
parser.add_option ('-v', '--verbose', action='store_true', default=False, help='verbose output')
(options, args) = parser.parse_args()
#if len(args) < 1:
# parser.error ('missing argument')
if options.verbose: print time.asctime()
main()
if options.verbose: print time.asctime()
if options.verbose: print 'TOTAL TIME IN MINUTES:',
if options.verbose: print (time.time() - start_time) / 60.0
sys.exit(0)
except KeyboardInterrupt, e: # Ctrl-C
raise e
except SystemExit, e: # sys.exit()
raise e
except Exception, e:
print 'ERROR, UNEXPECTED EXCEPTION'
print str(e)
traceback.print_exc()
os._exit(1)
With the above script, I get only the second file and related strings:
('file2.txt', 'string3', 'string4')

I think you want to use the action=append argument of the add_argument method
import argparse
parser= argparse.ArgumentParser()
parser.add_argument ('-f', '--file', nargs=3, action='append')
files = parser.parse_args('-f file1 string1 string2 -f file2 string3 string4 -f file3 string5 string6'.split()).file
for f in files:
print tuple(f)
gives you:
('file1', 'string1', 'string2')
('file2', 'string3', 'string4')
('file3', 'string5', 'string6')
Testing on cli:
with:
import argparse
parser= argparse.ArgumentParser(prog='Test', usage='%(prog)s -f Filename Option1 Option2 ')
parser.add_argument ('-f', '--file', nargs=3, action='append')
files = parser.parse_args().file
for f in files:
print tuple(f)
results:
python test.py -f file1 "foo bar" "baz" -f file2 foo bar
('file1', 'foo bar', 'baz')
('file2', 'foo', 'bar')
python test.py -f file1 "foo bar" "string2" -f file2 foo bar -f file3 "foo" "bar"
('file1', 'foo bar', 'string2')
('file2', 'foo', 'bar')
('file3', 'foo', 'bar')
python test.py -f file1 "foo bar"
usage: Test -f Filename Option1 Option2
Test: error: argument -f/--file: expected 3 argument(s)

argparse supports the notion of accumulators, which allow you to specify the same option more than once, which is probably more like what you want than anything optparse supports (your particular problem is that optparse doesn't know what to do with an argument specified multiple times, so it's "last one wins" at the command line). argparse is included in Python 2.7 and 3.2, but you should be able to download it for anything 2.6.x or later.

You weren't very clear on your constraints, but this will work for any number of -fs and other flags
import getopt
import sys
args = sys.argv[1:]
tuples = []
while args:
try:
opts, args = getopt.getopt(args, "f:v", ["file", "verbose"])
except getopt.GetoptError, err:
print str(err)
sys.exit(-1)
for o, a in opts:
if o in ("-f", "--file"):
tuples.append((a, args.pop(0), args.pop(0)))
if o in ("-v", "--verbose"):
print "yep, verbose"
print tuples

I would approach this similarly to the previous answer with a slight tweak:
import getopt
import sys
args = sys.argv[1:]
tuples = []
while args:
try:
(opts, args) = getopt.getopt(args, "f:v", ["file=", "verbose"])
except getopt.GetoptError, err:
print str(err)
sys.exit(2)
Now you can either require input in the following way:
-f file1.txt,string1,string2
And parse it in the following way:
for opt, arg in opts:
if opt in ("-f", "--file"):
tuples.append(tuple(arg.split(",")))
if opt in ("-v", "--verbose"):
print "yep, verbose"
print tuples
Or design the input as one string:
-f "file1.txt string1 string2"
and split on whatever strikes your fancy.

If you are using optparse because you want to remain compatible with python 2.6, the action='append' solution works, too:
import optparse
parser = optparse.OptionParser()
parser.add_option('-f', '--file', dest='filename', nargs=3, action='append')
Demonstration
>>> (opts, args) = parser.parse_args("-f file1.txt string1 string2 -f file2.txt string3 string4".split())
>>> for fn_tuple in opts.filename:
... print fn_tuple
('file1.txt', 'string1', 'string2')
('file2.txt', 'string3', 'string4')