Develop Reference

MPI with class & for-loop

I want to use MPI for parallel processing the calculation of hamiltonian paths in a graph.
So, I achieved this:
from mpi4py import MPI
import random,time
comm = MPI.COMM_WORLD
my_rank = comm.Get_rank()
p = comm.Get_size()
numOfNodes = 10
numOfProblems = 11
class Graph:
def __init__(self, numOfNodes):
if numOfNodes > 0:
self.numOfNodes = numOfNodes
else:
print("Error")
def calculateMaxPairs(self):
self.maxPairs = self.numOfNodes*(self.numOfNodes - 1)//2
def generatePairs(self):
self.calculateMaxPairs()
self.pairs = []
startRange = self.numOfNodes
endRange = (self.numOfNodes - 10)*3 + 18
numOfPairs = random.randint(startRange, endRange)
while len(self.pairs) != numOfPairs:
try:
startNode = random.randint(1, self.numOfNodes)
endNode = random.randint(1, self.numOfNodes)
if startNode == endNode:
raise ValueError
except ValueError:
pass
else:
pair = (startNode, endNode)
invertedPair = (endNode, startNode)
if pair not in self.pairs and invertedPair not in self.pairs:
self.pairs.append(pair)
self.hamiltonianPath = []
def generatePathLink(self):
self.graphLink = {}
for x in self.pairs:
x = str(x)
splitNode = x.split(', ')
a = int(splitNode[0][1:])
b = int(splitNode[1][:-1])
try:
if b not in self.graphLink[a]:
self.graphLink[a].append(b)
except KeyError:
self.graphLink[a] = []
self.graphLink[a].append(b)
finally:
try:
if a not in self.graphLink[b]:
self.graphLink[b].append(a)
except KeyError:
self.graphLink[b] = []
self.graphLink[b].append(a)
finally:
pass
def findPaths(self, start, end, path = []):
path = path + [start]
if start == end:
return [path]
if start not in self.graphLink:
return []
paths = []
for node in self.graphLink[start]:
if node not in path:
newpaths = self.findPaths(node, end, path)
for newpath in newpaths:
paths.append(newpath)
if (len(newpath) == self.numOfNodes):
self.hamiltonianPath = newpath
raise OverflowError
return paths
def exhaustiveSearch(self):
try:
allPaths = []
for startNode in self.graphLink:
for endNode in self.graphLink:
newPaths = self.findPaths(startNode, endNode)
for path in newPaths:
if (len(path) == self.numOfNodes):
allPaths.append(path)
return allPaths
except OverflowError:
return self.hamiltonianPath
else:
pass
def isHamiltonianPathExist(self):
time_start = time.clock()
self.generatePathLink()
if len(self.graphLink) != self.numOfNodes:
time_elapsed = (time.clock() - time_start)
return [[], time_elapsed]
else:
result = self.exhaustiveSearch()
time_elapsed = (time.clock() - time_start)
if len(result) == 0:
print("There isn't any Hamiltonian Path.")
else:
print("Computing time:", round(time_elapsed, 2), "seconds\n")
return [result, time_elapsed]
comm.send(result, dest=0)
yes = 0
no = 0
total_computing_time = 0
for x in range(1, numOfProblems + 1):
if my_rank !=0:
graph = Graph(numOfNodes)
graph.generatePairs()
output = graph.isHamiltonianPathExist()
else:
for procid in range(1,p):
result = comm.recv(source=procid)
time_elapsed = comm.recv(source=procid, tag=12)
total_computing_time += time_elapsed
if len(result) == 0:
no += 1
else:
yes += 1
print("Have Hamiltonian Path:", yes)
print("Don't Have Hamiltonian Path:", no)
print("Total computing time for %s problems in %s processes: %s s"%(numOfProblems, p, str(round(total_computing_time, 2))))
As you can see in this script, there's two sections.
The first one is where we will generate a Graph and calculate the hamiltonian paths.
The second one is where we tell the script to run this graph generating script in parallel in multiple processors.
The problem here is that it generates the graph and calculate paths in every processor, not dividing the job between them.
Where am I doing wrong?

loop instance method with list elements

I am trying to write a program (API testing) to loop through instance methods in a class. One of the methods "get_one_sale_license" has a variable "self.sale_sgid_in_test" I need to assign this variable each element in the list "sale_list_final" .The variable is used in the instance method as well as the main function. I put a for loop outside the class and it works fine with the output( this is modified part of a single file from the application using framework).
However, I want to remove the loop wrapping the class and try in a more pythonic way.
import json
from datetime import *
from N import NSession
from N import NArgParser
from i_data_provider import iDataProvider
sale_list = open("sales.txt").readlines()
sale_list_final = [s.rstrip() for s in sale_list]
tests_list = open("tests.txt").readlines()
tests_list_final = [t.rstrip() for t in tests_list]
cnt=0
cnt2=0
cnt3=0
cnt6=0
cnt11=0
for i in sale_list_final:
class iDeployChecks:
def __init__(self, badge, environment):
self.session = NSession(badge=badge,environment=environment)
self.logger = self.session.logger
# self.sale_sgid_list = []
self.sale_sgid_in_test = ''
self.i_session = iDataProvider(self.session)
def get_all_saleable_sales(self):
global cnt2
if cnt2 == 0:
self.i_session.get_sale_seller_info()
self.logger.info('SUCCESS..... Get all sales api\n')
cnt2 += 1
def get_all_sale_dashboard(self):
global cnt6
if cnt6 == 0:
self.i_session.get_All_sale()
self.logger.info('SUCCESS..... Get all sales on dashboard api\n')
cnt6 += 1
def get_current_user_sale_dashboard(self):
global cnt11
if cnt11 == 0:
self.i_session.get_current_user_sale()
self.logger.info('SUCCESS..... Get current sales on dashboard api\n')
cnt11 += 1
def get_one_sale_license(self):
self.logger.info('Getting sale details from:')
self.sale_sgid_list1 = [item.get('salesgid') for item in self.i_session.get_sale_seller_info()]
#self.sale_sgid_list =[item.get('salesgid') for item in self.i_session.get_sale_seller_info() if item.get('salesgid') == i]
print " for sale " + str(i)
self.sale_sgid_in_test = ''.join(i)
self.logger.info('\n')
self.logger.info('Get License for sale with sale sgid {}'.format(self.sale_sgid_in_test))
self.i_session.get_sale_license(self.sale_sgid_in_test)
self.logger.info('SUCCESS..... Get license api\n')
def get_weekly_stats_count(self):
global cnt
if cnt == 0:
self.i_session.get_weekly_statistics()
self.logger.info('SUCCESS..... Weekly statistics api\n')
cnt += 1
def get_sconfig_value_count(self):
self.i_session.get_sconfig_value(self.sale_sgid_in_test)
self.logger.info('SUCCESS..... sconfig api\n')
def main(self):
start = datetime.utcnow()
# check if the method in the list is present in the class, if yes run it.
for j in tests_list_final:
k = "self." + j + "()"
l= [x.strip() for x in j.split(',')]
m= "self."+l[0]+"(self,year)"
n= "self."+l[0]+"(self,year,month)"
if k == str("self.get_all_saleable_sales()"):
if cnt2 == 0:
self.logger.info('Get All sales')
self.get_all_saleable_sales()
self.logger.info('checking if sale with GSID ' + i + ' exists')
if i not in str(self.i_session.get_sale_seller_info()):
print "the sale with GSID " + i + " does not exist !!"
end1 = datetime.utcnow()
self.logger.info('i health check completed in {} seconds.'.format((end1 - start).seconds))
return
else:
print "sale exists !!"
continue
elif k == str("self.get_one_sale_license()"):
self.get_one_sale_license()
continue
elif k == str("self.get_sale_node_status_value()"):
try:
self.logger.info('Get sale node status for sale sgid {}'.format(self.sale_sgid_in_test))
self.get_sale_node_status_value()
except Exception as e: print e
else:
global cnt3
if (cnt3==0):
print '\n'
print " The testcase " +k + "test does not exist,please recheck test name !!"
print '\n'
cnt3 +=1
end = datetime.utcnow()
self.logger.info('IBDL health check completed in {} seconds.'.format((end - start).seconds))
if __name__ == '__main__':
parser = NArgParser(description='i Checks')
args = parser.parse_args()
iDeployChecks(args.badge, args.environment).main()

Remove the loop and include in the init
def __init__(self, badge, environment):
self.session = NSession(badge=badge,environment=environment)
self.logger = self.session.logger
self.sale_sgid_list = [i for i in sale_list_final]
self.sale_sgid_in_test = ''
self.i_session = iDataProvider(self.session)
Then loop for the same list element while calling the main
if __name__ == '__main__':
parser = NArgParser(description='i Checks')
args = parser.parse_args()
for i in sale_sgid_list:
iDeployChecks(args.badge, args.environment,i).main()

How to count elapsed time on Python

I have the following code:
def base(nomor)
day = localtime.tm_wday
time = localtime.tm_hour
no = str(nomor)
dosen = cek_dosen(no)
if dosen == 'null':
no_dosen()
elif dosen != 'null':
ada_dosen()
matkul = cek_jadwal(day,time,dosen)
if matkul == 'null':
no_jadwal()
elif matkul != 'null':
ada_jadwal()
pertemuan = cek_pertemuan(matkul)
print pertemuan
if pertemuan > 1:
cek_before(pertemuan)
filename = ''.join([dosen, matkul, str(pertemuan), ".pptx"])
else:
filename = ''.join([dosen, matkul, str(pertemuan), ".pptx"])
grabfile(filename)
os.system(''.join(["loimpress ",filename]))
pertemuan = pertemuan + 1
update_pertemuan(pertemuan,matkul)
mulai()
if __name__ == '__main__':
mulai()
while True:
data = port.read()
count += 1
if count == 1:
if str(data) != start:
nomor = ''
count = 0
elif 2 <= count <= 13:
nomor = nomor + str(data)
elif count == 16 and str(data) == stop:
base(nomor)
nomor = ''
count = 0
I want to count time elapse from after data = port.read() until after grabfile(filename). I've used start = time.time() after data = port.read and end = time.time() after grabfile, time = end - start, but it stuck after data = port.read() so I use Ctrl + C to stop that. If I put start = time.time() after no = str(nomor), I get Attribute Error : 'int' object has no attribute 'time'.
How do I count the elapsed time?

from time import clock
start = clock()
...
print "Time taken = %.5f" % (clock() - start)

Summarized:
import datetime
if __name__ == "__main__":
d1 = datetime.datetime.now()
data = port.read()
# Do more things ...
tdelta = datetime.datetime.now() - d1
print(tdelta.total_seconds()) # This is your answer

Take a look at the python timeit module.
Basic example:
>>> import timeit
>>> timeit.timeit('"-".join(str(n) for n in range(100))', number=10000)

Proper way to benchmark python code

I have the following modulo exponentiation code and I would like to benchmark a few lines in the function.
One line is:
temp = square(temp)
But python complains that global name 'square' is not defined. Also how can I benchmark the line with
ret = temp % n
Do I also need to write it into a function?
import math
import timeit
def int2baseTwo(x):
"""x is a positive integer. Convert it to base two as a list of integers
in reverse order as a list."""
# repeating x >>= 1 and x & 1 will do the trick
assert x >= 0
bitInverse = []
while x != 0:
bitInverse.append(x & 1)
x >>= 1
return bitInverse
def square(a):
return a ** 2
def modExp(a, d, n):
"""returns a ** d (mod n)"""
assert d >= 0
assert n >= 0
base2D = int2baseTwo(d)
#print 'base2D = ', base2D
base2DLength = len(base2D)
#print 'base2DLength = ', base2DLength
modArray = []
result = 1
temp = 1
for i in range(0, base2DLength):
if i == 0:
temp = a
continue
print(timeit.timeit("temp = square(temp)", setup="from __main__ import modExp"))
if base2D[i] == 1:
temp = temp * a
ret = temp % n
return ret
if __name__=="__main__":
print(timeit.timeit("modExp(1000,100,59)", setup="from __main__ import modExp"))

improving Boyer-Moore string search

I've been playing around with the Boyer-Moore sting search algorithm and starting with a base code set from Shriphani Palakodety I created 2 additional versions (v2 and v3) - each making some modifications such as removing len() function from the loop and than refactoring the while/if conditions. From v1 to v2 I see about a 10%-15% improvement and from v1 to v3 a 25%-30% improvement (significant).
My question is: does anyone have any additional mods that would improve performance even more (if you can submit as a v4) - keeping the base 'algorithm' true to Boyer-Moore.
#!/usr/bin/env python
import time
bcs = {} #the table
def goodSuffixShift(key):
for i in range(len(key)-1, -1, -1):
if key[i] not in bcs.keys():
bcs[key[i]] = len(key)-i-1
#---------------------- v1 ----------------------
def searchv1(text, key):
"""base from Shriphani Palakodety fixed for single char"""
i = len(key)-1
index = len(key) -1
j = i
while True:
if i < 0:
return j + 1
elif j > len(text):
return "not found"
elif text[j] != key[i] and text[j] not in bcs.keys():
j += len(key)
i = index
elif text[j] != key[i] and text[j] in bcs.keys():
j += bcs[text[j]]
i = index
else:
j -= 1
i -= 1
#---------------------- v2 ----------------------
def searchv2(text, key):
"""removed string len functions from loop"""
len_text = len(text)
len_key = len(key)
i = len_key-1
index = len_key -1
j = i
while True:
if i < 0:
return j + 1
elif j > len_text:
return "not found"
elif text[j] != key[i] and text[j] not in bcs.keys():
j += len_key
i = index
elif text[j] != key[i] and text[j] in bcs.keys():
j += bcs[text[j]]
i = index
else:
j -= 1
i -= 1
#---------------------- v3 ----------------------
def searchv3(text, key):
"""from v2 plus modified 3rd if condition
breaking down the comparison for efficiency,
modified the while loop to include the first
if condition (opposite of it)
"""
len_text = len(text)
len_key = len(key)
i = len_key-1
index = len_key -1
j = i
while i >= 0 and j <= len_text:
if text[j] != key[i]:
if text[j] not in bcs.keys():
j += len_key
i = index
else:
j += bcs[text[j]]
i = index
else:
j -= 1
i -= 1
if j > len_text:
return "not found"
else:
return j + 1
key_list = ["POWER", "HOUSE", "COMP", "SCIENCE", "SHRIPHANI", "BRUAH", "A", "H"]
text = "SHRIPHANI IS A COMPUTER SCIENCE POWERHOUSE"
t1 = time.clock()
for key in key_list:
goodSuffixShift(key)
#print searchv1(text, key)
searchv1(text, key)
bcs = {}
t2 = time.clock()
print('v1 took %0.5f ms' % ((t2-t1)*1000.0))
t1 = time.clock()
for key in key_list:
goodSuffixShift(key)
#print searchv2(text, key)
searchv2(text, key)
bcs = {}
t2 = time.clock()
print('v2 took %0.5f ms' % ((t2-t1)*1000.0))
t1 = time.clock()
for key in key_list:
goodSuffixShift(key)
#print searchv3(text, key)
searchv3(text, key)
bcs = {}
t2 = time.clock()
print('v3 took %0.5f ms' % ((t2-t1)*1000.0))

Using "in bcs.keys()" is creating a list and then doing an O(N) search of the list -- just use "in bcs".
Do the goodSuffixShift(key) thing inside the search function. Two benefits: the caller has only one API to use, and you avoid having bcs as a global (horrid ** 2).
Your indentation is incorrect in several places.
Update
This is not the Boyer-Moore algorithm (which uses TWO lookup tables). It looks more like the Boyer-Moore-Horspool algorithm, which uses only the first BM table.
A probable speedup: add the line 'bcsget = bcs.get' after setting up the bcs dict. Then replace:
if text[j] != key[i]:
if text[j] not in bcs.keys():
j += len_key
i = index
else:
j += bcs[text[j]]
i = index
with:
if text[j] != key[i]:
j += bcsget(text[j], len_key)
i = index
Update 2 -- back to basics, like getting the code correct before you optimise
Version 1 has some bugs which you have carried forward into versions 2 and 3. Some suggestions:
Change the not-found response from "not found" to -1. This makes it compatible with text.find(key), which you can use to check your results.
Get some more text values e.g. "R" * 20, "X" * 20, and "XXXSCIENCEYYY" for use with your existing key values.
Lash up a test harness, something like this:
func_list = [searchv1, searchv2, searchv3]
def test():
for text in text_list:
print '==== text is', repr(text)
for func in func_list:
for key in key_list:
try:
result = func(text, key)
except Exception, e:
print "EXCEPTION: %r expected:%d func:%s key:%r" % (e, expected, func.__name__, key)
continue
expected = text.find(key)
if result != expected:
print "ERROR actual:%d expected:%d func:%s key:%r" % (result, expected, func.__name__, key)
Run that, fix the errors in v1, carry those fixes forward, run the tests again until they're all OK. Then you can tidy up your timing harness along the same lines, and see what the performance is. Then you can report back here, and I'll give you my idea of what a searchv4 function should look like ;-)