So I have been searching far an wide for this. I want to run three functions in parallel, when one finishes return this value, stop the two others.
I have tried with asynciop, multiprocess and concurrent.futures, and I still can not get it to work.
As an example I have a function that returns 0 when above a certain threshold, and some number otherwise. To test my implementation I want to return the last non-zero value.
My problem is that regardless of whatever I try, all three methods complete every time. Instead of breaking when a match has been found.
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
def first_zero(f, values=[10, 10**2, 10**3]):
for i in values:
if f(i) == 0:
return i
return values[-1]
def bisect_search(f, first, last):
if f(first) == 0:
return first
middle = (last + first) // 2
while True:
# print(middle, "bisect")
if abs(last - first) <= 1:
return first
middle = (last + first) // 2
if f(middle) > 0:
first = middle
else:
last = middle
def linear_search(f, start, stop, increment):
i = start
minimum = min(start, stop)
maximum = max(start, stop)
if increment < 0:
while minimum <= i <= maximum:
if f(i) != 0:
return i
i += increment
else:
while minimum <= i <= maximum:
if f(i) == 0:
return i - increment
i += increment
return i
def linear_binary_search(f, start=0):
stop = first_zero(f)
tasks = [
(linear_search, [f, start, stop, 1]),
(linear_search, [f, stop, start, -1]),
(bisect_search, [f, start, stop]),
]
executor = ThreadPoolExecutor(max_workers=len(tasks))
futures = [executor.submit(task, *args) for (task, args) in tasks]
done, _ = wait(futures, return_when=FIRST_COMPLETED)
executor.shutdown(wait=False, cancel_futures=True)
return done.pop().result()
if __name__ == "__main__":
import random
import time
target = 995
def f(x):
time.sleep(0.01)
if x <= target:
return random.randint(1, 1000)
return 0
# Comment the following line to see
# how much slower the concurrent version is
print(linear_binary_search(f))
stop = first_zero(f)
print(linear_search(f, stop, 0, -1))
I want to use MPI for parallel processing the calculation of hamiltonian paths in a graph.
So, I achieved this:
from mpi4py import MPI
import random,time
comm = MPI.COMM_WORLD
my_rank = comm.Get_rank()
p = comm.Get_size()
numOfNodes = 10
numOfProblems = 11
class Graph:
def __init__(self, numOfNodes):
if numOfNodes > 0:
self.numOfNodes = numOfNodes
else:
print("Error")
def calculateMaxPairs(self):
self.maxPairs = self.numOfNodes*(self.numOfNodes - 1)//2
def generatePairs(self):
self.calculateMaxPairs()
self.pairs = []
startRange = self.numOfNodes
endRange = (self.numOfNodes - 10)*3 + 18
numOfPairs = random.randint(startRange, endRange)
while len(self.pairs) != numOfPairs:
try:
startNode = random.randint(1, self.numOfNodes)
endNode = random.randint(1, self.numOfNodes)
if startNode == endNode:
raise ValueError
except ValueError:
pass
else:
pair = (startNode, endNode)
invertedPair = (endNode, startNode)
if pair not in self.pairs and invertedPair not in self.pairs:
self.pairs.append(pair)
self.hamiltonianPath = []
def generatePathLink(self):
self.graphLink = {}
for x in self.pairs:
x = str(x)
splitNode = x.split(', ')
a = int(splitNode[0][1:])
b = int(splitNode[1][:-1])
try:
if b not in self.graphLink[a]:
self.graphLink[a].append(b)
except KeyError:
self.graphLink[a] = []
self.graphLink[a].append(b)
finally:
try:
if a not in self.graphLink[b]:
self.graphLink[b].append(a)
except KeyError:
self.graphLink[b] = []
self.graphLink[b].append(a)
finally:
pass
def findPaths(self, start, end, path = []):
path = path + [start]
if start == end:
return [path]
if start not in self.graphLink:
return []
paths = []
for node in self.graphLink[start]:
if node not in path:
newpaths = self.findPaths(node, end, path)
for newpath in newpaths:
paths.append(newpath)
if (len(newpath) == self.numOfNodes):
self.hamiltonianPath = newpath
raise OverflowError
return paths
def exhaustiveSearch(self):
try:
allPaths = []
for startNode in self.graphLink:
for endNode in self.graphLink:
newPaths = self.findPaths(startNode, endNode)
for path in newPaths:
if (len(path) == self.numOfNodes):
allPaths.append(path)
return allPaths
except OverflowError:
return self.hamiltonianPath
else:
pass
def isHamiltonianPathExist(self):
time_start = time.clock()
self.generatePathLink()
if len(self.graphLink) != self.numOfNodes:
time_elapsed = (time.clock() - time_start)
return [[], time_elapsed]
else:
result = self.exhaustiveSearch()
time_elapsed = (time.clock() - time_start)
if len(result) == 0:
print("There isn't any Hamiltonian Path.")
else:
print("Computing time:", round(time_elapsed, 2), "seconds\n")
return [result, time_elapsed]
comm.send(result, dest=0)
yes = 0
no = 0
total_computing_time = 0
for x in range(1, numOfProblems + 1):
if my_rank !=0:
graph = Graph(numOfNodes)
graph.generatePairs()
output = graph.isHamiltonianPathExist()
else:
for procid in range(1,p):
result = comm.recv(source=procid)
time_elapsed = comm.recv(source=procid, tag=12)
total_computing_time += time_elapsed
if len(result) == 0:
no += 1
else:
yes += 1
print("Have Hamiltonian Path:", yes)
print("Don't Have Hamiltonian Path:", no)
print("Total computing time for %s problems in %s processes: %s s"%(numOfProblems, p, str(round(total_computing_time, 2))))
As you can see in this script, there's two sections.
The first one is where we will generate a Graph and calculate the hamiltonian paths.
The second one is where we tell the script to run this graph generating script in parallel in multiple processors.
The problem here is that it generates the graph and calculate paths in every processor, not dividing the job between them.
Where am I doing wrong?
Anyone here with an idea of how to loop a constantly moving variable within a timeframe of 60 seconds to check it's highest, medium, lowest and average of the three?
Then creating more of a json objects of the past minutes
Tried using timer for custom threading and seems that it is not working for me
Below is my code. Kindly assist.
import sys, getopt
import time
from threading import Timer
import main.botlog as botlog
from main.settings import auth_client,product
class BotCandlestick(object):
def __init__(self, period=60, open=None, close=None, high=None, low=None, priceAverage=None):
self.current = None
self.open = open
self.close = close
self.high = high
self.low = low
self.startTime = time.time()
self.period = period
BotLog = botlog.BotLog
self.output = BotLog()
self.priceAverage = priceAverage
def tick(self, price):
# while True:
self.current = float(price)
if (self.open is None):
self.open = self.current
print(self.open)
if ((self.high is None) or (self.current > self.high)):
self.high = self.current
print(self.high)
if ((self.low is None) or (self.current < self.low)):
self.low = self.current
print(self.low)
if (time.time() >= (self.startTime + self.period)):
self.close = self.current
self.priceAverage = (self.high + self.low + self.close) / float(3)
print(self.priceAverage)
self.output.log(
"Open: " + str(self.open) + " Close: " + str(self.close) + " High: " + str(self.high) + " Low: " + str(
self.low) + " Current: " + str(self.current))
def isClosed(self):
if (self.close is not None):
return True
else:
return False
def main():
price = auth_client.get_product_ticker(product)
prices = float(price.get('price'))
price = BotCandlestick()
d = price.tick(prices)
t = Timer(10.0, d)
t.start()
if __name__ == '__main__':
main()
I did figure out a universal way to go about this problem. Kindly find the code below. It is more of a universal code to find value change within x time.
import gdax
import time
public = gdax.PublicClient()
while True:
endtime = time.time() + 60
x = []
while time.time() < endtime:
ticker = public.get_product_ticker('BTC-USD')
ticker = ticker['price']
ticker = float(ticker)
x.append(ticker)
change = ((x[0]-x[-1])/x[-1])
print(ticker)
time.sleep(1)
print("Open: {} High: {} Low: {} Close: {} Change: {}%".format(x[0], max(x), min(x), x[-1], change*100))
time.sleep(0)
I wrote this Python interpreter for a language called Self-modifying Brainf*** (SMBF). Today I discovered a bug where if the program dynamically creates code at the initial cell or after on the tape, it will not be executed. I wrote this interpreter to look as close as possible to the Ruby interpreter on the linked page. Note that this bug may exist in the original Ruby interpreter, too. I don't know, I haven't used it.
The way SMBF is different from normal BF is that the source code is placed on the tape to the left of the cell that the pointer starts at. So the program <. would print the last character of the source (a period). This works.
Note that I trimmed some code out so it's still runnable but takes less space in this post.
The interpreter:
from __future__ import print_function
import os, sys
class Tape(bytearray):
def __init__(self):
self.data = bytearray(b'\0' * 1000)
self.center = len(self.data) // 2
def __len__(self):
return len(self.data)
def __getitem__(self, index):
try:
return self.data[index + self.center]
except:
return 0
def __setitem__(self, index, val):
i = index + self.center
if i < 0 or i >= len(self.data):
# resize the data array to be large enough
new_size = len(self.data)
while True:
new_size *= 2
test_index = index + (new_size // 2)
if test_index >= 0 and test_index < new_size:
# array is big enough now
break
# generate the new array
new_data = bytearray(b'\0' * new_size)
new_center = new_size // 2
# copy old data into new array
for j in range(0, len(self.data)):
new_data[j - self.center + new_center] = self.data[j]
self.data = new_data
self.center = new_center
self.data[index + self.center] = val & 0xff
class Interpreter():
def __init__(self, data):
self.tape = Tape()
# copy the data into the tape
for i in range(0, len(data)):
self.tape[i - len(data)] = data[i]
# program start point
self.entrypoint = -len(data)
def call(self):
pc = self.entrypoint
ptr = 0
# same as -len(self.tape) // 2 <= pc + self.tape.center < len(self.tape) // 2
while -len(self.tape) <= pc < 0: # used to be "while pc < 0:"
c = chr(self.tape[pc])
if c == '>':
ptr += 1
elif c == '<':
ptr -= 1
elif c == '+':
self.tape[ptr] += 1
elif c == '-':
self.tape[ptr] -= 1
elif c == '.':
print(chr(self.tape[ptr]), end="")
elif c == ',':
sys.stdin.read(1)
elif c == '[':
if self.tape[ptr] == 0:
# advance to end of loop
loop_level = 1
while loop_level > 0:
pc += 1
if chr(self.tape[pc]) == '[': loop_level += 1
elif chr(self.tape[pc]) == ']': loop_level -= 1
elif c == ']':
# rewind to the start of the loop
loop_level = 1
while loop_level > 0:
pc -= 1
if chr(self.tape[pc]) == '[': loop_level -= 1
elif chr(self.tape[pc]) == ']': loop_level += 1
pc -= 1
pc += 1
# DEBUG
#print(pc, self.tape.data.find(b'.'))
def main():
# Working "Hello, World!" program.
#data = bytearray(b'<[.<]>>>>>>>>+\x00!dlroW ,olleH')
# Should print a period, but doesn't.
data = bytearray(b'>++++++++++++++++++++++++++++++++++++++++++++++')
intr = Interpreter(data)
intr.call()
#print(intr.tape.data.decode('ascii').strip('\0'))
if __name__ == "__main__":
main()
The problem:
This line is how I set the program (so I can run this on Ideone.com):
data = bytearray(b'++++++++++++++++++++++++++++++++++++++++++++++')
The program adds to the cell until it is 46, which is the decimal value for an ASCII ., which should print the current cell (a period). But for some reason, the program counter pc never gets to that cell. I want the program to run all code it finds until it hits the end of the tape, but I'm having a hard time getting the program counter to take into account the center of the tape, and ensure that it's still correct if the tape is resized in __setitem__.
The relevant line is (what I was trying out):
while -len(self.tape) <= pc < 0:
which was originally this:
while pc < 0:
So I think that the while line either needs to be adjusted, or I need to change it to while True: and just use a try/except while getting chr(self.tape[pc]) to determine if I've hit the end of the tape.
Does anyone see what is wrong or how to fix it?
Found a solution thanks to Sp3000.
self.end = 0 in Tape.__init__, self.end = max(self.end, index+1) in Tape.__setitem__ and replace the while in Interpreter.call with while pc < self.tape.end:.
I am currently working on my Python game, in ika, which uses python 2.5
I decided to use A* pathfinding for the AI. However, I find it too slow for my needs (3-4 enemies can lag the game, but I would like to supply up to 4-5 without problems). I know, that such complex search like A* is not mean to be scripted in python, but I am pretty sure, that my pathfinder is also implemented in the wrong way.
My question is: How can I speed up this algorithm?
I wrote my own binary heap, and there are some try: except: lines inside some functions. Those lines can create large overhead? Are there better methods maintaining the open list?
I supplied the algorithm with graphics interface, for testing purposes (when the pathfinder finishes searching, it will write the number of iterations and seconds it takes to find the path, inside the ika.txt file. Also, Pressing A will do a complete search, and S does that step by step.)
Graphical version:
http://data.hu/get/6084681/A_star.rar
Also, here is a pastebin version:
http://pastebin.com/9N8ybX5F
Here is the main code I use for pathfinding:
import ika
import time
class Node:
def __init__(self,x,y,parent=None,g=0,h=0):
self.x = x
self.y = y
self.parent = parent
self.g = g
self.h = h
def cost(self):
return self.g + self.h
def equal(self,node):
if self.x == node.x and self.y == node.y:
return True
else:
return False
class Emerald_Pathfinder:
def __init__(self):
pass
def setup(self,start,goal):
self.start = start
self.goal = goal
self.openlist = [None,start] # Implemented as binary heap
self.closedlist = {} # Implemented as hash
self.onopenlist = {} # Hash, for searching the openlist
self.found = False
self.current = None
self.iterations = 0
def lowest_cost(self):
pass
def add_nodes(self,current):
nodes = []
x = current.x
y = current.y
self.add_node(x+1,y,current,10,nodes)
self.add_node(x-1,y,current,10,nodes)
self.add_node(x,y+1,current,10,nodes)
self.add_node(x,y-1,current,10,nodes)
# Dont cut across corners
up = map.is_obstacle((x,y-1),x,y-1)
down = map.is_obstacle((x,y+1),x,y+1)
left = map.is_obstacle((x-1,y),x-1,y)
right = map.is_obstacle((x+1,y),x+1,y)
if right == False and down == False:
self.add_node(x+1,y+1,current,14,nodes)
if left == False and up == False:
self.add_node(x-1,y-1,current,14,nodes)
if right == False and up == False:
self.add_node(x+1,y-1,current,14,nodes)
if left == False and down == False:
self.add_node(x-1,y+1,current,14,nodes)
return nodes
def heuristic(self,x1,y1,x2,y2):
return (abs(x1-x2)+abs(y1-y2))*10
def add_node(self,x,y,parent,cost,list):
# If not obstructed
if map.is_obstacle((x,y),x,y) == False:
g = parent.g + cost
h = self.heuristic(x,y,self.goal.x,self.goal.y)
node = Node(x,y,parent,g,h)
list.append(node)
def ignore(self,node,current):
# If its on the closed list, or open list, ignore
try:
if self.closedlist[(node.x,node.y)] == True:
return True
except:
pass
# If the node is on the openlist, do the following
try:
# If its on the open list
if self.onopenlist[(node.x,node.y)] != None:
# Get the id number of the item on the real open list
index = self.openlist.index(self.onopenlist[(node.x,node.y)])
# If one of the coordinates equal, its not diagonal.
if node.x == current.x or node.y == current.y:
cost = 10
else:
cost = 14
# Check, is this items G cost is higher, than the current G + cost
if self.openlist[index].g > (current.g + cost):
# If so, then, make the list items parent, the current node.
self.openlist[index].g = current.g + cost
self.openlist[index].parent = current
# Now resort the binary heap, in the right order.
self.resort_binary_heap(index)
# And ignore the node
return True
except:
pass
return False
def resort_binary_heap(self,index):
m = index
while m > 1:
if self.openlist[m/2].cost() > self.openlist[m].cost():
temp = self.openlist[m/2]
self.openlist[m/2] = self.openlist[m]
self.openlist[m] = temp
m = m / 2
else:
break
def heap_add(self,node):
self.openlist.append(node)
# Add item to the onopenlist.
self.onopenlist[(node.x,node.y)] = node
m = len(self.openlist)-1
while m > 1:
if self.openlist[m/2].cost() > self.openlist[m].cost():
temp = self.openlist[m/2]
self.openlist[m/2] = self.openlist[m]
self.openlist[m] = temp
m = m / 2
else:
break
def heap_remove(self):
if len(self.openlist) == 1:
return
first = self.openlist[1]
# Remove the first item from the onopenlist
self.onopenlist[(self.openlist[1].x,self.openlist[1].y)] = None
last = self.openlist.pop(len(self.openlist)-1)
if len(self.openlist) == 1:
return last
else:
self.openlist[1] = last
v = 1
while True:
u = v
# If there is two children
if (2*u)+1 < len(self.openlist):
if self.openlist[2*u].cost() <= self.openlist[u].cost():
v = 2*u
if self.openlist[(2*u)+1].cost() <= self.openlist[v].cost():
v = (2*u)+1
# If there is only one children
elif 2*u < len(self.openlist):
if self.openlist[2*u].cost() <= self.openlist[u].cost():
v = 2*u
# If at least one child is smaller, than parent, swap them
if u != v:
temp = self.openlist[u]
self.openlist[u] = self.openlist[v]
self.openlist[v] = temp
else:
break
return first
def iterate(self):
# If the open list is empty, exit the game
if len(self.openlist) == 1:
ika.Exit("no path found")
# Expand iteration by one
self.iterations += 1
# Make the current node the lowest cost
self.current = self.heap_remove()
# Add it to the closed list
self.closedlist[(self.current.x,self.current.y)] = True
# Are we there yet?
if self.current.equal(self.goal) == True:
# Target reached
self.goal = self.current
self.found = True
print self.iterations
else:
# Add the adjacent nodes, and check them
nodes_around = self.add_nodes(self.current)
for na in nodes_around:
if self.ignore(na,self.current) == False:
self.heap_add(na)
def iterateloop(self):
time1 = time.clock()
while 1:
# If the open list is empty, exit the game
if len(self.openlist) == 1:
ika.Exit("no path found")
# Expand iteration by one
self.iterations += 1
# Make the current node the lowest cost
self.current = self.heap_remove()
# Add it to the closed list
self.closedlist[(self.current.x,self.current.y)] = True
# Are we there yet?
if self.current.equal(self.goal) == True:
# Target reached
self.goal = self.current
self.found = True
print "Number of iterations"
print self.iterations
break
else:
# Add the adjacent nodes, and check them
nodes_around = self.add_nodes(self.current)
for na in nodes_around:
if self.ignore(na,self.current) == False:
self.heap_add(na)
time2 = time.clock()
time3 = time2-time1
print "Seconds to find path:"
print time3
class Map:
def __init__(self):
self.map_size_x = 20
self.map_size_y = 15
self.obstructed = {} # Library, containing x,y couples
self.start = [2*40,3*40]
self.unit = [16*40,8*40]
def is_obstacle(self,couple,x,y):
if (x >= self.map_size_x or x < 0) or (y >= self.map_size_y or y < 0):
return True
try:
if self.obstructed[(couple)] != None:
return True
except:
return False
def render_screen():
# Draw the Character
ika.Video.DrawRect(map.start[0],map.start[1],map.start[0]+40,map.start[1]+40,ika.RGB(40,200,10),1)
# Draw walls
for x in range(0,map.map_size_x):
for y in range(0,map.map_size_y):
if map.is_obstacle((x,y),x,y) == True:
ika.Video.DrawRect(x*40,y*40,(x*40)+40,(y*40)+40,ika.RGB(168,44,0),1)
# Draw openlist items
for node in path.openlist:
if node == None:
continue
x = node.x
y = node.y
ika.Video.DrawRect(x*40,y*40,(x*40)+40,(y*40)+40,ika.RGB(100,100,100,50),1)
# Draw closedlist items
for x in range(0,map.map_size_x):
for y in range(0,map.map_size_y):
try:
if path.closedlist[(x,y)] == True:
ika.Video.DrawRect(x*40,y*40,(x*40)+20,(y*40)+20,ika.RGB(0,0,255))
except:
pass
# Draw the current square
try:
ika.Video.DrawRect(path.current.x*40,path.current.y*40,(path.current.x*40)+40,(path.current.y*40)+40,ika.RGB(128,128,128), 1)
except:
pass
ika.Video.DrawRect(mouse_x.Position(),mouse_y.Position(),mouse_x.Position()+8,mouse_y.Position()+8,ika.RGB(128,128,128), 1)
# Draw the path, if reached
if path.found == True:
node = path.goal
while node.parent:
ika.Video.DrawRect(node.x*40,node.y*40,(node.x*40)+40,(node.y*40)+40,ika.RGB(40,200,200),1)
node = node.parent
# Draw the Target
ika.Video.DrawRect(map.unit[0],map.unit[1],map.unit[0]+40,map.unit[1]+40,ika.RGB(128,40,200),1)
def mainloop():
while 1:
render_screen()
if mouse_middle.Pressed():
# Iterate pathfinder
if path.found == False:
path.iterateloop()
elif mouse_right.Pressed():
# Iterate pathfinder by one
if path.found == False:
path.iterate()
elif ika.Input.keyboard["A"].Pressed():
# Iterate pathfinder
if path.found == False:
path.iterateloop()
elif ika.Input.keyboard["S"].Pressed():
# Iterate pathfinder by one
if path.found == False:
path.iterate()
elif mouse_left.Position():
# Add a square to the map, to be obstructed
if path.iterations == 0:
x = mouse_x.Position()
y = mouse_y.Position()
map.obstructed[(int(x/40),int(y/40))] = True
# Mouse preview
x = mouse_x.Position()
y = mouse_y.Position()
mx = int(x/40)*40
my = int(y/40)*40
ika.Video.DrawRect(mx,my,mx+40,my+40,ika.RGB(150,150,150,70),1)
ika.Video.ShowPage()
ika.Input.Update()
map = Map()
path = Emerald_Pathfinder()
path.setup(Node(map.start[0]/40,map.start[1]/40),Node(map.unit[0]/40,map.unit[1]/40))
mouse_middle = ika.Input.mouse.middle
mouse_right = ika.Input.mouse.right
mouse_left = ika.Input.mouse.left
mouse_x = ika.Input.mouse.x
mouse_y = ika.Input.mouse.y
# Initialize loop
mainloop()
I appreciate any help!
(sorry for any spelling mistakes, English is not my native language)
I think a proper implementation in python will be fast enough for your purposes. But the boost library has an astar implementation and python bindings. https://github.com/erwinvaneijk/bgl-python