First off, I am brand new to the multiprocessing and threading world. I have two devices that generate data (gps and mca). the gps simulator is supposed to generate a location every 0.1 seconds. The mca is supposed to generate a random number every randomly generated time interval. When an event is registered by the mca, the count (cnt) is supposed to be sent to the count list.The same goes for the gps. The event handler is supposed to synchronize the count with the latest gps value is registered, and this should be printed to standard output. After 5 seconds, the mca should stop and send 'DONE' over a Queue to stop all of the other functions.I am also very new to Queue. It seems to me that my definitions start but don't do anything.
I would greatly appreciate it if someone could fix my code or let me know what is going wrong in it.
import random
from multiprocessing import Process, Queue
from time import sleep, time, clock
count = []
gps_data = []
def mca(q1):
print 'started'
t = 5
while True:
cnt = random.randint(0,30)
count.append(cnt)
dt = random.randint(0,3)
sleep(dt)
nt = t-dt
if nt <= 0:
break
q1.put('DONE')
def gps(q1):
print 'started2'
while q1.get() != 'DONE':
x = 0
dt = 0.1
sleep(dt)
y = x + 1
gps_data.append(y)
def event_handler(q1):
print 'started3'
size_i = len(count) #initial size of the mca list
while q1.get() != 'DONE':
size_f = len(count)
if size_f > size_i:
local_count = count[-1]
local_location = gps_data[-1]
data = local_count + local_location
print str(data)
size_i = size_f
else:
pass
if __name__ == '__main__':
q1 = Queue()
p_mca = Process(target = mca, args = (q1,))
p_gps = Process(target = gps, args = (q1,))
p_evh = Process(target = event_handler, args = (q1,))
p_evh.start()
p_gps.start()
p_mca.start()
p_evh.join()
p_gps.join()
p_mca.join()
Your variable t in mca() keeps getting set back to 5, thus
if nt <= 0:
is never True.
Like D_rez90 said,
if nt<=0:
is never true. You should change
nt=t-dt
to
t-=dt
if t<=0:
Related
I want to add a semaphore to where I commented down below the code but i couldnt.. so my code runs 4 threads that have linked list attached, and each linked list retrieves same items.. and they are sorted with a value from the items, then if there are no 2 biggest same numbers, winner is elected. if not nodework is called again for each thread with thr down below. so when there are 2 biggest same values and it rolls again, the linkedlist array from previous thread just mixes with the newly ones created, because the threads edit each other arrays as well. also the threads dont wait for each other to fill their arrays with data under with semaphoreAdding so they get wrong data set from the arrays as they keep executing. so i want to add a semaphore where with semaphoreAddingFinished is located in the code. so that, when each thread finishes .add() call, they can keep going. How can I add semaphore there ? I try everything but i get deadlock
import conset
import random
import threading
from threading import Timer
numberOfNodes = 4 #number of threads, (linked lists)
llList = []
roundCount = 1
roundCountCounter = 1
semaphore = threading.Semaphore(0)
semaphoreAdding = threading.Semaphore(1)
semaphoreAddingFinished = threading.Semaphore(1)
for x in range(0, numberOfNodes): #creating a global list with n number of ConSet instances (linked lists)
ll = conset.LinkedList()
llList.append(ll)
numb = 0 #thread counter
def nodeWork(nodeId, n):
global numb
randomInteger = random.randint(0, 3) #generate random number 0 to n^2
theItemTuple = (nodeId, randomInteger) #create the tuple for mailbox
print("Node", nodeId, "proposes value", randomInteger, "for round", roundCount)
semaphoreAddFinish = threading.Semaphore(0)
with semaphoreAdding:
for m in range(0, len(llList)): #add the tuple to all mailboxes of all nodes
llList[m].add(theItemTuple)
#print(nodeId, llList[nodeId].head.data)
with semaphoreAddingFinished: #i want to add a semaphore here
zzz =1
with semaphore:
for k in range(0, len(llList)):
node = llList[nodeId].head
if node.next:
if node.data[1] != node.next.data[1]:
print("Node", nodeId, "decide", node.data[0], "as the leader")
return
else:
print("Node", nodeId, "could not decide on the leader and moves to the round", roundCount+1)
llList[nodeId] = conset.LinkedList()
thr = threading.Thread(target=nodeWork, args=[nodeId, numberOfNodes])
thr.start()
numb = numb + 1
#print("numb", numb)
if numb == 4:
numb = 0
for l in range(0, numberOfNodes):
semaphoreAdding.release()
return
else:
print("Node", nodeId, "decide", node.data[0], "as the leader")
return
main_thread = threading.currentThread() # getting handle to the current thread
for num in range(0, numberOfNodes):
#threading.Lock.acquire()
t = threading.Thread(target=nodeWork, args=[num, numberOfNodes])
t.start()
if num == numberOfNodes-1:
semaphore.release()
semaphoreAdding.acquire()
#threading.Lock.release()
for t in threading.enumerate(): #waiting for all threads to end
if t is not main_thread:
t.join()
I am trying out multiprocessing for my Monty Hall game simulation for improved performance. The game is payed 10mm times and takes ~17 seconds when directly run, however, my multiprocessing implementation is taking significantly longer to run. I am clearly doing something wrong but I can't figure out what.
import multiprocessing
from MontyHall.game import Game
from MontyHall.player import Player
from Timer.timer import Timer
def doWork(input, output):
while True:
try:
f = input.get(timeout=1)
res = f()
output.put(res)
except:
break
def main():
# game setup
player_1 = Player(True) # always switch strategy
game_1 = Game(player_1)
input_queue = multiprocessing.Queue()
output_queue = multiprocessing.Queue()
# total simulations
for i in range(10000000):
input_queue.put(game_1.play_game)
with Timer('timer') as t:
# initialize 5 child processes
processes = []
for i in range(5):
p = multiprocessing.Process(target=doWork, args=(input_queue, output_queue))
processes.append(p)
p.start()
# terminate the processes
for p in processes:
p.join()
results = []
while len(results) != 10000000:
r = output_queue.get()
results.append(r)
win = results.count(True) / len(results)
loss = results.count(False) / len(results)
print(len(results))
print(win)
print(loss)
if __name__ == '__main__':
main()
This is my first post. Advice on posting etiquette is also appreciated. Thank you.
Code for the Classes:
class Player(object):
def __init__(self, switch_door=False):
self._switch_door = switch_door
#property
def switch_door(self):
return self._switch_door
#switch_door.setter
def switch_door(self, iswitch):
self._switch_door = iswitch
def choose_door(self):
return random.randint(0, 2)
class Game(object):
def __init__(self, player):
self.player = player
def non_prize_door(self, door_with_prize, player_choice):
"""Returns a door that doesn't contain the prize and that isn't the players original choice"""
x = 1
while x == door_with_prize or x == player_choice:
x = (x + 1) % 3 # assuming there are only 3 doors. Can be modified for more doors
return x
def switch_function(self, open_door, player_choice):
"""Returns the door that isn't the original player choice and isn't the opened door """
x = 1
while x == open_door or x == player_choice:
x = (x + 1) % 3 # assuming there are only 3 doors. Can be modified for more doors
return x
def play_game(self):
"""Game Logic"""
# randomly places the prize behind one of the three doors
door_with_prize = random.randint(0, 2)
# player chooses a door
player_choice = self.player.choose_door()
# host opens a door that doesn't contain the prize
open_door = self.non_prize_door(door_with_prize, player_choice)
# final player choice
if self.player.switch_door:
player_choice = self.switch_function(open_door, player_choice)
# Result
return player_choice == door_with_prize
Code for running it without multiprocessing:
from MontyHall.game import Game
from MontyHall.player import Player
from Timer.timer import Timer
def main():
# Setting up the game
player_2 = Player(True) # always switch
game_1 = Game(player_2)
# Testing out the hypothesis
with Timer('timer_1') as t:
results = []
for i in range(10000000):
results.append(game_1.play_game())
win = results.count(True) / len(results)
loss = results.count(False) / len(results)
print(
f'When switch strategy is {player_2.switch_door}, the win rate is {win:.2%} and the loss rate is {loss:.2%}')
if __name__ == '__main__':
main()
As you did not give the full code that we can run locally, I can only speculate. My guess is that you are passing an object(a method from your game) to other processes so pickling and unpickling took too much time. Unlike multithreading where you can "share" data, in multiprocessing, you need to pack the data and send to the other process.
However, there's a rule I always follow when I try to optimize my code - profile before optimizing! It would be much better to KNOW what's slow than GUESS.
It's a multiprocessing program so there are not a lot of options in the market. You could try viztracer which supports multiprocessing.
pip install viztracer
viztracer --log_multiprocess your_program.py
It will generate a result.html that you can open with chrome. Or you can just do
vizviewer result.html
I would suggest to reduce the iteration number so you can have a view of the whole picture(because viztracer uses a circular buffer and 10 million iterations will definitely overflow). But, you can still get the last piece of your code executing if you don't, which should be helpful enough for you to figure out what's going on.
I used viztracer as you gave the whole code.
This is one of your iteration in your worker process. As you can tell, the actual working part is very small(the yellow-ish slice in the middle p...). Most of the time has been spent on receiving and putting data, which eliminates the advantage of parallelization.
The correct way to do this is do it in batches. Also as this game does not actually require any data, you should just sent "I want to do it 1000 times" to the process, and let it do it, instead of sending the method one by one.
There's another interesting problem that you can easily find with viztracer:
This is the big picture of your worker process. Notice the large "nothing" in the end? Because your worker needs a timeout to finish, and that's when they are waiting. You should come up with a better idea to elegantly finish your worker process.
Updated my code. I fundamentally misunderstood the multiprocessing method.
def do_work(input, output):
"""Generic function that takes an input function and argument and runs it"""
while True:
try:
f, args = input.get(timeout=1)
results = f(*args)
output.put(results)
except:
output.put('Done')
break
def run_sim(game, num_sim):
"""Runs the game the given number of times"""
res = []
for i in range(num_sim):
res.append(game.play_game())
return res
def main():
input_queue = multiprocessing.Queue()
output_queue = multiprocessing.Queue()
g = Game(Player(False)) # set up game and player
num_sim = 2000000
for i in range(5):
input_queue.put((run_sim, (g, num_sim))) # run sim with game object and number of simulations passed into
# the queue
with Timer('Monty Hall Timer: ') as t:
processes = [] # list to save processes
for i in range(5):
p = multiprocessing.Process(target=do_work, args=(input_queue, output_queue))
processes.append(p)
p.start()
results = []
while True:
r = output_queue.get()
if r != 'Done':
results.append(r)
else:
break
# terminate processes
for p in processes:
p.terminate()
# combining the five returned list
flat_list = [item for sublist in results for item in sublist]
print(len(flat_list))
print(len(results))
Let says I have two parallel block loops. What is the best way to run them in parallel using python. Currently I am experimenting with multi-threading using following program
#!/usr/bin/env python
import time
import serial
import os
from threading import Thread
ser = serial.Serial(port='/dev/ttyUSB0', baudrate=38400, timeout=None)
ser.flushInput()
ser.flushOutput()
def getstrings(port):
buf = bytearray()
while True:
b = port.read(1)
if b == b'\x02':
del buf[:]
elif b == b'\x03':
yield buf.decode('ascii')
else:
buf.append(b)
def tester():
while 1:
print('testing')
def values():
count = ""
tem = ""
hv = ""
counti = 0
temi = 0
hvi = 0
while 1:
for item in getstrings(ser):
#if len(item) >= 10:
# continue
if item[1] == "C":
count = item.split('C')[1]
counti=int(count[0:5])
if item[1] == "T":
tem = item.split('T')[1]
temi=int(tem[0:5])
if item[1] == "H":
hv = item.split('H')[1]
hvi = int(hv[0:5])/10
print ("HV="+str(hvi)+" "+"Count="+str(counti)+" "+"Temp="+str(temi))
t1 = Thread(target=values)
t2 = Thread(target=tester)
t1.start()
t2.start()
Only the second thread works. It doesn't print the values from second. This is the first time I am experimenting with multi-threading. Once, I understood how this will function then I intend to use this to design a GUI using Tkinter libraries. I want to use loop of my program along Tkinter main loop. Any suggestion where I might be making a mistakes.
Update:
Yes it thread 2 not thread 1. My mistakes sorry about that. But individually both threads work if I comments t1.start() or t2.start(). However, together only thread 2 prints the output.
Given this code snippet:
import time
if __name__ == "__main__":
list = []
counter = 0
while True:
counter = counter + 1
time.sleep(0.0033)
list.append(counter)
I would like to create a thread which runs in the background doing some metadata calculations (finding the sum of the elements in the array) on the array "list" being populated in real time in the while loop.
import time
import threading
if __name__ == "__main__":
def print_sum(l):
while True:
total = sum(l)
print("the total is {}".format(total))
time.sleep(1)
#list = [] - should not use 'list'. - this shadows the built in object name list.
l = []
counter = 0
thread = threading.Thread(target=print_sum,args=(l,))
thread.daemon = True
thread.start()
while True:
counter = counter + 1
l.append(counter)
time.sleep(1)
This spins up a thread in the background running function print_sum to display the sum of the list.
i manipulate a sensor : HC SR04 to capture a distance.
I'm a newbie in Python and RPI. My code work, I capture distance during a time but one moment the script stop...
My code :
GPIO.setmode(GPIO.BCM)
GPIO_TRIGGER = 23
GPIO_ECHO = 24
GPIO.setup(GPIO_TRIGGER, GPIO.OUT)
GPIO.setup(GPIO_ECHO, GPIO.IN)
def main():
global state
print("ultrasonic")
while True:
print "1s second refresh.."
time.sleep(1)
i = 0
datas = []
average = 0
while i< 1:
GPIO.output(GPIO_TRIGGER, False)
time.sleep(C.time['count'])
GPIO.output(GPIO_TRIGGER, True)
time.sleep(0.00001)
GPIO.output(GPIO_TRIGGER, False)
while GPIO.input(GPIO_ECHO) == 0:
start = time.time()
while GPIO.input(GPIO_ECHO) == 1:
stop = time.time()
distance = (stop-start) * 17000
print "Distance : %.1f" % distance
average = F.getAverage(datas)
print "Average: %.1f" % average
GPIO.cleanup()
The code stop here
while GPIO.input(GPIO_ECHO) == 0:
start = time.time()
THE SOLUTION : with a sample timeout :
now = time()
while GPIO.input(self.gpio_echo) == 0 and time()-now<waitTime:
pass
I am also mucking about with this sensor. My code executes similar to yours and I need no timeout for it to work.
The one difference I can find is this:
while i< 1:
GPIO.output(GPIO_TRIGGER, False)
time.sleep(C.time['count'])
I don't know how long the sleep time is here, but it might be that that's causing the problem. If it would be similar to mine setting the Trigger to false would be directly after the setup of the in/out pins instead, and then there's a two second wait to eliminate noise. Your wait time might be lower, I can't tell. There should be no need to set the trigger to false again just before you send the pulse and, I don't know, but it might be causing a false start. I would change it to this to work similarly to mine and then remove the setting to false in the while loop.
GPIO.setup(GPIO_TRIGGER, GPIO.OUT)
GPIO.setup(GPIO_ECHO, GPIO.IN)
GPIO.output(GPIO_TRIGGER, False)
print("Waiting for sensor to settle\n")
time.sleep(2)
I'm not sure if this will solve the issue without the need for a timeout, but I don't seem to need one.
I've written a module for making an object of the sensor which then allows for some more readable scripting. I'm also quite new to python and not at all an experienced programmer so fun errors might be there somewhere, but it's here below if you want to use it or just compare code:
#! /usr/bin/python3
# dist.py this is a module for objectifying an ultrasonic distance sensor.
import RPi.GPIO as GPIO
import time
class Distancer(object):
#init takes an input of one GPIO for trigger and one for echo and creates the object,
#it searches for a calibration file in the working directory (name)Const.txt, if none
#is found it will initiate a calibration
def __init__(self, trig, cho, name):
self.trigger = trig
self.echo = cho
self.name = name
self.filename = self.name + 'Const.txt'
GPIO.setup(self.trigger, GPIO.OUT)
GPIO.setup(self.echo, GPIO.IN)
GPIO.output(self.trigger, False)
print("Waiting for sensor to calm down")
time.sleep(2)
try:
with open(self.filename, "r") as inConst:
self.theConst = int(inConst.read())
except (OSError, IOError) as e:
print("Not calibrated, initializing calibration")
self.calibrate()
with open(self.filename, "r") as inConst:
self.theConst = int(inConst.read())
#Returns the echo time
def measureTime(self):
GPIO.output(self.trigger, True)
time.sleep(0.00001)
GPIO.output(self.trigger, False)
while GPIO.input(self.echo) == 0:
pulse_start = time.time()
while GPIO.input(self.echo) == 1:
pulse_end = time.time()
pulse_duration = pulse_end - pulse_start
return pulse_duration
#Returns a distance in cm
def measure(self):
return self.measureTime() * self.theConst
#Makes you set up the sensor at 3 different distances in order to find the
#relation between pulse time and distance, it creates the file (name)Const.txt
#in the working directory and stores the constant there.
def calibrate(self):
ten = []
thirty = []
seventy = []
print("Make distance 10 cm, enter when ready")
input()
for i in range(30):
ten.append(10/self.measureTime())
time.sleep(0.2)
print("Make distance 30 cm, enter when ready")
input()
for i in range(30):
thirty.append(30/self.measureTime())
time.sleep(0.2)
print("Make distance 70 cm, enter when ready")
input()
for i in range(30):
seventy.append(70/self.measureTime())
time.sleep(0.2)
allTime = ten + thirty + seventy
theOne = 0.0
for i in range(90):
theOne = theOne + allTime[i]
theOne = theOne / 90
with open(self.filename, "w") as inConst:
inConst.write(str(round(theOne)))
#Will continually check distance with a given interval until something reaches the
#treshold (cm), takes an argument to set wether it should check for something being
#nearer(near) or farther(far) than the treashold. Returns True when treshold is reached.
def distWarn(self, nearfar, treashold):
if nearfar.lower() == "near":
while True:
if self.measure() < treashold:
return True
break
time.sleep(0.2)
if nearfar.lower() == "far":
while True:
if self.measure() > treashold:
return True
break
time.sleep(0.2)
#Will measure with a second interval and print the distance
def keepGoing(self):
while True:
try:
print(str(round(self.measure())) + ' cm')
time.sleep(1)
except KeyboardInterrupt:
print("Won't keep going")
break
I've run it with the code below to test it and everything seems to work. First time it's run it will prompt you to calibrate the sensor by putting it at different distances from something.
#! /usr/bin/python3
import RPi.GPIO as GPIO
import time
import dist as distancer
GPIO.setmode(GPIO.BOARD)
TRIG = 16
ECHO = 18
dist = distancer.Distancer(TRIG, ECHO, 'dist')
def main():
global dist
print(str(round(dist.measureTime(),5)) + ' s')
print(str(round(dist.measure())) + ' cm')
dist.distWarn('near', 10)
print('Warning, something nearer than 10 cm at ' + time.asctime( time.localtime(time.time()) ))
dist.distWarn('far', 10)
print('Warning, something further than 10 cm at ' + time.asctime( time.localtime(time.time()) ))
dist.keepGoing()
GPIO.cleanup()
print('Fin')
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
GPIO.cleanup()
print("Exiting")
time.sleep(1)
I am pretty sure you want
while GPIO.input(GPIO_ECHO)==GPIO.LOW:
start = time.time()
while GPIO.input(GPIO_ECHO) == GPIO.HIGH:
stop = time.time()
I don't think GPIO.input naturally returns zeros or ones, you can test that though.
Not really, I think that i lost the signal, i'll try a timeout in
while GPIO.input(GPIO_ECHO)==GPIO.LOW:
start = time.time()
I think that my program wait indefinitely a signal but he stay to 0
I know this is an old question. The cause of the problem was described in this question https://raspberrypi.stackexchange.com/questions/41159/...
The solution is to add a timeout, like the OP did, to the while loops similar to this:
# If a reschedule occurs or the object is very close
# the echo may already have been received in which case
# the following will loop continuously.
count=time.time()
while GPIO.input(GPIO_ECHO)==0 and time.time()-count<0.1:
start = time.time()
...
# if an object is not detected some devices do not
# lower the echo line in which case the following will
# loop continuously.
stop = time.time()
count=time.time()
while GPIO.input(GPIO_ECHO)==1 and time.time()-count<0.1:
stop = time.time()