Develop Reference

Shared variables in Python

I am trying to create multiple threads of bot and they share some variables, but I am failing miserably in getingt the shared variables to work.
Here is the code:
import requests
import sys
import threading
import signal
import time
class bot(threading.Thread):
terminate = False
#def __init__(self):
# threading.Thread.__init__(self)
# self.terminate = False
def getCode():
code_lock.acquire()
work_code = code
try:
code += 1
finally:
code_lock.release()
return work_code
def checkCode(code):
try:
#if(code % 1000000 == 0):
print("Code "+str(code)+" is being checked...\n")
html = requests.get(url+str(code))
html.encoding = 'utf-8'
return not 'Page Not Found' in html.text
except requests.exceptions.ConnectionError:
print("Connection Error! Retrying...\n")
time.sleep(0.5)
except KeyboardInterrupt:
logCode(code)
sys.exit()
def storeCode(code):
file_lock.acquire()
try:
file.write(code+'\n')
finally:
file_lock.release()
def logCode(code):
log_lock.acquire()
try:
log.write(code+'\n')
finally:
log_lock.release()
#def run(self):
# global bots
# global url
# global file
# global log
# global code_lock
# global file_lock
# global log_lock
while(not terminate):
code = getCode()
if(checkCode(code)):
storeCode(code)
logCode(code)
def main(code = 0, threads = 16):
#bots = [threading.Thread(target=bot) for bot in range(threads)]
bots = []
url = 'https://test.ing/codes/'
file = open("valid-codes.txt", "a")
log = open("log.txt", "a")
code_lock = threading.Lock()
file_lock = threading.Lock()
log_lock = threading.Lock()
def signal_handler(signal, frame):
print('Exiting...\n')
log_lock.acquire()
try:
log.write("\n\n"+str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))+"\n")
finally:
log_lock.release()
for bot in bots:
bot.terminate = True
for bot in bots:
bot.join()
sys.exit(0)
#for bot in bots:
# bot.start()
for i in range(threads):
t = bot()
bots.append(t)
t.start()
signal.signal(signal.SIGINT, signal_handler)
while True:
signal.pause()
main(736479509787350, 1)
With this code I get this error:
Traceback (most recent call last): File "bot.py", line 7, in
class bot(threading.Thread): File "bot.py", line 59, in bot
code = getCode() File "bot.py", line 14, in getCode
code_lock.acquire() NameError: name 'code_lock' is not defined
I don't know if I should override the run(self) method of bot, but when I tried that it never actually ran the method run and I also receive the same error from all the threads created: that int is not callable (and I can't see where I can possibly be using an int as object).
Additionaly I don't know if I am handling correctly the exit signal from keyboard, as you can see I am trying to deal with that using a terminate variable, but I don't think that this is the problem...
One last thing, the ConnectionError exception is not being appropriately handled, as it's saying "Retrying...", but in fact it will not retry, but I am aware of that and it should be ok, I'll fix it latter.
Worth mentioning that I'm not very used to deal with multi-threading and when I do deal with it, it is in C or C++.
Edit
I can make the code work by using global variables, but I do not want to do that, I prefer to avoid using globals. My attempts of passing the variables directly to the instances of the class bot or by passing an data-object to it weren't successful so far, whenever I pass the variables or the auxiliar object to bot I am unable to access them as attributes using self. and without self. Python claims that the variable was not defined.
Here is the updated code, without success yet:
import requests
import sys
import threading
import signal
import time
class Shared:
def __init__(self, code, url, file, log, code_lock, file_lock, log_lock):
self.code = code
self.url = url
self.file = file
self.log = log
self.code_lock = code_lock
self.file_lock = file_lock
self.log_lock = log_lock
class bot(threading.Thread):
def __init__(self, data):
threading.Thread.__init__(self)
self.terminate = False
self.data = data
#classmethod
def getCode(self):
self.data.code_lock.acquire()
work_code = self.data.code
try:
self.data.code += 1
finally:
self.data.code_lock.release()
return work_code
#classmethod
def checkCode(self, work_code):
try:
#if(code % 1000000 == 0):
print("Code "+str(work_code)+" is being checked...\n")
html = requests.get(self.data.url+str(work_code))
html.encoding = 'utf-8'
return not 'Page Not Found' in html.text
except requests.exceptions.ConnectionError:
print("Connection Error! Retrying...\n")
time.sleep(0.5)
except KeyboardInterrupt:
self.logCode(work_code)
sys.exit()
#classmethod
def storeCode(self, work_code):
self.data.file_lock.acquire()
try:
self.data.file.write(work_code+'\n')
finally:
self.data.file_lock.release()
#classmethod
def logCode(self, work_code):
self.data.log_lock.acquire()
try:
self.data.log.write(work_code+'\n')
finally:
self.data.log_lock.release()
#classmethod
def run(self):
while(not self.terminate):
work_code = self.getCode()
if(self.checkCode(work_code)):
self.storeCode(work_code)
self.logCode(work_code)
def main(code = 0, threads = 16):
#bots = [threading.Thread(target=bot) for bot in range(threads)]
bots = []
url = 'https://www.test.ing/codes/'
file = open("valid-codes.txt", "a")
log = open("log.txt", "a")
code_lock = threading.Lock()
file_lock = threading.Lock()
log_lock = threading.Lock()
data = Shared(code, url, file, log, code_lock, file_lock, log_lock)
def signal_handler(signal, frame):
print('Exiting...\n')
log_lock.acquire()
try:
log.write("\n\n"+str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))+"\n")
finally:
log_lock.release()
for bot in bots:
bot.terminate = True
for bot in bots:
bot.join()
sys.exit(0)
#for bot in bots:
# bot.start()
for i in range(threads):
t = bot(data)
bots.append(t)
t.start()
signal.signal(signal.SIGINT, signal_handler)
while True:
signal.pause()
main(736479509787350, 4)
Yet, the working code with global variables:
import requests
import sys
import threading
import signal
import time
code = 736479509787350
url = 'https://www.test.ing/codes/'
file = open("valid-codes.txt", "a")
log = open("log.txt", "a")
code_lock = threading.Lock()
file_lock = threading.Lock()
log_lock = threading.Lock()
terminate = False
class bot(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
#classmethod
def getCode(self):
global code
code_lock.acquire()
work_code = code
try:
code += 1
finally:
code_lock.release()
return work_code
#classmethod
def checkCode(self, work_code):
try:
if(code % 1000000 == 0):
print("Code "+str(work_code)+" is being checked...\n")
html = requests.get(url+str(work_code))
html.encoding = 'utf-8'
if(not 'Page Not Found' in html.text):
time.sleep(0.5)
html = requests.get(url+str(work_code)+":999999999")
html.encoding = 'utf-8'
return 'Page Not Found' in html.text
except requests.exceptions.ConnectionError:
#print("Connection Error! Retrying...\n")
time.sleep(1)
return self.checkCode(work_code)
except KeyboardInterrupt:
self.logCode(work_code)
sys.exit()
#classmethod
def storeCode(self, work_code):
global file_lock
global file
file_lock.acquire()
try:
file.write(str(work_code)+'\n')
finally:
file_lock.release()
#classmethod
def logCode(self, work_code):
global log_lock
global log
log_lock.acquire()
try:
log.write(str(work_code)+'\n')
finally:
log_lock.release()
#classmethod
def run(self):
global terminate
while(not terminate):
work_code = self.getCode()
if(self.checkCode(work_code)):
print("Code "+str(work_code)+" is a valid code!\n")
self.storeCode(work_code)
self.logCode(work_code)
def main(threads = 16):
#bots = [threading.Thread(target=bot) for bot in range(threads)]
bots = []
#url = 'https://www.facebook.com/leticia.m.demenezes/posts/'
#file = open("valid-codes.txt", "a")
#log = open("log.txt", "a")
#code_lock = threading.Lock()
#file_lock = threading.Lock()
#log_lock = threading.Lock()
def signal_handler(signal, frame):
global terminate
print('Exiting...\n')
log_lock.acquire()
try:
log.write("\n\n"+str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))+"\n")
finally:
log_lock.release()
terminate = True
for bot in bots:
bot.join()
sys.exit(0)
#for bot in bots:
# bot.start()
for i in range(threads):
t = bot()
bots.append(t)
t.start()
signal.signal(signal.SIGINT, signal_handler)
while True:
signal.pause()
main()

You could make the code_lock global as you're trying to do, but why not just pass it into each bot class?
t = bot(code_lock)
Next create a constructor for your class:
class bot(threading.Thread):
def __init__(self, code_lock):
threading.Thread.__init__(self)
self.code_lock = code_lock
Now, whenever you try to use code_lock inside your bot class, always prefix it with self (self.code_lock).
If you really insist on using global variables, then look into the global keyword.

It's clear that you are trying to access code_lock out of it's scope, may be you can follow #MartinKonecny suggestion to fix that.
I could see that even after fixing code_lock problem, your code has lot of problems. as soon as you fix code_lock problem you'll face similar issue with the variable code in the same function getCode.
After fixing all those compiled time issues, you'll face issues with your whole class implementation, this is not the way to implement Python classes.
It's better that you go through this to know more about python name spaces and classes.

How to communicate with multiple servers using the single class defined in python and run it parallel

I have a python class that communicates with a server. That python class has many functions, i was using few functions. But i want to access multiple servers data at same time using the python class i have.
Am trying something like this, but it'll run one after the other. And i wanted to get the data at same time.
import threading
from server_class import server
class runMonitor(threading.Thread):
def __init__(self,func):
self.func = func
threading.Thread.__init__(self)
def run(self):
self.func()
def monitorSB(ipAddr):
sb = server(ipAddr)
sb.readInfo()
print ('\nReading Registers...\n')
sb.read_rx()
sb.read_tx()
i = 0
while(1):
if i == 0:
print 'Monitoring Registers...'
i = 1
sb.monitor_tx()
sb.monitor_rx()
t = runMonitor(monitorSB('192.168.10.78'))
q = runMonitor(monitorSB('192.168.10.101'))
t.start()
q.start()
print ('\nTest Done...\n')
In the above code, i wanted to access both servers at same time. Help me how to run in parallel

The thing is by saying monitorSB('192.168.10.78') it will execute it before passing it to your thread. Try this:
import threading
from server_class import server
class runMonitor(threading.Thread):
def __init__(self,func, param):
self.func = func
self.param = param
threading.Thread.__init__(self)
def run(self):
self.func(self.param)
def monitorSB(ipAddr):
sb = server(ipAddr)
sb.readInfo()
print ('\nReading Registers...\n')
sb.read_rx()
sb.read_tx()
i = 0
while(1):
if i == 0:
print 'Monitoring Registers...'
i = 1
sb.monitor_tx()
sb.monitor_rx()
t = runMonitor(monitorSB, '192.168.10.78')
q = runMonitor(monitorSB, '192.168.10.101')
t.start()
q.start()
print ('\nTest Done...\n')

Multithreading (?): Manual interference in a loop

I've been looking into a way to directly change variables in a running module.
What I want to achieve is that a load test is being run and that I can manually adjust the call pace or whatsoever.
Below some code that I just created (not-tested e.d.), just to give you an idea.
class A():
def __init__(self):
self.value = 1
def runForever(self):
while(1):
print self.value
def setValue(self, value):
self.value = value
if __name__ == '__main__':
#Some code to create the A object and directly apply the value from an human's input
a = A()
#Some parallelism or something has to be applied.
a.runForever()
a.setValue(raw_input("New value: "))
Edit #1: Yes, I know that now I will never hit the a.setValue() :-)

Here is a multi-threaded example. This code will work with the python interpreter but not with the Python Shell of IDLE, because the raw_input function is not handled the same way.
from threading import Thread
from time import sleep
class A(Thread):
def __init__(self):
Thread.__init__(self)
self.value = 1
self.stop_flag = False
def run(self):
while not self.stop_flag:
sleep(1)
print(self.value)
def set_value(self, value):
self.value = value
def stop(self):
self.stop_flag = True
if __name__ == '__main__':
a = A()
a.start()
try:
while 1:
r = raw_input()
a.set_value(int(r))
except:
a.stop()

The pseudo code you wrote is quite similar to the way Threading / Multiprocessing works in python. You will want to start a (for example) thread that "runs forever" and then instead of modifying the internal rate value directly, you will probably just send a message through a Queue that gives the new value.
Check out this question.
Here is a demonstration of doing what you asked about. I prefer to use Queues to directly making calls on threads / processes.
import Queue # !!warning. if you use multiprocessing, use multiprocessing.Queue
import threading
import time
def main():
q = Queue.Queue()
tester = Tester(q)
tester.start()
while True:
user_input = raw_input("New period in seconds or (q)uit: ")
if user_input.lower() == 'q':
break
try:
new_speed = float(user_input)
except ValueError:
new_speed = None # ignore junk
if new_speed is not None:
q.put(new_speed)
q.put(Tester.STOP_TOKEN)
class Tester(threading.Thread):
STOP_TOKEN = '<<stop>>'
def __init__(self, q):
threading.Thread.__init__(self)
self.q = q
self.speed = 1
def run(self):
while True:
# get from the queue
try:
item = self.q.get(block=False) # don't hang
except Queue.Empty:
item = None # do nothing
if item:
# stop when requested
if item == self.STOP_TOKEN:
break # stop this thread loop
# otherwise check for a new speed
try:
self.speed = float(item)
except ValueError:
pass # whatever you like with unknown input
# do your thing
self.main_code()
def main_code(self):
time.sleep(self.speed) # or whatever you want to do
if __name__ == '__main__':
main()

Better keyboard interrupt detection for this threaded Spinner class

Ok, I've wrote this class based in a bunch of others Spinner classes that I've googled in Google Code Search.
It's working as intended, but I'm looking for a better way to handle KeyboardInterrupt and SystemExit exceptions. Is there better approaches?
Here's my code:
#!/usr/bin/env python
import itertools
import sys
import threading
class Spinner(threading.Thread):
'''Represent a random work indicator, handled in a separate thread'''
# Spinner glyphs
glyphs = ('|', '/', '-', '\\', '|', '/', '-')
# Output string format
output_format = '%-78s%-2s'
# Message to output while spin
spin_message = ''
# Message to output when done
done_message = ''
# Time between spins
spin_delay = 0.1
def __init__(self, *args, **kwargs):
'''Spinner constructor'''
threading.Thread.__init__(self, *args, **kwargs)
self.daemon = True
self.__started = False
self.__stopped = False
self.__glyphs = itertools.cycle(iter(self.glyphs))
def __call__(self, func, *args, **kwargs):
'''Convenient way to run a routine with a spinner'''
self.init()
skipped = False
try:
return func(*args, **kwargs)
except (KeyboardInterrupt, SystemExit):
skipped = True
finally:
self.stop(skipped)
def init(self):
'''Shows a spinner'''
self.__started = True
self.start()
def run(self):
'''Spins the spinner while do some task'''
while not self.__stopped:
self.spin()
def spin(self):
'''Spins the spinner'''
if not self.__started:
raise NotStarted('You must call init() first before using spin()')
if sys.stdin.isatty():
sys.stdout.write('\r')
sys.stdout.write(self.output_format % (self.spin_message,
self.__glyphs.next()))
sys.stdout.flush()
time.sleep(self.spin_delay)
def stop(self, skipped=None):
'''Stops the spinner'''
if not self.__started:
raise NotStarted('You must call init() first before using stop()')
self.__stopped = True
self.__started = False
if sys.stdin.isatty() and not skipped:
sys.stdout.write('\b%s%s\n' % ('\b' * len(self.done_message),
self.done_message))
sys.stdout.flush()
class NotStarted(Exception):
'''Spinner not started exception'''
pass
if __name__ == '__main__':
import time
# Normal example
spinner1 = Spinner()
spinner1.spin_message = 'Scanning...'
spinner1.done_message = 'DONE'
spinner1.init()
skipped = False
try:
time.sleep(5)
except (KeyboardInterrupt, SystemExit):
skipped = True
finally:
spinner1.stop(skipped)
# Callable example
spinner2 = Spinner()
spinner2.spin_message = 'Scanning...'
spinner2.done_message = 'DONE'
spinner2(time.sleep, 5)
Thank you in advance.

You probably don't need to worry about catching SystemExit as it is raised by sys.exit(). You might want to catch it to clean up some resources just before your program exits.
The other way to catch KeyboardInterrupt is to register a signal handler to catch SIGINT. However for your example using try..except makes more sense, so you're on the right track.
A few minor suggestions:
Perhaps rename the __call__ method to start, to make it more clear you're starting a job.
You might also want to make the Spinner class reusable by attaching a new thread within the start method, rather than in the constructor.
Also consider what happens when the user hits CTRL-C for the current spinner job -- can the next job be started, or should the app just exit?
You could also make the spin_message the first argument to start to associate it with the task about to be run.
For example, here is how someone might use Spinner:
dbproc = MyDatabaseProc()
spinner = Spinner()
spinner.done_message = 'OK'
try:
spinner.start("Dropping the database", dbproc.drop, "mydb")
spinner.start("Re-creating the database", dbproc.create, "mydb")
spinner.start("Inserting data into tables", dbproc.populate)
...
except (KeyboardInterrupt, SystemExit):
# stop the currently executing job
spinner.stop()
# do some cleanup if needed..
dbproc.cleanup()

python conditional lock

How can I implement conditional lock in threaded application, for instance I haw
30 threads that are calling function and for most off the time all threads can access is simultaneous, but depending on function input there can be condition when only one thread can do that one thing. (If value for input is repeated and some thread is still working then I need lock.)
I now that there is module threading with Rlock() but I don't now how to use it in a way that i described it in first part.
Edit: The question is actually about how to prevent any two threads from running the same function with the same argument at the same time. (Thanks to David for helping me formulate my question :) )

Try this: have a lock in the module where your function is, and if the input to the function is such that locking is required, acquire the lock inside the function. Otherwise don't.
l = threading.RLock()
def fn(arg):
if arg == arg_that_needs_lock:
l.acquire()
try:
# do stuff
finally:
l.release()
else:
# do other stuff
EDIT:
As far as I can tell now, the question is actually about how to prevent any two threads from running the same function with the same argument at the same time. There's no problem with two threads running the same function with different arguments at the same time, though. The simple method to do this, if all valid arguments to the function can be dictionary keys, is to create a dictionary of arguments to locks:
import threading
dict_lock = threading.RLock()
locks = {}
def fn_dict(arg):
dict_lock.acquire()
try:
if arg not in dict:
locks[arg] = threading.RLock()
l = locks[arg]
finally:
dict_lock.release()
l.acquire()
try:
# do stuff
finally:
l.release()
If your function can be called with many different arguments, though, that amounts to a lot of locks. Probably a better way is to have a set of all arguments with which the function is currently executing, and have the contents of that set protected by a lock. I think this should work:
set_condition = threading.Condition()
current_args = set()
def fn_set(arg):
set_condition.acquire()
try:
while arg in current_args:
set_condition.wait()
current_args.add(arg)
finally:
set_condition.release()
# do stuff
set_condition.acquire()
try:
current_args.remove(arg)
set_condition.notifyAll()
finally:
set_condition.release()

It sounds like you want something similar to a Readers-Writer lock.
This is probably not what you want, but might be a clue:
from __future__ import with_statement
import threading
def RWLock(readers = 1, writers = 1):
m = _Monitor(readers, writers)
return (_RWLock(m.r_increment, m.r_decrement), _RWLock(m.w_increment, m.w_decrement))
class _RWLock(object):
def __init__(self, inc, dec):
self.inc = inc
self.dec = dec
def acquire(self):
self.inc()
def release(self):
self.dec()
def __enter__(self):
self.inc()
def __exit__(self):
self.dec()
class _Monitor(object):
def __init__(self, max_readers, max_writers):
self.max_readers = max_readers
self.max_writers = max_writers
self.readers = 0
self.writers = 0
self.monitor = threading.Condition()
def r_increment(self):
with self.monitor:
while self.writers > 0 and self.readers < self.max_readers:
self.monitor.wait()
self.readers += 1
self.monitor.notify()
def r_decrement(self):
with self.monitor:
while self.writers > 0:
self.monitor.wait()
assert(self.readers > 0)
self.readers -= 1
self.monitor.notify()
def w_increment(self):
with self.monitor:
while self.readers > 0 and self.writers < self.max_writers:
self.monitor.wait()
self.writers += 1
self.monitor.notify()
def w_decrement(self):
with self.monitor:
assert(self.writers > 0)
self.writers -= 1
self.monitor.notify()
if __name__ == '__main__':
rl, wl = RWLock()
wl.acquire()
wl.release()
rl.acquire()
rl.release()
(Unfortunately not tested)