I wrote this script in order to extract metadata from data on specific ftp then save it on specific directory and waits a random time and extract again etc...
So there is 2 points :
First : to extract from specific ftp
Second : to save it and wait a random time which depends of the ftp used.
Sometime I have timeout issue from ftp. When it occurs there is no extract, save or waiting from the thread anymore. The others thread continue to work until they meet a timeout issue too.
I tried to catch exception in order to "restart" the thread which concerns by the ftp timeout issue. But nothing changes.
Someone please can help me to find a way to "restart" the thread?
Many thank's
class ftp_id(Thread):
def __init__(self, ftpID):
Thread.__init__(self)
self.ftpID = ftpID
def run(self):
while True:
with verrou:
siteID = self.ftpID
directory = str(datetime.now())[:-16].replace('-', '')
filename = siteID + '_' + str(datetime.now())[:-7].replace(':', '').replace(' ', '_').replace('-', '') + '.txt'
dictstr = myExtract(siteID)
myWriteFile(directory, filename, dictstr)
pendingtime = myWaiting(siteID, dictstr)
time.sleep(pendingtime)
except :
self.stop = True
self.stop = False
self.start()
self.join()
thread_01 = ftp_id("ftp-01")
thread_02 = ftp_id("ftp-02")
thread_03 = ftp_id("ftp-03")
thread_04 = ftp_id("ftp-04")
thread_01.start()
thread_02.start()
thread_03.start()
thread_04.start()
thread_01.join()
thread_02.join()
thread_03.join()
thread_04.join()
Threads cannot be restarted. So you should put the exception handling inside the while-block, and never exit the loop:
def download(server):
while True:
try:
with verrou:
now = datetime.now()
directory = '{:%Y%m%d}'.format(now)
filename = '{}_{:%Y%m%d_%H%M%S}.txt'.format(server, now)
dictstr = myExtract(server)
myWriteFile(directory, filename, dictstr)
pendingtime = myWaiting(server, dictstr)
time.sleep(pendingtime)
except:
# restart
pass
threads = [
Thread(target=download, args=(server,))
for server in ["ftp-01", "ftp-02", "ftp-03", "ftp-04"
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
threads cannot be restarted in python ... of coarse you could just implement your own "thread" class thats not really a thread
class ftp_id(object): # this is a bad name for a class please see pep8
def __init__(self, ftpID):
super(ftp_id,self)
self.ftpID = ftpID
self.thread = None
self.thread_stop = False
def __getattr__(self):
return getattr(self.thread)
def start(self):
if self.thread:
self.thread_stop = True
self.thread.terminate()
self.thread.join()
self.thread = threading.Thread(target = self.run)
self.thread.start()
def run(self):
self.thread_stop = False
while not self.thread_stop:
with verrou:
siteID = self.ftpID
directory = str(datetime.now())[:-16].replace('-', '')
filename = siteID + '_' + str(datetime.now())[:-7].replace(':', '').replace(' ', '_').replace('-', '') + '.txt'
dictstr = myExtract(siteID)
myWriteFile(directory, filename, dictstr)
pendingtime = myWaiting(siteID, dictstr)
time.sleep(pendingtime)
although i dont know if this actually answers your question
Related
I have RabbitMQ server running in Docker and two python clients that connect to the server and send messages to each other using headers exchange. Message rate is about 10/s. After some amount of time (most of the time after 300-500 messages have been exchanged) one of the exchange become unresponsive. channel.basic_publish call passes without any exception but receiver doesn't receive any messages. Also on rabbitmq dashboard there's no any activity on this exchange. rabbitmq dashboard screenshot
Here is the code example:
import pika
import threading
import time
import sys
class Test:
def __init__(
self,
p_username,
p_password,
p_host,
p_port,
p_virtualHost,
p_outgoingExchange,
p_incomingExchange
):
self.__outgoingExch = p_outgoingExchange
self.__incomingExch = p_incomingExchange
self.__headers = {'topic': 'test'}
self.__queueName = ''
self.__channelConsumer = None
self.__channelProducer = None
self.__isRun = False
l_credentials = pika.PlainCredentials(p_username, p_password)
l_parameters = pika.ConnectionParameters(
host=p_host,
port=p_port,
virtual_host=p_virtualHost,
credentials=l_credentials,
socket_timeout=30,
connection_attempts=5,
)
self.__connection = pika.SelectConnection(
parameters=l_parameters,
on_open_callback=self.__on_connection_open,
on_open_error_callback=self.__on_connection_open_error,
on_close_callback=self.__on_connection_closed
)
def __on_connection_open(self, _conn):
print("Connection opened")
self.__connection.channel(on_open_callback=self.__on_consume_channel_open)
self.__connection.channel(on_open_callback=self.__on_produce_channel_open)
def __on_connection_open_error(self, _conn, _exception):
print("Failed to open connection")
def __on_connection_closed(self, _conn, p_exception):
print("Connection closed: {}".format(p_exception))
def __on_consume_channel_open(self, p_ch):
print("Consumer channel opened")
self.__channelConsumer = p_ch
self.__channelConsumer.exchange_declare(
exchange=self.__incomingExch,
exchange_type="headers",
callback=self.__on_consume_exchange_declared
)
def __on_consume_exchange_declared(self, p_method):
print("Consumer exchange declared")
self.__channelConsumer.queue_declare(
queue='',
callback=self.__on_queue_declare
)
def __on_queue_declare(self, p_method):
print("Consumer queue declared")
self.__queueName = p_method.method.queue
self.__channelConsumer.queue_bind(
queue=self.__queueName,
exchange=self.__incomingExch,
arguments=self.__headers,
)
self.__channelConsumer.basic_consume(self.__queueName, self.__onMessageReceived)
def __on_produce_channel_open(self, p_ch):
print("Producer channel opened")
self.__channelProducer = p_ch
self.__channelProducer.exchange_declare(
exchange=self.__outgoingExch,
exchange_type="headers",
callback=self.__on_produce_exchange_declared
)
def __on_produce_exchange_declared(self, p_method):
print("Producer exchange declared")
l_publisher = threading.Thread(target=self.__publishProcedure)
l_publisher.start()
def __onMessageReceived(self, p_channel, p_method, p_properties, p_body):
p_channel.basic_ack(p_method.delivery_tag)
print("Message received: {}".format(p_body))
def __publishProcedure(self):
print("Start publishing")
l_msgCounter = 0
while self.__isRun:
l_msgCounter += 1
self.__publish(l_msgCounter)
time.sleep(0.1)
def __publish(self, p_msgCounter):
self.__channelProducer.basic_publish(
exchange=self.__outgoingExch,
routing_key="#",
body=str(p_msgCounter),
properties=pika.BasicProperties(headers=self.__headers)
)
def run(self):
self.__isRun = True
try:
self.__connection.ioloop.start()
except KeyboardInterrupt:
self.__isRun = False
self.__connection.close()
print("Exit...")
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Provide node name [node1 | node2]")
exit(-1)
l_outgoingExch = ''
l_incomingExch = ''
if sys.argv[1] == 'node1':
l_outgoingExch = 'node2.headers'
l_incomingExch = 'node1.headers'
elif sys.argv[1] == 'node2':
l_outgoingExch = 'node1.headers'
l_incomingExch = 'node2.headers'
else:
print("Wrong node name")
exit(-1)
l_testInstance = Test(
p_username='admin',
p_password='admin',
p_host='localhost',
p_port=5672,
p_virtualHost='/',
p_incomingExchange=l_incomingExch,
p_outgoingExchange=l_outgoingExch
)
l_testInstance.run()
I run two instances as two nodes (node1 and node2) so they should communicate with each other.
Also sometimes I have the issue described here:
Stream connection lost: AssertionError(('_AsyncTransportBase._produce() tx buffer size underflow', -275, 1),)
I found that I misused pika. As pika documentation states, it's not safe to share connection across multiple threads. The only way you can interact with connection from other threads is to use add_callback_threadsafe function. In my example it should look like this:
def __publishProcedure(self):
print("Start publishing")
l_msgCounter = 0
while self.__isRun:
l_msgCounter += 1
l_cb = functools.partial(self.__publish, l_msgCounter)
self.__connection.ioloop.add_callback_threadsafe(l_cb)
time.sleep(0.1)
def __publish(self, p_msgCounter):
self.__channelProducer.basic_publish(
exchange=self.__outgoingExch,
routing_key="#",
body=str(p_msgCounter),
properties=pika.BasicProperties(headers=self.__headers)
)
I have problem with multiprocessing in python. In code below I call 7 workers (multiprocessing.Process) and one result threading.Thread. Before and after processing of data (extracting some metadata from files), I run:
lsof | grep ' <user> ' | grep 'python3'
And I get some open handles as:
python3 17291 ivo DEL REG 0,20 5288943 /dev/shm/ZMcs2H
python3 17291 ivo DEL REG 0,20 5288942 /dev/shm/3iMR4q
python3 17291 ivo DEL REG 0,20 5288941 /dev/shm/XPYh79
and when running multiprocessing many times in loop (processing some continuous messages) I get
OSError: [Errno 24] Too many open files
Is there something wrong with dealing with multiprocessing package?
def worker_process_results(meta_queue, res_dict):
while True:
try:
(path, meta) = meta_queue.get()
res_dict[path] = meta
finally:
meta_queue.task_done()
def multiprocess_get_metadata(paths, thread_count = 7):
""" Scan files for metadata (multiprocessing). """
file_queue = multiprocessing.JoinableQueue()
meta_queue = multiprocessing.JoinableQueue()
res_dict = dict()
# result thread
meta_thread = threading.Thread(target = lambda: worker_process_results(meta_queue, res_dict))
meta_thread.daemon = True
meta_thread.start()
workers = []
for _ in range(0, min(thread_count, len(paths))):
worker = MetaDataWorker(file_queue, meta_queue)
worker.daemon = True
worker.start()
workers.append(worker)
for path in paths:
file_queue.put(path)
file_queue.join()
meta_queue.join()
for x in workers:
x.terminate()
return res_dict
class MetaDataWorker(multiprocessing.Process):
''' Use library to get meta data from file. '''
def __init__(self, file_queue, meta_queue):
''' Constructor. '''
super().__init__()
self.file_queue = file_queue
self.meta_queue = meta_queue
def run(self):
""" Run. """
while True:
try:
path = self.file_queue.get()
meta = getmetadata(path)
meta = None
self.meta_queue.put((path, meta))
except Exception as err:
print("Thread end.")
print("{0}".format(err))
finally:
self.file_queue.task_done()
Already solved, I needed to send some ending signals to workers and result thread to stop never-ending loop
I am about to code something and thought I'd put my idea out here first to see if anyone has comments, etc.
I want to create a class in python for simultaneously monitoring (and merging) multiple log files that will be used as part of an automated test. I am hoping to simply execute 'tail -f' (over SSH with paramiko, maybe) on each file using a separate thread. Then, every few seconds, get the stdout from each thread and merge it into one file with a suffix added to each line that identifies the source. This way I can write tests of distributed systems and monitor the logs of maybe a dozen machines at once (numerous of which have the same purpose and are behind a load balancer, etc)
Startup:
for machine, logfile in config_list:
create thread running tail -f on logfile on machine
create accumulator thread that:
wakes up each second and
gets all config_list stdout and merges it into one in-memory list
Test_API:
method to get/query data from the in memory accumulator.
in memory list would be the only data item needed to be synchronized
So, I am wondering: is paramiko the correct choice? any caveats, etc about handling the threading (have never done anything with threading in python)? any additional ideas that come to mind?
thanks in advance!
feel free to post code snippets. i will update this post with a working solution once it is finished. i anticipate it will be pretty small
Just found this:
Creating multiple SSH connections at a time using Paramiko
EDIT
From looking at a couple other posts, I have this so far. It is just doing a tail, not a tail -f and does not have the polling I need.
from someplace import TestLogger
import threading
import paramiko
def start_watching():
logger = TestLogger().get()
logs_to_watch = [('somemachine1', '/var/log/foo'),
('somemachine2', '/var/log/bar')]
threads = []
for machine, filename in logs_to_watch:
logger.info(machine)
logger.info(filename)
t = threading.Thread(target=workon, args=(machine, filename))
t.start()
threads.append(t)
for t in threads:
t.join()
for merge_line in merged_log:
logger.info(merge_line.dump())
outlock = threading.Lock()
merged_log = []
def workon(host, logfile):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username='yourusername', allow_agent=True, look_for_keys=True)
stdin, stdout, stderr = ssh.exec_command('sudo tail ' + logfile)
stdin.flush()
with outlock:
line = stdout.readline()
while line:
line = stdout.readline()
merged_log.append(MergeLogLine(line, host, logfile))
class MergeLogLine():
def __init__(self, line, host, logfile):
self._line = line
self._host = host
self._logfile = logfile
def line(self):
return self._line
def host(self):
return self._host
def logfile(self):
return self._logfile
def dump(self):
return self._line + '(from host = ' + self._host + ', log = ' + self._logfile + ')'
This turned out to be pretty hard. Here is a working sample:
Sample 'client code':
import sys
import traceback
import tail_accumulate as ta
import time
def main(argv):
user = 'cmead'
logs_to_watch = [('somemachine1', '/var/log/bar/sample.log'),
('somemachine2', '/var/log/foo')]
tac = ta.TailAccumulateConfig(logs_to_watch, user)
try:
ta.start_watching(tac)
time.sleep(10)
for merge_line in ta.get_merged_log():
print merge_line.dump()
except Exception as e:
print traceback.format_exc()
ta.stop()
if __name__ == "__main__":
main(sys.argv[1:])
Tail accumulate package:
import threading
import paramiko
import select
import time
threads = []
stopFlag = None
class TailAccumulateConfig():
def __init__(self, log_list, user):
self._log_list = log_list
self._user = user
def user(self):
return self._user
def log_list(self):
return self._log_list
def start_watching(tail_accumulate_config):
global stopFlag
stopFlag = threading.Event()
for machine, filename in tail_accumulate_config.log_list():
t = LogListenWorker(stopFlag, machine, filename, tail_accumulate_config.user())
t.start()
global threads
threads.append(t)
def stop():
global stopFlag
stopFlag.set()
def get_merged_log():
with outlock:
global merged_log
temp = merged_log[:]
del merged_log[:]
return temp
outlock = threading.Lock()
merged_log = []
class LogListenWorker(threading.Thread):
def __init__(self, event, host, logfile, username):
threading.Thread.__init__(self)
self.stopped = event
self.host = host
self.logfile = logfile
self.username = username
def run(self):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(self.host, username=self.username)
transport = ssh.get_transport()
channel = transport.open_session()
channel.exec_command('sudo tail -f ' + self.logfile)
while not self.stopped.isSet():
try:
rl, wl, xl = select.select([channel],[],[],3.0)
if len(rl) > 0:
# Must be stdout
line = channel.recv(1024)
else:
time.sleep(1.0)
continue
except Exception as e:
break
if line:
with outlock:
sublines = line.split('\n')
for subline in sublines:
merged_log.append(MergeLogLine(subline, self.host, self.logfile))
ssh.close()
class MergeLogLine():
def __init__(self, line, host, logfile):
self._line = line
self._host = host
self._logfile = logfile
def line(self):
return self._line
def host(self):
return self._host
def logfile(self):
return self._logfile
def dump(self):
return self._line + ' ---> (from host = ' + self._host + ', log = ' + self._logfile + ')'
I have been working with pyinotify and I am having issues with it where after multiple changes to a folder it simply stops receiving notifications. I have a feeling it is something to do with the the fact that two threads are running; namely the notifier thread and the wxpython thread.
The purpose of the app is to essentially load a picture to the screen when it detects an ip connection, monitor a folder for the file 'Checklist' and based on that file do some processing i.e move files around.
It works intermittently but being a python newbie Im not exactly sure what the issue might be as I have basically taken the threaded example and worked around it. Sometimes, it only gets one notification and stops receiving file change notifications.
Additionally, if I restart the linux box and try again, it works for a good number of file changes and then stops receiving notifications again which makes me think that perhaps its not releasing the watches properly?
Any help would be greatly appreciated as well as optimizations and improvements are very welcome. I'm sure I could learn a lot from the feedback. The code is below
import pyinotify
import os.path
import shutil
import errno
import subprocess
import logging
import wx
import time
import signal
import sys
#update CHECKLIST name
CHECKLIST = 'Checklist' #this must exist in the update archive
#static values
DIR_UPDATE = 'd'
FILE_UPDATE = 'f'
PING_IP = ' localhost' # change in production
#paths
WATCH_PATH = '/home/test'
LOG_PATH = '/home/test/update.log'
CONNECTED_IMG = 'update.jpg'
UPDATING_IMG = 'updating.png'
#msgs
UPDATEFOUND_MSG = ' Update Found '
UPDATEUNZIPPEDSTART_MSG = ' Update unzipping '
UPDATEUNZIPPED_MSG = ' Update unzipped '
UPDATEFILE_MSG = ' Update file '
UPDATEFILEMSG_CONT = ' moved into path '
REMOVEFILEMSG_CONT = ' removed from update folder '
UPDATECOMPLETE_CONT = ' Update complete'
ROADANGELRESTART_MSG = ' Update restarting app '
DIRCREATED_MSG = ' Directory created at path '
#errors
UPDATEFAILED_MSG = ' Update process failed on '
BADLYFORMED_MSG = ' Badly formed src/dest combination '
UPDATESRCUNAVAILABLE = ' Invalid update file specified '
UPDATEDESTUNAVAILABLE = ' Invalid update destination specified '
INVALIDUPDATEFORMAT = ' Invalid format string '
#on startup create the watchfolder if it doesnt exist
WM = pyinotify.WatchManager() # Watch Manager
WM_MASK = pyinotify.IN_CLOSE_WRITE # watched events
#setup logger
LOGGER = logging.getLogger('Updater')
LOG_HANDLE = logging.FileHandler(LOG_PATH)
FORMATTER = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
LOG_HANDLE.setFormatter(FORMATTER)
LOGGER.addHandler(LOG_HANDLE)
LOGGER.setLevel(logging.INFO)
#Global values used primarily in the main function loop
HANDLER = None
NOTIFIER = None
WDD = None
UPDATE_UI = None
WINDOW = None
CURR_IMG = None
LAST_CURRIMG = None
class EventHandler(pyinotify.ProcessEvent):
VERBOSE = False
""" Main class to monitor file events and process accordingly"""
def process_IN_CLOSE_WRITE(self, event):
""" Only executes when a Checklist has finished being written to"""
path = event.pathname
print 'evt'
#look for the update_ready file before processing
if (os.path.basename(path) == 'Checklist'):
EventHandler.parse_updates(WATCH_PATH)
global CURR_IMG
CURR_IMG = os.path.join(WATCH_PATH, UPDATING_IMG)
show_window()
print 'update completed'
time.sleep(1000)
#classmethod
def parse_updates(cls, path):
""" parses update files """
#handle errors for opening the file
file_path = os.path.join(path, CHECKLIST)
print file_path
files = open(file_path)
#handle errors for malformed tuples-done
#handle errors for unavailable files-done
#handle errors for unavailable dests-done #created automatically
#handle permission errors
for line in files:
#remove linebreaks etc and ensure its not empty
if line.strip():
array = line.split('=')
length = len(array)
if length == 3:
EventHandler.process_line(path, array)
else:
if length > 0:
EventHandler.print_bad_msg(array[0])
else:
EventHandler.print_bad_msg()
print 'removing ', file_path
os.remove(file_path) #remove the checklist file
#classmethod
def mkdir(cls, path):
""" makes a directory from a path"""
try:
os.mkdir(path)
print DIRCREATED_MSG, path
except OSError, err:
print err
if err.errno != errno.EEXIST: #thrown when the dir already exists
return False
#classmethod
def move_file(cls, src, dest):
""" moves a file from src to dest and remove after
expects that the dest already exists at this point
otherwise ignores the move"""
#print 'moving from', src, 'to ', dest
if os.path.isfile(dest):
shutil.copy2(src, dest)
else:
print UPDATEDESTUNAVAILABLE
#remove the src file when done
os.remove(src)
#classmethod
def process_line(cls, path, array):
""" process a line from the checklist"""
#remove newlines etc
update_file = array[0].strip()
update_src = os.path.join(path, update_file)
update_dest = array[1].strip()
update_type = array[2].strip()
#ensure we have valid values in all three fields
if update_file and update_dest and update_type:
#ensure the src file exists
if os.path.isfile(update_src):
#check if destination is directory and
#copy the file into the directory
if update_type == DIR_UPDATE:
EventHandler.mkdir(update_dest)
dest = os.path.join(update_dest, update_file)
EventHandler.move_file(update_src, dest)
else:
EventHandler.move_file(update_src, update_dest)
else:
print UPDATESRCUNAVAILABLE
else:
print INVALIDUPDATEFORMAT
#classmethod
def print_bad_msg(cls, msg = ''):
""" print a badly formed message with optional value"""
if msg:
print BADLYFORMED_MSG, msg
else:
print BADLYFORMED_MSG
class UpdateFrame(wx.Frame):
""" Displays update images to screen"""
def __init__(self, path):
wx.Frame.__init__(self, None, wx.ID_ANY)
image_file = path
image = wx.Bitmap(image_file)
image_size = image.GetSize()
# set the frame size to fit the screen size
self.SetClientSize(wx.DisplaySize())
# bitmap's upper left corner is in frame position (x, y)
# by default pos=(0, 0)
wx.StaticBitmap(self, wx.ID_ANY, image, size = image_size)
# the parent is the frame
self.SetTitle('Update Mode')
def ping_ip():
""" ping once to establish connection """
ret = subprocess.call("ping -c 1 %s" % PING_IP,
shell=True,
stdout=open('/dev/null', 'w'),
stderr=subprocess.STDOUT)
if ret == 0:
return True
else:
return False
def show_window():
""" update screen window when currimage changes is set """
global UPDATE_UI
global WINDOW
global CURR_IMG
global LAST_CURRIMG
if LAST_CURRIMG != CURR_IMG:
if not UPDATE_UI:
UPDATE_UI = wx.App()
if not WINDOW:
WINDOW = UpdateFrame(CURR_IMG)
UPDATE_UI.ExitMainLoop()
while(UPDATE_UI.IsMainLoopRunning()):
pass
WINDOW.Destroy()
WINDOW = UpdateFrame(CURR_IMG)
WINDOW.Show(True)
UPDATE_UI.MainLoop()
LAST_CURRIMG = CURR_IMG
print 'changed'
def in_updatemode():
return ping_ip()
while True:
try:
if not in_updatemode():
print 'waiting for connect'
time.sleep(3)
if NOTIFIER:
NOTIFIER.stop()
else:
if not HANDLER:
HANDLER = EventHandler()
if not NOTIFIER:
NOTIFIER = pyinotify.ThreadedNotifier(WM, HANDLER)
NOTIFIER.start()
if not WDD:
WDD = WM.add_watch(WATCH_PATH, WM_MASK, rec=True,quiet=False)
# ip is active so show the image and start the notifier
# state = ip active
CURR_IMG = os.path.join(WATCH_PATH, CONNECTED_IMG)
show_window()
print 'here'
except KeyboardInterrupt:
print 'out'
NOTIFIER.stop()
break
I basically found out that that the issue was indeed the fact that pyinotify's threaded notifier and wxPython's main loop dont play nice together.
The solution was to create a custom main loop (something I didn't know you could do in the first place) and place the pyinotify's threaded notifier within that loop. That way it ran as part of the wxPython main loop.
I got the idea from
http://www.java2s.com/Open-Source/Python/GUI/wxPython/wxPython-src-2.8.11.0/wxPython/samples/mainloop/mainloop.py.htm
Code below explains the concept
class CustomApp(wx.App):
def MainLoop(self):
global HANDLER
global WM
global NOTIFIER
global WDD
global UPDATE_UI
global PING_TIMER
# Create an event loop and make it active. If you are
# only going to temporarily have a nested event loop then
# you should get a reference to the old one and set it as
# the active event loop when you are done with this one...
evtloop = wx.EventLoop()
old = wx.EventLoop.GetActive()
wx.EventLoop.SetActive(evtloop)
# This outer loop determines when to exit the application,
# for this example we let the main frame reset this flag
# when it closes.
while self.keepGoing:
# At this point in the outer loop you could do
# whatever you implemented your own MainLoop for. It
# should be quick and non-blocking, otherwise your GUI
# will freeze.
# call_your_code_here()
if not HANDLER:
HANDLER = EventHandler()
if not WM:
WM = pyinotify.WatchManager() # Watch Manager
if not NOTIFIER:
NOTIFIER = pyinotify.ThreadedNotifier(WM, HANDLER)
NOTIFIER.start()
print 'notifier started'
if not WDD:
WDD = WM.add_watch(WATCH_PATH, WM_MASK, rec=True,quiet=False)
# This inner loop will process any GUI events
# until there are no more waiting.
while evtloop.Pending():
evtloop.Dispatch()
# Send idle events to idle handlers. You may want to
# throttle this back a bit somehow so there is not too
# much CPU time spent in the idle handlers. For this
# example, I'll just snooze a little...
time.sleep(0.10)
self.ProcessIdle()
wx.EventLoop.SetActive(old)
def OnInit(self):
global UPDATE_UI
if not UPDATE_UI:
UPDATE_UI = Updater()
UPDATE_UI.Show()
self.SetTopWindow(UPDATE_UI)
self.keepGoing = True
return True
"--------------------------------------------------------------------------------------"
Watcher()
app = CustomApp(False)
Additionally, I wanted to catch SIGINT in the program and I solved it using the recipe from this url
http://code.activestate.com/recipes/496735-workaround-for-missed-sigint-in-multithreaded-prog/
Hope this helps another python newbie or oldie :)
I have a list.
symbols = ('GGP', 'JPM', 'AIG', 'AMZN','GGP', 'rx', 'jnj', 'osip')
URL = "http://www.Xxxx_symbol=%s"
def fetch(symbols):
try:
url = URL % '+'.join(symbols)
fp = urllib2.urlopen(url)
try:
data = fp.read()
finally:
fp.close()
return data
except Exception as e:
print "No Internet Access"
I am trying to multithread (with 4 threads) the fetch process, not multi-process and not use twisted. The output file of Url fetch is csv with 7 lines of header info that I want to get rid of. I would like to loop each symbol in it own file. I used this fetch code before. I can get a symbol list which has one element.
This should get you started:
from threading import Thread, Lock
data = {}
data_lock = Lock()
class Fetcher(Thread):
def __init__(self, symbol):
super(Thread, self).__init__()
Thread.__init__(self)
self.symbol = symbol
def run(self):
# put the code from fetch() here
# replace 'data = fp.read()' with the following
tmp = fp.read()
data_lock.acquire()
data[self.symbol] = tmp
data_lock.release()
# Start a new Fetcher thread like this:
fetcher = Fetcher(symbol)
fetcher.start()
# To wait for the thread to finish, use Thread.join():
fetcher.join()