I am having an issue with accessing the queue object within my class in multiple functions, the following code might show you what i am trying to do;
import MySQLdb
import socket, sys
from struct import *
import threading
import sched, time
from datetime import datetime
from copy import deepcopy
import Queue
IP = {}
class QP:
def __init__(self):
self.jobs = Queue.Queue()
# this function runs every 10 seconds
# and processes any queued data.
def processQueue(self):
print(self.jobs.qsize())
time.sleep(5)
if self.jobs.empty():
print("No items in queue")
else:
IP_TEMP = {}
IP_TEMP = self.jobs.get()
IP_TEMP_QUEUE = {}
IP_TEMP_QUEUE = IP_TEMP
try:
cnx = #connect to database
cursor = cnx.cursor()
for k, v in IP_TEMP.iteritems():
#there is some code here, but its not the issue
try:
cursor.execute(add_packet, data_packet)
cnx.commit()
print("Task Done")
except:
print("Query failed, skipping")
break
except:
self.queueJobs(IP_TEMP_QUEUE)
IP = {}
self.jobs.task_done()
self.processQueue()
# this function is called by other modules to add data
def queueJobs(self, data):
self.jobs.put(data)
print(self.jobs.qsize())
return True
This is how i call the queueJobs method in the other modules:
self.process.queueJobs(IP_TEMP_QUEUE)
Now, the main issue is that the processQueue() function always returns that there is no jobs in the queue, even though they have been added below in the queueJobs() function.
Any help would be great!
Related
I have a python programs that gets memory leaks when use an third-party SO.
I simplify my code like this:
import time
import sys
import threading
import codecs
import ctypes
sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
class TestThirdPartySo(object):
def __init__(self):
# this so uses thread-specific data
self.call_stat_so = ctypes.CDLL("./third_party_fun.so")
self.handle = self.call_stat_so._handle
def test_fun(self):
self.call_stat_so.fun_xxx()
def thread_fun():
TestThirdPartySo().test_fun()
def test_main(num):
count = 0
while True:
# create 3 * num threads
thread_num = 3
thread_list = []
for _ in range(thread_num):
thread_list.append(threading.Thread(target=thread_fun))
for thread in thread_list:
thread.start()
for thread in thread_list:
thread.join()
count += thread_num
time.sleep(0.01)
if count % 100 == 0:
print("finied %s" % count)
if count > num:
break
print("end !!!!")
if __name__ == '__main__':
num = sys.argv[1]
test_main(int(num))
Now, I know this shared object uses thread-specific data.And I have tried to close the SO after called it like this:
class TestThirdPartySo(object):
def __init__(self):
# this so uses thread-specific data
self.call_stat_so = ctypes.CDLL("./third_party_fun.so")
self.handle = self.call_stat_so._handle
def test_fun(self):
self.call_stat_so.fun_xxx()
def __del__(self):
dlclose_func(self.handle)
def dlclose_func(_handle):
dlclose_func_tmp = ctypes.cdll.LoadLibrary('libdl.so').dlclose
dlclose_func_tmp.argtypes = [ctypes.c_void_p]
dlclose_func_tmp(_handle)
But I failed to close the so. And I'm also not sure if the leaked memory will be freed after closing the so.
If the program not uses multi-threads or creates a fixed number of threads(threadpool), it works ok.
For some reason,I need create threads constantly in my program. What can I do to prevent this memory leaks?
I've been looking into the multiprocessing module to figure this out but I'm not entirely sure of all the components I need or how to structure them. The basic structure/logic that I'm trying to get goes something like this though:
import datetime
import time
import multiprocessing
class So_Classy():
def __init__(self):
self.value = 0
def update_values(self):
print('In the process doing stuff!')
while True:
self.value = self.value + 1
time.sleep(0.1)
print("Self.value = {self.value}")
def run(self):
# Constantly update old value to a new one
try:
if __name__ == '__main__':
p = multiprocessing.Process(target=self.update_values)
p.start()
print("Process started!")
except Exception as e:
print(str(e))
def get_result(self, arg):
return self.value*arg
##### MAIN PROGRAM #####
# Initialize process given certain parameters
sc = So_Classy()
# This spawns a process running an infinite while loop
sc.run()
one_second = datetime.datetime.now() + datetime.timedelta(seconds=1)
while True:
if datetime.datetime.now() > one_second:
# Pass an arg to sc and do a calc with it
print(sc.get_result(5))
one_second = datetime.datetime.now() + datetime.timedelta(seconds=1)
The run() function is making it through to the end without causing an exception but it doesn't appear to actually be entering the process. No idea why. :\
The real process I will be using will be computationally intensive so it has to run as a separate process.
I'm trying to make two processes and make them communicate to each other. One of them gets values through a COM library using win32com, and the other just gets the values from the first process via a queue and prints them out. I think the code below has no problem, but it doesn't work(p2 process doesn't show values at all). If I just make the first process print queue values in the same process by
item = self.q.get()
print(item)
it shows values in the queue. So, I think putting values in the queue has no problem, and therefore, there could be some problems in exchanging values via the queue, using win32com
import win32com.client
import os
import multiprocessing as mp
from PyQt4.QtGui import QApplication
from datetime import datetime, timedelta
global q
q = mp.Queue() # A queue is used to send values from p1 to p2
class RealTrEventHandler(object):
def __init__(self):
self.q = q
def OnReceiveRealData(self,szTrCode):
date = datetime.utcnow() + timedelta(hours=3)
type = self.GetFieldData("OutBlock", "cgubun")
appending_line = date + ', ' + type
self.q.put(appending_line)
#item = self.q.get() # it prints values out if these are not comments
#print(item)
def ticker():
loop = QApplication([])
global instXASession, instXAReal
print('TICKER: ', os.getpid() )
# When an event occurs, it calls RealTrEventHandler class
instXAReal = win32com.client.DispatchWithEvents("XA_DataSet.XAReal", RealTrEventHandler)
instXAReal.LoadFromResFile("C:\\eBEST\\xingAPI\\Res\\OVC.res")
instXAReal.SetFieldData("InBlock", "symbol", "CLX17")
loop.exec_()
class listener(mp.Process): # What listener does is only to get values via the queue and prints them out
def __init__(self):
mp.Process.__init__(self)
self.q = q
def run(self):
print('CSM PID: ', os.getpid() )
while True:
item = self.q.get()
print(item)
if __name__ == '__main__':
loop = QApplication([])
print('MAIN: ', os.getpid() )
p1 = mp.Process( target = ticker, args=() )
p1.start()
p2 = listener()
p2.start()
mp.freeze_support()
loop.exec_()
Could anyone give me some advice?
Have you tried to explicitly declare q as global in your listener and RealTrEventHandler class? E.g.:
class listener(mp.Process):
global q
def __init__(self):
mp.Process.__init__(self)
self.q=q
Another way of passing variables between (at least) threads is using the buildins-module, but I'm not sure if multiprocessing is so much different.
I wrote a Threading class which tests whether a webserver is up or not.
import urllib
import threading
import time
import Queue
class Thread_CheckDeviceState(threading.Thread):
def __init__(self, device_ip, queue, inter=0.1):
self._run = True
self._codes = {}
self._queue = queue
self._device_ip = device_ip
self._inter = inter
self._elapsed = 0
threading.Thread.__init__(self)
def stop(self):
self._run = False
def run(self):
start = time.time()
while self._run:
try:
code = urllib.urlopen(self._device_ip).getcode()
except Exception:
code = "nope"
finally:
measure = time.time()
self._elapsed += measure-start
print self._elapsed, code
self._codes.update(
{self._elapsed:code}
)
time.sleep(self._inter)
self._queue.put(self._codes)
q = Queue.Queue()
thread = Thread_CheckDeviceState("http://192.168.1.3", q)
thread.start()
time.sleep(10)
thread.stop()
print q.get()
It works fine - until I disconnect my pc from the network. From that moment on the thread just does nothing until it is stopped. I would expect it to just continue and set the code to "nope", like I wrote it in the exception handler. Why doesn't it work
You need to use urllib2 instead, and specify a timeout parameter when you call urlopen().
I have a python class that communicates with a server. That python class has many functions, i was using few functions. But i want to access multiple servers data at same time using the python class i have.
Am trying something like this, but it'll run one after the other. And i wanted to get the data at same time.
import threading
from server_class import server
class runMonitor(threading.Thread):
def __init__(self,func):
self.func = func
threading.Thread.__init__(self)
def run(self):
self.func()
def monitorSB(ipAddr):
sb = server(ipAddr)
sb.readInfo()
print ('\nReading Registers...\n')
sb.read_rx()
sb.read_tx()
i = 0
while(1):
if i == 0:
print 'Monitoring Registers...'
i = 1
sb.monitor_tx()
sb.monitor_rx()
t = runMonitor(monitorSB('192.168.10.78'))
q = runMonitor(monitorSB('192.168.10.101'))
t.start()
q.start()
print ('\nTest Done...\n')
In the above code, i wanted to access both servers at same time. Help me how to run in parallel
The thing is by saying monitorSB('192.168.10.78') it will execute it before passing it to your thread. Try this:
import threading
from server_class import server
class runMonitor(threading.Thread):
def __init__(self,func, param):
self.func = func
self.param = param
threading.Thread.__init__(self)
def run(self):
self.func(self.param)
def monitorSB(ipAddr):
sb = server(ipAddr)
sb.readInfo()
print ('\nReading Registers...\n')
sb.read_rx()
sb.read_tx()
i = 0
while(1):
if i == 0:
print 'Monitoring Registers...'
i = 1
sb.monitor_tx()
sb.monitor_rx()
t = runMonitor(monitorSB, '192.168.10.78')
q = runMonitor(monitorSB, '192.168.10.101')
t.start()
q.start()
print ('\nTest Done...\n')